vault: move audit-PVC autoresizer annotations to kubernetes_annotations
Background: 2026-05-10 someone added `server.auditStorage.annotations`
to vault/main.tf attempting to enable pvc-autoresizer on audit-vault-N
PVCs. The vault helm chart maps that block into the StatefulSet's
volumeClaimTemplates, which is immutable post-creation on existing
StatefulSets. Result: 4 consecutive helm upgrade attempts (rev 16-19)
all rejected with "StatefulSet spec: Forbidden", leaving the release
stuck in failed state since 22:47 UTC that day. Live PVCs were
hand-annotated via `kubectl annotate` as a workaround, but the IaC
declared a path that couldn't be applied — every subsequent tg apply
on the vault stack would re-fail.
Fix:
* Remove `annotations` block from `server.auditStorage` values
(with a comment recording why it can't live there).
* Add `kubernetes_annotations` resources for audit-vault-{0,1,2}
with `force = true`, so Terraform adopts the existing annotations
and tracks the desired-state in IaC going forward. The autoresizer
cares about PVC annotations, not StatefulSet template annotations,
so this is functionally equivalent.
Done out-of-band before commit (helm state was already corrupted):
`helm rollback vault 15 -n vault` → revision 20 deployed (clean).
Verified: helm status vault = deployed; audit-vault-0 still has
threshold=10% storage_limit=10Gi annotations; cluster healthcheck
no longer reports vault/vault=failed.
This commit is contained in:
parent
18a17891c4
commit
a699d5bedf
1 changed files with 30 additions and 8 deletions
|
|
@ -63,14 +63,13 @@ resource "helm_release" "vault" {
|
|||
enabled = true
|
||||
size = "2Gi"
|
||||
storageClass = "proxmox-lvm-encrypted" # Migrated 2026-04-25 from nfs-proxmox
|
||||
# Vault audit logs grow unbounded per request; let pvc-autoresizer
|
||||
# expand the volume up to 10Gi rather than ride a stuck-Pending
|
||||
# vault-0 the moment the PVC fills.
|
||||
annotations = {
|
||||
"resize.topolvm.io/threshold" = "10%"
|
||||
"resize.topolvm.io/increase" = "100%"
|
||||
"resize.topolvm.io/storage_limit" = "10Gi"
|
||||
}
|
||||
# Note: pvc-autoresizer annotations on audit-vault-{0,1,2} are
|
||||
# NOT declared here. The chart maps `annotations` into the
|
||||
# StatefulSet's volumeClaimTemplates, which is immutable
|
||||
# post-creation — every helm upgrade with this block set fails
|
||||
# with "StatefulSet spec: Forbidden" (rev 16-19 on 2026-05-10).
|
||||
# Instead, the annotations are applied directly to the live
|
||||
# PVCs via the kubernetes_annotations resources below.
|
||||
}
|
||||
|
||||
standalone = { enabled = false }
|
||||
|
|
@ -166,6 +165,29 @@ resource "helm_release" "vault" {
|
|||
})]
|
||||
}
|
||||
|
||||
# pvc-autoresizer annotations on the audit PVCs. Applied here (not via
|
||||
# the chart's `server.auditStorage.annotations`) because StatefulSet
|
||||
# volumeClaimTemplates are immutable post-creation — the chart-mediated
|
||||
# path fails the helm upgrade with "spec: Forbidden". Audit logs grow
|
||||
# unbounded per request; allow the volume to expand to 10Gi rather
|
||||
# than ride a stuck-Pending vault-N the moment the PVC fills.
|
||||
resource "kubernetes_annotations" "audit_vault_autoresizer" {
|
||||
for_each = toset(["0", "1", "2"])
|
||||
api_version = "v1"
|
||||
kind = "PersistentVolumeClaim"
|
||||
metadata {
|
||||
name = "audit-vault-${each.key}"
|
||||
namespace = "vault"
|
||||
}
|
||||
annotations = {
|
||||
"resize.topolvm.io/threshold" = "10%"
|
||||
"resize.topolvm.io/increase" = "100%"
|
||||
"resize.topolvm.io/storage_limit" = "10Gi"
|
||||
}
|
||||
force = true
|
||||
depends_on = [helm_release.vault]
|
||||
}
|
||||
|
||||
# --- Self-read: Vault's own OIDC credentials from KV ---
|
||||
|
||||
data "vault_kv_secret_v2" "vault" {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue