From a699d5bedfc9e7709cf2bedaba3b78b45fb92ff0 Mon Sep 17 00:00:00 2001
From: Viktor Barzin <vbarzin@gmail.com>
Date: Mon, 11 May 2026 19:41:35 +0000
Subject: [PATCH] vault: move audit-PVC autoresizer annotations to
 kubernetes_annotations
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Background: 2026-05-10 someone added `server.auditStorage.annotations`
to vault/main.tf attempting to enable pvc-autoresizer on audit-vault-N
PVCs. The vault helm chart maps that block into the StatefulSet's
volumeClaimTemplates, which is immutable post-creation on existing
StatefulSets. Result: 4 consecutive helm upgrade attempts (rev 16-19)
all rejected with "StatefulSet spec: Forbidden", leaving the release
stuck in failed state since 22:47 UTC that day. Live PVCs were
hand-annotated via `kubectl annotate` as a workaround, but the IaC
declared a path that couldn't be applied — every subsequent tg apply
on the vault stack would re-fail.

Fix:
  * Remove `annotations` block from `server.auditStorage` values
    (with a comment recording why it can't live there).
  * Add `kubernetes_annotations` resources for audit-vault-{0,1,2}
    with `force = true`, so Terraform adopts the existing annotations
    and tracks the desired-state in IaC going forward. The autoresizer
    cares about PVC annotations, not StatefulSet template annotations,
    so this is functionally equivalent.

Done out-of-band before commit (helm state was already corrupted):
  `helm rollback vault 15 -n vault` → revision 20 deployed (clean).

Verified: helm status vault = deployed; audit-vault-0 still has
threshold=10% storage_limit=10Gi annotations; cluster healthcheck
no longer reports vault/vault=failed.
---
 stacks/vault/main.tf | 38 ++++++++++++++++++++++++++++++--------
 1 file changed, 30 insertions(+), 8 deletions(-)

diff --git a/stacks/vault/main.tf b/stacks/vault/main.tf
index 85b7a7f0..4433be3e 100644
--- a/stacks/vault/main.tf
+++ b/stacks/vault/main.tf
@@ -63,14 +63,13 @@ resource "helm_release" "vault" {
         enabled      = true
         size         = "2Gi"
         storageClass = "proxmox-lvm-encrypted" # Migrated 2026-04-25 from nfs-proxmox
-        # Vault audit logs grow unbounded per request; let pvc-autoresizer
-        # expand the volume up to 10Gi rather than ride a stuck-Pending
-        # vault-0 the moment the PVC fills.
-        annotations = {
-          "resize.topolvm.io/threshold"     = "10%"
-          "resize.topolvm.io/increase"      = "100%"
-          "resize.topolvm.io/storage_limit" = "10Gi"
-        }
+        # Note: pvc-autoresizer annotations on audit-vault-{0,1,2} are
+        # NOT declared here. The chart maps `annotations` into the
+        # StatefulSet's volumeClaimTemplates, which is immutable
+        # post-creation — every helm upgrade with this block set fails
+        # with "StatefulSet spec: Forbidden" (rev 16-19 on 2026-05-10).
+        # Instead, the annotations are applied directly to the live
+        # PVCs via the kubernetes_annotations resources below.
       }
 
       standalone = { enabled = false }
@@ -166,6 +165,29 @@ resource "helm_release" "vault" {
   })]
 }
 
+# pvc-autoresizer annotations on the audit PVCs. Applied here (not via
+# the chart's `server.auditStorage.annotations`) because StatefulSet
+# volumeClaimTemplates are immutable post-creation — the chart-mediated
+# path fails the helm upgrade with "spec: Forbidden". Audit logs grow
+# unbounded per request; allow the volume to expand to 10Gi rather
+# than ride a stuck-Pending vault-N the moment the PVC fills.
+resource "kubernetes_annotations" "audit_vault_autoresizer" {
+  for_each    = toset(["0", "1", "2"])
+  api_version = "v1"
+  kind        = "PersistentVolumeClaim"
+  metadata {
+    name      = "audit-vault-${each.key}"
+    namespace = "vault"
+  }
+  annotations = {
+    "resize.topolvm.io/threshold"     = "10%"
+    "resize.topolvm.io/increase"      = "100%"
+    "resize.topolvm.io/storage_limit" = "10Gi"
+  }
+  force      = true
+  depends_on = [helm_release.vault]
+}
+
 # --- Self-read: Vault's own OIDC credentials from KV ---
 
 data "vault_kv_secret_v2" "vault" {