From 6f042ee239f4f3103cb5b2564430e0b26077681f Mon Sep 17 00:00:00 2001
From: Viktor Barzin <vbarzin@gmail.com>
Date: Sun, 28 Jun 2026 16:14:42 +0000
Subject: [PATCH] fix(fire-planner): grafana fire-planner-pg datasource
 survives pw rotation

The fire-planner-pg Grafana datasource baked the rotating fire_planner DB
password into its provisioning ConfigMap at terraform plan-time, so on every
7-day static-role rotation the password went stale and ALL fire-planner-pg
dashboards (fire-planner, cost-of-living, and the new wealth FIRE Countdown)
silently failed with "password authentication failed for user fire_planner"
until the next stack apply.

Switch to the same live-env pattern wealth-pg / payslips-pg already use:
- new ExternalSecret grafana-fire-planner-pg-creds (monitoring ns, Reloader
  match) mirrors the rotating Vault static-creds/pg-fire-planner password
- datasource ConfigMap now references $__env{FIRE_PLANNER_PG_PASSWORD}
- Grafana mounts it via envFromSecrets; reloader (auto) restarts Grafana on
  rotation so the provisioned datasource never goes stale

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 stacks/fire-planner/main.tf                   | 60 +++++++++++++++----
 .../monitoring/grafana_chart_values.yaml      |  2 +
 2 files changed, 52 insertions(+), 10 deletions(-)

diff --git a/stacks/fire-planner/main.tf b/stacks/fire-planner/main.tf
index 58d3df05..67d851cc 100644
--- a/stacks/fire-planner/main.tf
+++ b/stacks/fire-planner/main.tf
@@ -660,16 +660,53 @@ module "ingress_api" {
   auth = "none"
 }
 
-# Plan-time read of the ESO-created K8s Secret for Grafana datasource
-# password. First-apply gotcha: must
-# `terragrunt apply -target=kubernetes_manifest.db_external_secret` so
-# the Secret exists before this data source plans.
-data "kubernetes_secret" "fire_planner_db_creds" {
-  metadata {
-    name      = "fire-planner-db-creds"
-    namespace = kubernetes_namespace.fire_planner.metadata[0].name
+# ExternalSecret in the monitoring namespace mirroring the rotating
+# fire_planner DB password. Grafana mounts this via envFromSecrets in
+# monitoring/grafana_chart_values.yaml; the datasource ConfigMap below
+# references it as $__env{FIRE_PLANNER_PG_PASSWORD}. Reloader restarts
+# Grafana whenever ESO updates this secret (on the 7d static-role
+# rotation), so the provisioned datasource never goes stale — replaces
+# the old plan-time `data.kubernetes_secret` bake that broke weekly.
+# Mirrors the wealth-pg / payslips-pg pattern.
+resource "kubernetes_manifest" "grafana_fire_planner_pg_creds" {
+  field_manager {
+    force_conflicts = true
+  }
+  manifest = {
+    apiVersion = "external-secrets.io/v1"
+    kind       = "ExternalSecret"
+    metadata = {
+      name      = "grafana-fire-planner-pg-creds"
+      namespace = "monitoring"
+    }
+    spec = {
+      refreshInterval = "15m"
+      secretStoreRef = {
+        name = "vault-database"
+        kind = "ClusterSecretStore"
+      }
+      target = {
+        name = "grafana-fire-planner-pg-creds"
+        template = {
+          metadata = {
+            annotations = {
+              "reloader.stakater.com/match" = "true"
+            }
+          }
+          data = {
+            FIRE_PLANNER_PG_PASSWORD = "{{ .password }}"
+          }
+        }
+      }
+      data = [{
+        secretKey = "password"
+        remoteRef = {
+          key      = "static-creds/pg-fire-planner"
+          property = "password"
+        }
+      }]
+    }
   }
-  depends_on = [kubernetes_manifest.db_external_secret]
 }
 
 # Grafana datasource for fire_planner PostgreSQL DB.
@@ -706,12 +743,15 @@ resource "kubernetes_config_map" "grafana_fire_planner_datasource" {
           timescaledb     = false
         }
         secureJsonData = {
-          password = data.kubernetes_secret.fire_planner_db_creds.data["DB_PASSWORD"]
+          # Live env from grafana-fire-planner-pg-creds (above), injected into
+          # Grafana via envFromSecrets; reloader refreshes it on rotation.
+          password = "$__env{FIRE_PLANNER_PG_PASSWORD}"
         }
         editable = true
       }]
     })
   }
+  depends_on = [kubernetes_manifest.grafana_fire_planner_pg_creds]
 }
 
 # CI retrigger 2026-05-16T13:42:57+00:00 — bulk enrollment apply (pipeline #689 killed)
diff --git a/stacks/monitoring/modules/monitoring/grafana_chart_values.yaml b/stacks/monitoring/modules/monitoring/grafana_chart_values.yaml
index 50ae668b..ca0562eb 100644
--- a/stacks/monitoring/modules/monitoring/grafana_chart_values.yaml
+++ b/stacks/monitoring/modules/monitoring/grafana_chart_values.yaml
@@ -86,6 +86,8 @@ envFromSecrets:
     optional: true
   - name: grafana-job-hunter-pg-creds
     optional: true
+  - name: grafana-fire-planner-pg-creds
+    optional: true
 env:
   GF_SERVER_ROOT_URL: https://grafana.viktorbarzin.me