grafana: env-var datasources + reloader so Vault rotations stop breaking dashboards

Wealth, Payslips, and Job-Hunter Grafana datasources all baked the
rotating PG password into their ConfigMap at TF-apply time, so every
7-day Vault static-role rotation silently broke the panels until a
manual `terragrunt apply`. Same family as the recurring grafana-mysql
backend bug — Grafana caches creds at startup and never picks up the
new ESO-synced password without a restart.

Fix:
- Each source stack now creates an ExternalSecret in `monitoring`
  exposing the rotating password as `<NAME>_PG_PASSWORD` env-var.
- Grafana mounts those via `envFromSecrets` (optional=true so a
  missing source stack doesn't block boot) and the datasource
  ConfigMaps reference `$__env{<NAME>_PG_PASSWORD}` instead of a
  literal password.
- `reloader.stakater.com/auto: "true"` on the Grafana pod restarts
  it whenever any of the four DB-cred Secrets is updated.

Tested end-to-end: forced `vault write -force database/rotate-role/
pg-wealthfolio-sync` → ESO synced (~30s) → reloader fired →
Grafana booted with new env in ~50s total → all three /api/datasources
/uid/*/health endpoints return "Database Connection OK".

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

stacks/job-hunter/main.tf

@@ -294,18 +294,52 @@ resource "kubernetes_service" "job_hunter" {
   }
 }
 
-# Plan-time read of the ESO-created DB creds Secret for Grafana datasource.
-# First apply: -target=kubernetes_manifest.db_external_secret first so the Secret exists.
-data "kubernetes_secret" "job_hunter_db_creds" {
-  metadata {
-    name      = "job-hunter-db-creds"
-    namespace = kubernetes_namespace.job_hunter.metadata[0].name
+# ExternalSecret in the monitoring namespace mirroring the rotating
+# job_hunter DB password. Grafana mounts this via envFromSecrets in
+# monitoring/grafana_chart_values.yaml; the datasource ConfigMap below
+# references it as $__env{JOB_HUNTER_PG_PASSWORD}. Reloader restarts
+# Grafana whenever ESO updates this secret (every 7d on rotation).
+resource "kubernetes_manifest" "grafana_job_hunter_db_external_secret" {
+  manifest = {
+    apiVersion = "external-secrets.io/v1beta1"
+    kind       = "ExternalSecret"
+    metadata = {
+      name      = "grafana-job-hunter-pg-creds"
+      namespace = "monitoring"
+    }
+    spec = {
+      refreshInterval = "15m"
+      secretStoreRef = {
+        name = "vault-database"
+        kind = "ClusterSecretStore"
+      }
+      target = {
+        name = "grafana-job-hunter-pg-creds"
+        template = {
+          metadata = {
+            annotations = {
+              "reloader.stakater.com/match" = "true"
+            }
+          }
+          data = {
+            JOB_HUNTER_PG_PASSWORD = "{{ .password }}"
+          }
+        }
+      }
+      data = [{
+        secretKey = "password"
+        remoteRef = {
+          key      = "static-creds/pg-job-hunter"
+          property = "password"
+        }
+      }]
+    }
   }
-  depends_on = [kubernetes_manifest.db_external_secret]
 }
 
 # Grafana datasource for the job_hunter Postgres DB. Lives in the monitoring
 # namespace so the grafana sidecar (label grafana_datasource=1) picks it up.
+# Password is injected via $__env{...} from grafana-job-hunter-pg-creds (above).
 resource "kubernetes_config_map" "grafana_job_hunter_datasource" {
   metadata {
     name      = "grafana-job-hunter-datasource"
@@ -333,10 +367,11 @@ resource "kubernetes_config_map" "grafana_job_hunter_datasource" {
           timescaledb     = false
         }
         secureJsonData = {
-          password = data.kubernetes_secret.job_hunter_db_creds.data["DB_PASSWORD"]
+          password = "$__env{JOB_HUNTER_PG_PASSWORD}"
         }
         editable = true
       }]
     })
   }
+  depends_on = [kubernetes_manifest.grafana_job_hunter_db_external_secret]
 }

stacks/monitoring/modules/monitoring/grafana_chart_values.yaml

@@ -22,6 +22,7 @@ topologySpreadConstraints:
         app.kubernetes.io/name: grafana
 podAnnotations:
   dependency.kyverno.io/wait-for: "mysql.dbaas:3306"
+  reloader.stakater.com/auto: "true"
 podDisruptionBudget:
   maxUnavailable: 1
 persistence:
@@ -72,6 +73,19 @@ dashboardProviders:
 envFromSecrets:
   - name: grafana-db-creds
     optional: false
+  # Cross-namespace passwords for provisioned datasources backed by
+  # rotating Vault static-roles. Each source stack creates the secret
+  # via its own ExternalSecret in `monitoring`. `optional: true` lets
+  # Grafana boot if a stack hasn't applied yet; reloader (podAnnotation
+  # above) restarts Grafana when any of these secrets is created or
+  # rotated, so $__env{...} substitution in datasource ConfigMaps stays
+  # current.
+  - name: grafana-wealth-pg-creds
+    optional: true
+  - name: grafana-payslips-pg-creds
+    optional: true
+  - name: grafana-job-hunter-pg-creds
+    optional: true
 env:
   GF_SERVER_ROOT_URL: https://grafana.viktorbarzin.me
 

stacks/n8n/workflows/instagram-approval.json

@@ -173,7 +173,7 @@
         },
         "sendBody": true,
         "specifyBody": "json",
-        "jsonBody": "={{ JSON.stringify({ chat_id: $json.chat_id, message_id: $json.message_id, reply_markup: { inline_keyboard: [] } }) }}",
+        "jsonBody": "={{ JSON.stringify({ chat_id: $('Parse callback_data').item.json.chat_id, message_id: $('Parse callback_data').item.json.message_id, reply_markup: { inline_keyboard: [] } }) }}",
         "options": {"timeout": 30000}
       },
       "id": "edit-reply-markup",
@@ -181,7 +181,7 @@
       "type": "n8n-nodes-base.httpRequest",
       "typeVersion": 4.2,
       "position": [1570, 400],
-      "notes": "Strip the inline buttons from the original DM."
+      "notes": "Strip the inline buttons from the original DM. Refers back to Parse callback_data because the previous Telegram HTTP call replaced $json with its API response (which has result.chat.id, not chat_id at root)."
     },
     {
       "parameters": {
@@ -195,7 +195,7 @@
         },
         "sendBody": true,
         "specifyBody": "json",
-        "jsonBody": "={{ JSON.stringify({ callback_query_id: $json.callback_query_id, text: 'Recorded' }) }}",
+        "jsonBody": "={{ JSON.stringify({ callback_query_id: $('Parse callback_data').item.json.callback_query_id, text: 'Recorded' }) }}",
         "options": {"timeout": 15000}
       },
       "id": "answer-callback",
@@ -203,7 +203,7 @@
       "type": "n8n-nodes-base.httpRequest",
       "typeVersion": 4.2,
       "position": [1790, 400],
-      "notes": "Dismiss the spinner on the user's tap."
+      "notes": "Dismiss the spinner on the user's tap. callback_query_id from Parse callback_data (upstream HTTP responses don't carry it)."
     },
     {
       "parameters": {

stacks/payslip-ingest/main.tf

@@ -404,18 +404,52 @@ resource "kubernetes_cron_job_v1" "actualbudget_payroll_sync" {
   ]
 }
 
-# Plan-time read of the ESO-created K8s Secret for Grafana datasource password.
-# First apply: -target=kubernetes_manifest.db_external_secret first so the Secret exists.
-data "kubernetes_secret" "payslip_ingest_db_creds" {
-  metadata {
-    name      = "payslip-ingest-db-creds"
-    namespace = kubernetes_namespace.payslip_ingest.metadata[0].name
+# ExternalSecret in the monitoring namespace mirroring the rotating
+# payslip-ingest DB password. Grafana mounts this via envFromSecrets in
+# monitoring/grafana_chart_values.yaml; the datasource ConfigMap below
+# references it as $__env{PAYSLIPS_PG_PASSWORD}. Reloader restarts
+# Grafana whenever ESO updates this secret (every 7d on rotation).
+resource "kubernetes_manifest" "grafana_payslips_db_external_secret" {
+  manifest = {
+    apiVersion = "external-secrets.io/v1beta1"
+    kind       = "ExternalSecret"
+    metadata = {
+      name      = "grafana-payslips-pg-creds"
+      namespace = "monitoring"
+    }
+    spec = {
+      refreshInterval = "15m"
+      secretStoreRef = {
+        name = "vault-database"
+        kind = "ClusterSecretStore"
+      }
+      target = {
+        name = "grafana-payslips-pg-creds"
+        template = {
+          metadata = {
+            annotations = {
+              "reloader.stakater.com/match" = "true"
+            }
+          }
+          data = {
+            PAYSLIPS_PG_PASSWORD = "{{ .password }}"
+          }
+        }
+      }
+      data = [{
+        secretKey = "password"
+        remoteRef = {
+          key      = "static-creds/pg-payslip-ingest"
+          property = "password"
+        }
+      }]
+    }
   }
-  depends_on = [kubernetes_manifest.db_external_secret]
 }
 
 # Grafana datasource for payslip_ingest PostgreSQL DB.
 # Lives in the monitoring namespace so the grafana sidecar (label grafana_datasource=1) picks it up.
+# Password is injected via $__env{...} from grafana-payslips-pg-creds (above).
 resource "kubernetes_config_map" "grafana_payslips_datasource" {
   metadata {
     name      = "grafana-payslips-datasource"
@@ -445,10 +479,11 @@ resource "kubernetes_config_map" "grafana_payslips_datasource" {
           timescaledb     = false
         }
         secureJsonData = {
-          password = data.kubernetes_secret.payslip_ingest_db_creds.data["DB_PASSWORD"]
+          password = "$__env{PAYSLIPS_PG_PASSWORD}"
         }
         editable = true
       }]
     })
   }
+  depends_on = [kubernetes_manifest.grafana_payslips_db_external_secret]
 }

stacks/wealthfolio/main.tf

@@ -601,18 +601,52 @@ resource "kubernetes_cron_job_v1" "wealthfolio_sync" {
   }
 }
 
-# Plan-time read of the ESO-created K8s Secret for Grafana datasource password.
-# First apply: -target=kubernetes_manifest.wealthfolio_sync_db_external_secret first.
-data "kubernetes_secret" "wealthfolio_sync_db_creds" {
-  metadata {
-    name      = "wealthfolio-sync-db-creds"
-    namespace = kubernetes_namespace.wealthfolio.metadata[0].name
+# ExternalSecret in the monitoring namespace mirroring the rotating
+# wealthfolio_sync DB password. Grafana mounts this via envFromSecrets
+# in monitoring/grafana_chart_values.yaml; the datasource ConfigMap
+# below references it as $__env{WEALTH_PG_PASSWORD}. Reloader restarts
+# Grafana whenever ESO updates this secret (every 7d on rotation).
+resource "kubernetes_manifest" "grafana_wealth_db_external_secret" {
+  manifest = {
+    apiVersion = "external-secrets.io/v1beta1"
+    kind       = "ExternalSecret"
+    metadata = {
+      name      = "grafana-wealth-pg-creds"
+      namespace = "monitoring"
+    }
+    spec = {
+      refreshInterval = "15m"
+      secretStoreRef = {
+        name = "vault-database"
+        kind = "ClusterSecretStore"
+      }
+      target = {
+        name = "grafana-wealth-pg-creds"
+        template = {
+          metadata = {
+            annotations = {
+              "reloader.stakater.com/match" = "true"
+            }
+          }
+          data = {
+            WEALTH_PG_PASSWORD = "{{ .password }}"
+          }
+        }
+      }
+      data = [{
+        secretKey = "password"
+        remoteRef = {
+          key      = "static-creds/pg-wealthfolio-sync"
+          property = "password"
+        }
+      }]
+    }
   }
-  depends_on = [kubernetes_manifest.wealthfolio_sync_db_external_secret]
 }
 
 # Grafana datasource for wealthfolio_sync PostgreSQL DB.
 # Lives in the monitoring namespace so the Grafana sidecar (grafana_datasource=1) picks it up.
+# Password is injected via $__env{...} from grafana-wealth-pg-creds (above).
 resource "kubernetes_config_map" "grafana_wealth_datasource" {
   metadata {
     name      = "grafana-wealth-datasource"
@@ -640,12 +674,13 @@ resource "kubernetes_config_map" "grafana_wealth_datasource" {
           timescaledb     = false
         }
         secureJsonData = {
-          password = data.kubernetes_secret.wealthfolio_sync_db_creds.data["PGPASSWORD"]
+          password = "$__env{WEALTH_PG_PASSWORD}"
         }
         editable = true
       }]
     })
   }
+  depends_on = [kubernetes_manifest.grafana_wealth_db_external_secret]
 }
 
 ############################################################################