From 8c619278d3540fdbe52164bb75f7e35540e5f1ac Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Sat, 9 May 2026 17:38:38 +0000 Subject: [PATCH] grafana: env-var datasources + reloader so Vault rotations stop breaking dashboards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wealth, Payslips, and Job-Hunter Grafana datasources all baked the rotating PG password into their ConfigMap at TF-apply time, so every 7-day Vault static-role rotation silently broke the panels until a manual `terragrunt apply`. Same family as the recurring grafana-mysql backend bug — Grafana caches creds at startup and never picks up the new ESO-synced password without a restart. Fix: - Each source stack now creates an ExternalSecret in `monitoring` exposing the rotating password as `_PG_PASSWORD` env-var. - Grafana mounts those via `envFromSecrets` (optional=true so a missing source stack doesn't block boot) and the datasource ConfigMaps reference `$__env{_PG_PASSWORD}` instead of a literal password. - `reloader.stakater.com/auto: "true"` on the Grafana pod restarts it whenever any of the four DB-cred Secrets is updated. Tested end-to-end: forced `vault write -force database/rotate-role/ pg-wealthfolio-sync` → ESO synced (~30s) → reloader fired → Grafana booted with new env in ~50s total → all three /api/datasources /uid/*/health endpoints return "Database Connection OK". Co-Authored-By: Claude Opus 4.7 --- stacks/job-hunter/main.tf | 51 ++++++++++++++++--- .../monitoring/grafana_chart_values.yaml | 14 +++++ stacks/n8n/workflows/instagram-approval.json | 8 +-- stacks/payslip-ingest/main.tf | 51 ++++++++++++++++--- stacks/wealthfolio/main.tf | 51 ++++++++++++++++--- 5 files changed, 147 insertions(+), 28 deletions(-) diff --git a/stacks/job-hunter/main.tf b/stacks/job-hunter/main.tf index 8f849c33..46f208ff 100644 --- a/stacks/job-hunter/main.tf +++ b/stacks/job-hunter/main.tf @@ -294,18 +294,52 @@ resource "kubernetes_service" "job_hunter" { } } -# Plan-time read of the ESO-created DB creds Secret for Grafana datasource. -# First apply: -target=kubernetes_manifest.db_external_secret first so the Secret exists. -data "kubernetes_secret" "job_hunter_db_creds" { - metadata { - name = "job-hunter-db-creds" - namespace = kubernetes_namespace.job_hunter.metadata[0].name +# ExternalSecret in the monitoring namespace mirroring the rotating +# job_hunter DB password. Grafana mounts this via envFromSecrets in +# monitoring/grafana_chart_values.yaml; the datasource ConfigMap below +# references it as $__env{JOB_HUNTER_PG_PASSWORD}. Reloader restarts +# Grafana whenever ESO updates this secret (every 7d on rotation). +resource "kubernetes_manifest" "grafana_job_hunter_db_external_secret" { + manifest = { + apiVersion = "external-secrets.io/v1beta1" + kind = "ExternalSecret" + metadata = { + name = "grafana-job-hunter-pg-creds" + namespace = "monitoring" + } + spec = { + refreshInterval = "15m" + secretStoreRef = { + name = "vault-database" + kind = "ClusterSecretStore" + } + target = { + name = "grafana-job-hunter-pg-creds" + template = { + metadata = { + annotations = { + "reloader.stakater.com/match" = "true" + } + } + data = { + JOB_HUNTER_PG_PASSWORD = "{{ .password }}" + } + } + } + data = [{ + secretKey = "password" + remoteRef = { + key = "static-creds/pg-job-hunter" + property = "password" + } + }] + } } - depends_on = [kubernetes_manifest.db_external_secret] } # Grafana datasource for the job_hunter Postgres DB. Lives in the monitoring # namespace so the grafana sidecar (label grafana_datasource=1) picks it up. +# Password is injected via $__env{...} from grafana-job-hunter-pg-creds (above). resource "kubernetes_config_map" "grafana_job_hunter_datasource" { metadata { name = "grafana-job-hunter-datasource" @@ -333,10 +367,11 @@ resource "kubernetes_config_map" "grafana_job_hunter_datasource" { timescaledb = false } secureJsonData = { - password = data.kubernetes_secret.job_hunter_db_creds.data["DB_PASSWORD"] + password = "$__env{JOB_HUNTER_PG_PASSWORD}" } editable = true }] }) } + depends_on = [kubernetes_manifest.grafana_job_hunter_db_external_secret] } diff --git a/stacks/monitoring/modules/monitoring/grafana_chart_values.yaml b/stacks/monitoring/modules/monitoring/grafana_chart_values.yaml index 6bd8d8f4..6af2661c 100644 --- a/stacks/monitoring/modules/monitoring/grafana_chart_values.yaml +++ b/stacks/monitoring/modules/monitoring/grafana_chart_values.yaml @@ -22,6 +22,7 @@ topologySpreadConstraints: app.kubernetes.io/name: grafana podAnnotations: dependency.kyverno.io/wait-for: "mysql.dbaas:3306" + reloader.stakater.com/auto: "true" podDisruptionBudget: maxUnavailable: 1 persistence: @@ -72,6 +73,19 @@ dashboardProviders: envFromSecrets: - name: grafana-db-creds optional: false + # Cross-namespace passwords for provisioned datasources backed by + # rotating Vault static-roles. Each source stack creates the secret + # via its own ExternalSecret in `monitoring`. `optional: true` lets + # Grafana boot if a stack hasn't applied yet; reloader (podAnnotation + # above) restarts Grafana when any of these secrets is created or + # rotated, so $__env{...} substitution in datasource ConfigMaps stays + # current. + - name: grafana-wealth-pg-creds + optional: true + - name: grafana-payslips-pg-creds + optional: true + - name: grafana-job-hunter-pg-creds + optional: true env: GF_SERVER_ROOT_URL: https://grafana.viktorbarzin.me diff --git a/stacks/n8n/workflows/instagram-approval.json b/stacks/n8n/workflows/instagram-approval.json index 72d40cfc..425db970 100644 --- a/stacks/n8n/workflows/instagram-approval.json +++ b/stacks/n8n/workflows/instagram-approval.json @@ -173,7 +173,7 @@ }, "sendBody": true, "specifyBody": "json", - "jsonBody": "={{ JSON.stringify({ chat_id: $json.chat_id, message_id: $json.message_id, reply_markup: { inline_keyboard: [] } }) }}", + "jsonBody": "={{ JSON.stringify({ chat_id: $('Parse callback_data').item.json.chat_id, message_id: $('Parse callback_data').item.json.message_id, reply_markup: { inline_keyboard: [] } }) }}", "options": {"timeout": 30000} }, "id": "edit-reply-markup", @@ -181,7 +181,7 @@ "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [1570, 400], - "notes": "Strip the inline buttons from the original DM." + "notes": "Strip the inline buttons from the original DM. Refers back to Parse callback_data because the previous Telegram HTTP call replaced $json with its API response (which has result.chat.id, not chat_id at root)." }, { "parameters": { @@ -195,7 +195,7 @@ }, "sendBody": true, "specifyBody": "json", - "jsonBody": "={{ JSON.stringify({ callback_query_id: $json.callback_query_id, text: 'Recorded' }) }}", + "jsonBody": "={{ JSON.stringify({ callback_query_id: $('Parse callback_data').item.json.callback_query_id, text: 'Recorded' }) }}", "options": {"timeout": 15000} }, "id": "answer-callback", @@ -203,7 +203,7 @@ "type": "n8n-nodes-base.httpRequest", "typeVersion": 4.2, "position": [1790, 400], - "notes": "Dismiss the spinner on the user's tap." + "notes": "Dismiss the spinner on the user's tap. callback_query_id from Parse callback_data (upstream HTTP responses don't carry it)." }, { "parameters": { diff --git a/stacks/payslip-ingest/main.tf b/stacks/payslip-ingest/main.tf index 82fc3543..f5f42d32 100644 --- a/stacks/payslip-ingest/main.tf +++ b/stacks/payslip-ingest/main.tf @@ -404,18 +404,52 @@ resource "kubernetes_cron_job_v1" "actualbudget_payroll_sync" { ] } -# Plan-time read of the ESO-created K8s Secret for Grafana datasource password. -# First apply: -target=kubernetes_manifest.db_external_secret first so the Secret exists. -data "kubernetes_secret" "payslip_ingest_db_creds" { - metadata { - name = "payslip-ingest-db-creds" - namespace = kubernetes_namespace.payslip_ingest.metadata[0].name +# ExternalSecret in the monitoring namespace mirroring the rotating +# payslip-ingest DB password. Grafana mounts this via envFromSecrets in +# monitoring/grafana_chart_values.yaml; the datasource ConfigMap below +# references it as $__env{PAYSLIPS_PG_PASSWORD}. Reloader restarts +# Grafana whenever ESO updates this secret (every 7d on rotation). +resource "kubernetes_manifest" "grafana_payslips_db_external_secret" { + manifest = { + apiVersion = "external-secrets.io/v1beta1" + kind = "ExternalSecret" + metadata = { + name = "grafana-payslips-pg-creds" + namespace = "monitoring" + } + spec = { + refreshInterval = "15m" + secretStoreRef = { + name = "vault-database" + kind = "ClusterSecretStore" + } + target = { + name = "grafana-payslips-pg-creds" + template = { + metadata = { + annotations = { + "reloader.stakater.com/match" = "true" + } + } + data = { + PAYSLIPS_PG_PASSWORD = "{{ .password }}" + } + } + } + data = [{ + secretKey = "password" + remoteRef = { + key = "static-creds/pg-payslip-ingest" + property = "password" + } + }] + } } - depends_on = [kubernetes_manifest.db_external_secret] } # Grafana datasource for payslip_ingest PostgreSQL DB. # Lives in the monitoring namespace so the grafana sidecar (label grafana_datasource=1) picks it up. +# Password is injected via $__env{...} from grafana-payslips-pg-creds (above). resource "kubernetes_config_map" "grafana_payslips_datasource" { metadata { name = "grafana-payslips-datasource" @@ -445,10 +479,11 @@ resource "kubernetes_config_map" "grafana_payslips_datasource" { timescaledb = false } secureJsonData = { - password = data.kubernetes_secret.payslip_ingest_db_creds.data["DB_PASSWORD"] + password = "$__env{PAYSLIPS_PG_PASSWORD}" } editable = true }] }) } + depends_on = [kubernetes_manifest.grafana_payslips_db_external_secret] } diff --git a/stacks/wealthfolio/main.tf b/stacks/wealthfolio/main.tf index 4cbdb010..3c1ea23f 100644 --- a/stacks/wealthfolio/main.tf +++ b/stacks/wealthfolio/main.tf @@ -601,18 +601,52 @@ resource "kubernetes_cron_job_v1" "wealthfolio_sync" { } } -# Plan-time read of the ESO-created K8s Secret for Grafana datasource password. -# First apply: -target=kubernetes_manifest.wealthfolio_sync_db_external_secret first. -data "kubernetes_secret" "wealthfolio_sync_db_creds" { - metadata { - name = "wealthfolio-sync-db-creds" - namespace = kubernetes_namespace.wealthfolio.metadata[0].name +# ExternalSecret in the monitoring namespace mirroring the rotating +# wealthfolio_sync DB password. Grafana mounts this via envFromSecrets +# in monitoring/grafana_chart_values.yaml; the datasource ConfigMap +# below references it as $__env{WEALTH_PG_PASSWORD}. Reloader restarts +# Grafana whenever ESO updates this secret (every 7d on rotation). +resource "kubernetes_manifest" "grafana_wealth_db_external_secret" { + manifest = { + apiVersion = "external-secrets.io/v1beta1" + kind = "ExternalSecret" + metadata = { + name = "grafana-wealth-pg-creds" + namespace = "monitoring" + } + spec = { + refreshInterval = "15m" + secretStoreRef = { + name = "vault-database" + kind = "ClusterSecretStore" + } + target = { + name = "grafana-wealth-pg-creds" + template = { + metadata = { + annotations = { + "reloader.stakater.com/match" = "true" + } + } + data = { + WEALTH_PG_PASSWORD = "{{ .password }}" + } + } + } + data = [{ + secretKey = "password" + remoteRef = { + key = "static-creds/pg-wealthfolio-sync" + property = "password" + } + }] + } } - depends_on = [kubernetes_manifest.wealthfolio_sync_db_external_secret] } # Grafana datasource for wealthfolio_sync PostgreSQL DB. # Lives in the monitoring namespace so the Grafana sidecar (grafana_datasource=1) picks it up. +# Password is injected via $__env{...} from grafana-wealth-pg-creds (above). resource "kubernetes_config_map" "grafana_wealth_datasource" { metadata { name = "grafana-wealth-datasource" @@ -640,12 +674,13 @@ resource "kubernetes_config_map" "grafana_wealth_datasource" { timescaledb = false } secureJsonData = { - password = data.kubernetes_secret.wealthfolio_sync_db_creds.data["PGPASSWORD"] + password = "$__env{WEALTH_PG_PASSWORD}" } editable = true }] }) } + depends_on = [kubernetes_manifest.grafana_wealth_db_external_secret] } ############################################################################