From 94717dcd328ccacfd3df2cc739ec5ccd9772f7b7 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Tue, 17 Mar 2026 07:39:29 +0000 Subject: [PATCH] fix DB password rotation desync in 5 stacks Vault DB engine rotates passwords weekly but 5 stacks baked passwords at Terraform plan time, causing stale credentials until next apply. - real-estate-crawler: add vault-database ESO, use secret_key_ref in 3 deployments - nextcloud: switch Helm chart to existingSecret for DB password - grafana: add vault-database ESO, use envFromSecrets in Helm values - woodpecker: use extraSecretNamesForEnvFrom, remove plan-time data source chain - affine: add vault-database ESO, use secret_key_ref in deployment + init container --- stacks/affine/main.tf | 58 +++++++++++++-- stacks/nextcloud/chart_values.yaml | 4 +- stacks/nextcloud/main.tf | 10 ++- stacks/platform/main.tf | 1 - stacks/platform/modules/monitoring/grafana.tf | 38 +++++++++- .../monitoring/grafana_chart_values.yaml | 4 +- stacks/platform/modules/monitoring/main.tf | 4 -- stacks/real-estate-crawler/main.tf | 72 +++++++++++++++---- stacks/woodpecker/main.tf | 11 +-- stacks/woodpecker/values.yaml | 5 +- 10 files changed, 166 insertions(+), 41 deletions(-) diff --git a/stacks/affine/main.tf b/stacks/affine/main.tf index 29b622cd..26c3324f 100644 --- a/stacks/affine/main.tf +++ b/stacks/affine/main.tf @@ -39,6 +39,42 @@ data "kubernetes_secret" "eso_secrets" { depends_on = [kubernetes_manifest.external_secret] } +# DB credentials from Vault database engine (rotated automatically) +# Provides DATABASE_URL that auto-updates when password rotates +resource "kubernetes_manifest" "db_external_secret" { + manifest = { + apiVersion = "external-secrets.io/v1beta1" + kind = "ExternalSecret" + metadata = { + name = "affine-db-creds" + namespace = "affine" + } + spec = { + refreshInterval = "15m" + secretStoreRef = { + name = "vault-database" + kind = "ClusterSecretStore" + } + target = { + name = "affine-db-creds" + template = { + data = { + DATABASE_URL = "postgresql://affine:{{ .password }}@${var.postgresql_host}:5432/affine" + } + } + } + data = [{ + secretKey = "password" + remoteRef = { + key = "static-creds/pg-affine" + property = "password" + } + }] + } + } + depends_on = [kubernetes_namespace.affine] +} + locals { mailserver_accounts = jsondecode(data.kubernetes_secret.eso_secrets.data["mailserver_accounts"]) } @@ -64,10 +100,6 @@ module "tls_secret" { locals { common_env = [ - { - name = "DATABASE_URL" - value = "postgresql://affine:${data.kubernetes_secret.eso_secrets.data["db_password"]}@${var.postgresql_host}:5432/affine" - }, { name = "REDIS_SERVER_HOST" value = var.redis_host @@ -163,6 +195,15 @@ resource "kubernetes_deployment" "affine" { value = env.value.value } } + env { + name = "DATABASE_URL" + value_from { + secret_key_ref { + name = "affine-db-creds" + key = "DATABASE_URL" + } + } + } volume_mount { name = "data" @@ -200,6 +241,15 @@ resource "kubernetes_deployment" "affine" { value = env.value.value } } + env { + name = "DATABASE_URL" + value_from { + secret_key_ref { + name = "affine-db-creds" + key = "DATABASE_URL" + } + } + } volume_mount { name = "data" diff --git a/stacks/nextcloud/chart_values.yaml b/stacks/nextcloud/chart_values.yaml index 8007019e..4638e866 100644 --- a/stacks/nextcloud/chart_values.yaml +++ b/stacks/nextcloud/chart_values.yaml @@ -61,8 +61,10 @@ externalDatabase: type: mysql host: ${mysql_host} user: nextcloud - password: ${db_password} database: nextcloud + existingSecret: + secretName: nextcloud-db-creds + passwordKey: DB_PASSWORD persistence: enabled: true diff --git a/stacks/nextcloud/main.tf b/stacks/nextcloud/main.tf index b2855039..24324c77 100644 --- a/stacks/nextcloud/main.tf +++ b/stacks/nextcloud/main.tf @@ -62,10 +62,7 @@ resource "kubernetes_manifest" "external_secret" { } # DB credentials from Vault database engine (rotated every 24h) -# NOTE: Nextcloud Helm values use plan-time db_password from KV — the Helm -# release will use the KV snapshot until the next terragrunt apply. This -# ExternalSecret provides runtime-refreshed credentials for any future -# migration to envFrom-based secret injection. +# Nextcloud Helm chart reads password at runtime via existingSecret reference resource "kubernetes_manifest" "db_external_secret" { manifest = { apiVersion = "external-secrets.io/v1beta1" @@ -146,8 +143,9 @@ resource "helm_release" "nextcloud" { atomic = true version = "8.8.1" - values = [templatefile("${path.module}/chart_values.yaml", { tls_secret_name = var.tls_secret_name, db_password = data.vault_kv_secret_v2.secrets.data["db_password"], redis_host = var.redis_host, mysql_host = var.mysql_host })] - timeout = 6000 + values = [templatefile("${path.module}/chart_values.yaml", { tls_secret_name = var.tls_secret_name, redis_host = var.redis_host, mysql_host = var.mysql_host })] + timeout = 6000 + depends_on = [kubernetes_manifest.db_external_secret] } resource "kubernetes_config_map" "apache_tuning" { diff --git a/stacks/platform/main.tf b/stacks/platform/main.tf index e96d1156..490b4aca 100644 --- a/stacks/platform/main.tf +++ b/stacks/platform/main.tf @@ -220,7 +220,6 @@ module "monitoring" { tiny_tuya_service_secret = data.vault_kv_secret_v2.secrets.data["tiny_tuya_service_secret"] haos_api_token = data.vault_kv_secret_v2.secrets.data["haos_api_token"] pve_password = data.vault_kv_secret_v2.secrets.data["pve_password"] - grafana_db_password = data.vault_kv_secret_v2.secrets.data["grafana_db_password"] grafana_admin_password = data.vault_kv_secret_v2.secrets.data["grafana_admin_password"] tier = local.tiers.cluster } diff --git a/stacks/platform/modules/monitoring/grafana.tf b/stacks/platform/modules/monitoring/grafana.tf index d74ad4ac..6fa3113a 100644 --- a/stacks/platform/modules/monitoring/grafana.tf +++ b/stacks/platform/modules/monitoring/grafana.tf @@ -67,6 +67,41 @@ resource "kubernetes_persistent_volume" "alertmanager_pv" { # } # } +# DB credentials from Vault database engine (rotated automatically) +# Provides GF_DATABASE_PASSWORD that auto-updates when password rotates +resource "kubernetes_manifest" "grafana_db_creds" { + manifest = { + apiVersion = "external-secrets.io/v1beta1" + kind = "ExternalSecret" + metadata = { + name = "grafana-db-creds" + namespace = kubernetes_namespace.monitoring.metadata[0].name + } + spec = { + refreshInterval = "15m" + secretStoreRef = { + name = "vault-database" + kind = "ClusterSecretStore" + } + target = { + name = "grafana-db-creds" + template = { + data = { + GF_DATABASE_PASSWORD = "{{ .password }}" + } + } + } + data = [{ + secretKey = "password" + remoteRef = { + key = "static-creds/mysql-grafana" + property = "password" + } + }] + } + } +} + resource "kubernetes_config_map" "grafana_dashboards" { for_each = fileset("${path.module}/dashboards", "*.json") @@ -92,5 +127,6 @@ resource "helm_release" "grafana" { repository = "https://grafana.github.io/helm-charts" chart = "grafana" - values = [templatefile("${path.module}/grafana_chart_values.yaml", { db_password = var.grafana_db_password, grafana_admin_password = var.grafana_admin_password, mysql_host = var.mysql_host })] + values = [templatefile("${path.module}/grafana_chart_values.yaml", { grafana_admin_password = var.grafana_admin_password, mysql_host = var.mysql_host })] + depends_on = [kubernetes_manifest.grafana_db_creds] } diff --git a/stacks/platform/modules/monitoring/grafana_chart_values.yaml b/stacks/platform/modules/monitoring/grafana_chart_values.yaml index 64aac761..ebf649cb 100644 --- a/stacks/platform/modules/monitoring/grafana_chart_values.yaml +++ b/stacks/platform/modules/monitoring/grafana_chart_values.yaml @@ -64,8 +64,10 @@ dashboardProviders: # editable: "true" options: path: "/var/lib/grafana/dashboards/default" +envFromSecrets: + - name: grafana-db-creds + optional: false env: - GF_DATABASE_PASSWORD: "${db_password}" GF_SERVER_ROOT_URL: https://grafana.viktorbarzin.me grafana.ini: diff --git a/stacks/platform/modules/monitoring/main.tf b/stacks/platform/modules/monitoring/main.tf index 9492be74..b420662b 100644 --- a/stacks/platform/modules/monitoring/main.tf +++ b/stacks/platform/modules/monitoring/main.tf @@ -23,10 +23,6 @@ variable "pve_password" { type = string sensitive = true } -variable "grafana_db_password" { - type = string - sensitive = true -} variable "grafana_admin_password" { type = string sensitive = true diff --git a/stacks/real-estate-crawler/main.tf b/stacks/real-estate-crawler/main.tf index 6210bfdf..ee37a25b 100644 --- a/stacks/real-estate-crawler/main.tf +++ b/stacks/real-estate-crawler/main.tf @@ -33,6 +33,42 @@ resource "kubernetes_manifest" "external_secret" { depends_on = [kubernetes_namespace.realestate-crawler] } +# DB credentials from Vault database engine (rotated automatically) +# Provides DB_CONNECTION_STRING that auto-updates when password rotates +resource "kubernetes_manifest" "db_external_secret" { + manifest = { + apiVersion = "external-secrets.io/v1beta1" + kind = "ExternalSecret" + metadata = { + name = "realestate-crawler-db-creds" + namespace = "realestate-crawler" + } + spec = { + refreshInterval = "15m" + secretStoreRef = { + name = "vault-database" + kind = "ClusterSecretStore" + } + target = { + name = "realestate-crawler-db-creds" + template = { + data = { + DB_CONNECTION_STRING = "mysql://wrongmove:{{ .password }}@${var.mysql_host}:3306/wrongmove" + } + } + } + data = [{ + secretKey = "password" + remoteRef = { + key = "static-creds/mysql-wrongmove" + property = "password" + } + }] + } + } + depends_on = [kubernetes_namespace.realestate-crawler] +} + data "kubernetes_secret" "eso_secrets" { metadata { name = "real-estate-crawler-secrets" @@ -189,18 +225,14 @@ resource "kubernetes_deployment" "realestate-crawler-api" { value = "prod" } env { - name = "DB_CONNECTION_STRING" - value = "mysql://wrongmove:${data.kubernetes_secret.eso_secrets.data["db_password"]}@${var.mysql_host}:3306/wrongmove" - + name = "DB_CONNECTION_STRING" + value_from { + secret_key_ref { + name = "realestate-crawler-db-creds" + key = "DB_CONNECTION_STRING" + } + } } - # env { - # name = "HTTP_PROXY" - # value = "http://tor-proxy.tor-proxy:8118" - # } - # env { - # name = "HTTPS_PROXY" - # value = "http://tor-proxy.tor-proxy:8118" - # } env { name = "CELERY_BROKER_URL" value = "redis://${var.redis_host}:6379/0" @@ -384,8 +416,13 @@ resource "kubernetes_deployment" "realestate-crawler-celery" { value = "prod" } env { - name = "DB_CONNECTION_STRING" - value = "mysql://wrongmove:${data.kubernetes_secret.eso_secrets.data["db_password"]}@${var.mysql_host}:3306/wrongmove" + name = "DB_CONNECTION_STRING" + value_from { + secret_key_ref { + name = "realestate-crawler-db-creds" + key = "DB_CONNECTION_STRING" + } + } } env { name = "CELERY_BROKER_URL" @@ -498,8 +535,13 @@ resource "kubernetes_deployment" "realestate-crawler-celery-beat" { value = "prod" } env { - name = "DB_CONNECTION_STRING" - value = "mysql://wrongmove:${data.kubernetes_secret.eso_secrets.data["db_password"]}@${var.mysql_host}:3306/wrongmove" + name = "DB_CONNECTION_STRING" + value_from { + secret_key_ref { + name = "realestate-crawler-db-creds" + key = "DB_CONNECTION_STRING" + } + } } env { name = "CELERY_BROKER_URL" diff --git a/stacks/woodpecker/main.tf b/stacks/woodpecker/main.tf index c5b64fc7..61450dc6 100644 --- a/stacks/woodpecker/main.tf +++ b/stacks/woodpecker/main.tf @@ -85,9 +85,12 @@ resource "kubernetes_manifest" "external_secret" { } # DB credentials from Vault database engine (rotated every 24h) -# Updated: ExternalSecret now provides DATABASE_DATASOURCE -# which gets injected via envFrom and auto-updates when password rotates +# ExternalSecret provides WOODPECKER_DATABASE_DATASOURCE injected via +# server.extraSecretNamesForEnvFrom — auto-updates when password rotates resource "kubernetes_manifest" "db_external_secret" { + field_manager { + force_conflicts = true + } manifest = { apiVersion = "external-secrets.io/v1beta1" kind = "ExternalSecret" @@ -105,8 +108,7 @@ resource "kubernetes_manifest" "db_external_secret" { name = "woodpecker-db-creds" template = { data = { - # Key matches the Woodpecker Helm chart env var name - DATABASE_DATASOURCE = "postgres://woodpecker:{{ .password }}@${var.postgresql_host}:5432/woodpecker?sslmode=disable" + WOODPECKER_DATABASE_DATASOURCE = "postgres://woodpecker:{{ .password }}@${var.postgresql_host}:5432/woodpecker?sslmode=disable" } } } @@ -215,7 +217,6 @@ resource "helm_release" "woodpecker" { github_client_id = data.vault_kv_secret_v2.secrets.data["github_client_id"] github_client_secret = data.vault_kv_secret_v2.secrets.data["github_client_secret"] agent_secret = data.vault_kv_secret_v2.secrets.data["agent_secret"] - postgresql_host = var.postgresql_host forgejo_client_id = data.vault_kv_secret_v2.secrets.data["forgejo_client_id"] forgejo_client_secret = data.vault_kv_secret_v2.secrets.data["forgejo_client_secret"] forgejo_url = var.woodpecker_forgejo_url diff --git a/stacks/woodpecker/values.yaml b/stacks/woodpecker/values.yaml index e7296462..33d524a0 100644 --- a/stacks/woodpecker/values.yaml +++ b/stacks/woodpecker/values.yaml @@ -8,6 +8,8 @@ server: registry: docker.io repository: woodpeckerci/woodpecker-server tag: "v3.13.0" + extraSecretNamesForEnvFrom: + - woodpecker-db-creds env: WOODPECKER_HOST: "https://ci.viktorbarzin.me" WOODPECKER_ADMIN: "${woodpecker_admins}" @@ -25,9 +27,6 @@ server: WOODPECKER_FORGEJO_CLIENT: "${forgejo_client_id}" WOODPECKER_FORGEJO_SECRET: "${forgejo_client_secret}" WOODPECKER_FORGEJO_URL: "${forgejo_url}" - envFrom: - - secretRef: - name: woodpecker-db-creds service: type: ClusterIP port: 80