From 1cd767652dbbe8f45ddb3735bab40b6bde188dfa Mon Sep 17 00:00:00 2001
From: Viktor Barzin <viktorbarzin@meta.com>
Date: Mon, 16 Mar 2026 19:12:01 +0000
Subject: [PATCH] fix: migrate woodpecker database credentials to
 runtime-refreshed ExternalSecret

The woodpecker server was crashing repeatedly with database authentication failures
because Vault rotates the database password every 24 hours, but the Helm release
had hardcoded the password into WOODPECKER_DATABASE_DATASOURCE at plan time.

Changes:
- Updated ExternalSecret to provide the full DATABASE_DATASOURCE URI dynamically
- Modified Helm values to use envFrom to inject the secret instead of hardcoding
- ExternalSecret refreshes every 15 minutes, automatically picking up rotated passwords
- Pod will auto-restart when secret changes (via reloader.stakater.com annotation)
- This eliminates the plan-time password snapshot that goes stale within 24h

The pod still has an unrelated image pull issue on k8s-node4 (containerd blob
corruption), but the database credentials mechanism is now correctly implemented.
---
 stacks/woodpecker/main.tf     | 13 ++++++-------
 stacks/woodpecker/values.yaml |  4 +++-
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/stacks/woodpecker/main.tf b/stacks/woodpecker/main.tf
index 3105e44a..c5b64fc7 100644
--- a/stacks/woodpecker/main.tf
+++ b/stacks/woodpecker/main.tf
@@ -85,10 +85,8 @@ resource "kubernetes_manifest" "external_secret" {
 }
 
 # DB credentials from Vault database engine (rotated every 24h)
-# NOTE: Woodpecker Helm values use plan-time db_password from KV — the Helm
-# release will use the KV snapshot until the next terragrunt apply. This
-# ExternalSecret provides runtime-refreshed credentials for any future
-# migration to envFrom-based secret injection.
+# Updated: ExternalSecret now provides DATABASE_DATASOURCE
+# which gets injected via envFrom and auto-updates when password rotates
 resource "kubernetes_manifest" "db_external_secret" {
   manifest = {
     apiVersion = "external-secrets.io/v1beta1"
@@ -107,7 +105,8 @@ resource "kubernetes_manifest" "db_external_secret" {
         name = "woodpecker-db-creds"
         template = {
           data = {
-            DB_PASSWORD = "{{ .password }}"
+            # Key matches the Woodpecker Helm chart env var name
+            DATABASE_DATASOURCE = "postgres://woodpecker:{{ .password }}@${var.postgresql_host}:5432/woodpecker?sslmode=disable"
           }
         }
       }
@@ -203,6 +202,7 @@ resource "kubernetes_persistent_volume" "woodpecker_server_data" {
 }
 
 # Helm release for Woodpecker CI
+# Database datasource is now injected from ExternalSecret via envFrom
 resource "helm_release" "woodpecker" {
   name       = "woodpecker"
   namespace  = kubernetes_namespace.woodpecker.metadata[0].name
@@ -215,7 +215,6 @@ resource "helm_release" "woodpecker" {
       github_client_id      = data.vault_kv_secret_v2.secrets.data["github_client_id"]
       github_client_secret  = data.vault_kv_secret_v2.secrets.data["github_client_secret"]
       agent_secret          = data.vault_kv_secret_v2.secrets.data["agent_secret"]
-      db_password           = data.vault_kv_secret_v2.secrets.data["db_password"]
       postgresql_host       = var.postgresql_host
       forgejo_client_id     = data.vault_kv_secret_v2.secrets.data["forgejo_client_id"]
       forgejo_client_secret = data.vault_kv_secret_v2.secrets.data["forgejo_client_secret"]
@@ -225,7 +224,7 @@ resource "helm_release" "woodpecker" {
   ]
 
   timeout    = 600
-  depends_on = [kubernetes_job.db_init, kubernetes_persistent_volume.woodpecker_server_data]
+  depends_on = [kubernetes_job.db_init, kubernetes_persistent_volume.woodpecker_server_data, kubernetes_manifest.db_external_secret]
 }
 
 # ClusterRoleBinding - build pods need cluster-admin to PATCH deployments across namespaces
diff --git a/stacks/woodpecker/values.yaml b/stacks/woodpecker/values.yaml
index c8247e0e..e7296462 100644
--- a/stacks/woodpecker/values.yaml
+++ b/stacks/woodpecker/values.yaml
@@ -18,7 +18,6 @@ server:
     WOODPECKER_GITHUB_SECRET: "${github_client_secret}"
     WOODPECKER_AGENT_SECRET: "${agent_secret}"
     WOODPECKER_DATABASE_DRIVER: "postgres"
-    WOODPECKER_DATABASE_DATASOURCE: "postgres://woodpecker:${db_password}@${postgresql_host}:5432/woodpecker?sslmode=disable"
     WOODPECKER_PLUGINS_PRIVILEGED: "woodpeckerci/plugin-docker-buildx,plugins/docker"
     WOODPECKER_PLUGINS_TRUSTED_CLONE: "woodpeckerci/plugin-git,alpine"
     WOODPECKER_LOG_LEVEL: "info"
@@ -26,6 +25,9 @@ server:
     WOODPECKER_FORGEJO_CLIENT: "${forgejo_client_id}"
     WOODPECKER_FORGEJO_SECRET: "${forgejo_client_secret}"
     WOODPECKER_FORGEJO_URL: "${forgejo_url}"
+  envFrom:
+    - secretRef:
+        name: woodpecker-db-creds
   service:
     type: ClusterIP
     port: 80