From b958935ee0584a42d6da76303b83a060cb39e876 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Fri, 5 Jun 2026 09:17:47 +0000 Subject: [PATCH] woodpecker: reload server on Vault PG password rotation [ci skip] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit woodpecker-server sets reloader.stakater.com/search="true" but the woodpecker-db-creds ExternalSecret never carried the matching reloader.stakater.com/match="true", so Stakater Reloader never restarted the server when Vault rotated the pg-woodpecker password (7-day static role). The DB DSN is injected via envFrom, which does not hot-reload a running pod — so after each rotation the server kept using the revoked password until some unrelated restart (Keel bump, drain, manual) recreated it inside the window. A latent weekly DB-outage masked by incidental restarts. Add the match annotation to the ESO target template and correct the stale "rotated every 24h" comment (actual rotation_period is 604800s = 7 days). Verified end-to-end: forced 'vault write -f database/rotate-role/pg-woodpecker', ESO updated the secret in ~3s, Reloader auto-restarted woodpecker-server in ~36s, new pod reconnected with zero DB errors. [ci skip] because the change was already applied via scripts/tg apply. Co-Authored-By: Claude Opus 4.8 --- stacks/woodpecker/main.tf | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/stacks/woodpecker/main.tf b/stacks/woodpecker/main.tf index 2b389cd8..afe7d009 100644 --- a/stacks/woodpecker/main.tf +++ b/stacks/woodpecker/main.tf @@ -32,7 +32,7 @@ resource "kubernetes_namespace" "woodpecker" { labels = { "resource-governance/custom-quota" = "true" tier = local.tiers.edge - "keel.sh/enrolled" = "true" + "keel.sh/enrolled" = "true" } } lifecycle { @@ -89,9 +89,15 @@ resource "kubernetes_manifest" "external_secret" { depends_on = [kubernetes_namespace.woodpecker] } -# DB credentials from Vault database engine (rotated every 24h) -# ExternalSecret provides WOODPECKER_DATABASE_DATASOURCE injected via -# server.extraSecretNamesForEnvFrom — auto-updates when password rotates +# DB credentials from Vault database engine (rotated every 7 days — static +# role pg-woodpecker, rotation_period 604800 in stacks/vault). ExternalSecret +# provides WOODPECKER_DATABASE_DATASOURCE injected via +# server.extraSecretNamesForEnvFrom. envFrom does NOT hot-reload a running pod, +# so the target secret carries reloader.stakater.com/match="true" and the +# server sets reloader.stakater.com/search="true" (values.yaml); together they +# make Stakater Reloader restart the server when the rotated password lands — +# without it the pod kept using the revoked password until an unrelated restart +# (latent weekly rotation outage, fixed 2026-06-05). resource "kubernetes_manifest" "db_external_secret" { field_manager { force_conflicts = true @@ -112,6 +118,11 @@ resource "kubernetes_manifest" "db_external_secret" { target = { name = "woodpecker-db-creds" template = { + metadata = { + annotations = { + "reloader.stakater.com/match" = "true" + } + } data = { WOODPECKER_DATABASE_DATASOURCE = "postgres://woodpecker:{{ .password }}@${var.postgresql_host}:5432/woodpecker?sslmode=disable" }