add pod dependency management via Kyverno init container injection

Kyverno ClusterPolicy reads dependency.kyverno.io/wait-for annotation
and injects busybox init containers that block until each dependency
is reachable (nc -z). Annotations added to 18 stacks (24 deployments).

Includes graceful-db-maintenance.sh script for planned DB maintenance
(scales dependents to 0, saves replica counts, restores on startup).

stacks/platform/modules/kyverno/dependency-init-containers.tf

@@ -0,0 +1,72 @@
+
+# =============================================================================
+# Pod Dependency Init Container Injection
+# =============================================================================
+# Reads the annotation dependency.kyverno.io/wait-for from pods and injects
+# init containers that wait for each listed dependency to be reachable.
+#
+# Usage:
+#   annotations:
+#     dependency.kyverno.io/wait-for: "postgresql.dbaas:5432,redis.redis:6379"
+#
+# Each comma-separated entry becomes a busybox init container that runs
+# `nc -z <host> <port>` in a loop until the dependency is reachable.
+# Existing init containers are preserved — Kyverno appends to the array.
+
+resource "kubernetes_manifest" "inject_dependency_init_containers" {
+  manifest = {
+    apiVersion = "kyverno.io/v1"
+    kind       = "ClusterPolicy"
+    metadata = {
+      name = "inject-dependency-init-containers"
+      annotations = {
+        "policies.kyverno.io/title"       = "Inject Dependency Init Containers"
+        "policies.kyverno.io/description" = "Injects wait-for init containers based on dependency.kyverno.io/wait-for pod annotation. Each comma-separated host:port entry becomes a busybox init container that blocks until the dependency is reachable via nc -z."
+      }
+    }
+    spec = {
+      rules = [
+        {
+          name = "wait-for-dependencies"
+          match = {
+            any = [
+              {
+                resources = {
+                  kinds      = ["Pod"]
+                  operations = ["CREATE"]
+                }
+              }
+            ]
+          }
+          preconditions = {
+            all = [
+              {
+                key      = "{{ request.object.metadata.annotations.\"dependency.kyverno.io/wait-for\" || '' }}"
+                operator = "NotEquals"
+                value    = ""
+              }
+            ]
+          }
+          mutate = {
+            foreach = [
+              {
+                list = "request.object.metadata.annotations.\"dependency.kyverno.io/wait-for\" | split(@, ',')"
+                patchStrategicMerge = {
+                  spec = {
+                    initContainers = [
+                      {
+                        name    = "wait-for-{{ element | split(@, ':') | [0] | replace_all(@, '.', '-') }}"
+                        image   = "busybox:1.37"
+                        command = ["sh", "-c", "until nc -z {{ element | split(@, ':') | [0] }} {{ element | split(@, ':') | [1] }}; do echo waiting for {{ element }}; sleep 2; done"]
+                      }
+                    ]
+                  }
+                }
+              }
+            ]
+          }
+        }
+      ]
+    }
+  }
+}

stacks/platform/modules/monitoring/grafana_chart_values.yaml

@@ -18,6 +18,8 @@ topologySpreadConstraints:
     labelSelector:
       matchLabels:
         app.kubernetes.io/name: grafana
+podAnnotations:
+  dependency.kyverno.io/wait-for: "mysql.dbaas:3306"
 podDisruptionBudget:
   maxUnavailable: 1
 persistence: