viktor/infra
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

add pod dependency management via Kyverno init container injection

Browse source 
Kyverno ClusterPolicy reads dependency.kyverno.io/wait-for annotation
and injects busybox init containers that block until each dependency
is reachable (nc -z). Annotations added to 18 stacks (24 deployments).

Includes graceful-db-maintenance.sh script for planned DB maintenance
(scales dependents to 0, saves replica counts, restores on startup).
This commit is contained in:
Viktor Barzin 2026-03-15 19:17:44 +00:00
parent dc274ab413
commit 0f262ceda3
22 changed files with 282 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

143
scripts/graceful-db-maintenance.sh Executable file
View file

@ -0,0 +1,143 @@
#!/usr/bin/env bash
# graceful-db-maintenance.sh — Scale down/up dependents of a service
# based on the dependency.kyverno.io/wait-for pod annotation.
#
# Usage:
# ./scripts/graceful-db-maintenance.sh shutdown mysql.dbaas
# # ... perform maintenance ...
# ./scripts/graceful-db-maintenance.sh startup mysql.dbaas
#
# The shutdown action saves original replica counts to a state file
# so startup can restore them exactly.
set -euo pipefail
ACTION="${1:-}"
SERVICE="${2:-}"
STATE_DIR="/tmp"
usage() {
echo "Usage: $0 <shutdown|startup> <service>"
echo ""
echo "Examples:"
echo " $0 shutdown mysql.dbaas # Scale down all MySQL dependents"
echo " $0 startup mysql.dbaas # Restore all MySQL dependents"
echo " $0 shutdown postgresql.dbaas # Scale down all PostgreSQL dependents"
echo " $0 shutdown redis.redis # Scale down all Redis dependents"
exit 1
}
[[ -z "$ACTION" || -z "$SERVICE" ]] && usage
[[ "$ACTION" != "shutdown" && "$ACTION" != "startup" ]] && usage
STATE_FILE="${STATE_DIR}/dep-maintenance-$(echo "$SERVICE" | tr '.' '-').json"
KUBECONFIG="${KUBECONFIG:-$(dirname "$0")/../config}"
export KUBECONFIG
# Find all pods with the dependency annotation containing our service
find_dependent_owners() {
local service="$1"
kubectl get pods --all-namespaces \
-o jsonpath='{range .items[*]}{.metadata.namespace}{"\t"}{.metadata.annotations.dependency\.kyverno\.io/wait-for}{"\t"}{.metadata.ownerReferences[0].kind}{"\t"}{.metadata.ownerReferences[0].name}{"\n"}{end}' \
2>/dev/null | \
grep "$service" | \
while IFS=$'\t' read -r ns annotation owner_kind owner_name; do
[[ -z "$owner_kind" || -z "$owner_name" ]] && continue
# Resolve ReplicaSet -> Deployment
if [[ "$owner_kind" == "ReplicaSet" ]]; then
deploy_name=$(kubectl get replicaset "$owner_name" -n "$ns" \
-o jsonpath='{.metadata.ownerReferences[0].name}' 2>/dev/null || true)
if [[ -n "$deploy_name" ]]; then
echo "Deployment/${deploy_name}/${ns}"
fi
elif [[ "$owner_kind" == "StatefulSet" ]]; then
echo "StatefulSet/${owner_name}/${ns}"
fi
done | sort -u
}
do_shutdown() {
echo "Finding dependents of $SERVICE..."
local owners
owners=$(find_dependent_owners "$SERVICE")
if [[ -z "$owners" ]]; then
echo "No dependents found for $SERVICE"
exit 0
fi
echo "Dependents found:"
echo "$owners" | while IFS='/' read -r kind name ns; do
echo " $ns/$kind/$name"
done
# Save current replica counts
local state="[]"
while IFS='/' read -r kind name ns; do
replicas=$(kubectl get "$kind" "$name" -n "$ns" \
-o jsonpath='{.spec.replicas}' 2>/dev/null || echo "1")
state=$(echo "$state" | jq --arg kind "$kind" --arg name "$name" \
--arg ns "$ns" --argjson replicas "${replicas:-1}" \
'. + [{"kind": $kind, "name": $name, "namespace": $ns, "replicas": $replicas}]')
done <<< "$owners"
echo "$state" > "$STATE_FILE"
echo "Saved replica state to $STATE_FILE"
# Scale down
while IFS='/' read -r kind name ns; do
echo "Scaling $ns/$kind/$name to 0..."
kubectl scale "$kind" "$name" -n "$ns" --replicas=0
done <<< "$owners"
echo ""
echo "Waiting for pods to terminate..."
while IFS='/' read -r kind name ns; do
kubectl rollout status "$kind" "$name" -n "$ns" --timeout=120s 2>/dev/null || true
done <<< "$owners"
echo ""
echo "All dependents of $SERVICE scaled to 0."
echo "Run '$0 startup $SERVICE' after maintenance to restore."
}
do_startup() {
if [[ ! -f "$STATE_FILE" ]]; then
echo "Error: No state file found at $STATE_FILE"
echo "Did you run '$0 shutdown $SERVICE' first?"
exit 1
fi
echo "Restoring dependents of $SERVICE from $STATE_FILE..."
local count
count=$(jq length "$STATE_FILE")
for ((i = 0; i < count; i++)); do
kind=$(jq -r ".[$i].kind" "$STATE_FILE")
name=$(jq -r ".[$i].name" "$STATE_FILE")
ns=$(jq -r ".[$i].namespace" "$STATE_FILE")
replicas=$(jq -r ".[$i].replicas" "$STATE_FILE")
echo "Scaling $ns/$kind/$name to $replicas..."
kubectl scale "$kind" "$name" -n "$ns" --replicas="$replicas"
done
echo ""
echo "Waiting for rollouts..."
for ((i = 0; i < count; i++)); do
kind=$(jq -r ".[$i].kind" "$STATE_FILE")
name=$(jq -r ".[$i].name" "$STATE_FILE")
ns=$(jq -r ".[$i].namespace" "$STATE_FILE")
kubectl rollout status "$kind" "$name" -n "$ns" --timeout=300s 2>/dev/null || true
done
rm -f "$STATE_FILE"
echo ""
echo "All dependents of $SERVICE restored."
}
case "$ACTION" in
shutdown) do_shutdown ;;
startup) do_startup ;;
esac

3
stacks/affine/main.tf
View file

@ -112,6 +112,9 @@ resource "kubernetes_deployment" "affine" {
labels = {
app = "affine"
}
annotations = {
"dependency.kyverno.io/wait-for" = "postgresql.dbaas:5432,redis.redis:6379"
}
}
spec {
# Init container to run database migrations

3
stacks/claude-memory/main.tf
View file

@ -117,6 +117,9 @@ resource "kubernetes_deployment" "claude-memory" {
labels = {
app = "claude-memory"
}
annotations = {
"dependency.kyverno.io/wait-for" = "postgresql.dbaas:5432"
}
}
spec {
affinity {

1
stacks/dawarich/main.tf
View file

@ -84,6 +84,7 @@ resource "kubernetes_deployment" "dawarich" {
annotations = {
# "diun.enable" = "true"
# "diun.include_tags" = "latest"
"dependency.kyverno.io/wait-for" = "postgresql.dbaas:5432,redis.redis:6379"
}
}
spec {

3
stacks/grampsweb/main.tf
View file

@ -137,6 +137,9 @@ resource "kubernetes_deployment" "grampsweb" {
labels = {
app = "grampsweb"
}
annotations = {
"dependency.kyverno.io/wait-for" = "redis.redis:6379"
}
}
spec {
container {

3
stacks/hackmd/main.tf
View file

@ -57,6 +57,9 @@ resource "kubernetes_deployment" "hackmd" {
app = "hackmd"
"kubernetes.io/cluster-service" = "true"
}
annotations = {
"dependency.kyverno.io/wait-for" = "mysql.dbaas:3306"
}
}
spec {
# container {

3
stacks/health/main.tf
View file

@ -52,6 +52,9 @@ resource "kubernetes_deployment" "health" {
labels = {
app = "health"
}
annotations = {
"dependency.kyverno.io/wait-for" = "postgresql.dbaas:5432"
}
}
spec {
container {

3
stacks/immich/frame.tf
View file

@ -58,6 +58,9 @@ resource "kubernetes_deployment" "immich-frame" {
labels = {
app = "immich-frame"
}
annotations = {
"dependency.kyverno.io/wait-for" = "immich-server.immich:2283"
}
}
spec {
container {

5
stacks/linkwarden/main.tf
View file

@ -87,8 +87,9 @@ resource "kubernetes_deployment" "linkwarden" {
app = "linkwarden"
}
annotations = {
"diun.enable" = "false"
"diun.include_tags" = "latest"
"diun.enable" = "false"
"diun.include_tags" = "latest"
"dependency.kyverno.io/wait-for" = "postgresql.dbaas:5432"
}
}
spec {

3
stacks/matrix/main.tf
View file

@ -50,6 +50,9 @@ resource "kubernetes_deployment" "matrix" {
labels = {
app = "matrix"
}
annotations = {
"dependency.kyverno.io/wait-for" = "postgresql.dbaas:5432"
}
}
spec {
init_container {

3
stacks/n8n/main.tf
View file

@ -122,6 +122,9 @@ resource "kubernetes_deployment" "n8n" {
labels = {
app = "n8n"
}
annotations = {
"dependency.kyverno.io/wait-for" = "postgresql.dbaas:5432"
}
}
spec {
service_account_name = kubernetes_service_account.n8n.metadata[0].name

1
stacks/nextcloud/chart_values.yaml
View file

@ -98,6 +98,7 @@ readinessProbe:
podAnnotations:
diun.enable: "true"
diun.include_tags: "^[0-9]+(?:.[0-9]+)?(?:.[0-9]+)?.*"
dependency.kyverno.io/wait-for: "mysql.dbaas:3306,redis.redis:6379"
collabora:
enabled: false # Using onlyoffice instead

3
stacks/ollama/main.tf
View file

@ -248,6 +248,9 @@ resource "kubernetes_deployment" "ollama-ui" {
labels = {
app = "ollama-ui"
}
annotations = {
"dependency.kyverno.io/wait-for" = "ollama.ollama:11434"
}
}
spec {
container {

3
stacks/onlyoffice/main.tf
View file

@ -123,6 +123,9 @@ resource "kubernetes_deployment" "onlyoffice-document-server" {
labels = {
app = "onlyoffice-document-server"
}
annotations = {
"dependency.kyverno.io/wait-for" = "mysql.dbaas:3306,redis.redis:6379"
}
}
spec {
container {

5
stacks/paperless-ngx/main.tf
View file

@ -97,8 +97,9 @@ resource "kubernetes_deployment" "paperless-ngx" {
app = "paperless-ngx"
}
annotations = {
"diun.enable" = "false"
"diun.include_tags" = "^\\d+(?:\\.\\d+)?(?:\\.\\d+)?$"
"diun.enable" = "false"
"diun.include_tags" = "^\\d+(?:\\.\\d+)?(?:\\.\\d+)?$"
"dependency.kyverno.io/wait-for" = "mysql.dbaas:3306,redis.redis:6379"
}
}
spec {

72
stacks/platform/modules/kyverno/dependency-init-containers.tf Normal file
View file

@ -0,0 +1,72 @@
# =============================================================================
# Pod Dependency Init Container Injection
# =============================================================================
# Reads the annotation dependency.kyverno.io/wait-for from pods and injects
# init containers that wait for each listed dependency to be reachable.
#
# Usage:
# annotations:
# dependency.kyverno.io/wait-for: "postgresql.dbaas:5432,redis.redis:6379"
#
# Each comma-separated entry becomes a busybox init container that runs
# `nc -z <host> <port>` in a loop until the dependency is reachable.
# Existing init containers are preserved — Kyverno appends to the array.
resource "kubernetes_manifest" "inject_dependency_init_containers" {
manifest = {
apiVersion = "kyverno.io/v1"
kind = "ClusterPolicy"
metadata = {
name = "inject-dependency-init-containers"
annotations = {
"policies.kyverno.io/title" = "Inject Dependency Init Containers"
"policies.kyverno.io/description" = "Injects wait-for init containers based on dependency.kyverno.io/wait-for pod annotation. Each comma-separated host:port entry becomes a busybox init container that blocks until the dependency is reachable via nc -z."
}
}
spec = {
rules = [
{
name = "wait-for-dependencies"
match = {
any = [
{
resources = {
kinds = ["Pod"]
operations = ["CREATE"]
}
}
]
}
preconditions = {
all = [
{
key = "{{ request.object.metadata.annotations.\"dependency.kyverno.io/wait-for\" || '' }}"
operator = "NotEquals"
value = ""
}
]
}
mutate = {
foreach = [
{
list = "request.object.metadata.annotations.\"dependency.kyverno.io/wait-for\" | split(@, ',')"
patchStrategicMerge = {
spec = {
initContainers = [
{
name = "wait-for-{{ element | split(@, ':') | [0] | replace_all(@, '.', '-') }}"
image = "busybox:1.37"
command = ["sh", "-c", "until nc -z {{ element | split(@, ':') | [0] }} {{ element | split(@, ':') | [1] }}; do echo waiting for {{ element }}; sleep 2; done"]
}
]
}
}
}
]
}
}
]
}
}
}

2
stacks/platform/modules/monitoring/grafana_chart_values.yaml
View file

@ -18,6 +18,8 @@ topologySpreadConstraints:
labelSelector:
matchLabels:
app.kubernetes.io/name: grafana
podAnnotations:
dependency.kyverno.io/wait-for: "mysql.dbaas:3306"
podDisruptionBudget:
maxUnavailable: 1
persistence:

9
stacks/real-estate-crawler/main.tf
View file

@ -142,6 +142,9 @@ resource "kubernetes_deployment" "realestate-crawler-api" {
app = "realestate-crawler-api"
"kubernetes.io/cluster-service" = "true"
}
annotations = {
"dependency.kyverno.io/wait-for" = "mysql.dbaas:3306,redis.redis:6379"
}
}
spec {
container {
@ -316,6 +319,9 @@ resource "kubernetes_deployment" "realestate-crawler-celery" {
labels = {
app = "realestate-crawler-celery"
}
annotations = {
"dependency.kyverno.io/wait-for" = "mysql.dbaas:3306,redis.redis:6379"
}
}
spec {
container {
@ -430,6 +436,9 @@ resource "kubernetes_deployment" "realestate-crawler-celery-beat" {
labels = {
app = "realestate-crawler-celery-beat"
}
annotations = {
"dependency.kyverno.io/wait-for" = "mysql.dbaas:3306,redis.redis:6379"
}
}
spec {
container {

6
stacks/rybbit/main.tf
View file

@ -252,6 +252,9 @@ resource "kubernetes_deployment" "rybbit" {
labels = {
app = "rybbit"
}
annotations = {
"dependency.kyverno.io/wait-for" = "postgresql.dbaas:5432,clickhouse.rybbit:8123"
}
}
spec {
container {
@ -404,6 +407,9 @@ resource "kubernetes_deployment" "rybbit-client" {
labels = {
app = "rybbit-client"
}
annotations = {
"dependency.kyverno.io/wait-for" = "rybbit.rybbit:3001"
}
}
spec {
container {

3
stacks/speedtest/main.tf
View file

@ -83,6 +83,9 @@ resource "kubernetes_deployment" "speedtest" {
labels = {
app = "speedtest"
}
annotations = {
"dependency.kyverno.io/wait-for" = "mysql.dbaas:3306"
}
}
spec {
container {

3
stacks/tandoor/main.tf
View file

@ -95,6 +95,9 @@ resource "kubernetes_deployment" "tandoor" {
labels = {
app = "tandoor"
}
annotations = {
"dependency.kyverno.io/wait-for" = "postgresql.dbaas:5432"
}
}
spec {
container {

6
stacks/trading-bot/main.tf
View file

@ -207,6 +207,9 @@ resource "kubernetes_deployment" "trading-bot-frontend" {
labels = {
app = "trading-bot-frontend"
}
annotations = {
"dependency.kyverno.io/wait-for" = "postgresql.dbaas:5432,redis.redis:6379"
}
}
spec {
container {
@ -299,6 +302,9 @@ resource "kubernetes_deployment" "trading-bot-workers" {
labels = {
app = "trading-bot-workers"
}
annotations = {
"dependency.kyverno.io/wait-for" = "postgresql.dbaas:5432,redis.redis:6379"
}
}
spec {
container {