diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index 1003f4d2..37f81406 100755 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -73,7 +73,7 @@ Violations cause state drift, which causes future applies to break or silently r - **LimitRange**: Tier-based defaults silently apply to pods with `resources: {}`. Always set explicit resources on containers needing more than defaults. Tier 3-edge and 4-aux now use Burstable QoS (request < limit) to reduce scheduler pressure. - **Democratic-CSI sidecars**: Must set explicit resources (32-80Mi) in Helm values — 17 sidecars default to 256Mi each via LimitRange. `csiProxy` is a TOP-LEVEL chart key, not nested under controller/node. - **ResourceQuota blocks rolling updates**: When quota is tight, scale to 0 then back to 1 instead of RollingUpdate. Or use Recreate strategy. -- **Kyverno ndots drift**: Kyverno injects dns_config on all pods. Add `lifecycle { ignore_changes = [spec[0].template[0].spec[0].dns_config] }` to kubernetes_deployment resources to prevent perpetual TF plan drift. +- **Kyverno ndots drift**: Kyverno injects dns_config on all pods. Every `kubernetes_deployment`, `kubernetes_stateful_set`, and `kubernetes_cron_job_v1` MUST include `lifecycle { ignore_changes = [spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 }` (use `spec[0].job_template[0].spec[0].template[0].spec[0].dns_config` for CronJobs). The `# KYVERNO_LIFECYCLE_V1` marker is the canonical discoverability tag — grep for it to locate every site. A shared Terraform module was considered but `ignore_changes` only accepts static attribute paths (not module outputs, locals, or expressions), so the snippet convention is the only viable path. Full rationale and copy-paste snippets in `AGENTS.md` → "Kyverno Drift Suppression". - **NVIDIA GPU operator resources**: dcgm-exporter and cuda-validator resources configurable via `dcgmExporter.resources` and `validator.resources` in nvidia values.yaml. - **Pin database versions**: Disable Diun (image update monitoring) for MySQL, PostgreSQL, Redis. - **Quarterly right-sizing**: Check Goldilocks dashboard. Compare VPA upperBound to current request. Also check for under-provisioned (VPA upper > request x 0.8). diff --git a/AGENTS.md b/AGENTS.md index 0662fc99..a726c628 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -75,6 +75,28 @@ Terragrunt-based homelab managing a Kubernetes cluster (5 nodes, v1.34.2) on Pro ## Shared Variables (never hardcode) `var.nfs_server` (192.168.1.127), `var.redis_host`, `var.postgresql_host`, `var.mysql_host`, `var.ollama_host`, `var.mail_host` +## Kyverno Drift Suppression (`# KYVERNO_LIFECYCLE_V1`) + +Kyverno's admission webhook mutates every pod with a `dns_config { option { name = "ndots"; value = "2" } }` block (fixes NxDomain search-domain floods — see `k8s-ndots-search-domain-nxdomain-flood` skill). Terraform does not manage that field, so without suppression every pod-owning resource shows perpetual `spec[0].template[0].spec[0].dns_config` drift. + +**Rule**: every `kubernetes_deployment`, `kubernetes_stateful_set`, `kubernetes_daemon_set`, and `kubernetes_cron_job_v1` MUST include the following `lifecycle` block, tagged with the `# KYVERNO_LIFECYCLE_V1` marker so every site is greppable: + +```hcl +# kubernetes_deployment / kubernetes_stateful_set / kubernetes_daemon_set +lifecycle { + ignore_changes = [spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 +} + +# kubernetes_cron_job_v1 (extra job_template nesting) +lifecycle { + ignore_changes = [spec[0].job_template[0].spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 +} +``` + +**Why not a shared module?** Terraform's `ignore_changes` meta-argument only accepts static attribute paths. It rejects module outputs, locals, variables, and any expression. A DRY module is therefore impossible — the canonical pattern IS the snippet + marker. When `kubernetes_manifest` resources get Kyverno `generate.kyverno.io/*` annotations mutated, a sibling convention `# KYVERNO_MANIFEST_V1` will be introduced (Phase B). + +**Audit**: `rg "KYVERNO_LIFECYCLE_V1" stacks/ | wc -l` — should grow (never shrink). Add the marker to every new pod-owning resource. The `_template/main.tf.example` stub shows the canonical form. + ## Tier System `0-core` | `1-cluster` | `2-gpu` | `3-edge` | `4-aux` — Kyverno auto-generates LimitRange + ResourceQuota per namespace based on tier label. - Containers without explicit `resources {}` get default limits (256Mi for edge/aux — causes OOMKill for heavy apps) diff --git a/stacks/_template/main.tf.example b/stacks/_template/main.tf.example index c668292e..6f52337a 100644 --- a/stacks/_template/main.tf.example +++ b/stacks/_template/main.tf.example @@ -63,7 +63,7 @@ resource "kubernetes_deployment" "app" { } } lifecycle { - ignore_changes = [spec[0].template[0].spec[0].dns_config] + ignore_changes = [spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 } } diff --git a/stacks/beads-server/main.tf b/stacks/beads-server/main.tf index b8af8c66..fb3a2e36 100644 --- a/stacks/beads-server/main.tf +++ b/stacks/beads-server/main.tf @@ -151,7 +151,7 @@ resource "kubernetes_deployment" "dolt" { } lifecycle { ignore_changes = [ - spec[0].template[0].spec[0].dns_config + spec[0].template[0].spec[0].dns_config # KYVERNO_LIFECYCLE_V1 ] } } @@ -355,7 +355,7 @@ resource "kubernetes_deployment" "workbench" { } lifecycle { ignore_changes = [ - spec[0].template[0].spec[0].dns_config + spec[0].template[0].spec[0].dns_config # KYVERNO_LIFECYCLE_V1 ] } } @@ -626,7 +626,7 @@ resource "kubernetes_deployment" "beadboard" { } lifecycle { ignore_changes = [ - spec[0].template[0].spec[0].dns_config + spec[0].template[0].spec[0].dns_config # KYVERNO_LIFECYCLE_V1 ] } } diff --git a/stacks/claude-agent-service/main.tf b/stacks/claude-agent-service/main.tf index cdb9140a..53a3d695 100644 --- a/stacks/claude-agent-service/main.tf +++ b/stacks/claude-agent-service/main.tf @@ -430,7 +430,7 @@ resource "kubernetes_deployment" "claude_agent" { } lifecycle { - ignore_changes = [spec[0].template[0].spec[0].dns_config] + ignore_changes = [spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 } } diff --git a/stacks/dbaas/modules/dbaas/main.tf b/stacks/dbaas/modules/dbaas/main.tf index adedfb8b..802fbf09 100644 --- a/stacks/dbaas/modules/dbaas/main.tf +++ b/stacks/dbaas/modules/dbaas/main.tf @@ -549,7 +549,7 @@ resource "kubernetes_stateful_set_v1" "mysql_standalone" { } lifecycle { - ignore_changes = [spec[0].template[0].spec[0].dns_config] + ignore_changes = [spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 } } diff --git a/stacks/diun/main.tf b/stacks/diun/main.tf index 04af08e6..4389ca5a 100644 --- a/stacks/diun/main.tf +++ b/stacks/diun/main.tf @@ -226,6 +226,6 @@ resource "kubernetes_deployment" "diun" { } } lifecycle { - ignore_changes = [spec[0].template[0].spec[0].dns_config] + ignore_changes = [spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 } } diff --git a/stacks/ebooks/main.tf b/stacks/ebooks/main.tf index 31858dd2..d8cd6d1d 100644 --- a/stacks/ebooks/main.tf +++ b/stacks/ebooks/main.tf @@ -346,7 +346,7 @@ resource "kubernetes_deployment" "calibre-web-automated" { } } lifecycle { - ignore_changes = [spec[0].template[0].spec[0].dns_config] + ignore_changes = [spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 } } @@ -466,7 +466,7 @@ resource "kubernetes_deployment" "annas-archive-stacks" { } } lifecycle { - ignore_changes = [spec[0].template[0].spec[0].dns_config] + ignore_changes = [spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 } } @@ -615,7 +615,7 @@ resource "kubernetes_deployment" "audiobookshelf" { } } lifecycle { - ignore_changes = [spec[0].template[0].spec[0].dns_config] + ignore_changes = [spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 } } @@ -876,7 +876,7 @@ resource "kubernetes_deployment" "book_search" { } } lifecycle { - ignore_changes = [spec[0].template[0].spec[0].dns_config] + ignore_changes = [spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 } } diff --git a/stacks/freedify/factory/main.tf b/stacks/freedify/factory/main.tf index 353646e0..c66b9029 100755 --- a/stacks/freedify/factory/main.tf +++ b/stacks/freedify/factory/main.tf @@ -193,7 +193,7 @@ resource "kubernetes_deployment" "freedify" { } } lifecycle { - ignore_changes = [spec[0].template[0].spec[0].dns_config] + ignore_changes = [spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 } } diff --git a/stacks/hermes-agent/main.tf b/stacks/hermes-agent/main.tf index cd8e840a..92188462 100644 --- a/stacks/hermes-agent/main.tf +++ b/stacks/hermes-agent/main.tf @@ -374,7 +374,7 @@ resource "kubernetes_deployment" "hermes_agent" { } } lifecycle { - ignore_changes = [spec[0].template[0].spec[0].dns_config] + ignore_changes = [spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 } } diff --git a/stacks/immich/main.tf b/stacks/immich/main.tf index 2cd804d2..f931f14a 100644 --- a/stacks/immich/main.tf +++ b/stacks/immich/main.tf @@ -145,7 +145,7 @@ resource "kubernetes_deployment" "immich_server" { lifecycle { ignore_changes = [ - spec[0].template[0].spec[0].dns_config, + spec[0].template[0].spec[0].dns_config, # KYVERNO_LIFECYCLE_V1 ] } @@ -373,7 +373,7 @@ resource "kubernetes_deployment" "immich-postgres" { lifecycle { ignore_changes = [ - spec[0].template[0].spec[0].dns_config, + spec[0].template[0].spec[0].dns_config, # KYVERNO_LIFECYCLE_V1 ] } @@ -532,7 +532,7 @@ resource "kubernetes_deployment" "immich-machine-learning" { lifecycle { ignore_changes = [ - spec[0].template[0].spec[0].dns_config, + spec[0].template[0].spec[0].dns_config, # KYVERNO_LIFECYCLE_V1 ] } diff --git a/stacks/insta2spotify/main.tf b/stacks/insta2spotify/main.tf index 9dd86586..f5971e0a 100644 --- a/stacks/insta2spotify/main.tf +++ b/stacks/insta2spotify/main.tf @@ -198,7 +198,7 @@ resource "kubernetes_deployment" "insta2spotify" { } } lifecycle { - ignore_changes = [spec[0].template[0].spec[0].dns_config] + ignore_changes = [spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 } } diff --git a/stacks/k8s-portal/modules/k8s-portal/main.tf b/stacks/k8s-portal/modules/k8s-portal/main.tf index c44b06fc..cd1692c5 100644 --- a/stacks/k8s-portal/modules/k8s-portal/main.tf +++ b/stacks/k8s-portal/modules/k8s-portal/main.tf @@ -115,7 +115,7 @@ resource "kubernetes_deployment" "k8s_portal" { lifecycle { # DRIFT_WORKAROUND: CI pipeline owns image tag (kubectl set image from Woodpecker/GHA); Kyverno mutates dns_config for ndots. Reviewed 2026-04-18. ignore_changes = [ - spec[0].template[0].spec[0].dns_config, + spec[0].template[0].spec[0].dns_config, # KYVERNO_LIFECYCLE_V1 spec[0].template[0].spec[0].container[0].image, # CI updates image tag ] } diff --git a/stacks/openclaw/main.tf b/stacks/openclaw/main.tf index ae26459b..47932ec3 100644 --- a/stacks/openclaw/main.tf +++ b/stacks/openclaw/main.tf @@ -1175,7 +1175,7 @@ resource "kubernetes_deployment" "openlobster" { } } lifecycle { - ignore_changes = [spec[0].template[0].spec[0].dns_config] + ignore_changes = [spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 } } diff --git a/stacks/phpipam/main.tf b/stacks/phpipam/main.tf index 0bf0bc79..f2c2e567 100644 --- a/stacks/phpipam/main.tf +++ b/stacks/phpipam/main.tf @@ -197,7 +197,7 @@ resource "kubernetes_deployment" "phpipam_web" { } } lifecycle { - ignore_changes = [spec[0].template[0].spec[0].dns_config] + ignore_changes = [spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 } } diff --git a/stacks/priority-pass/main.tf b/stacks/priority-pass/main.tf index 7e5aaded..2910d2fc 100644 --- a/stacks/priority-pass/main.tf +++ b/stacks/priority-pass/main.tf @@ -89,7 +89,7 @@ resource "kubernetes_deployment" "priority-pass" { } } lifecycle { - ignore_changes = [spec[0].template[0].spec[0].dns_config] + ignore_changes = [spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 } } diff --git a/stacks/status-page/main.tf b/stacks/status-page/main.tf index 49f062f7..6c943d6c 100644 --- a/stacks/status-page/main.tf +++ b/stacks/status-page/main.tf @@ -611,6 +611,6 @@ PYEOF } } lifecycle { - ignore_changes = [spec[0].job_template[0].spec[0].template[0].spec[0].dns_config] + ignore_changes = [spec[0].job_template[0].spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 } } diff --git a/stacks/traefik/modules/traefik/error-pages.tf b/stacks/traefik/modules/traefik/error-pages.tf index ff56e554..54346a07 100644 --- a/stacks/traefik/modules/traefik/error-pages.tf +++ b/stacks/traefik/modules/traefik/error-pages.tf @@ -89,7 +89,7 @@ resource "kubernetes_deployment" "error_pages" { } lifecycle { - ignore_changes = [spec[0].template[0].spec[0].dns_config] + ignore_changes = [spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 } } diff --git a/stacks/uptime-kuma/modules/uptime-kuma/main.tf b/stacks/uptime-kuma/modules/uptime-kuma/main.tf index 174ca0f0..3a6506a7 100644 --- a/stacks/uptime-kuma/modules/uptime-kuma/main.tf +++ b/stacks/uptime-kuma/modules/uptime-kuma/main.tf @@ -517,7 +517,7 @@ PYEOF } } lifecycle { - ignore_changes = [spec[0].job_template[0].spec[0].template[0].spec[0].dns_config] + ignore_changes = [spec[0].job_template[0].spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 } } @@ -707,6 +707,6 @@ PYEOF } } lifecycle { - ignore_changes = [spec[0].job_template[0].spec[0].template[0].spec[0].dns_config] + ignore_changes = [spec[0].job_template[0].spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 } } diff --git a/stacks/wealthfolio/main.tf b/stacks/wealthfolio/main.tf index 5d37d5b7..bd27d056 100644 --- a/stacks/wealthfolio/main.tf +++ b/stacks/wealthfolio/main.tf @@ -76,7 +76,7 @@ resource "kubernetes_persistent_volume_claim" "data_proxmox" { resource "kubernetes_deployment" "wealthfolio" { lifecycle { - ignore_changes = [spec[0].template[0].spec[0].dns_config] + ignore_changes = [spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 } metadata { name = "wealthfolio" diff --git a/stacks/webhook_handler/main.tf b/stacks/webhook_handler/main.tf index 3c822780..0ecb73f0 100644 --- a/stacks/webhook_handler/main.tf +++ b/stacks/webhook_handler/main.tf @@ -230,7 +230,7 @@ resource "kubernetes_deployment" "webhook_handler" { } } lifecycle { - ignore_changes = [spec[0].template[0].spec[0].dns_config] + ignore_changes = [spec[0].template[0].spec[0].dns_config] # KYVERNO_LIFECYCLE_V1 } }