kyverno(wave1): swap kubernetes_manifest → kubectl_manifest + flip 3 security policies to Enforce
## Resolves code-e2dp (Kyverno TF apply blocked)
Root cause: terraform-provider-kubernetes v3.1.0 panics on plan/refresh of
kubernetes_manifest resources holding Kyverno ClusterPolicy CRDs (large
CEL/foreach schemas). Workaround: swap to gavinbunney/kubectl_manifest which
treats manifests as opaque YAML strings.
## Migration mechanics
- Root terragrunt.hcl: added gavinbunney/kubectl provider declaration so all
stacks get it generated in providers.tf.
- stacks/kyverno/modules/kyverno/versions.tf (new): module-level provider source
declaration (required for kubectl_manifest in a child module).
- Converted 17 kubernetes_manifest resources across 7 files to kubectl_manifest
with yaml_body = yamlencode({...}). depends_on chains preserved.
- terraform state rm for all 17 old kubernetes_manifest entries.
- stacks/kyverno/imports.tf (new): TF 1.5+ import blocks mapping each
kubectl_manifest to its live cluster resource by apiVersion//Kind//name ID.
- One resource (policy_inject_keel_annotations) needed kubectl delete + recreate
because the kubectl provider couldn't patch it cleanly (resourceVersion=0
invalid for update — gotcha when adopting a resource previously
kubernetes_manifest-owned).
## W1.4 — security policies Audit → Enforce (LIVE)
Three policies flipped: deny-privileged-containers, deny-host-namespaces,
restrict-sys-admin. Verified live via kubectl. failurePolicy=Ignore preserved.
## Shared exclude list (35 namespaces)
local.security_policy_exclude_namespaces in security-policies.tf.
- 31 critical from memory id=1970 (Keel rollout list)
- + frigate (camera HW transcoding needs host access)
- + kured (privileged DaemonSet for node reboots)
- + default (etcd backup/defrag CronJobs use hostNetwork)
- + changedetection (uses SYS_ADMIN for chromium sandbox)
## W1.5 — require-trusted-registries stays Audit
Pattern */* allows anything-with-a-slash; Enforce would be a no-op for supply
chain. Tracked under beads code-8ywc as follow-up.
## TF import-blocks
The imports.tf file should be removed in a follow-up cleanup commit once
verified — TF doesn't auto-clean these.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
Closes: code-e2dp
This commit is contained in:
parent
83079758bb
commit
1082cba0fb
10 changed files with 148 additions and 53 deletions
|
|
@ -13,8 +13,8 @@
|
|||
# `nc -z <host> <port>` in a loop until the dependency is reachable.
|
||||
# Existing init containers are preserved — Kyverno appends to the array.
|
||||
|
||||
resource "kubernetes_manifest" "inject_dependency_init_containers" {
|
||||
manifest = {
|
||||
resource "kubectl_manifest" "inject_dependency_init_containers" {
|
||||
yaml_body = yamlencode({
|
||||
apiVersion = "kyverno.io/v1"
|
||||
kind = "ClusterPolicy"
|
||||
metadata = {
|
||||
|
|
@ -68,5 +68,5 @@ resource "kubernetes_manifest" "inject_dependency_init_containers" {
|
|||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,8 +16,8 @@
|
|||
# (used by the rollback runbook). The keel namespace itself is always
|
||||
# excluded (design decision #11 — supervisor must not auto-update).
|
||||
|
||||
resource "kubernetes_manifest" "policy_inject_keel_annotations" {
|
||||
manifest = {
|
||||
resource "kubectl_manifest" "policy_inject_keel_annotations" {
|
||||
yaml_body = yamlencode({
|
||||
apiVersion = "kyverno.io/v1"
|
||||
kind = "ClusterPolicy"
|
||||
metadata = {
|
||||
|
|
@ -192,7 +192,7 @@ resource "kubernetes_manifest" "policy_inject_keel_annotations" {
|
|||
}
|
||||
}]
|
||||
}
|
||||
}
|
||||
})
|
||||
depends_on = [helm_release.kyverno]
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -136,8 +136,8 @@ resource "helm_release" "kyverno" {
|
|||
#
|
||||
# Uses namespaceSelector to match tiers — no API call needed.
|
||||
# One rule per tier so Kyverno resolves the tier value from its informer cache.
|
||||
resource "kubernetes_manifest" "mutate_tier_from_namespace" {
|
||||
manifest = {
|
||||
resource "kubectl_manifest" "mutate_tier_from_namespace" {
|
||||
yaml_body = yamlencode({
|
||||
apiVersion = "kyverno.io/v1"
|
||||
kind = "ClusterPolicy"
|
||||
metadata = {
|
||||
|
|
@ -180,10 +180,10 @@ resource "kubernetes_manifest" "mutate_tier_from_namespace" {
|
|||
}
|
||||
}]
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
# resource "kubernetes_manifest" "enforce_pod_tier_label" {
|
||||
# resource "kubectl_manifest" "enforce_pod_tier_label" {
|
||||
# manifest = {
|
||||
# apiVersion = "kyverno.io/v1"
|
||||
# kind = "ClusterPolicy"
|
||||
|
|
|
|||
|
|
@ -81,8 +81,8 @@ resource "kubernetes_cluster_role_binding" "kyverno_background_secret_manager" {
|
|||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_manifest" "sync_registry_credentials" {
|
||||
manifest = {
|
||||
resource "kubectl_manifest" "sync_registry_credentials" {
|
||||
yaml_body = yamlencode({
|
||||
apiVersion = "kyverno.io/v1"
|
||||
kind = "ClusterPolicy"
|
||||
metadata = {
|
||||
|
|
@ -124,7 +124,7 @@ resource "kubernetes_manifest" "sync_registry_credentials" {
|
|||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
depends_on = [
|
||||
helm_release.kyverno,
|
||||
|
|
|
|||
|
|
@ -88,8 +88,8 @@ resource "kubernetes_priority_class" "tier_4_aux" {
|
|||
# Creates a LimitRange in each namespace based on its tier label.
|
||||
# Only affects containers WITHOUT explicit resource requests/limits.
|
||||
|
||||
resource "kubernetes_manifest" "generate_limitrange_by_tier" {
|
||||
manifest = {
|
||||
resource "kubectl_manifest" "generate_limitrange_by_tier" {
|
||||
yaml_body = yamlencode({
|
||||
apiVersion = "kyverno.io/v1"
|
||||
kind = "ClusterPolicy"
|
||||
metadata = {
|
||||
|
|
@ -450,7 +450,7 @@ resource "kubernetes_manifest" "generate_limitrange_by_tier" {
|
|||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
|
@ -463,10 +463,10 @@ resource "kubernetes_manifest" "generate_limitrange_by_tier" {
|
|||
# IMPORTANT: LimitRange (Layer 2) must exist before ResourceQuota takes effect,
|
||||
# because ResourceQuota requires all pods to have resource requests set.
|
||||
|
||||
resource "kubernetes_manifest" "generate_resourcequota_by_tier" {
|
||||
depends_on = [kubernetes_manifest.generate_limitrange_by_tier]
|
||||
resource "kubectl_manifest" "generate_resourcequota_by_tier" {
|
||||
depends_on = [kubectl_manifest.generate_limitrange_by_tier]
|
||||
|
||||
manifest = {
|
||||
yaml_body = yamlencode({
|
||||
apiVersion = "kyverno.io/v1"
|
||||
kind = "ClusterPolicy"
|
||||
metadata = {
|
||||
|
|
@ -721,7 +721,7 @@ resource "kubernetes_manifest" "generate_resourcequota_by_tier" {
|
|||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
|
@ -731,8 +731,8 @@ resource "kubernetes_manifest" "generate_resourcequota_by_tier" {
|
|||
# Skips pods that already have a priorityClassName set.
|
||||
# Uses namespaceSelector instead of API calls — no round-trip to the API server.
|
||||
|
||||
resource "kubernetes_manifest" "mutate_priority_from_tier" {
|
||||
manifest = {
|
||||
resource "kubectl_manifest" "mutate_priority_from_tier" {
|
||||
yaml_body = yamlencode({
|
||||
apiVersion = "kyverno.io/v1"
|
||||
kind = "ClusterPolicy"
|
||||
metadata = {
|
||||
|
|
@ -797,7 +797,7 @@ resource "kubernetes_manifest" "mutate_priority_from_tier" {
|
|||
}
|
||||
}]
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -806,8 +806,8 @@ resource "kubernetes_manifest" "mutate_priority_from_tier" {
|
|||
# external DNS lookup (search domain expansion). This policy injects ndots:2
|
||||
# on all pods to reduce NxDomain flood while still allowing short-name service
|
||||
# resolution (e.g. "redis.redis" has 1 dot, so it still expands).
|
||||
resource "kubernetes_manifest" "mutate_ndots" {
|
||||
manifest = {
|
||||
resource "kubectl_manifest" "mutate_ndots" {
|
||||
yaml_body = yamlencode({
|
||||
apiVersion = "kyverno.io/v1"
|
||||
kind = "ClusterPolicy"
|
||||
metadata = {
|
||||
|
|
@ -865,7 +865,7 @@ resource "kubernetes_manifest" "mutate_ndots" {
|
|||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
|
@ -876,8 +876,8 @@ resource "kubernetes_manifest" "mutate_ndots" {
|
|||
# non-GPU pods on the GPU node, regardless of namespace tier.
|
||||
# Runs after Layer 4 (tier injection), so it overrides the tier-based priority.
|
||||
|
||||
resource "kubernetes_manifest" "mutate_gpu_priority" {
|
||||
manifest = {
|
||||
resource "kubectl_manifest" "mutate_gpu_priority" {
|
||||
yaml_body = yamlencode({
|
||||
apiVersion = "kyverno.io/v1"
|
||||
kind = "ClusterPolicy"
|
||||
metadata = {
|
||||
|
|
@ -946,7 +946,7 @@ resource "kubernetes_manifest" "mutate_gpu_priority" {
|
|||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
|
@ -991,8 +991,8 @@ resource "kubernetes_cluster_role_binding_v1" "kyverno_cleanup_pods" {
|
|||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_manifest" "cleanup_failed_pods" {
|
||||
manifest = {
|
||||
resource "kubectl_manifest" "cleanup_failed_pods" {
|
||||
yaml_body = yamlencode({
|
||||
apiVersion = "kyverno.io/v2"
|
||||
kind = "ClusterCleanupPolicy"
|
||||
metadata = {
|
||||
|
|
@ -1023,7 +1023,7 @@ resource "kubernetes_manifest" "cleanup_failed_pods" {
|
|||
}
|
||||
schedule = "15 * * * *"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
|
@ -1042,8 +1042,8 @@ resource "kubernetes_manifest" "cleanup_failed_pods" {
|
|||
# JSON6902 remove op fails on missing paths — per-element precondition gates
|
||||
# the mutation so pods without CPU limits pass through untouched.
|
||||
|
||||
resource "kubernetes_manifest" "mutate_strip_cpu_limits" {
|
||||
manifest = {
|
||||
resource "kubectl_manifest" "mutate_strip_cpu_limits" {
|
||||
yaml_body = yamlencode({
|
||||
apiVersion = "kyverno.io/v1"
|
||||
kind = "ClusterPolicy"
|
||||
metadata = {
|
||||
|
|
@ -1151,5 +1151,5 @@ resource "kubernetes_manifest" "mutate_strip_cpu_limits" {
|
|||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,8 +29,8 @@ locals {
|
|||
]
|
||||
}
|
||||
|
||||
resource "kubernetes_manifest" "policy_deny_privileged" {
|
||||
manifest = {
|
||||
resource "kubectl_manifest" "policy_deny_privileged" {
|
||||
yaml_body = yamlencode({
|
||||
apiVersion = "kyverno.io/v1"
|
||||
kind = "ClusterPolicy"
|
||||
metadata = {
|
||||
|
|
@ -80,13 +80,13 @@ resource "kubernetes_manifest" "policy_deny_privileged" {
|
|||
}
|
||||
}]
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
depends_on = [helm_release.kyverno]
|
||||
}
|
||||
|
||||
resource "kubernetes_manifest" "policy_deny_host_namespaces" {
|
||||
manifest = {
|
||||
resource "kubectl_manifest" "policy_deny_host_namespaces" {
|
||||
yaml_body = yamlencode({
|
||||
apiVersion = "kyverno.io/v1"
|
||||
kind = "ClusterPolicy"
|
||||
metadata = {
|
||||
|
|
@ -129,13 +129,13 @@ resource "kubernetes_manifest" "policy_deny_host_namespaces" {
|
|||
}
|
||||
}]
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
depends_on = [helm_release.kyverno]
|
||||
}
|
||||
|
||||
resource "kubernetes_manifest" "policy_restrict_capabilities" {
|
||||
manifest = {
|
||||
resource "kubectl_manifest" "policy_restrict_capabilities" {
|
||||
yaml_body = yamlencode({
|
||||
apiVersion = "kyverno.io/v1"
|
||||
kind = "ClusterPolicy"
|
||||
metadata = {
|
||||
|
|
@ -180,7 +180,7 @@ resource "kubernetes_manifest" "policy_restrict_capabilities" {
|
|||
}
|
||||
}]
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
depends_on = [helm_release.kyverno]
|
||||
}
|
||||
|
|
@ -193,8 +193,8 @@ resource "kubernetes_manifest" "policy_restrict_capabilities" {
|
|||
# when the pull-through cache at 10.0.20.10 has transient failures.
|
||||
# For :latest or untagged images, set to Always so stale images don't persist.
|
||||
|
||||
resource "kubernetes_manifest" "policy_set_image_pull_policy" {
|
||||
manifest = {
|
||||
resource "kubectl_manifest" "policy_set_image_pull_policy" {
|
||||
yaml_body = yamlencode({
|
||||
apiVersion = "kyverno.io/v1"
|
||||
kind = "ClusterPolicy"
|
||||
metadata = {
|
||||
|
|
@ -271,13 +271,13 @@ resource "kubernetes_manifest" "policy_set_image_pull_policy" {
|
|||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
depends_on = [helm_release.kyverno]
|
||||
}
|
||||
|
||||
resource "kubernetes_manifest" "policy_require_trusted_registries" {
|
||||
manifest = {
|
||||
resource "kubectl_manifest" "policy_require_trusted_registries" {
|
||||
yaml_body = yamlencode({
|
||||
apiVersion = "kyverno.io/v1"
|
||||
kind = "ClusterPolicy"
|
||||
metadata = {
|
||||
|
|
@ -325,7 +325,7 @@ resource "kubernetes_manifest" "policy_require_trusted_registries" {
|
|||
}
|
||||
}]
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
depends_on = [helm_release.kyverno]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@ resource "kubernetes_secret" "tls_secret" {
|
|||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_manifest" "sync_tls_secret" {
|
||||
manifest = {
|
||||
resource "kubectl_manifest" "sync_tls_secret" {
|
||||
yaml_body = yamlencode({
|
||||
apiVersion = "kyverno.io/v1"
|
||||
kind = "ClusterPolicy"
|
||||
metadata = {
|
||||
|
|
@ -61,7 +61,7 @@ resource "kubernetes_manifest" "sync_tls_secret" {
|
|||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
depends_on = [
|
||||
helm_release.kyverno,
|
||||
|
|
|
|||
11
stacks/kyverno/modules/kyverno/versions.tf
Normal file
11
stacks/kyverno/modules/kyverno/versions.tf
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
# kubectl provider — used by kubectl_manifest resources (swapped from
|
||||
# hashicorp/kubernetes kubernetes_manifest due to provider crash on Kyverno
|
||||
# ClusterPolicy CRDs, beads code-e2dp).
|
||||
terraform {
|
||||
required_providers {
|
||||
kubectl = {
|
||||
source = "gavinbunney/kubectl"
|
||||
version = "~> 1.14"
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue