viktor/infra
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[ci skip] kyverno: scale to 2 replicas, eliminate API calls from policies

Browse source 
- Scale admission controller to 2 replicas with topology spread across nodes
- Rewrite inject-priority-class-from-tier: use namespaceSelector instead of
  API call per pod admission (eliminates Kyverno→API server round-trip)
- Rewrite sync-tier-label-from-namespace: same namespaceSelector approach
- Extract governance_tiers local to DRY up tier definitions
This commit is contained in:
Viktor Barzin 2026-02-24 23:09:56 +00:00
parent dcb465a7e5
commit 7bc975aa16
2 changed files with 100 additions and 93 deletions
Download patch file Download diff file Expand all files Collapse all files

82
stacks/platform/modules/kyverno/main.tf
View file

@ -28,6 +28,8 @@ resource "helm_release" "kyverno" {
}
admissionController = {
replicas = 2
container = {
resources = {
limits = {
@ -53,12 +55,30 @@ resource "helm_release" "kyverno" {
failureThreshold = 4
successThreshold = 1
}
# Spread replicas across nodes for HA
topologySpreadConstraints = [
{
maxSkew = 1
topologyKey = "kubernetes.io/hostname"
whenUnsatisfiable = "DoNotSchedule"
labelSelector = {
matchLabels = {
"app.kubernetes.io/component" = "admission-controller"
"app.kubernetes.io/instance" = "kyverno"
}
}
}
]
}
})]
}
# To unlabel all:
# kubectl label deployment,statefulset,daemonset --all-namespaces -l tier tier-
#
# Uses namespaceSelector to match tiers — no API call needed.
# One rule per tier so Kyverno resolves the tier value from its informer cache.
resource "kubernetes_manifest" "mutate_tier_from_namespace" {
manifest = {
apiVersion = "kyverno.io/v1"
@ -67,49 +87,41 @@ resource "kubernetes_manifest" "mutate_tier_from_namespace" {
name = "sync-tier-label-from-namespace"
}
spec = {
rules = [
{
name = "lookup-and-add-tier"
match = {
any = [
{
resources = {
kinds = ["Deployment", "StatefulSet", "DaemonSet"]
}
}
]
}
exclude = {
any = [
{
resources = {
namespaces = ["kube-system", "metallb-system", "n8n"]
}
}
]
}
# Context allows us to perform an API call to get Namespace metadata
context = [
rules = [for tier in local.governance_tiers : {
name = "sync-tier-${tier}"
match = {
any = [
{
name = "namespaceLabel"
apiCall = {
urlPath = "/api/v1/namespaces/{{request.namespace}}"
jmesPath = "metadata.labels.tier || 'default'"
resources = {
kinds = ["Deployment", "StatefulSet", "DaemonSet"]
namespaceSelector = {
matchLabels = {
tier = tier
}
}
}
}
]
mutate = {
patchStrategicMerge = {
metadata = {
labels = {
# Injects the variable discovered in the context above
"+(tier)" = "{{namespaceLabel}}"
}
}
exclude = {
any = [
{
resources = {
namespaces = ["kube-system", "metallb-system", "n8n"]
}
}
]
}
mutate = {
patchStrategicMerge = {
metadata = {
labels = {
"+(tier)" = tier
}
}
}
}
]
}]
}
}
}

111
stacks/platform/modules/kyverno/resource-governance.tf
View file

@ -8,6 +8,11 @@
# 3. ResourceQuotas (Kyverno generate) - hard ceiling on namespace resource consumption
# 4. Priority injection (Kyverno mutate) - set priorityClassName based on namespace tier label
locals {
governance_tiers = ["0-core", "1-cluster", "2-gpu", "3-edge", "4-aux"]
excluded_namespaces = ["kube-system", "metallb-system", "kyverno", "calico-system", "calico-apiserver"]
}
# -----------------------------------------------------------------------------
# Layer 1: PriorityClasses
# -----------------------------------------------------------------------------
@ -661,6 +666,7 @@ resource "kubernetes_manifest" "generate_resourcequota_by_tier" {
# -----------------------------------------------------------------------------
# Automatically sets priorityClassName on Pods based on their namespace's tier label.
# Skips pods that already have a priorityClassName set.
# Uses namespaceSelector instead of API calls — no round-trip to the API server.
resource "kubernetes_manifest" "mutate_priority_from_tier" {
manifest = {
@ -674,69 +680,58 @@ resource "kubernetes_manifest" "mutate_priority_from_tier" {
}
}
spec = {
rules = [
{
name = "inject-priority-class"
match = {
any = [
{
resources = {
kinds = ["Pod"]
}
}
]
}
exclude = {
any = [
{
resources = {
namespaces = ["kube-system", "metallb-system", "kyverno", "calico-system", "calico-apiserver"]
}
}
]
}
context = [
rules = [for tier in local.governance_tiers : {
name = "inject-priority-${tier}"
match = {
any = [
{
name = "tierLabel"
apiCall = {
urlPath = "/api/v1/namespaces/{{request.namespace}}"
jmesPath = "metadata.labels.tier || ''"
resources = {
kinds = ["Pod"]
namespaceSelector = {
matchLabels = {
tier = tier
}
}
}
}
]
preconditions = {
all = [
{
key = "{{request.object.spec.priorityClassName || ''}}"
operator = "Equals"
value = ""
},
{
key = "{{tierLabel}}"
operator = "NotEquals"
value = ""
}
]
}
mutate = {
patchesJson6902 = yamlencode([
{
op = "remove"
path = "/spec/priority"
},
{
op = "remove"
path = "/spec/preemptionPolicy"
},
{
op = "add"
path = "/spec/priorityClassName"
value = "tier-{{tierLabel}}"
}
])
}
}
]
exclude = {
any = [
{
resources = {
namespaces = local.excluded_namespaces
}
}
]
}
preconditions = {
all = [
{
key = "{{request.object.spec.priorityClassName || ''}}"
operator = "Equals"
value = ""
}
]
}
mutate = {
patchesJson6902 = yamlencode([
{
op = "remove"
path = "/spec/priority"
},
{
op = "remove"
path = "/spec/preemptionPolicy"
},
{
op = "add"
path = "/spec/priorityClassName"
value = "tier-${tier}"
}
])
}
}]
}
}
}