From 7bc975aa16d0da09a06e53f8f0a6203fd9fd03e6 Mon Sep 17 00:00:00 2001
From: Viktor Barzin <viktorbarzin@meta.com>
Date: Tue, 24 Feb 2026 23:09:56 +0000
Subject: [PATCH] [ci skip] kyverno: scale to 2 replicas, eliminate API calls
 from policies
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Scale admission controller to 2 replicas with topology spread across nodes
- Rewrite inject-priority-class-from-tier: use namespaceSelector instead of
  API call per pod admission (eliminates Kyverno→API server round-trip)
- Rewrite sync-tier-label-from-namespace: same namespaceSelector approach
- Extract governance_tiers local to DRY up tier definitions
---
 stacks/platform/modules/kyverno/main.tf       |  82 +++++++------
 .../modules/kyverno/resource-governance.tf    | 111 +++++++++---------
 2 files changed, 100 insertions(+), 93 deletions(-)

diff --git a/stacks/platform/modules/kyverno/main.tf b/stacks/platform/modules/kyverno/main.tf
index ecfde9e7..801db91a 100644
--- a/stacks/platform/modules/kyverno/main.tf
+++ b/stacks/platform/modules/kyverno/main.tf
@@ -28,6 +28,8 @@ resource "helm_release" "kyverno" {
     }
 
     admissionController = {
+      replicas = 2
+
       container = {
         resources = {
           limits = {
@@ -53,12 +55,30 @@ resource "helm_release" "kyverno" {
         failureThreshold    = 4
         successThreshold    = 1
       }
+
+      # Spread replicas across nodes for HA
+      topologySpreadConstraints = [
+        {
+          maxSkew           = 1
+          topologyKey       = "kubernetes.io/hostname"
+          whenUnsatisfiable = "DoNotSchedule"
+          labelSelector = {
+            matchLabels = {
+              "app.kubernetes.io/component" = "admission-controller"
+              "app.kubernetes.io/instance"  = "kyverno"
+            }
+          }
+        }
+      ]
     }
   })]
 }
 
 # To unlabel all:
 # kubectl label deployment,statefulset,daemonset --all-namespaces -l tier tier-
+#
+# Uses namespaceSelector to match tiers — no API call needed.
+# One rule per tier so Kyverno resolves the tier value from its informer cache.
 resource "kubernetes_manifest" "mutate_tier_from_namespace" {
   manifest = {
     apiVersion = "kyverno.io/v1"
@@ -67,49 +87,41 @@ resource "kubernetes_manifest" "mutate_tier_from_namespace" {
       name = "sync-tier-label-from-namespace"
     }
     spec = {
-      rules = [
-        {
-          name = "lookup-and-add-tier"
-          match = {
-            any = [
-              {
-                resources = {
-                  kinds = ["Deployment", "StatefulSet", "DaemonSet"]
-                }
-              }
-            ]
-          }
-          exclude = {
-            any = [
-              {
-                resources = {
-                  namespaces = ["kube-system", "metallb-system", "n8n"]
-                }
-              }
-            ]
-          }
-          # Context allows us to perform an API call to get Namespace metadata
-          context = [
+      rules = [for tier in local.governance_tiers : {
+        name = "sync-tier-${tier}"
+        match = {
+          any = [
             {
-              name = "namespaceLabel"
-              apiCall = {
-                urlPath  = "/api/v1/namespaces/{{request.namespace}}"
-                jmesPath = "metadata.labels.tier || 'default'"
+              resources = {
+                kinds = ["Deployment", "StatefulSet", "DaemonSet"]
+                namespaceSelector = {
+                  matchLabels = {
+                    tier = tier
+                  }
+                }
               }
             }
           ]
-          mutate = {
-            patchStrategicMerge = {
-              metadata = {
-                labels = {
-                  # Injects the variable discovered in the context above
-                  "+(tier)" = "{{namespaceLabel}}"
-                }
+        }
+        exclude = {
+          any = [
+            {
+              resources = {
+                namespaces = ["kube-system", "metallb-system", "n8n"]
+              }
+            }
+          ]
+        }
+        mutate = {
+          patchStrategicMerge = {
+            metadata = {
+              labels = {
+                "+(tier)" = tier
               }
             }
           }
         }
-      ]
+      }]
     }
   }
 }
diff --git a/stacks/platform/modules/kyverno/resource-governance.tf b/stacks/platform/modules/kyverno/resource-governance.tf
index 403b64e1..5aaabf17 100644
--- a/stacks/platform/modules/kyverno/resource-governance.tf
+++ b/stacks/platform/modules/kyverno/resource-governance.tf
@@ -8,6 +8,11 @@
 # 3. ResourceQuotas (Kyverno generate) - hard ceiling on namespace resource consumption
 # 4. Priority injection (Kyverno mutate) - set priorityClassName based on namespace tier label
 
+locals {
+  governance_tiers    = ["0-core", "1-cluster", "2-gpu", "3-edge", "4-aux"]
+  excluded_namespaces = ["kube-system", "metallb-system", "kyverno", "calico-system", "calico-apiserver"]
+}
+
 # -----------------------------------------------------------------------------
 # Layer 1: PriorityClasses
 # -----------------------------------------------------------------------------
@@ -661,6 +666,7 @@ resource "kubernetes_manifest" "generate_resourcequota_by_tier" {
 # -----------------------------------------------------------------------------
 # Automatically sets priorityClassName on Pods based on their namespace's tier label.
 # Skips pods that already have a priorityClassName set.
+# Uses namespaceSelector instead of API calls — no round-trip to the API server.
 
 resource "kubernetes_manifest" "mutate_priority_from_tier" {
   manifest = {
@@ -674,69 +680,58 @@ resource "kubernetes_manifest" "mutate_priority_from_tier" {
       }
     }
     spec = {
-      rules = [
-        {
-          name = "inject-priority-class"
-          match = {
-            any = [
-              {
-                resources = {
-                  kinds = ["Pod"]
-                }
-              }
-            ]
-          }
-          exclude = {
-            any = [
-              {
-                resources = {
-                  namespaces = ["kube-system", "metallb-system", "kyverno", "calico-system", "calico-apiserver"]
-                }
-              }
-            ]
-          }
-          context = [
+      rules = [for tier in local.governance_tiers : {
+        name = "inject-priority-${tier}"
+        match = {
+          any = [
             {
-              name = "tierLabel"
-              apiCall = {
-                urlPath  = "/api/v1/namespaces/{{request.namespace}}"
-                jmesPath = "metadata.labels.tier || ''"
+              resources = {
+                kinds = ["Pod"]
+                namespaceSelector = {
+                  matchLabels = {
+                    tier = tier
+                  }
+                }
               }
             }
           ]
-          preconditions = {
-            all = [
-              {
-                key      = "{{request.object.spec.priorityClassName || ''}}"
-                operator = "Equals"
-                value    = ""
-              },
-              {
-                key      = "{{tierLabel}}"
-                operator = "NotEquals"
-                value    = ""
-              }
-            ]
-          }
-          mutate = {
-            patchesJson6902 = yamlencode([
-              {
-                op   = "remove"
-                path = "/spec/priority"
-              },
-              {
-                op   = "remove"
-                path = "/spec/preemptionPolicy"
-              },
-              {
-                op    = "add"
-                path  = "/spec/priorityClassName"
-                value = "tier-{{tierLabel}}"
-              }
-            ])
-          }
         }
-      ]
+        exclude = {
+          any = [
+            {
+              resources = {
+                namespaces = local.excluded_namespaces
+              }
+            }
+          ]
+        }
+        preconditions = {
+          all = [
+            {
+              key      = "{{request.object.spec.priorityClassName || ''}}"
+              operator = "Equals"
+              value    = ""
+            }
+          ]
+        }
+        mutate = {
+          patchesJson6902 = yamlencode([
+            {
+              op   = "remove"
+              path = "/spec/priority"
+            },
+            {
+              op   = "remove"
+              path = "/spec/preemptionPolicy"
+            },
+            {
+              op    = "add"
+              path  = "/spec/priorityClassName"
+              value = "tier-${tier}"
+            }
+          ])
+        }
+      }]
     }
   }
 }