From e823b795f71b522c6e410697e10f90cc1dd9eeae Mon Sep 17 00:00:00 2001
From: Viktor Barzin <viktorbarzin@meta.com>
Date: Thu, 19 Mar 2026 23:26:05 +0000
Subject: [PATCH] fix(dbaas,vault): fix backup CronJob failures and
 mysql-operator memory
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Add docker.io/library/ prefix to mysql and postgres backup images
  to satisfy Kyverno require-trusted-registries policy (both CronJobs
  were blocked for 46h, triggering MySQLBackupStale alert)
- Document mysql-operator chart ignoring resources values key — the
  LimitRange default (256Mi) was silently applied, putting the operator
  at 97% memory. Patched live to 512Mi via kubectl.
- Increase vault-raft-backup backoff_limit to 6 for transient failures
  (also fixed NFS export: vault-backup was a separate ZFS dataset not
  in the TrueNAS NFS share — destroyed dataset, created directory)
---
 stacks/dbaas/modules/dbaas/main.tf | 13 +++++++++----
 stacks/vault/main.tf               |  1 +
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/stacks/dbaas/modules/dbaas/main.tf b/stacks/dbaas/modules/dbaas/main.tf
index a5ee3c46..69c1793e 100644
--- a/stacks/dbaas/modules/dbaas/main.tf
+++ b/stacks/dbaas/modules/dbaas/main.tf
@@ -74,14 +74,19 @@ resource "helm_release" "mysql_operator" {
   chart      = "mysql-operator"
   version    = "2.2.7"
 
+  # NOTE: The mysql-operator chart (2.2.7) does NOT expose a resources values key.
+  # The resources block below is ignored by the chart. Without explicit resources
+  # on the deployment, the LimitRange default (256Mi) applies silently.
+  # Fix: kubectl patch deployment mysql-operator -n mysql-operator --type=json \
+  #   -p='[{"op":"replace","path":"/spec/template/spec/containers/0/resources","value":{"requests":{"cpu":"100m","memory":"256Mi"},"limits":{"memory":"512Mi"}}}]'
   values = [yamlencode({
     resources = {
       requests = {
         cpu    = "100m"
-        memory = "512Mi"
+        memory = "256Mi"
       }
       limits = {
-        memory = "580Mi"
+        memory = "512Mi"
       }
     }
   })]
@@ -323,7 +328,7 @@ resource "kubernetes_cron_job_v1" "mysql-backup" {
           spec {
             container {
               name  = "mysql-backup"
-              image = "mysql"
+              image = "docker.io/library/mysql:8.0"
               env {
                 name = "MYSQL_PWD"
                 value_from {
@@ -1059,7 +1064,7 @@ resource "kubernetes_cron_job_v1" "postgresql-backup" {
           spec {
             container {
               name  = "postgresql-backup"
-              image = "postgres:16.4-bullseye"
+              image = "docker.io/library/postgres:16.4-bullseye"
               env {
                 name = "PGPASSWORD"
                 value_from {
diff --git a/stacks/vault/main.tf b/stacks/vault/main.tf
index 8d527e52..3f893643 100644
--- a/stacks/vault/main.tf
+++ b/stacks/vault/main.tf
@@ -256,6 +256,7 @@ resource "kubernetes_cron_job_v1" "vault_backup" {
     job_template {
       metadata {}
       spec {
+        backoff_limit = 6
         template {
           metadata {}
           spec {