From a73f3fcb6b17deb0d58b4ea0331697e672e23c70 Mon Sep 17 00:00:00 2001
From: Viktor Barzin <viktorbarzin@meta.com>
Date: Sun, 15 Feb 2026 17:20:47 +0000
Subject: [PATCH] Cluster health remediation: cleanup CronJob, disable
 Collabora, fix GPU probe, add NFS exports [ci skip]

- Add daily CronJob to auto-clean Failed/Evicted pods cluster-wide (infra-maintenance)
- Disable Collabora in Nextcloud (broken HPA caused scaling storm; using OnlyOffice instead)
- Increase gpu-pod-exporter liveness probe timeout from 1s to 5s
- Add osm-routing NFS exports (osrm-data, otp-data)
---
 modules/kubernetes/infra-maintenance/main.tf  |  69 ++++++++++++++++++
 .../kubernetes/nextcloud/chart_values.yaml    |   6 +-
 modules/kubernetes/nvidia/main.tf             |   1 +
 secrets/nfs_directories.txt                   | Bin 1623 -> 1666 bytes
 4 files changed, 71 insertions(+), 5 deletions(-)

diff --git a/modules/kubernetes/infra-maintenance/main.tf b/modules/kubernetes/infra-maintenance/main.tf
index 4625ba92..27a92a96 100644
--- a/modules/kubernetes/infra-maintenance/main.tf
+++ b/modules/kubernetes/infra-maintenance/main.tf
@@ -141,3 +141,72 @@ resource "kubernetes_cron_job_v1" "backup-etcd" {
     }
   }
 }
+
+# Clean up evicted/failed pods cluster-wide daily
+resource "kubernetes_cron_job_v1" "cleanup-failed-pods" {
+  metadata {
+    name      = "cleanup-failed-pods"
+    namespace = "default"
+  }
+  spec {
+    schedule                      = "0 2 * * *"
+    successful_jobs_history_limit = 1
+    failed_jobs_history_limit     = 1
+    concurrency_policy            = "Forbid"
+    job_template {
+      metadata {
+        name = "cleanup-failed-pods"
+      }
+      spec {
+        template {
+          metadata {
+            name = "cleanup-failed-pods"
+          }
+          spec {
+            service_account_name = kubernetes_service_account.cleanup_sa.metadata[0].name
+            container {
+              name    = "cleanup"
+              image   = "bitnami/kubectl:latest"
+              command = ["/bin/sh", "-c", "kubectl delete pods -A --field-selector=status.phase=Failed --ignore-not-found"]
+            }
+            restart_policy = "Never"
+          }
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_service_account" "cleanup_sa" {
+  metadata {
+    name      = "failed-pod-cleanup"
+    namespace = "default"
+  }
+}
+
+resource "kubernetes_cluster_role" "cleanup_role" {
+  metadata {
+    name = "failed-pod-cleanup"
+  }
+  rule {
+    api_groups = [""]
+    resources  = ["pods"]
+    verbs      = ["list", "delete"]
+  }
+}
+
+resource "kubernetes_cluster_role_binding" "cleanup_binding" {
+  metadata {
+    name = "failed-pod-cleanup"
+  }
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "ClusterRole"
+    name      = kubernetes_cluster_role.cleanup_role.metadata[0].name
+  }
+  subject {
+    kind      = "ServiceAccount"
+    name      = kubernetes_service_account.cleanup_sa.metadata[0].name
+    namespace = "default"
+  }
+}
diff --git a/modules/kubernetes/nextcloud/chart_values.yaml b/modules/kubernetes/nextcloud/chart_values.yaml
index 861eaf1a..cda04812 100644
--- a/modules/kubernetes/nextcloud/chart_values.yaml
+++ b/modules/kubernetes/nextcloud/chart_values.yaml
@@ -59,11 +59,7 @@ podAnnotations:
   diun.include_tags: "^[0-9]+(?:.[0-9]+)?(?:.[0-9]+)?.*"
 
 collabora:
-  enabled: true # Currently the app is disabled as using onlyoffice instead
-
-  autoscaling:
-    # enable autocaling, please check collabora README.md first
-    enabled: true
+  enabled: false # Using onlyoffice instead
 
 cronjob:
   enabled: true
diff --git a/modules/kubernetes/nvidia/main.tf b/modules/kubernetes/nvidia/main.tf
index 8f46839c..b9356cc2 100644
--- a/modules/kubernetes/nvidia/main.tf
+++ b/modules/kubernetes/nvidia/main.tf
@@ -605,6 +605,7 @@ resource "kubernetes_daemonset" "gpu_pod_exporter" {
             }
             initial_delay_seconds = 30
             period_seconds        = 30
+            timeout_seconds       = 5
           }
         }
 
diff --git a/secrets/nfs_directories.txt b/secrets/nfs_directories.txt
index d99f3aacdafb397a9b299cf35fcff9e30844636e..893c5531d55ed4f3bdcd94e63d07e6ea7e33ff26 100644
GIT binary patch
literal 1666
zcmV-|27UPeM@dveQdv+`06#9nxdyQIC5u<%Qsy?4rMO+<+`K(cmXA*YR9F1F0+n=5
zpcpEJm2AI#x{<ykUaxNqu5Q^kBl`Xmpe)Vib@C**ds^^ztb<y9-)#<R+W_H$b8mRK
zF{m$l$M;Z{@RmQ){%;0&>kn$QRlu9BzuKzZrL*q_yAeY`_Erg5oetcA-=Ocqn^`R@
zGRg4L%T4;09NfuK^=u$dMxsWI-&*F*v&RMVB#Z75L+U~4T@H~`AAtoLOPUp;FUt}o
z0avvKcP-aw6o1v?wRUt6%6s&EOBib?_0+xVktH7_o^`R2utO!1(a6NazQ!&jL|Gag
zNx-|XXF;60_L+Ld=$%wHfAmWQ{O5Q{9G%kX+tjQd<`Xt2```Ns3)$bHGKr9niwmk~
zr(5*rEQdaDcseKwQE|We{+2c1T0rJ~88SKw6EF3%9)L^z%fK=8V)|?Ia%N=Z=E645
z7@{P0JLrg$u3pIO1ZPDa@p594$M{q7u3zgAlo!;*1B~Zf(SzHw+skD2#ko#Ef@or~
zAmc65_I)4}$czE2_k%DlZR^$R)j-b+mz($jy#G4R@fu0%snBn8DWVDyUdXW7!A&AN
zc4W2sPPA0-YoF_Bs<?a)Uu8y6+g1);{jScZNOKDmW|{E(($w69lQfQ#+JUFQpg(4(
zKPV5^Wg3IaC{nkW=#*>5H5Lwg^$p;CHT~)|n@B;|p#%}lZl6^Y#qq@pJmEJbR+`ug
zil8haEC!j`rCgJrk)!Vz<Bz0X?Tz}oVHH<Z)hsPCv2|b;w`ZmDxsCi+i=9=B`B6SI
z!k=R?W2AuPrZ|%U>lsxT-ytMUQ%q_-4gy*dN{;*S9miZWxyt%%f3W;ddmHl+w2J{N
zPGh(|NUu@g&J{;9Q&R8KjB`YA;S%>?%Yd*mtue6`k#5TbE|X4!Hdk{N^wI3Tx{Pi%
z33}bOKCn%6?Tp3KnaFc4Bh#0VNTHYTZoBbasTWcl`F1_gYvi|%Hd59hk7*CS!|1YW
zJ+10uDmL85CC6$8oO6$H!X7tbq5-@7@EB&ZiMtCiD!_|UEXN4Fmjzn2>T*=Fr1_w%
zhg_8TS|56rp|%Bc48Fx!^-=z9`;TT@T?>|D13#S=E>|bwnFiD8)?)YKUWd6(?0=9K
zkoK&m*l74SJHIdy#@!fSCV2*r&@ih9<JidSpy1wBP%v{`-hD=99Advob@gAgDddvx
zTaND+=rf6e2E99~MDo^NKG6Xq_o@PbCR3fG9Z2}vpw*bOxJQdujqDYK)#}$n$Oqn%
zBCUtm`d6m|;Y*C;eb@pwE7`|)-5s_%8}`O~1GJQp92UZbn}vZd9PaHzXk|pTsJbRs
ztgq;<Q*G?>e9{-DVD}ii2&wI09lygCkG;y^BII5XNU5cB`~VTH$K<JP7cniOUC>*P
z<fi;IC0d$OxP+*62lm3ry|&*Hlpsq`$D(de1%XOg7V=u#uQ)8DkXz@#H`2H6<Ecxn
zgKn6Mre+QU6MZ9&`uyoU2`cdDA4trOc5NY*B)8{&6?GXCEGb`mryb#)v!hZU&t?*0
zVsKSAST*h;VgGd8zMKu?Y87;LLlEE$hCz^?<Vx^$DKJ-35<YFynDO6!nd%cYEu!*7
zx#*^L_quAr-i!(lz@S}>7WNJ}2j>mk$ykKd_v$&84^roOfz}X%fFHw*RJK;Nfi;RG
zP|xEeVdw&?QjoKKZ%!&KwDcjQexg^vqO~7CIDD`TsV_dDB$y8siB$-E4JqwbXd}v{
zSO~w<cNEkkCW9fiRC(=(#@*u2Cf~Hq9tiLsXN8=nV!kU6M0R>;-w7P*>8gJP$`bQ(
z3oBG=7znPlym0@UpK%|JCY3^g)51aQFV#wYer|?z0{+Ho2lUU{$bRd(%;A^6<uXgX
zH3=a=SNam?QXt=?*z2s6y6nVWYG)s6wfG|aQ2~Q>X|87_QXE=rEJ2Vkkrm88%TCfq
zhRKE=$h~GkIE&+@8rHS?qL<2CxTZNry{dvyUb&h)6|M?lj-+D|-fH^NhEnY33=>_v
zTh5_Q%;Uy~#1mE_pb+Md?yEZlsJ_Cei-(uyDHL+A4Jr7NzmK>XRwCnx?g!RCv{bla
z+)rQHtAb>=E_jN9lWDxQD<7F-7_y(`{BB|)Q9uo#*K&^4n@eJn7$Zj&aLBdU##WuX
M=$6rmpUh=$3iI<u0{{R3

literal 1623
zcmV-d2B`S}M@dveQdv+`0AH;WV;xtw7t+}OT{YaiK`!^}<RYCiDK>Nagh3OIMNvdv
zrM&71bfq{7u~-Bw5^}wu@!T5PnLiPC(EYS2@P+|4Hz1YI_wOO#Dm}cJP;w?%{ipvq
z{d-}67ZwAkJvg0(19U>UaXJsJryPp7-YVe4#(^B2oooPdNiN-SFof5o6p0}^8&Y6m
zfCpTjo#(9%c{5-;?bWIgCpArLnqM&x=R>LP>+r$?BJFPE=z;<Tu4J})@ul4Ywp+>q
zh!F`OAnSh%A}~fe<_f1$TsSi<^nNUxGNF1!9?6<#oz)Rg6!>}j;rrkW+F#bnk>bEV
z#E%39?r?r<5-(~+(9eCxKTJ@<28rC3`BwR4_`&?v_!|(2u6{l{<2LX|$J8Md5i*Wq
zKOzvf^lwuaKa3bpL*~&djr#7q*cQVn+QE+Yu_h^FQR$ny+>B*^X*7hK5qZXMf*XFt
z)Bnt_dwYocSgH7*JGyM@IHgBr+?q3X?8jV9IkYj5rSx48;LC8&hcobc!^Y58!9VdN
znpt+s)uoQk^jsjG@83<n^YG*d90#-=FfiJPHJ~tY@L7f;?&k^XUc6?P<O@A=b|Jg^
z>?AxuBiRJ1tVgU)+RKGAg7e%(S1*?Gah=a8cnJ~2C4SaXoje{m_^K}YkWAft4zw-U
z6P3T2P=3bXvQ(SUiz`$oBv@v6T)rtQV*1x&o@Y>Fu$hysvq&>n|2FYS79(eF69pup
z2)M<}{};MtVQ3ybnz$=(nuD7!_TlgP!7M2jRr@43UY+XG3-@jKyt*aQ!b&CCNh;nc
z<Zp-Ti92N${#1(X)AL13#p4Q^=gTkmlbe<gz`ZDyzki>BXQzFMcBctK)V^R~!p5E9
zA;5G|k$s|$9OXQMy|JG~RogUL81y5T`rPDnpEjIM$yOZs;k!X4wYu(cFUP!hg=O&!
z$jM@xlN%*C;*i9f0@1_6=w-eQKqowvIPQdBcs9wDHYt~mync)fa9j6LM+GABkqVh2
zN75jzDzl1NKYh-hh(V_XFFuW0?r%If*em;-C<UYu(UXSRWnsf20usKxjNl4JFFf4+
z$!J#Eb9WbTMeN>82*Ufs(LZJoZN4Ggx!J<3d=`X<Wv#;=Y)0`%WS^oRGo;Ev+HDdB
zk#rcDzX?EISC*7Kc8;k$U;H7Wu89$~oiN_TXW}e#U~E?;fxAi#iW`nFp5DOszNTx(
z$;7QL`2{ybs&VZaRTALU-4YaA7|{7cswEW+U<!G!YXLe>(BYi}jFx>k`EXkWeeb@3
z3b?jS(lsZv+b?f?P@d&Ep6InV|7U5urU-fH62w}MrUI(!V&i8eOi?&hzt5Cril|!w
zdZO|xCMRRcute_NxzfIf#@WhO-S`ei=@>&h=fjEMdkI2<#u{Cv-jaJU4%%EYgUTmF
z<t!*&a?Mk8*=2nL?c=Ra^Zr1HxBU%f9iefv_RHRlsT<Tb?#`5*xy*I$fI8me48Y3B
zD4l<WdP2gNTpgh?9F7O@-zo20giEE(2gEd!Ypm1^0Uew>d)5wn<WC{Kd8v~x^?)?*
z;m(YHgvN)B-`W5NOF?4!j6;&M7hPJW(uktCCp9YwlyHI<u`HH#fr5ZC_<~`~YY^Jf
zcLtrICA{nT%QkO9VR272YyGE|WIr9oFgsN2!QC~ht_xz&6tJpKftI0n`!9c=^xrP@
zzAOw-<3Ve;l&WpzVr@NI{1b{BqK$ZYaFN&mSyDOmRc`5$A*zqqJ{K!tb0Re;iJjTM
z#elCrWkG*g{IH>aYyvKR`f(Y|BRbX$GTv<zUHO8@45Zg`K!_BP((i7#y%hu4r{aD%
zPGCN&7sOw)s~wY&j+_Rdg%nIeMjlsPBkC5di47v4!>+!N&@6-KH#j7UhBd@OOyzT?
zj1c<1qH|}_e27ke<?>7NdjnHvqgoD5jisc`!k5@}TVXB<Sgzs*GBiDWZH*{{=i9PH
za^o4ieO4jT4=`>uh^@J<B`%hSy}Ey6PK#qwd@T;^lc-2jfW_yf(ZWSeKoVPy;%3!3
zB{=l1S9E(e_byryi{b%;=xv`vK8iNQ)&uuabjQ1UFV<6`avzq7E&3YfHA>cCTUYaW
VVQV?B_3XpZ5bIcYR6?SDV6XPYAS?g?