From 51d77369de0a8b45ca7c2ae2880f1157fbcacf78 Mon Sep 17 00:00:00 2001
From: Viktor Barzin <viktorbarzin@meta.com>
Date: Mon, 2 Mar 2026 20:23:36 +0000
Subject: [PATCH] [ci skip] fix: add mount_options to all NFS PVs
 (soft,timeo=30,retrans=3)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Critical fix: StorageClass mountOptions only apply during dynamic
provisioning. Our static PVs (created by Terraform) were missing
mount_options, so all NFS mounts defaulted to hard,timeo=600 —
the exact stale mount behavior we were trying to eliminate.

Adds mount_options directly to the nfs_volume module PV spec and
to the monitoring PVs (prometheus, loki, alertmanager).

Requires re-applying all stacks to propagate to existing PVs.
---
 stacks/platform/modules/monitoring/grafana.tf    | 6 ++++++
 stacks/platform/modules/monitoring/loki.tf       | 6 ++++++
 stacks/platform/modules/monitoring/prometheus.tf | 6 ++++++
 3 files changed, 18 insertions(+)

diff --git a/stacks/platform/modules/monitoring/grafana.tf b/stacks/platform/modules/monitoring/grafana.tf
index 0212be60..19363222 100644
--- a/stacks/platform/modules/monitoring/grafana.tf
+++ b/stacks/platform/modules/monitoring/grafana.tf
@@ -43,6 +43,12 @@ resource "kubernetes_persistent_volume" "alertmanager_pv" {
         }
       }
     }
+    mount_options = [
+      "soft",
+      "timeo=30",
+      "retrans=3",
+      "actimeo=5",
+    ]
     storage_class_name = "nfs-truenas"
   }
 }
diff --git a/stacks/platform/modules/monitoring/loki.tf b/stacks/platform/modules/monitoring/loki.tf
index 91d5cf9d..384e9c36 100644
--- a/stacks/platform/modules/monitoring/loki.tf
+++ b/stacks/platform/modules/monitoring/loki.tf
@@ -33,6 +33,12 @@ resource "kubernetes_persistent_volume" "loki" {
         }
       }
     }
+    mount_options = [
+      "soft",
+      "timeo=30",
+      "retrans=3",
+      "actimeo=5",
+    ]
     storage_class_name               = "nfs-truenas"
     persistent_volume_reclaim_policy = "Retain"
     volume_mode                      = "Filesystem"
diff --git a/stacks/platform/modules/monitoring/prometheus.tf b/stacks/platform/modules/monitoring/prometheus.tf
index b288b21e..4fbc07f1 100644
--- a/stacks/platform/modules/monitoring/prometheus.tf
+++ b/stacks/platform/modules/monitoring/prometheus.tf
@@ -38,6 +38,12 @@ resource "kubernetes_persistent_volume" "prometheus_server_pvc" {
         }
       }
     }
+    mount_options = [
+      "soft",
+      "timeo=30",
+      "retrans=3",
+      "actimeo=5",
+    ]
     storage_class_name               = "nfs-truenas"
     persistent_volume_reclaim_policy = "Retain"
     volume_mode                      = "Filesystem"