From 5b5b855528177a470f5e8c7daa3fa97083749287 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Fri, 5 Jun 2026 17:44:47 +0000 Subject: [PATCH] monitoring(alloy): drop goflow2 + vpa logs from Loki to cut sdc write wear goflow2 emits ~8 GB/day of per-flow NetFlow JSON to stdout (~64% of all cluster log volume) but only its Prometheus aggregate metrics are used; vpa is ~1.3 GB/day of Goldilocks/VPA recommender chatter. Both are low-value and were landing in Loki (PVC on the contended sdc HDD). Drop them at the Alloy relabel. Reversible (remove the drop rule). Loki ingestion drops ~73%. Co-Authored-By: Claude Opus 4.8 --- stacks/monitoring/modules/monitoring/alloy.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/stacks/monitoring/modules/monitoring/alloy.yaml b/stacks/monitoring/modules/monitoring/alloy.yaml index a938a29a..d2f9602d 100644 --- a/stacks/monitoring/modules/monitoring/alloy.yaml +++ b/stacks/monitoring/modules/monitoring/alloy.yaml @@ -37,6 +37,19 @@ alloy: discovery.relabel "pod_logs" { targets = discovery.kubernetes.pod.targets + // Drop high-volume, low-value producers from Loki to cut sdc write wear + // (the log PVC is on the contended sdc HDD). goflow2 emits one JSON line + // per NetFlow record to stdout (~8 GB/day, ~64% of all cluster logs) but + // we only use its Prometheus aggregate metrics, not the per-flow logs; + // vpa = Goldilocks/VPA recommender chatter (~1.3 GB/day). Both reversible + // — remove this rule to ship them again. (Added 2026-06-05.) + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_name"] + separator = "/" + regex = "monitoring/goflow2-.*|vpa/.*" + action = "drop" + } + // Label creation - "namespace" field from "__meta_kubernetes_namespace" rule { source_labels = ["__meta_kubernetes_namespace"]