From 9b4970da613b9382316a9c7f69aba8c615454085 Mon Sep 17 00:00:00 2001
From: Viktor Barzin <vbarzin@gmail.com>
Date: Tue, 21 Apr 2026 22:29:15 +0000
Subject: [PATCH] =?UTF-8?q?monitoring:=20alert=20hygiene=20=E2=80=94=20dis?=
 =?UTF-8?q?ambiguate,=20rename,=20tune,=20fix=20inhibits?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- HighPowerUsage: add subsystem:gpu (line 724) + subsystem:r730 (line 775)
  labels so the two same-named alerts are distinguishable in routing.
- HeadscaleDown (deployment-replicas flavor, line 1414) → rename to
  HeadscaleReplicasMismatch. Line 2039 keeps HeadscaleDown as the real
  up-metric critical check. NodeDown inhibit rule updated to suppress
  the renamed alert too.
- EmailRoundtripStale (line 1816): for 10m → 20m. Survives one missed
  20-min probe cycle before firing, cuts flapping (12 short-burst fires
  over last 24h).

ATSOverload tuning skipped: 24h fire-count is 0, it's continuously
firing not flapping — already-known sustained 83% ATS load, tuning
would not change behavior.

8 backup *NeverSucceeded rules audited: all 7 using
kube_cronjob_status_last_successful_time target real K8s CronJobs with
active metrics (not Pushgateway-sourced). PrometheusBackupNeverRun
already uses absent() correctly. No fixes needed.
---
 .../modules/monitoring/prometheus_chart_values.tpl        | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/stacks/monitoring/modules/monitoring/prometheus_chart_values.tpl b/stacks/monitoring/modules/monitoring/prometheus_chart_values.tpl
index 8142727f..87903ff6 100755
--- a/stacks/monitoring/modules/monitoring/prometheus_chart_values.tpl
+++ b/stacks/monitoring/modules/monitoring/prometheus_chart_values.tpl
@@ -73,7 +73,7 @@ alertmanager:
       - source_matchers:
           - alertname = NodeDown
         target_matchers:
-          - alertname =~ "NodeNotReady|NodeConditionBad|PodCrashLooping|ContainerOOMKilled|DeploymentReplicasMismatch|StatefulSetReplicasMismatch|DaemonSetMissingPods|ScrapeTargetDown|NodeLowFreeMemory|PostgreSQLDown|RedisDown|HeadscaleDown|AuthentikDown|PoisonFountainDown|HackmdDown|PrivatebinDown|MailServerDown|EmailRoundtripFailing|EmailRoundtripStale|NodeExporterDown|DockerRegistryDown|HomeAssistantDown|CloudflaredDown|TechnitiumDNSDown|iDRACRedfishMetricsMissing|iDRACSNMPMetricsMissing|HomeAssistantMetricsMissing"
+          - alertname =~ "NodeNotReady|NodeConditionBad|PodCrashLooping|ContainerOOMKilled|DeploymentReplicasMismatch|StatefulSetReplicasMismatch|DaemonSetMissingPods|ScrapeTargetDown|NodeLowFreeMemory|PostgreSQLDown|RedisDown|HeadscaleDown|HeadscaleReplicasMismatch|AuthentikDown|PoisonFountainDown|HackmdDown|PrivatebinDown|MailServerDown|EmailRoundtripFailing|EmailRoundtripStale|NodeExporterDown|DockerRegistryDown|HomeAssistantDown|CloudflaredDown|TechnitiumDNSDown|iDRACRedfishMetricsMissing|iDRACSNMPMetricsMissing|HomeAssistantMetricsMissing"
       # NFS down causes mass pod failures and NFS-dependent service outages
       - source_matchers:
           - alertname = NFSServerUnresponsive
@@ -726,6 +726,7 @@ serverFiles:
             for: 30m
             labels:
               severity: info
+              subsystem: gpu
             annotations:
               summary: "GPU power: {{ $value | printf \"%.0f\" }}W (threshold: 50W)"
           - alert: HighUtilization
@@ -777,6 +778,7 @@ serverFiles:
             for: 60m
             labels:
               severity: info
+              subsystem: r730
             annotations:
               summary: "Server power: {{ $value | printf \"%.0f\" }}W (threshold: 300W)"
           - alert: UsingInverterEnergyForTooLong
@@ -1411,7 +1413,7 @@ serverFiles:
               severity: warning
             annotations:
               summary: "Redis master {{ $labels.pod }} has only {{ $value }} connected replicas (expected 2)"
-          - alert: HeadscaleDown
+          - alert: HeadscaleReplicasMismatch
             expr: (kube_deployment_status_replicas_available{namespace="headscale"} or on() vector(0)) < 1
             for: 5m
             labels:
@@ -1815,7 +1817,7 @@ serverFiles:
               summary: "Email round-trip probe failing. Check MX DNS, Postfix, Mailgun API, and IMAP."
           - alert: EmailRoundtripStale
             expr: (time() - email_roundtrip_last_success_timestamp{job="email-roundtrip-monitor"}) > 3600
-            for: 10m
+            for: 20m
             labels:
               severity: warning
             annotations: