diff --git a/stacks/monitoring/modules/monitoring/prometheus_chart_values.tpl b/stacks/monitoring/modules/monitoring/prometheus_chart_values.tpl index ce35bfb7..a86c832f 100755 --- a/stacks/monitoring/modules/monitoring/prometheus_chart_values.tpl +++ b/stacks/monitoring/modules/monitoring/prometheus_chart_values.tpl @@ -2840,15 +2840,22 @@ serverFiles: annotations: summary: "MAM ratio is {{ $value | printf \"%.2f\" }} for 24h (target: >= 1.0)" - alert: MAMFarmingStuck + # Heartbeat-based: fires only when the grabber CronJob has not COMPLETED + # a run in >4h (the original failure mode: Forbid-blocked / wedged in + # ContainerCreating). The grabber heartbeats mam_grabber_last_run_timestamp + # on every completed run — including legit dry runs that grab 0 (its random + # search offset lands on an empty/over-filtered page, which is normal). The + # old increase(mam_farming_grabbed[4h])==0 could not tell "didn't run" from + # "ran, nothing to grab" (Pushgateway serves the last value forever), so a + # dry freeleech period false-fired. Cookie-expiry and qBittorrent-down have + # their own alerts (MAM session cookie / QBittorrentDisconnected). expr: | - increase(mam_farming_grabbed[4h]) == 0 - and mam_farming_total_seeding < 150 - and mam_ratio >= 1.2 - for: 4h + time() - mam_grabber_last_run_timestamp > 4 * 3600 + for: 15m labels: severity: warning annotations: - summary: "Grabber has added 0 torrents in 4h despite healthy ratio ({{ $value | printf \"%.2f\" }})" + summary: "MAM freeleech grabber has not completed a run in {{ $value | humanizeDuration }} — CronJob stuck/blocked" - alert: MAMJanitorStuckBacklog expr: mam_janitor_skipped_active > 400 for: 6h diff --git a/stacks/servarr/mam-farming/files/freeleech-grabber.py b/stacks/servarr/mam-farming/files/freeleech-grabber.py index 3c7064c7..160a160e 100644 --- a/stacks/servarr/mam-farming/files/freeleech-grabber.py +++ b/stacks/servarr/mam-farming/files/freeleech-grabber.py @@ -134,6 +134,7 @@ def main(): profile_metrics + f'mam_grabber_skipped_reason{{reason="{reason}"}} 1\n' + f"mam_farming_grabbed 0\n" + + f"mam_grabber_last_run_timestamp {int(time.time())}\n" ) return @@ -153,7 +154,11 @@ def main(): ).json() except Exception as e: print(f"qBittorrent unreachable: {e}", file=sys.stderr) - push(profile_metrics + "mam_farming_grabbed 0\n") + push( + profile_metrics + + "mam_farming_grabbed 0\n" + + f"mam_grabber_last_run_timestamp {int(time.time())}\n" + ) sys.exit(1) farming = [t for t in all_torrents if t.get("category") == "mam-farming"] @@ -264,6 +269,7 @@ def main(): + f"mam_farming_grabbed {grabbed}\n" + f"mam_farming_total_seeding {len(farming) + grabbed}\n" + f"mam_farming_size_bytes {total_size}\n" + + f"mam_grabber_last_run_timestamp {int(time.time())}\n" ) push(metrics) print(f"Done: grabbed={grabbed}")