fix: restore tree dropped by 6d224861; land stem95su gdrive-sync (10m) [ci skip]
6d224861 came from a --no-checkout worktree whose empty index made the
commit drop every file except two. This restores 05b50d2b's full tree and
correctly adds stacks/stem95su/gdrive-sync.tf + the service-catalog stem95su
entry. Forward-only (parent=6d224861, no force-push); [ci skip] since the
live infra was never applied from the broken commit.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>This commit is contained in:
Viktor Barzin
parent
6d224861c4
commit
fd0f4a0365
1166 changed files with 358546 additions and 0 deletions
163
stacks/servarr/mam-farming/files/bp-spender.py
Normal file
163
stacks/servarr/mam-farming/files/bp-spender.py
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
"""
|
||||
MAM bonus-point spender — tier-aware, pay-what-we-owe.
|
||||
|
||||
MAM's bonusBuy.php API enforces a hard 50 GiB minimum per purchase
|
||||
("Automated spenders are limited to buying at least 50 GB... due to log
|
||||
spam"). Valid API tiers are 50, 100, 200, 500 GiB (@ 500 BP/GiB). That
|
||||
means the "pay exactly what we owe" approach from the recovery plan
|
||||
rounds UP to 50 GiB for the first purchase — small buys can only be done
|
||||
via the web UI, not the API.
|
||||
|
||||
Logic: pick the smallest valid tier that both (a) satisfies the ratio
|
||||
deficit and (b) we can afford without burning the BP reserve. Skip if
|
||||
nothing fits; the cron will retry in 6 h once BP grows.
|
||||
"""
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
||||
PUSHGW = "http://prometheus-prometheus-pushgateway.monitoring:9091"
|
||||
COOKIE_FILE = "/data/mam_id"
|
||||
|
||||
TARGET_RATIO = float(os.environ.get("TARGET_RATIO", "2.0"))
|
||||
RESERVE_BP = int(os.environ.get("RESERVE_BP", "500"))
|
||||
BP_PER_GB = int(os.environ.get("BP_PER_GB", "500"))
|
||||
# MAM-enforced minimum purchase for API callers: 50 GiB.
|
||||
API_TIERS_GIB = (50, 100, 200, 500)
|
||||
|
||||
CLASS_CODES = {
|
||||
"Mouse": 0,
|
||||
"Vole": 1,
|
||||
"User": 2,
|
||||
"Power User": 3,
|
||||
"Elite": 4,
|
||||
"Torrent Master": 5,
|
||||
"Power TM": 6,
|
||||
"Elite TM": 7,
|
||||
"VIP": 8,
|
||||
}
|
||||
|
||||
|
||||
def save_cookie(resp):
|
||||
for c in resp.cookies:
|
||||
if c.name == "mam_id":
|
||||
fd, tmp = tempfile.mkstemp(dir="/data")
|
||||
os.write(fd, c.value.encode())
|
||||
os.close(fd)
|
||||
os.rename(tmp, COOKIE_FILE)
|
||||
return
|
||||
|
||||
|
||||
def push(metrics):
|
||||
try:
|
||||
requests.post(
|
||||
f"{PUSHGW}/metrics/job/mam-bp-spender", data=metrics, timeout=10
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"pushgateway error: {e}", file=sys.stderr)
|
||||
|
||||
|
||||
def load_cookie():
|
||||
if os.path.exists(COOKIE_FILE):
|
||||
return open(COOKIE_FILE).read().strip()
|
||||
return os.environ.get("MAM_ID", "")
|
||||
|
||||
|
||||
def main():
|
||||
mam_id = load_cookie()
|
||||
if not mam_id:
|
||||
print("No mam_id available", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
s = requests.Session()
|
||||
s.cookies.set("mam_id", mam_id, domain=".myanonamouse.net")
|
||||
|
||||
r = s.get("https://www.myanonamouse.net/jsonLoad.php", timeout=15)
|
||||
if r.status_code != 200:
|
||||
push("mam_farming_cookie_expired 1\n")
|
||||
print(f"Cookie expired: {r.status_code}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
save_cookie(r)
|
||||
|
||||
profile = r.json()
|
||||
ratio = float(profile.get("ratio", 0) or 0)
|
||||
classname = profile.get("classname", "Mouse")
|
||||
class_code = CLASS_CODES.get(classname, 0)
|
||||
# MAM returns `downloaded`/`uploaded` as pretty strings ("715.55 MiB");
|
||||
# `*_bytes` are the authoritative integer fields.
|
||||
downloaded = int(profile.get("downloaded_bytes", 0) or 0)
|
||||
uploaded = int(profile.get("uploaded_bytes", 0) or 0)
|
||||
bp = int(float(profile.get("seedbonus", 0) or 0))
|
||||
|
||||
deficit_bytes = max(0, int(downloaded * TARGET_RATIO) - uploaded)
|
||||
needed_gib = math.ceil(deficit_bytes / (1024**3)) + 1 if deficit_bytes > 0 else 0
|
||||
affordable_gib = max(0, (bp - RESERVE_BP) // BP_PER_GB)
|
||||
|
||||
# Pick the smallest API tier that satisfies the deficit AND fits the
|
||||
# budget. If even the smallest tier is too expensive, skip — the cron
|
||||
# will retry in 6 h once BP has grown.
|
||||
buy_gib = 0
|
||||
for tier in API_TIERS_GIB:
|
||||
if tier >= needed_gib and tier <= affordable_gib:
|
||||
buy_gib = tier
|
||||
break
|
||||
if buy_gib == 0 and needed_gib > 0 and affordable_gib >= API_TIERS_GIB[0]:
|
||||
# Deficit exceeds all tiers we can afford — buy the largest
|
||||
# tier that fits to make progress.
|
||||
for tier in reversed(API_TIERS_GIB):
|
||||
if tier <= affordable_gib:
|
||||
buy_gib = tier
|
||||
break
|
||||
|
||||
print(
|
||||
f"Profile: ratio={ratio} class={classname} "
|
||||
f"DL={downloaded / 1024**3:.2f} GiB UL={uploaded / 1024**3:.2f} GiB "
|
||||
f"BP={bp} | deficit={deficit_bytes / 1024**3:.2f} GiB "
|
||||
f"needed={needed_gib} affordable={affordable_gib} buy={buy_gib}"
|
||||
)
|
||||
|
||||
spent_gib = 0
|
||||
if buy_gib >= API_TIERS_GIB[0]:
|
||||
time.sleep(3)
|
||||
url = (
|
||||
"https://www.myanonamouse.net/json/bonusBuy.php"
|
||||
f"?spendtype=upload&amount={buy_gib}"
|
||||
)
|
||||
r2 = s.get(url, timeout=15)
|
||||
save_cookie(r2)
|
||||
try:
|
||||
body = r2.json()
|
||||
except ValueError:
|
||||
body = {}
|
||||
ok = r2.status_code == 200 and body.get("success") is True
|
||||
print(
|
||||
f"Buy {buy_gib} GiB -> {r2.status_code} "
|
||||
f"success={body.get('success')} {r2.text[:160]}"
|
||||
)
|
||||
if ok:
|
||||
spent_gib = buy_gib
|
||||
|
||||
metrics = (
|
||||
"mam_farming_cookie_expired 0\n"
|
||||
f"mam_ratio {ratio}\n"
|
||||
f'mam_class_code{{classname="{classname}"}} {class_code}\n'
|
||||
f"mam_downloaded_bytes {downloaded}\n"
|
||||
f"mam_uploaded_bytes {uploaded}\n"
|
||||
f"mam_bp_balance {bp}\n"
|
||||
f"mam_bp_spent_gb {spent_gib}\n"
|
||||
f"mam_bp_needed_gib {needed_gib}\n"
|
||||
f"mam_bp_affordable_gib {affordable_gib}\n"
|
||||
)
|
||||
push(metrics)
|
||||
print(
|
||||
f"Done: BP={bp}, spent={spent_gib} GiB (needed={needed_gib}, "
|
||||
f"affordable={affordable_gib})"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
273
stacks/servarr/mam-farming/files/freeleech-grabber.py
Normal file
273
stacks/servarr/mam-farming/files/freeleech-grabber.py
Normal file
|
|
@ -0,0 +1,273 @@
|
|||
"""
|
||||
MAM freeleech grabber — demand-first, ratio-guarded.
|
||||
|
||||
Selects small-but-popular freeleech titles to grow the account's upload
|
||||
credit. Refuses to grab while the account is in Mouse class or ratio is
|
||||
below 1.2, because MAM rejects peer-list announces under those conditions
|
||||
and new grabs only deepen the ratio hole.
|
||||
|
||||
Cleanup is handled by `mam-farming-janitor.py`, which runs unconditionally.
|
||||
"""
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
||||
QB_URL = "http://qbittorrent.servarr.svc.cluster.local"
|
||||
PUSHGW = "http://prometheus-prometheus-pushgateway.monitoring:9091"
|
||||
COOKIE_FILE = "/data/mam_id"
|
||||
GRABBED_IDS_FILE = "/data/grabbed_ids.txt"
|
||||
|
||||
MIN_MB = int(os.environ.get("MIN_MB", "50"))
|
||||
MAX_MB = int(os.environ.get("MAX_MB", "1024"))
|
||||
LEECHER_FLOOR = int(os.environ.get("LEECHER_FLOOR", "1"))
|
||||
# MAM's catalogue is well-seeded by design — a ceiling of 50 rejected ~99%
|
||||
# of candidates in live testing. 200 still filters out truly oversupplied
|
||||
# swarms while keeping enough working-set to grab 3-5 titles per run.
|
||||
SEEDER_CEILING = int(os.environ.get("SEEDER_CEILING", "200"))
|
||||
GRAB_PER_RUN = int(os.environ.get("GRAB_PER_RUN", "5"))
|
||||
MAX_TORRENTS = int(os.environ.get("MAX_TORRENTS", "500"))
|
||||
# The guard's real job is to prevent the Mouse-class death spiral (see RC1
|
||||
# in the original recovery plan). Once class > Mouse, MAM serves peer
|
||||
# lists normally and demand-first filtering (leechers>=1) keeps new grabs
|
||||
# upload-positive. Keep a low floor as a tripwire for catastrophic dips
|
||||
# rather than a steady-state block.
|
||||
RATIO_FLOOR = float(os.environ.get("RATIO_FLOOR", "0.5"))
|
||||
REQUEST_SLEEP = float(os.environ.get("REQUEST_SLEEP", "3"))
|
||||
|
||||
CLASS_CODES = {
|
||||
"Mouse": 0,
|
||||
"Vole": 1,
|
||||
"User": 2,
|
||||
"Power User": 3,
|
||||
"Elite": 4,
|
||||
"Torrent Master": 5,
|
||||
"Power TM": 6,
|
||||
"Elite TM": 7,
|
||||
"VIP": 8,
|
||||
}
|
||||
|
||||
|
||||
def parse_size(s):
|
||||
# MAM pretty-prints sizes with thousands separators (e.g. "1,002.9 MiB").
|
||||
units = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3, "TiB": 1024**4}
|
||||
parts = s.replace(",", "").split()
|
||||
if len(parts) != 2:
|
||||
return 0
|
||||
return int(float(parts[0]) * units.get(parts[1], 1))
|
||||
|
||||
|
||||
def save_cookie(resp):
|
||||
for c in resp.cookies:
|
||||
if c.name == "mam_id":
|
||||
fd, tmp = tempfile.mkstemp(dir="/data")
|
||||
os.write(fd, c.value.encode())
|
||||
os.close(fd)
|
||||
os.rename(tmp, COOKIE_FILE)
|
||||
return
|
||||
|
||||
|
||||
def push(metrics):
|
||||
try:
|
||||
requests.post(
|
||||
f"{PUSHGW}/metrics/job/mam-freeleech-grabber", data=metrics, timeout=10
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"pushgateway error: {e}", file=sys.stderr)
|
||||
|
||||
|
||||
def load_cookie():
|
||||
if os.path.exists(COOKIE_FILE):
|
||||
return open(COOKIE_FILE).read().strip()
|
||||
return os.environ.get("MAM_ID", "")
|
||||
|
||||
|
||||
def exit_cookie_expired(status):
|
||||
push("mam_farming_cookie_expired 1\n")
|
||||
print(f"Cookie expired: {status}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def main():
|
||||
mam_id = load_cookie()
|
||||
if not mam_id:
|
||||
print("No mam_id available", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
s = requests.Session()
|
||||
s.cookies.set("mam_id", mam_id, domain=".myanonamouse.net")
|
||||
|
||||
r = s.get("https://www.myanonamouse.net/jsonLoad.php", timeout=15)
|
||||
if r.status_code != 200:
|
||||
exit_cookie_expired(r.status_code)
|
||||
save_cookie(r)
|
||||
|
||||
profile = r.json()
|
||||
ratio = float(profile.get("ratio", 0) or 0)
|
||||
classname = profile.get("classname", "Mouse")
|
||||
# `*_bytes` are authoritative integers; `downloaded`/`uploaded` are
|
||||
# pretty strings like "715.55 MiB".
|
||||
downloaded = int(profile.get("downloaded_bytes", 0) or 0)
|
||||
uploaded = int(profile.get("uploaded_bytes", 0) or 0)
|
||||
class_code = CLASS_CODES.get(classname, 0)
|
||||
|
||||
profile_metrics = (
|
||||
f"mam_farming_cookie_expired 0\n"
|
||||
f"mam_ratio {ratio}\n"
|
||||
f'mam_class_code{{classname="{classname}"}} {class_code}\n'
|
||||
f"mam_downloaded_bytes {downloaded}\n"
|
||||
f"mam_uploaded_bytes {uploaded}\n"
|
||||
)
|
||||
|
||||
if ratio < RATIO_FLOOR or classname == "Mouse":
|
||||
reason = "mouse_class" if classname == "Mouse" else "low_ratio"
|
||||
print(
|
||||
f"Skip grab: ratio={ratio} class={classname} (floor={RATIO_FLOOR}) "
|
||||
f"reason={reason}"
|
||||
)
|
||||
push(
|
||||
profile_metrics
|
||||
+ f'mam_grabber_skipped_reason{{reason="{reason}"}} 1\n'
|
||||
+ f"mam_farming_grabbed 0\n"
|
||||
)
|
||||
return
|
||||
|
||||
time.sleep(REQUEST_SLEEP)
|
||||
r = s.get("https://t.myanonamouse.net/json/dynamicSeedbox.php", timeout=15)
|
||||
save_cookie(r)
|
||||
print(f"Seedbox: {r.text[:80]}")
|
||||
|
||||
grabbed_ids = set()
|
||||
if os.path.exists(GRABBED_IDS_FILE):
|
||||
raw = open(GRABBED_IDS_FILE).read().strip()
|
||||
grabbed_ids = set(raw.split("\n")) if raw else set()
|
||||
|
||||
try:
|
||||
all_torrents = requests.get(
|
||||
f"{QB_URL}/api/v2/torrents/info", timeout=10
|
||||
).json()
|
||||
except Exception as e:
|
||||
print(f"qBittorrent unreachable: {e}", file=sys.stderr)
|
||||
push(profile_metrics + "mam_farming_grabbed 0\n")
|
||||
sys.exit(1)
|
||||
|
||||
farming = [t for t in all_torrents if t.get("category") == "mam-farming"]
|
||||
all_names_lower = {t["name"].lower() for t in all_torrents}
|
||||
total_size = sum(t.get("size", 0) for t in farming)
|
||||
|
||||
print(
|
||||
f"Profile: ratio={ratio} class={classname} | "
|
||||
f"Farming: {len(farming)}, {total_size / (1024**3):.1f} GiB, "
|
||||
f"tracked IDs: {len(grabbed_ids)}"
|
||||
)
|
||||
|
||||
grabbed = 0
|
||||
if len(farming) >= MAX_TORRENTS:
|
||||
print(f"At max torrents ({MAX_TORRENTS}), skipping grab")
|
||||
else:
|
||||
time.sleep(REQUEST_SLEEP)
|
||||
offset = random.randint(0, 1400)
|
||||
params = {
|
||||
"tor[searchType]": "fl",
|
||||
"tor[searchIn]": "torrents",
|
||||
"tor[perpage]": "50",
|
||||
"tor[startNumber]": str(offset),
|
||||
}
|
||||
r = s.get(
|
||||
"https://www.myanonamouse.net/tor/js/loadSearchJSONbasic.php",
|
||||
params=params,
|
||||
timeout=15,
|
||||
)
|
||||
save_cookie(r)
|
||||
data = r.json()
|
||||
results = data.get("data", []) or []
|
||||
print(
|
||||
f"Search offset={offset}, found={data.get('found', 0)}, "
|
||||
f"page_results={len(results)}"
|
||||
)
|
||||
|
||||
candidates = []
|
||||
for t in results:
|
||||
tid = str(t.get("id", ""))
|
||||
if tid in grabbed_ids:
|
||||
continue
|
||||
title = t.get("title", "")
|
||||
if any(title.lower() in n for n in all_names_lower):
|
||||
grabbed_ids.add(tid)
|
||||
continue
|
||||
size = parse_size(t.get("size", "0 B"))
|
||||
if size < MIN_MB * 1024**2 or size > MAX_MB * 1024**2:
|
||||
continue
|
||||
seeders = int(t.get("seeders", 999) or 999)
|
||||
leechers = int(t.get("leechers", 0) or 0)
|
||||
if leechers < LEECHER_FLOOR:
|
||||
continue
|
||||
if seeders > SEEDER_CEILING:
|
||||
continue
|
||||
wedge_bonus = (
|
||||
200 if (t.get("free") == 1 or t.get("personal_freeleech") == 1) else 0
|
||||
)
|
||||
score = leechers * 3 - seeders * 0.5 + wedge_bonus
|
||||
candidates.append((score, t))
|
||||
|
||||
candidates.sort(key=lambda x: -x[0])
|
||||
|
||||
for score, t in candidates[:GRAB_PER_RUN]:
|
||||
time.sleep(REQUEST_SLEEP)
|
||||
tid = t["id"]
|
||||
r = s.get(
|
||||
f"https://www.myanonamouse.net/tor/download.php?tid={tid}", timeout=15
|
||||
)
|
||||
save_cookie(r)
|
||||
if not r.content.startswith(b"d"):
|
||||
print(f"Bad torrent body for tid={tid}")
|
||||
grabbed_ids.add(str(tid))
|
||||
continue
|
||||
add_resp = requests.post(
|
||||
f"{QB_URL}/api/v2/torrents/add",
|
||||
files={
|
||||
"torrents": (
|
||||
f"{tid}.torrent",
|
||||
r.content,
|
||||
"application/x-bittorrent",
|
||||
)
|
||||
},
|
||||
data={
|
||||
"savepath": "/downloads/mam-farming",
|
||||
"category": "mam-farming",
|
||||
"tags": "mam,freeleech",
|
||||
},
|
||||
timeout=20,
|
||||
)
|
||||
ok = add_resp.status_code == 200 and add_resp.text.strip() != "Fails."
|
||||
print(
|
||||
f"{'Added' if ok else 'FAILED'} (score={score:.1f}): "
|
||||
f"{t['title'][:60]} ({t['size']}, S:{t.get('seeders')} "
|
||||
f"L:{t.get('leechers')}) -> {add_resp.status_code}"
|
||||
)
|
||||
grabbed_ids.add(str(tid))
|
||||
if ok:
|
||||
grabbed += 1
|
||||
|
||||
fd, tmp = tempfile.mkstemp(dir="/data")
|
||||
os.write(fd, "\n".join(grabbed_ids).encode())
|
||||
os.close(fd)
|
||||
os.rename(tmp, GRABBED_IDS_FILE)
|
||||
|
||||
metrics = (
|
||||
profile_metrics
|
||||
+ f"mam_farming_grabbed {grabbed}\n"
|
||||
+ f"mam_farming_total_seeding {len(farming) + grabbed}\n"
|
||||
+ f"mam_farming_size_bytes {total_size}\n"
|
||||
)
|
||||
push(metrics)
|
||||
print(f"Done: grabbed={grabbed}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
177
stacks/servarr/mam-farming/files/mam-farming-janitor.py
Normal file
177
stacks/servarr/mam-farming/files/mam-farming-janitor.py
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
"""
|
||||
MAM farming janitor — H&R-aware cleanup.
|
||||
|
||||
Runs every 15 minutes independently of the grabber's ratio guard: stuck
|
||||
torrents accumulate fastest precisely when the grabber is skipping. Never
|
||||
deletes a torrent that's inside MAM's 72-hour Hit-and-Run window.
|
||||
|
||||
Set DRY_RUN=1 to log candidates without deleting (used for the first
|
||||
24 hours after rollout to sanity-check the rules against live state).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
||||
QB_URL = "http://qbittorrent.servarr.svc.cluster.local"
|
||||
PUSHGW = "http://prometheus-prometheus-pushgateway.monitoring:9091"
|
||||
|
||||
DRY_RUN = os.environ.get("DRY_RUN", "0") == "1"
|
||||
HNR_SEED_SECONDS = int(os.environ.get("HNR_SEED_SECONDS", str(72 * 3600)))
|
||||
NEVER_STARTED_AGE = int(os.environ.get("NEVER_STARTED_AGE", str(24 * 3600)))
|
||||
STALLED_AGE = int(os.environ.get("STALLED_AGE", str(3 * 86400)))
|
||||
SATISFIED_SEED_AGE = int(os.environ.get("SATISFIED_SEED_AGE", str(3 * 86400)))
|
||||
SATISFIED_SEEDER_FLOOR = int(os.environ.get("SATISFIED_SEEDER_FLOOR", "5"))
|
||||
GRACEFUL_SEED_AGE = int(os.environ.get("GRACEFUL_SEED_AGE", str(14 * 86400)))
|
||||
ZERO_DEMAND_AGE = int(os.environ.get("ZERO_DEMAND_AGE", str(7 * 86400)))
|
||||
UNREG_KEYWORDS = ("unregistered", "torrent not found", "info hash not authorized")
|
||||
|
||||
REASONS = (
|
||||
"never_started",
|
||||
"stalled_old",
|
||||
"satisfied_redundant",
|
||||
"graceful_retire",
|
||||
"zero_demand",
|
||||
"unregistered",
|
||||
)
|
||||
|
||||
|
||||
def classify(t, now, tracker_msg):
|
||||
age = now - int(t.get("added_on", 0) or 0)
|
||||
progress = float(t.get("progress", 0) or 0)
|
||||
downloaded = int(t.get("downloaded", 0) or 0)
|
||||
uploaded = int(t.get("uploaded", 0) or 0)
|
||||
seed_time = int(t.get("seeding_time", 0) or 0)
|
||||
state = t.get("state", "")
|
||||
num_complete = int(t.get("num_complete", 0) or 0)
|
||||
|
||||
if tracker_msg and any(k in tracker_msg.lower() for k in UNREG_KEYWORDS):
|
||||
return "unregistered"
|
||||
|
||||
if progress < 1.0:
|
||||
if age > NEVER_STARTED_AGE and downloaded == 0:
|
||||
return "never_started"
|
||||
if state == "stalledDL" and age > STALLED_AGE:
|
||||
return "stalled_old"
|
||||
return None
|
||||
|
||||
if seed_time < HNR_SEED_SECONDS:
|
||||
return "hnr_window"
|
||||
|
||||
if seed_time > GRACEFUL_SEED_AGE:
|
||||
return "graceful_retire"
|
||||
if (
|
||||
seed_time >= HNR_SEED_SECONDS
|
||||
and uploaded == 0
|
||||
and age > ZERO_DEMAND_AGE
|
||||
):
|
||||
return "zero_demand"
|
||||
if seed_time > SATISFIED_SEED_AGE and num_complete > SATISFIED_SEEDER_FLOOR:
|
||||
return "satisfied_redundant"
|
||||
return None
|
||||
|
||||
|
||||
def fetch_tracker_msg(hash_):
|
||||
try:
|
||||
resp = requests.get(
|
||||
f"{QB_URL}/api/v2/torrents/trackers",
|
||||
params={"hash": hash_},
|
||||
timeout=10,
|
||||
)
|
||||
trackers = resp.json() or []
|
||||
except Exception:
|
||||
return ""
|
||||
for tr in trackers:
|
||||
url = tr.get("url", "")
|
||||
if url.startswith("** ["):
|
||||
continue
|
||||
msg = tr.get("msg", "")
|
||||
if msg:
|
||||
return msg
|
||||
return ""
|
||||
|
||||
|
||||
def push(metrics):
|
||||
try:
|
||||
requests.post(
|
||||
f"{PUSHGW}/metrics/job/mam-farming-janitor", data=metrics, timeout=10
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"pushgateway error: {e}", file=sys.stderr)
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
all_torrents = requests.get(
|
||||
f"{QB_URL}/api/v2/torrents/info", timeout=15
|
||||
).json()
|
||||
except Exception as e:
|
||||
print(f"qBittorrent unreachable: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
farming = [t for t in all_torrents if t.get("category") == "mam-farming"]
|
||||
now = int(time.time())
|
||||
|
||||
deleted = {r: 0 for r in REASONS}
|
||||
preserved_hnr = 0
|
||||
skipped_active = 0
|
||||
delete_hashes = []
|
||||
|
||||
# Only inspect tracker msg on torrents with a peer problem — avoids
|
||||
# hundreds of extra API calls when things are healthy.
|
||||
for t in farming:
|
||||
state = t.get("state", "")
|
||||
progress = float(t.get("progress", 0) or 0)
|
||||
tracker_msg = ""
|
||||
if progress < 1.0 and state in ("stalledDL", "metaDL", "missingFiles"):
|
||||
tracker_msg = fetch_tracker_msg(t["hash"])
|
||||
|
||||
verdict = classify(t, now, tracker_msg)
|
||||
if verdict is None:
|
||||
skipped_active += 1
|
||||
elif verdict == "hnr_window":
|
||||
preserved_hnr += 1
|
||||
else:
|
||||
deleted[verdict] += 1
|
||||
delete_hashes.append((t["hash"], verdict, t.get("name", "")[:60]))
|
||||
|
||||
for hash_, reason, name in delete_hashes:
|
||||
if DRY_RUN:
|
||||
print(f"[DRY_RUN] would delete ({reason}): {name}")
|
||||
continue
|
||||
try:
|
||||
requests.post(
|
||||
f"{QB_URL}/api/v2/torrents/delete",
|
||||
data={"hashes": hash_, "deleteFiles": "true"},
|
||||
timeout=20,
|
||||
)
|
||||
print(f"Deleted ({reason}): {name}")
|
||||
except Exception as e:
|
||||
print(f"Delete failed for {name}: {e}", file=sys.stderr)
|
||||
|
||||
for reason in REASONS:
|
||||
push(
|
||||
f'mam_janitor_deleted_per_run{{reason="{reason}"}} '
|
||||
f"{deleted[reason] if not DRY_RUN else 0}\n"
|
||||
f'mam_janitor_dry_run_candidates{{reason="{reason}"}} '
|
||||
f"{deleted[reason] if DRY_RUN else 0}\n"
|
||||
)
|
||||
push(
|
||||
f"mam_janitor_preserved_hnr {preserved_hnr}\n"
|
||||
f"mam_janitor_skipped_active {skipped_active}\n"
|
||||
f"mam_janitor_dry_run {1 if DRY_RUN else 0}\n"
|
||||
f"mam_janitor_last_run_timestamp {now}\n"
|
||||
)
|
||||
|
||||
total = sum(deleted.values())
|
||||
print(
|
||||
f"Done: deleted={total} preserved_hnr={preserved_hnr} "
|
||||
f"skipped_active={skipped_active} dry_run={DRY_RUN}"
|
||||
)
|
||||
print(f" per reason: {deleted}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
270
stacks/servarr/mam-farming/main.tf
Normal file
270
stacks/servarr/mam-farming/main.tf
Normal file
|
|
@ -0,0 +1,270 @@
|
|||
variable "namespace" {
|
||||
type = string
|
||||
default = "servarr"
|
||||
}
|
||||
|
||||
locals {
|
||||
python_image = "docker.io/library/python:3.12-alpine"
|
||||
pip_prefix = "pip install -q requests > /dev/null 2>&1; python3 /tmp/script.py"
|
||||
|
||||
# Dry-run window was satisfied by a one-shot test on 2026-04-19 that
|
||||
# produced 466 `never_started` candidates and 0 matches in any other
|
||||
# reason bucket — consistent with Phase B's expected 495 stuck torrents.
|
||||
# Enforcing from here on.
|
||||
janitor_dry_run = "0"
|
||||
}
|
||||
|
||||
# ----------------------------- NFS data volume -----------------------
|
||||
# Migrated off proxmox-lvm (2026-06-04): the cookie + grabbed-ID dedup list
|
||||
# are two plain-text files (no embedded DB), so NFS is safe and removes this
|
||||
# volume from the per-VM SCSI-LUN hotplug path entirely — a stuck `query-pci`
|
||||
# on a disk-heavy node VM used to wedge the grabber in ContainerCreating (the
|
||||
# disk never enumerated, Forbid blocked every run → MAMFarmingStuck). NFS
|
||||
# mounts over the network, consumes zero LUN slots, and is RWX so the grabber
|
||||
# and bp-spender can co-schedule on any node. See docs/architecture/storage.md
|
||||
# "Per-VM SCSI-LUN cap" lever #1.
|
||||
module "mam_data_nfs" {
|
||||
source = "../../../modules/kubernetes/nfs_volume"
|
||||
name = "servarr-mam-farming-data"
|
||||
namespace = var.namespace
|
||||
nfs_server = "192.168.1.127"
|
||||
nfs_path = "/srv/nfs/servarr/mam-farming"
|
||||
storage = "1Gi"
|
||||
}
|
||||
|
||||
# --------------------------- Grabber ---------------------------------
|
||||
# Every 30 minutes: skip while ratio < 1.2 or class == Mouse; otherwise
|
||||
# grab up to 5 small-but-popular freeleech torrents. Existing ConfigMap
|
||||
# + CronJob are adopted via imports in the parent stack.
|
||||
|
||||
resource "kubernetes_config_map" "grabber_script" {
|
||||
metadata {
|
||||
name = "mam-freeleech-grabber-script"
|
||||
namespace = var.namespace
|
||||
}
|
||||
data = {
|
||||
"script.py" = file("${path.module}/files/freeleech-grabber.py")
|
||||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_cron_job_v1" "grabber" {
|
||||
metadata {
|
||||
name = "mam-freeleech-grabber"
|
||||
namespace = var.namespace
|
||||
}
|
||||
spec {
|
||||
schedule = "*/30 * * * *"
|
||||
concurrency_policy = "Forbid"
|
||||
successful_jobs_history_limit = 3
|
||||
failed_jobs_history_limit = 3
|
||||
job_template {
|
||||
metadata {}
|
||||
spec {
|
||||
backoff_limit = 2
|
||||
ttl_seconds_after_finished = 300
|
||||
template {
|
||||
metadata {}
|
||||
spec {
|
||||
restart_policy = "Never"
|
||||
container {
|
||||
name = "freeleech-grabber"
|
||||
image = local.python_image
|
||||
command = ["/bin/sh", "-c", local.pip_prefix]
|
||||
env {
|
||||
name = "MAM_ID"
|
||||
value_from {
|
||||
secret_key_ref {
|
||||
name = "servarr-secrets"
|
||||
key = "mam_id"
|
||||
}
|
||||
}
|
||||
}
|
||||
resources {
|
||||
requests = { memory = "64Mi", cpu = "10m" }
|
||||
limits = { memory = "128Mi" }
|
||||
}
|
||||
volume_mount {
|
||||
name = "script"
|
||||
mount_path = "/tmp/script.py"
|
||||
sub_path = "script.py"
|
||||
}
|
||||
volume_mount {
|
||||
name = "data"
|
||||
mount_path = "/data"
|
||||
}
|
||||
}
|
||||
volume {
|
||||
name = "script"
|
||||
config_map {
|
||||
name = kubernetes_config_map.grabber_script.metadata[0].name
|
||||
}
|
||||
}
|
||||
volume {
|
||||
name = "data"
|
||||
persistent_volume_claim {
|
||||
claim_name = module.mam_data_nfs.claim_name
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
lifecycle {
|
||||
# KYVERNO_LIFECYCLE_V1: Kyverno admission webhook mutates dns_config with ndots=2
|
||||
ignore_changes = [spec[0].job_template[0].spec[0].template[0].spec[0].dns_config]
|
||||
}
|
||||
}
|
||||
|
||||
# --------------------------- BP Spender ------------------------------
|
||||
# Every 6 hours: compute the upload deficit against TARGET_RATIO and buy
|
||||
# exactly what we need (+1 GiB margin), capped by BP reserve. Existing
|
||||
# ConfigMap + CronJob are adopted via imports in the parent stack.
|
||||
|
||||
resource "kubernetes_config_map" "bp_spender_script" {
|
||||
metadata {
|
||||
name = "mam-bp-spender-script"
|
||||
namespace = var.namespace
|
||||
}
|
||||
data = {
|
||||
"script.py" = file("${path.module}/files/bp-spender.py")
|
||||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_cron_job_v1" "bp_spender" {
|
||||
metadata {
|
||||
name = "mam-bp-spender"
|
||||
namespace = var.namespace
|
||||
}
|
||||
spec {
|
||||
schedule = "0 */6 * * *"
|
||||
concurrency_policy = "Forbid"
|
||||
successful_jobs_history_limit = 3
|
||||
failed_jobs_history_limit = 3
|
||||
job_template {
|
||||
metadata {}
|
||||
spec {
|
||||
backoff_limit = 2
|
||||
ttl_seconds_after_finished = 300
|
||||
template {
|
||||
metadata {}
|
||||
spec {
|
||||
restart_policy = "Never"
|
||||
container {
|
||||
name = "bp-spender"
|
||||
image = local.python_image
|
||||
command = ["/bin/sh", "-c", local.pip_prefix]
|
||||
env {
|
||||
name = "MAM_ID"
|
||||
value_from {
|
||||
secret_key_ref {
|
||||
name = "servarr-secrets"
|
||||
key = "mam_id"
|
||||
}
|
||||
}
|
||||
}
|
||||
resources {
|
||||
requests = { memory = "64Mi", cpu = "10m" }
|
||||
limits = { memory = "128Mi" }
|
||||
}
|
||||
volume_mount {
|
||||
name = "script"
|
||||
mount_path = "/tmp/script.py"
|
||||
sub_path = "script.py"
|
||||
}
|
||||
volume_mount {
|
||||
name = "data"
|
||||
mount_path = "/data"
|
||||
}
|
||||
}
|
||||
volume {
|
||||
name = "script"
|
||||
config_map {
|
||||
name = kubernetes_config_map.bp_spender_script.metadata[0].name
|
||||
}
|
||||
}
|
||||
volume {
|
||||
name = "data"
|
||||
persistent_volume_claim {
|
||||
claim_name = module.mam_data_nfs.claim_name
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
lifecycle {
|
||||
# KYVERNO_LIFECYCLE_V1: Kyverno admission webhook mutates dns_config with ndots=2
|
||||
ignore_changes = [spec[0].job_template[0].spec[0].template[0].spec[0].dns_config]
|
||||
}
|
||||
}
|
||||
|
||||
# ----------------------------- Janitor -------------------------------
|
||||
# New: every 15 minutes, independent of grabber ratio guard. Deletes
|
||||
# stuck/unregistered/redundant torrents in category=mam-farming while
|
||||
# preserving torrents inside the 72h H&R window.
|
||||
|
||||
resource "kubernetes_config_map" "janitor_script" {
|
||||
metadata {
|
||||
name = "mam-farming-janitor-script"
|
||||
namespace = var.namespace
|
||||
}
|
||||
data = {
|
||||
"script.py" = file("${path.module}/files/mam-farming-janitor.py")
|
||||
}
|
||||
}
|
||||
|
||||
resource "kubernetes_cron_job_v1" "janitor" {
|
||||
metadata {
|
||||
name = "mam-farming-janitor"
|
||||
namespace = var.namespace
|
||||
}
|
||||
spec {
|
||||
schedule = "*/15 * * * *"
|
||||
concurrency_policy = "Forbid"
|
||||
successful_jobs_history_limit = 3
|
||||
failed_jobs_history_limit = 3
|
||||
job_template {
|
||||
metadata {}
|
||||
spec {
|
||||
backoff_limit = 2
|
||||
ttl_seconds_after_finished = 300
|
||||
template {
|
||||
metadata {}
|
||||
spec {
|
||||
restart_policy = "Never"
|
||||
container {
|
||||
name = "farming-janitor"
|
||||
image = local.python_image
|
||||
command = ["/bin/sh", "-c", local.pip_prefix]
|
||||
env {
|
||||
name = "DRY_RUN"
|
||||
value = local.janitor_dry_run
|
||||
}
|
||||
resources {
|
||||
requests = { memory = "64Mi", cpu = "10m" }
|
||||
limits = { memory = "128Mi" }
|
||||
}
|
||||
volume_mount {
|
||||
name = "script"
|
||||
mount_path = "/tmp/script.py"
|
||||
sub_path = "script.py"
|
||||
}
|
||||
}
|
||||
volume {
|
||||
name = "script"
|
||||
config_map {
|
||||
name = kubernetes_config_map.janitor_script.metadata[0].name
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
lifecycle {
|
||||
# KYVERNO_LIFECYCLE_V1: Kyverno admission webhook mutates dns_config with ndots=2
|
||||
ignore_changes = [spec[0].job_template[0].spec[0].template[0].spec[0].dns_config]
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue