rybbit: sync excludes CAPI blocklist + fix CF items per_page (500)
The edge CF IP List can't hold the ~31k CAPI community blocklist (already enforced in-kernel by the firewall-bouncer), so the sync now skips origin=CAPI and carries only high-signal local/curated decisions (+ a 9000 safety cap). Also fixes the list-items GET: per_page=1000 returned a misleading CF 400 'invalid or expired cursor' (10027); the endpoint max is 500. Verified live: crowdsec_ban populates (4 IPs) and the sync exits 0. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
6d5d3726d6
commit
f55bb6c422
1 changed files with 22 additions and 1 deletions
|
|
@ -138,9 +138,18 @@ def fetch_decisions():
|
||||||
headers={"X-Api-Key": LAPI_KEY, "Accept": "application/json"},
|
headers={"X-Api-Key": LAPI_KEY, "Accept": "application/json"},
|
||||||
)
|
)
|
||||||
block = set()
|
block = set()
|
||||||
|
skipped_capi = 0
|
||||||
for d in data or []:
|
for d in data or []:
|
||||||
if (d.get("scope") or "").lower() != "ip":
|
if (d.get("scope") or "").lower() != "ip":
|
||||||
continue
|
continue
|
||||||
|
# EXCLUDE the CAPI community blocklist: ~31k IPs, far over a CF IP
|
||||||
|
# List's capacity, and ALREADY enforced in-kernel for direct hosts by
|
||||||
|
# the cs-firewall-bouncer DaemonSet. The edge list carries only our
|
||||||
|
# HIGH-SIGNAL local + curated decisions (own scenarios, cscli-import,
|
||||||
|
# subscribed lists).
|
||||||
|
if (d.get("origin") or "").upper() == "CAPI":
|
||||||
|
skipped_capi += 1
|
||||||
|
continue
|
||||||
ip = d.get("value")
|
ip = d.get("value")
|
||||||
if not ip:
|
if not ip:
|
||||||
continue
|
continue
|
||||||
|
|
@ -149,6 +158,16 @@ def fetch_decisions():
|
||||||
block.add(ip)
|
block.add(ip)
|
||||||
# captcha / throttle / other remediation types are ignored at the edge
|
# captcha / throttle / other remediation types are ignored at the edge
|
||||||
# (ban-only enforcement — see the docstring above)
|
# (ban-only enforcement — see the docstring above)
|
||||||
|
if skipped_capi:
|
||||||
|
print(f"[info] excluded {skipped_capi} CAPI decisions (enforced at L3 by "
|
||||||
|
f"the firewall-bouncer; too many for a CF list)")
|
||||||
|
# Safety cap: a CF IP List can't hold unbounded entries. Never push more
|
||||||
|
# than this — keep a bounded, deterministic subset and warn.
|
||||||
|
MAX_ITEMS = 9000
|
||||||
|
if len(block) > MAX_ITEMS:
|
||||||
|
print(f"[warn] desired {len(block)} exceeds {MAX_ITEMS} cap; truncating "
|
||||||
|
f"(consider a CF plan with a higher list limit)", file=sys.stderr)
|
||||||
|
block = set(sorted(block)[:MAX_ITEMS])
|
||||||
return block
|
return block
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -160,7 +179,9 @@ def cf_list_items(list_id):
|
||||||
out = {}
|
out = {}
|
||||||
cursor = ""
|
cursor = ""
|
||||||
while True:
|
while True:
|
||||||
url = f"{CF_API}/accounts/{CF_ACCOUNT_ID}/rules/lists/{list_id}/items?per_page=1000"
|
# per_page max for the list-items endpoint is 500; 1000 returns a
|
||||||
|
# misleading HTTP 400 "invalid or expired cursor" (CF error 10027).
|
||||||
|
url = f"{CF_API}/accounts/{CF_ACCOUNT_ID}/rules/lists/{list_id}/items?per_page=500"
|
||||||
if cursor:
|
if cursor:
|
||||||
url += f"&cursor={urllib.parse.quote(cursor)}"
|
url += f"&cursor={urllib.parse.quote(cursor)}"
|
||||||
res = _cf(url)
|
res = _cf(url)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue