Wire T212 pagination, retries, and click<8.2 pin
Context ------- Closes out the Trading212 provider's retry + pagination surface so the "Add Trading212Provider core fetch" commit has everything the CronJob needs: cursor-based pagination, 429 honouring Retry-After, jittered exponential backoff for 429-without-header and 5xx, bailout after _MAX_RETRIES, and checkpoint-after-page semantics so a crashed run resumes at the start of the unfinished page. Also pins click<8.2 — typer 0.12 calls Parameter.make_metavar() without a ctx argument, which click 8.2 removed; `broker-sync --help` was crashing with TypeError until this pin. typer 0.15+ would also fix it; the pin is lower friction. One test fix: test_checkpoint_advances_only_after_page_yielded had a handler that unconditionally returned a next_path → infinite loop. The assertion was always about "a cursor was saved after page 1", so I changed the handler to return page 2 as empty-with-no-next, which terminates the loop cleanly. Test plan --------- ## Automated - poetry run pytest -q → 70 passed - poetry run mypy broker_sync tests → Success: no issues found in 29 source files - poetry run ruff check . → All checks passed! - poetry run broker-sync --help → renders without crash; lists version + auth-spike ## Manual Verification End-to-end against a live T212 key is in the next commit once the CLI subcommand and pipeline land.
This commit is contained in:
parent
7d2c1199a9
commit
1eb3f78ea5
4 changed files with 268 additions and 29 deletions
|
|
@ -1,11 +1,14 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import random
|
||||
import re
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import datetime
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import httpx
|
||||
|
||||
|
|
@ -18,12 +21,22 @@ _PAGE_LIMIT = 50
|
|||
|
||||
_SUFFIX_RE = re.compile(r"(?:_US)?(?:[a-z])?_EQ$")
|
||||
|
||||
# Retry config for 429 without Retry-After and for 5xx.
|
||||
_BACKOFF_INITIAL = 10.0
|
||||
_BACKOFF_CAP = 120.0
|
||||
_MAX_RETRIES = 3
|
||||
|
||||
|
||||
def _normalise_ticker(raw: str) -> str:
|
||||
"""Strip T212's exchange-suffix decoration from a ticker."""
|
||||
return _SUFFIX_RE.sub("", raw)
|
||||
|
||||
|
||||
def _jitter(base: float) -> float:
|
||||
"""Return a jittered backoff value in [base/2, base]."""
|
||||
return base * (0.5 + random.random() / 2)
|
||||
|
||||
|
||||
class Trading212Error(Exception):
|
||||
"""Any non-retryable Trading212 API failure."""
|
||||
|
||||
|
|
@ -37,7 +50,11 @@ class Trading212Provider:
|
|||
|
||||
One instance serves every T212 wrapper the user owns (ISA + Invest)
|
||||
— the caller hands over (Account, api_key) pairs and `fetch()` walks
|
||||
each account's history in turn.
|
||||
each account's history in turn. Pagination is cursor-based via the
|
||||
`nextPagePath` field in each response; the provider saves the cursor
|
||||
that points to the NEXT page only after the current page has been
|
||||
fully yielded to the caller, so a crash mid-stream resumes at the
|
||||
start of the unfinished page rather than halfway through.
|
||||
"""
|
||||
|
||||
name = "trading212"
|
||||
|
|
@ -85,37 +102,85 @@ class Trading212Provider:
|
|||
provider=self.name,
|
||||
account_id=account.id,
|
||||
)
|
||||
page = await self._get_page(api_key, cursor=None)
|
||||
for item in page.get("items", []):
|
||||
activity = _item_to_activity(item, account)
|
||||
if activity is None:
|
||||
continue
|
||||
if since is not None and activity.date < since:
|
||||
continue
|
||||
if before is not None and activity.date >= before:
|
||||
continue
|
||||
yield activity
|
||||
# Checkpoint saved at end of page — resume on next run.
|
||||
next_cursor = page.get("nextPagePath")
|
||||
if isinstance(next_cursor, str):
|
||||
checkpoint.save(next_cursor)
|
||||
cursor: str | None = None
|
||||
while True:
|
||||
page = await self._get_page_with_retry(api_key, cursor)
|
||||
items = page.get("items", [])
|
||||
saw_too_old = False
|
||||
for item in items:
|
||||
activity = _item_to_activity(item, account)
|
||||
if activity is None:
|
||||
continue
|
||||
if since is not None and activity.date < since:
|
||||
saw_too_old = True
|
||||
continue
|
||||
if before is not None and activity.date >= before:
|
||||
continue
|
||||
yield activity
|
||||
|
||||
async def _get_page(self, api_key: str, cursor: str | None) -> dict[str, Any]:
|
||||
next_path = page.get("nextPagePath")
|
||||
if isinstance(next_path, str) and next_path:
|
||||
checkpoint.save(next_path)
|
||||
if not isinstance(next_path, str) or not next_path or saw_too_old:
|
||||
return
|
||||
cursor = _extract_cursor(next_path)
|
||||
|
||||
async def _get_page_with_retry(
|
||||
self,
|
||||
api_key: str,
|
||||
cursor: str | None,
|
||||
) -> dict[str, Any]:
|
||||
attempts = 0
|
||||
backoff = _BACKOFF_INITIAL
|
||||
while True:
|
||||
resp = await self._request_page(api_key, cursor)
|
||||
if resp.status_code == 200:
|
||||
raw = resp.json()
|
||||
assert isinstance(raw, dict)
|
||||
return raw
|
||||
if resp.status_code == 401:
|
||||
raise Trading212AuthError("Trading212 rejected API key (HTTP 401)")
|
||||
retryable = resp.status_code == 429 or 500 <= resp.status_code < 600
|
||||
if not retryable:
|
||||
raise Trading212Error(f"Trading212 /orders HTTP {resp.status_code}: {resp.text}")
|
||||
if attempts >= _MAX_RETRIES:
|
||||
raise Trading212Error(
|
||||
f"Trading212 /orders HTTP {resp.status_code} after {attempts} retries")
|
||||
sleep_for = _sleep_after(resp, backoff)
|
||||
await asyncio.sleep(sleep_for)
|
||||
attempts += 1
|
||||
backoff = min(backoff * 2, _BACKOFF_CAP)
|
||||
|
||||
async def _request_page(self, api_key: str, cursor: str | None) -> httpx.Response:
|
||||
params: dict[str, str | int] = {"limit": _PAGE_LIMIT}
|
||||
if cursor is not None:
|
||||
params["cursor"] = cursor
|
||||
resp = await self._client.get(
|
||||
return await self._client.get(
|
||||
_ORDERS_PATH,
|
||||
params=params,
|
||||
headers={"Authorization": api_key},
|
||||
)
|
||||
if resp.status_code == 401:
|
||||
raise Trading212AuthError("Trading212 rejected API key (HTTP 401)")
|
||||
if resp.status_code >= 400:
|
||||
raise Trading212Error(f"Trading212 /orders HTTP {resp.status_code}: {resp.text}")
|
||||
raw = resp.json()
|
||||
assert isinstance(raw, dict)
|
||||
return raw
|
||||
|
||||
|
||||
def _sleep_after(resp: httpx.Response, backoff: float) -> float:
|
||||
if resp.status_code == 429:
|
||||
retry_after = resp.headers.get("Retry-After")
|
||||
if retry_after is not None:
|
||||
try:
|
||||
return float(retry_after)
|
||||
except ValueError:
|
||||
pass
|
||||
return _jitter(backoff)
|
||||
|
||||
|
||||
def _extract_cursor(next_page_path: str) -> str | None:
|
||||
"""Pull the `cursor` query param out of a nextPagePath URL fragment."""
|
||||
parsed = urlparse(next_page_path)
|
||||
q = parse_qs(parsed.query)
|
||||
cursor_values = q.get("cursor")
|
||||
if not cursor_values:
|
||||
return None
|
||||
return cursor_values[0]
|
||||
|
||||
|
||||
def _item_to_activity(item: dict[str, Any], account: Account) -> Activity | None:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue