Add sync pipeline + trading212 CLI subcommand
Some checks are pending
CI / test (push) Waiting to run
CI / build (push) Blocked by required conditions
CI / deploy (push) Blocked by required conditions

Context
-------
Closes the gap between "Trading212 provider yields Activities" and
"activities land in Wealthfolio with dedup". One generic pipeline
function works for every provider (Phase 2 IMAP ingest and Phase 3
CSV drop will reuse it).

This change
-----------
- `broker_sync/pipeline.py` — sync_provider_to_wealthfolio():
  ensure accounts exist in Wealthfolio, fetch, dedup against the local
  SQLite store, batch into Wealthfolio's CSV import at 200 rows each,
  record successful imports in the dedup store with the returned
  Wealthfolio activity id. Failed batches don't touch the dedup store
  so the next run retries.
- Notes field stamped with `sync:<provider>:<external_id>` for human
  auditability — NOT used for dedup (the SQLite store owns that).
- `broker_sync/cli.py` — new `trading212` subcommand driven by
  T212_API_KEYS_JSON + WF_* + BROKER_SYNC_DATA_DIR env vars. Two modes:
  `steady` fetches last 7 days; `backfill` pulls all history. Exits 0
  on clean run, 1 if any batch failed, 2 on config errors.
- Pipeline tests with MockTransport: dedup-skip-then-import happy path
  (verifies imported CSV contains only the unseen rows and all three
  are recorded after the run); import-rejected path (verifies the
  failed row is NOT recorded so the next run retries).

Test plan
---------
## Automated
- poetry run pytest -q  →  70 passed
- poetry run mypy broker_sync tests  →  Success: no issues found in 29 source files
- poetry run ruff check .  →  All checks passed!
- poetry run broker-sync trading212 --help  →  shows all env vars + mode flag

## Manual Verification
Live smoke test blocked on:
1. Vault secret/broker-sync seeded (wf_base_url, wf_username, wf_password,
   trading212_api_keys).
2. Terraform stack applied (infra/stacks/broker-sync/ — staged, not yet applied).
3. Image pushed to viktorbarzin/broker-sync on DockerHub via GHA.

Once those land:
    kubectl -n broker-sync create job t212-backfill \
      --from=cronjob/broker-sync-trading212 -- \
      broker-sync trading212 --mode=backfill
This commit is contained in:
Viktor Barzin 2026-04-17 19:45:43 +00:00
parent 1eb3f78ea5
commit 6fc2ac5322
3 changed files with 411 additions and 5 deletions

View file

@ -1,10 +1,19 @@
from __future__ import annotations
import asyncio
import json
import logging
import os
import sys
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import TYPE_CHECKING
import typer
if TYPE_CHECKING:
from broker_sync.models import Account
app = typer.Typer(help="broker-sync: pull brokerage activity into Wealthfolio")
@ -22,9 +31,7 @@ def auth_spike(
wf_password: str = typer.Option(..., envvar="WF_PASSWORD"),
session_path: str = typer.Option("/data/wealthfolio_session.json", envvar="WF_SESSION_PATH"),
) -> None:
"""Phase 0.5 — prove end-to-end auth + 1 activity import against live Wealthfolio."""
import asyncio
"""Phase 0.5 — prove end-to-end auth against live Wealthfolio."""
from broker_sync.sinks.wealthfolio import WealthfolioSink
async def _run() -> None:
@ -48,12 +55,119 @@ def auth_spike(
sys.exit(1)
@app.command("trading212")
def trading212(
wf_base_url: str = typer.Option(..., envvar="WF_BASE_URL"),
wf_username: str = typer.Option(..., envvar="WF_USERNAME"),
wf_password: str = typer.Option(..., envvar="WF_PASSWORD"),
wf_session_path: str = typer.Option("/data/wealthfolio_session.json", envvar="WF_SESSION_PATH"),
t212_api_keys_json: str = typer.Option(..., envvar="T212_API_KEYS_JSON"),
data_dir: str = typer.Option("/data", envvar="BROKER_SYNC_DATA_DIR"),
mode: str = typer.Option("steady", help="steady = last-7-days; backfill = full history"),
) -> None:
"""Phase 1 — sync Trading212 accounts into Wealthfolio.
T212_API_KEYS_JSON is a JSON array of
{id, name, account_type, currency, api_key}
objects one entry per T212 account (ISA, Invest).
"""
from broker_sync.dedup import SyncRecordStore
from broker_sync.pipeline import sync_provider_to_wealthfolio
from broker_sync.providers.trading212 import Trading212Provider
from broker_sync.sinks.wealthfolio import WealthfolioSink
_setup_logging()
accounts = _parse_t212_accounts(t212_api_keys_json)
if not accounts:
typer.echo("No accounts configured in T212_API_KEYS_JSON — nothing to do.", err=True)
sys.exit(2)
data = Path(data_dir)
checkpoint_dir = data / "watermarks"
checkpoint_dir.mkdir(parents=True, exist_ok=True)
if mode == "steady":
since: datetime | None = datetime.now(UTC) - timedelta(days=7)
elif mode == "backfill":
since = None
else:
typer.echo(f"Unknown mode: {mode!r}. Use 'steady' or 'backfill'.", err=True)
sys.exit(2)
async def _run() -> None:
sink = WealthfolioSink(
base_url=wf_base_url,
username=wf_username,
password=wf_password,
session_path=wf_session_path,
)
provider = Trading212Provider(
accounts=accounts,
checkpoint_dir=checkpoint_dir,
)
dedup = SyncRecordStore(data / "sync.db")
try:
# Ensure cookie upfront so a first-run with no session file still works.
if not Path(wf_session_path).exists():
await sink.login()
result = await sync_provider_to_wealthfolio(
provider=provider,
sink=sink,
dedup=dedup,
since=since,
)
finally:
await provider.close()
await sink.close()
typer.echo(f"trading212: fetched={result.fetched} "
f"new={result.new_after_dedup} "
f"imported={result.imported} "
f"failed={result.failed}")
if result.failed > 0:
sys.exit(1)
asyncio.run(_run())
def _setup_logging() -> None:
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s %(message)s",
)
def _parse_t212_accounts(raw: str) -> list[tuple[Account, str]]:
"""Parse T212_API_KEYS_JSON into (Account, api_key) pairs."""
from broker_sync.models import Account, AccountType
parsed = json.loads(raw)
if not isinstance(parsed, list):
raise typer.BadParameter("T212_API_KEYS_JSON must be a JSON array")
pairs: list[tuple[Account, str]] = []
for entry in parsed:
if not isinstance(entry, dict):
raise typer.BadParameter("Each T212 entry must be an object")
try:
account = Account(
id=entry["id"],
name=entry["name"],
account_type=AccountType(entry["account_type"]),
currency=entry.get("currency", "GBP"),
provider="trading212",
)
api_key = entry["api_key"]
except KeyError as e:
raise typer.BadParameter(f"T212 entry missing required key: {e}") from None
pairs.append((account, api_key))
return pairs
def main() -> None:
# Entry point called by the console-script in pyproject.toml.
app()
if __name__ == "__main__":
# Guard env for readability when running under `python -m broker_sync.cli`.
os.environ.setdefault("COLUMNS", "120")
main()