Context: Phase 2b wiring — hand the bearer-token InvestEngineProvider
into the existing sync pipeline (sync_provider_to_wealthfolio), mirroring
the trading212 subcommand.
Environment contract:
WF_BASE_URL, WF_USERNAME, WF_PASSWORD, WF_SESSION_PATH (shared with trading212)
IE_BEARER_TOKEN (devtools-pasted)
IE_TOKEN_EXPIRES_AT (ISO-8601; Viktor sets on paste)
BROKER_SYNC_DATA_DIR (sync.db + checkpoint state)
Exit codes:
0 = clean run
1 = some rows failed to import (mirrors trading212 behaviour)
2 = token already expired per IE_TOKEN_EXPIRES_AT, or malformed ISO
timestamp, or live 401 response from IE (InvestEngineTokenExpiredError),
or unknown --mode flag
The pre-request expiry check is deliberate: a CronJob that runs during
the refresh window would otherwise waste a request on a dead token and
get the same 401 that we already know about from the clock. Exit 2
from the clock-only path also separates "token is old" from "wealthfolio
rejected a batch" in the CronJob alert pipeline.
Mode defaults:
--mode steady → since = now - 30d (bigger window than T212's 7d
because the IE sync only runs once a month in steady
state; 30d guarantees no gap even after a missed run)
--mode backfill → since = None (full history)
This change:
- `invest-engine` subcommand added to broker_sync/cli.py
- Token-expiry pre-check (clock), IE_TOKEN_EXPIRES_AT ISO parsing with
a UTC default for naive timestamps, and graceful handling of
InvestEngineTokenExpiredError surfaced during pipeline run
- 3 new tests in tests/test_cli.py covering the 3 exit-2 paths
## Automated
poetry run pytest tests/test_cli.py -v
======================== 4 passed in 0.28s =========================
poetry run pytest -q
98 passed, 1 skipped in 0.85s
poetry run mypy --strict .
Success: no issues found in 34 source files
poetry run ruff check .
All checks passed!
## Manual Verification
1. Populate Vault keys per the docstring in
broker_sync/providers/invest_engine.py (Viktor pastes token + sets
expires_at to the Monday morning of next month).
2. Set env:
export WF_BASE_URL=https://wealthfolio.viktorbarzin.me
export WF_USERNAME=viktor
export WF_PASSWORD=<from Vault>
export IE_BEARER_TOKEN=<from Vault>
export IE_TOKEN_EXPIRES_AT=<from Vault>
export BROKER_SYNC_DATA_DIR=/tmp/ie-smoke
3. poetry run broker-sync invest-engine --mode backfill
Expected: single line "invest-engine: fetched=N new=M imported=M failed=0"
on success; exit 2 with "InvestEngine token expired..." if the clock
or server disagrees; exit 2 with "IE_TOKEN_EXPIRES_AT not a valid
ISO-8601 timestamp..." if the env var is malformed.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
273 lines
9.3 KiB
Python
273 lines
9.3 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING
|
|
|
|
import typer
|
|
|
|
if TYPE_CHECKING:
|
|
from broker_sync.models import Account
|
|
|
|
app = typer.Typer(
|
|
help="broker-sync: pull brokerage activity into Wealthfolio",
|
|
# CRITICAL: rich tracebacks print all local variables on crash, which
|
|
# includes env-sourced credentials (WF_PASSWORD, T212_API_KEYS_JSON).
|
|
# Kubernetes pod logs are world-readable — leaking creds there is a
|
|
# security incident. Plain tracebacks only.
|
|
pretty_exceptions_enable=False,
|
|
)
|
|
|
|
|
|
@app.command("version")
|
|
def version() -> None:
|
|
"""Print version and exit — used by the no-op Phase 0 CronJob as a liveness check."""
|
|
from broker_sync import __version__
|
|
typer.echo(f"broker-sync {__version__}")
|
|
|
|
|
|
@app.command("auth-spike")
|
|
def auth_spike(
|
|
wf_base_url: str = typer.Option(..., envvar="WF_BASE_URL", help="Wealthfolio base URL"),
|
|
wf_username: str = typer.Option(..., envvar="WF_USERNAME"),
|
|
wf_password: str = typer.Option(..., envvar="WF_PASSWORD"),
|
|
session_path: str = typer.Option("/data/wealthfolio_session.json", envvar="WF_SESSION_PATH"),
|
|
) -> None:
|
|
"""Phase 0.5 — prove end-to-end auth against live Wealthfolio."""
|
|
from broker_sync.sinks.wealthfolio import WealthfolioSink
|
|
|
|
async def _run() -> None:
|
|
sink = WealthfolioSink(
|
|
base_url=wf_base_url,
|
|
username=wf_username,
|
|
password=wf_password,
|
|
session_path=session_path,
|
|
)
|
|
try:
|
|
await sink.login()
|
|
accounts = await sink.list_accounts()
|
|
typer.echo(f"Logged in. {len(accounts)} account(s) visible.")
|
|
finally:
|
|
await sink.close()
|
|
|
|
try:
|
|
asyncio.run(_run())
|
|
except Exception as e:
|
|
typer.echo(f"auth-spike failed: {e}", err=True)
|
|
sys.exit(1)
|
|
|
|
|
|
@app.command("trading212")
|
|
def trading212(
|
|
wf_base_url: str = typer.Option(..., envvar="WF_BASE_URL"),
|
|
wf_username: str = typer.Option(..., envvar="WF_USERNAME"),
|
|
wf_password: str = typer.Option(..., envvar="WF_PASSWORD"),
|
|
wf_session_path: str = typer.Option("/data/wealthfolio_session.json", envvar="WF_SESSION_PATH"),
|
|
t212_api_keys_json: str = typer.Option(..., envvar="T212_API_KEYS_JSON"),
|
|
data_dir: str = typer.Option("/data", envvar="BROKER_SYNC_DATA_DIR"),
|
|
mode: str = typer.Option("steady", help="steady = last-7-days; backfill = full history"),
|
|
) -> None:
|
|
"""Phase 1 — sync Trading212 accounts into Wealthfolio.
|
|
|
|
T212_API_KEYS_JSON is a JSON array of
|
|
{id, name, account_type, currency, api_key}
|
|
objects — one entry per T212 account (ISA, Invest).
|
|
"""
|
|
from broker_sync.dedup import SyncRecordStore
|
|
from broker_sync.pipeline import sync_provider_to_wealthfolio
|
|
from broker_sync.providers.trading212 import Trading212Provider
|
|
from broker_sync.sinks.wealthfolio import WealthfolioSink
|
|
|
|
_setup_logging()
|
|
accounts = _parse_t212_accounts(t212_api_keys_json)
|
|
if not accounts:
|
|
typer.echo("No accounts configured in T212_API_KEYS_JSON — nothing to do.", err=True)
|
|
sys.exit(2)
|
|
|
|
data = Path(data_dir)
|
|
checkpoint_dir = data / "watermarks"
|
|
checkpoint_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
if mode == "steady":
|
|
since: datetime | None = datetime.now(UTC) - timedelta(days=7)
|
|
elif mode == "backfill":
|
|
since = None
|
|
else:
|
|
typer.echo(f"Unknown mode: {mode!r}. Use 'steady' or 'backfill'.", err=True)
|
|
sys.exit(2)
|
|
|
|
async def _run() -> None:
|
|
sink = WealthfolioSink(
|
|
base_url=wf_base_url,
|
|
username=wf_username,
|
|
password=wf_password,
|
|
session_path=wf_session_path,
|
|
)
|
|
provider = Trading212Provider(
|
|
accounts=accounts,
|
|
checkpoint_dir=checkpoint_dir,
|
|
)
|
|
dedup = SyncRecordStore(data / "sync.db")
|
|
try:
|
|
# Ensure cookie upfront so a first-run with no session file still works.
|
|
if not Path(wf_session_path).exists():
|
|
await sink.login()
|
|
result = await sync_provider_to_wealthfolio(
|
|
provider=provider,
|
|
sink=sink,
|
|
dedup=dedup,
|
|
since=since,
|
|
)
|
|
finally:
|
|
await provider.close()
|
|
await sink.close()
|
|
|
|
typer.echo(f"trading212: fetched={result.fetched} "
|
|
f"new={result.new_after_dedup} "
|
|
f"imported={result.imported} "
|
|
f"failed={result.failed}")
|
|
if result.failed > 0:
|
|
sys.exit(1)
|
|
|
|
asyncio.run(_run())
|
|
|
|
|
|
@app.command("invest-engine")
|
|
def invest_engine(
|
|
wf_base_url: str = typer.Option(..., envvar="WF_BASE_URL"),
|
|
wf_username: str = typer.Option(..., envvar="WF_USERNAME"),
|
|
wf_password: str = typer.Option(..., envvar="WF_PASSWORD"),
|
|
wf_session_path: str = typer.Option("/data/wealthfolio_session.json", envvar="WF_SESSION_PATH"),
|
|
ie_bearer_token: str = typer.Option(..., envvar="IE_BEARER_TOKEN"),
|
|
ie_token_expires_at: str = typer.Option(..., envvar="IE_TOKEN_EXPIRES_AT"),
|
|
data_dir: str = typer.Option("/data", envvar="BROKER_SYNC_DATA_DIR"),
|
|
mode: str = typer.Option("steady", help="steady = last-30-days; backfill = full history"),
|
|
) -> None:
|
|
"""Phase 2b — sync InvestEngine activity into Wealthfolio via Bearer token.
|
|
|
|
The Bearer token is pasted from browser devtools by Viktor (MFA blocks
|
|
scripted login). IE_TOKEN_EXPIRES_AT is the ISO-8601 timestamp he sets
|
|
when he pastes it; we fail fast with exit=2 if that moment has passed
|
|
so a CronJob that runs past the refresh window doesn't burn a request
|
|
on a known-dead token.
|
|
"""
|
|
from broker_sync.dedup import SyncRecordStore
|
|
from broker_sync.pipeline import sync_provider_to_wealthfolio
|
|
from broker_sync.providers.invest_engine import (
|
|
InvestEngineProvider,
|
|
InvestEngineTokenExpiredError,
|
|
)
|
|
from broker_sync.sinks.wealthfolio import WealthfolioSink
|
|
|
|
_setup_logging()
|
|
|
|
try:
|
|
expires_at = datetime.fromisoformat(ie_token_expires_at)
|
|
except ValueError as e:
|
|
typer.echo(f"IE_TOKEN_EXPIRES_AT not a valid ISO-8601 timestamp: {e}", err=True)
|
|
sys.exit(2)
|
|
if expires_at.tzinfo is None:
|
|
expires_at = expires_at.replace(tzinfo=UTC)
|
|
if expires_at <= datetime.now(UTC):
|
|
typer.echo(
|
|
f"InvestEngine token expired at {expires_at.isoformat()} — "
|
|
f"Viktor must paste a fresh Bearer into Vault.",
|
|
err=True,
|
|
)
|
|
sys.exit(2)
|
|
|
|
data = Path(data_dir)
|
|
data.mkdir(parents=True, exist_ok=True)
|
|
|
|
if mode == "steady":
|
|
since: datetime | None = datetime.now(UTC) - timedelta(days=30)
|
|
elif mode == "backfill":
|
|
since = None
|
|
else:
|
|
typer.echo(f"Unknown mode: {mode!r}. Use 'steady' or 'backfill'.", err=True)
|
|
sys.exit(2)
|
|
|
|
async def _run() -> None:
|
|
sink = WealthfolioSink(
|
|
base_url=wf_base_url,
|
|
username=wf_username,
|
|
password=wf_password,
|
|
session_path=wf_session_path,
|
|
)
|
|
provider = InvestEngineProvider(
|
|
bearer_token=ie_bearer_token,
|
|
token_expires_at=expires_at,
|
|
)
|
|
dedup = SyncRecordStore(data / "sync.db")
|
|
try:
|
|
if not Path(wf_session_path).exists():
|
|
await sink.login()
|
|
result = await sync_provider_to_wealthfolio(
|
|
provider=provider,
|
|
sink=sink,
|
|
dedup=dedup,
|
|
since=since,
|
|
)
|
|
except InvestEngineTokenExpiredError as e:
|
|
typer.echo(f"InvestEngine auth failed: {e}", err=True)
|
|
sys.exit(2)
|
|
finally:
|
|
await provider.close()
|
|
await sink.close()
|
|
|
|
typer.echo(f"invest-engine: fetched={result.fetched} "
|
|
f"new={result.new_after_dedup} "
|
|
f"imported={result.imported} "
|
|
f"failed={result.failed}")
|
|
if result.failed > 0:
|
|
sys.exit(1)
|
|
|
|
asyncio.run(_run())
|
|
|
|
|
|
def _setup_logging() -> None:
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
|
)
|
|
|
|
|
|
def _parse_t212_accounts(raw: str) -> list[tuple[Account, str]]:
|
|
"""Parse T212_API_KEYS_JSON into (Account, api_key) pairs."""
|
|
from broker_sync.models import Account, AccountType
|
|
|
|
parsed = json.loads(raw)
|
|
if not isinstance(parsed, list):
|
|
raise typer.BadParameter("T212_API_KEYS_JSON must be a JSON array")
|
|
|
|
pairs: list[tuple[Account, str]] = []
|
|
for entry in parsed:
|
|
if not isinstance(entry, dict):
|
|
raise typer.BadParameter("Each T212 entry must be an object")
|
|
try:
|
|
account = Account(
|
|
id=entry["id"],
|
|
name=entry["name"],
|
|
account_type=AccountType(entry["account_type"]),
|
|
currency=entry.get("currency", "GBP"),
|
|
provider="trading212",
|
|
)
|
|
api_key = entry["api_key"]
|
|
except KeyError as e:
|
|
raise typer.BadParameter(f"T212 entry missing required key: {e}") from None
|
|
pairs.append((account, api_key))
|
|
return pairs
|
|
|
|
|
|
def main() -> None:
|
|
app()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
os.environ.setdefault("COLUMNS", "120")
|
|
main()
|