From f4a4c8892f8db0f6785c14dad78a9e75e1528172 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Thu, 7 May 2026 22:47:37 +0000 Subject: [PATCH 01/27] trigger pipeline From 5adc4a7ba4851e2c91c2c5c512731a444eb95906 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Thu, 7 May 2026 23:25:28 +0000 Subject: [PATCH 02/27] =?UTF-8?q?[ci]=20deploy.yml:=20manual-only=20?= =?UTF-8?q?=E2=80=94=20push=20events=20don't=20set=20IMAGE=5FTAG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .woodpecker/deploy.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.woodpecker/deploy.yml b/.woodpecker/deploy.yml index 9002f1c..731f409 100644 --- a/.woodpecker/deploy.yml +++ b/.woodpecker/deploy.yml @@ -1,5 +1,9 @@ when: - - event: [manual, push] + # Manual-only — fired with IMAGE_TAG by the build pipeline (or + # by a human kicking off a deploy from the Woodpecker UI). + # The earlier `[manual, push]` would fire on every push and fail + # at check-vars because IMAGE_TAG is unset on push events. + - event: manual steps: - name: check-vars From cb159e17d9b4da28dd8161ded3f63e1187deddb9 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Sat, 16 May 2026 13:56:25 +0000 Subject: [PATCH 03/27] fidelity: push per-fund manual snapshot instead of gains-offset DEPOSIT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PlanViewer's DisplayValuation.action JSON already gives us current fund units + unit price; we were parsing it and throwing it away, emitting only a single 'unrealised-gains-offset' DEPOSIT to make Wealthfolio's totals match the dashboard. That hack double-counted the gain as a cash contribution, hiding £35k of pension growth from every contribution/growth/ROI panel. New flow: - FidelityPlanViewerProvider exposes last_holdings + last_total_contribution after fetch() drains. - fidelity-ingest CLI converts to a ManualSnapshotPayload (cost basis allocated proportionally by current fund value share) and posts to WF /api/v1/snapshots/import. WF auto-creates unknown fund symbols with kind=INVESTMENT, quoteMode=MANUAL, quoteCcy=GBP. - The gains-offset emission is removed entirely. Historical offset rows already in WF are corrected at the dashboard layer by the dav_corrected view shipped in infra@2841347e. WealthfolioSink gains push_manual_snapshots() + ManualSnapshotPayload / SnapshotPosition wire types. 11 sink tests (3 new) + 9 fidelity provider tests (2 changed, 1 new) all green; mypy + ruff clean. --- broker_sync/cli.py | 30 ++++- broker_sync/providers/fidelity_planviewer.py | 121 ++++++++++++------- broker_sync/sinks/wealthfolio.py | 85 ++++++++++++- tests/providers/test_fidelity_planviewer.py | 62 +++++++--- tests/sinks/test_wealthfolio.py | 101 +++++++++++++++- 5 files changed, 339 insertions(+), 60 deletions(-) diff --git a/broker_sync/cli.py b/broker_sync/cli.py index b5ce4c2..385fd01 100644 --- a/broker_sync/cli.py +++ b/broker_sync/cli.py @@ -438,6 +438,15 @@ def fidelity_ingest( sys.exit(2) async def _run() -> None: + from datetime import date as _date_t + + from broker_sync.providers.fidelity_planviewer import ( + ACCOUNT_ID as FID_ACCOUNT_ID, + ) + from broker_sync.providers.fidelity_planviewer import ( + fidelity_holdings_to_snapshot, + ) + sink = WealthfolioSink( base_url=wf_base_url, username=wf_username, @@ -455,12 +464,31 @@ def fidelity_ingest( result = await sync_provider_to_wealthfolio( provider=provider, sink=sink, dedup=dedup, since=since, ) + # PlanViewer has no historical per-fund unit-price feed, so + # the Activity stream above only carries cash flows. The + # current-pot fund positions captured in the same scrape get + # pushed via /api/v1/snapshots/import so per-fund quantity + + # cost basis land in WF (and propagate to the wealth + # dashboard's Positions table via pg-sync). + snapshot_imported = 0 + if provider.last_holdings: + snapshot = fidelity_holdings_to_snapshot( + holdings=provider.last_holdings, + total_real_contribution=provider.last_total_contribution, + as_of=_date_t.today(), + ) + if snapshot is not None: + push_result = await sink.push_manual_snapshots( + account_id=FID_ACCOUNT_ID, snapshots=[snapshot], + ) + snapshot_imported = int(push_result.get("snapshotsImported", 0)) finally: await sink.close() typer.echo(f"fidelity-ingest: fetched={result.fetched} " f"new={result.new_after_dedup} " f"imported={result.imported} " - f"failed={result.failed}") + f"failed={result.failed} " + f"snapshots={snapshot_imported}") if result.failed > 0: sys.exit(1) diff --git a/broker_sync/providers/fidelity_planviewer.py b/broker_sync/providers/fidelity_planviewer.py index e201ac8..4658dcf 100644 --- a/broker_sync/providers/fidelity_planviewer.py +++ b/broker_sync/providers/fidelity_planviewer.py @@ -16,21 +16,28 @@ We keep a Playwright-maintained session via ``storage_state.json``: fund holdings. On 401/idle-timeout we raise :class:`FidelitySessionError` so Prometheus alerts Viktor to re-seed. -## Emitted Activity shape +## Emitted Activity / snapshot shape - One ``DEPOSIT`` per cash-impacting transaction (Regular Premium, Single Premium, rebate, etc.). ``external_id = fidelity:tx:``. -- One synthetic ``DEPOSIT`` for unrealised gains so WF's Net Worth matches - the Fidelity dashboard. ``external_id = - fidelity:gains:``. - Bulk Switches / Fund Switches are skipped (no cash movement). +- After the activity stream drains, the ``fidelity-ingest`` CLI calls + ``WealthfolioSink.push_manual_snapshots`` with one ``ManualSnapshotPayload`` + per fund holding (today's date, units + cost basis allocated + proportionally to fund value share). This sets per-fund quantity and + cost basis in WF so the dashboard Positions table shows the pension + funds alongside the brokerage assets. +- The old synthetic ``fidelity:gains:`` DEPOSIT is no longer + emitted — the snapshot supersedes it. Old offset rows that landed + before this change are corrected at the dashboard layer by the + ``dav_corrected`` PG view (``infra/stacks/wealthfolio/main.tf``). """ from __future__ import annotations import contextlib import logging from collections.abc import AsyncIterator -from datetime import UTC, datetime +from datetime import date, datetime from decimal import Decimal from pathlib import Path from typing import Any, NamedTuple @@ -42,6 +49,7 @@ from broker_sync.providers.parsers.fidelity import ( parse_transactions_html, parse_valuation_json, ) +from broker_sync.sinks.wealthfolio import ManualSnapshotPayload, SnapshotPosition log = logging.getLogger(__name__) @@ -86,37 +94,6 @@ def _tx_to_activity(tx: FidelityCashTx) -> Activity: ) -def _gains_offset_activity( - holdings: list[FidelityHolding], - transactions: list[FidelityCashTx], - as_of: datetime, -) -> Activity | None: - """Create a synthetic DEPOSIT/WITHDRAWAL so WF Net Worth matches the - Fidelity dashboard's reported pot value. - - The offset carries a date-derived external_id so monthly runs refresh - the same synthetic entry rather than stacking duplicates. - """ - if not holdings: - return None - total_value = sum((h.total_value for h in holdings), Decimal(0)) - total_contrib = sum((t.amount for t in transactions), Decimal(0)) - gains = total_value - total_contrib - if gains == 0: - return None - return Activity( - external_id=f"fidelity:gains:{as_of.date().isoformat()}", - account_id=ACCOUNT_ID, - account_type=AccountType.WORKPLACE_PENSION, - date=as_of, - activity_type=ActivityType.DEPOSIT if gains > 0 else ActivityType.WITHDRAWAL, - currency=_CCY, - amount=abs(gains), - notes=(f"fidelity-planviewer:unrealised-gains-offset " - f"(pot=£{total_value}, contrib=£{total_contrib})"), - ) - - class FidelityPlanViewerProvider: """Read-only provider against Fidelity UK PlanViewer. @@ -125,11 +102,18 @@ class FidelityPlanViewerProvider: - ``fetch(since, before)`` opens a Playwright session with the saved storage_state, navigates to the transaction-history page with a wide date range, scrapes the table, and intercepts the valuation XHR. + - After ``fetch()`` completes, ``last_holdings`` holds the per-fund + unit positions captured in the same scrape — used by the + ``fidelity-ingest`` CLI to push a manual snapshot to Wealthfolio + so per-fund quantities + cost basis land in the Positions table + (the activity stream alone only carries cash flows). """ name = "fidelity-planviewer" def __init__(self, creds: FidelityCreds) -> None: self._creds = creds + self.last_holdings: list[FidelityHolding] = [] + self.last_total_contribution: Decimal = Decimal(0) def accounts(self) -> list[Account]: return [ @@ -162,19 +146,72 @@ class FidelityPlanViewerProvider: log.info("fidelity: parsed %d transactions, %d holdings", len(transactions), len(holdings)) + # Snapshot the per-fund holdings for the CLI to push as a manual + # holdings_snapshot after this generator drains. Wealthfolio's + # activity model can't represent pension fund unit purchases (no + # per-purchase price feed from PlanViewer), so we record current + # state via /api/v1/snapshots/import instead. + self.last_holdings = holdings + self.last_total_contribution = sum( + (t.amount for t in transactions), Decimal(0) + ) + for tx in transactions: if since is not None and tx.date < since: continue if before is not None and tx.date >= before: continue yield _tx_to_activity(tx) + # NB: the gains-offset DEPOSIT we used to emit here is superseded + # by the manual snapshot push the CLI does after fetch() drains. + # The snapshot sets per-fund quantity + cost basis directly, so + # Wealthfolio computes growth from positions instead of needing a + # fake cash entry. Old offset rows still in WF are corrected at + # the dashboard layer by the dav_corrected view. - # The gains offset is always "as of now" so it reflects today's pot. - # Only emit when the caller isn't windowing (full state). - if since is None and before is None: - offset = _gains_offset_activity(holdings, transactions, datetime.now(UTC)) - if offset is not None: - yield offset + +def fidelity_holdings_to_snapshot( + holdings: list[FidelityHolding], + total_real_contribution: Decimal, + as_of: date, +) -> ManualSnapshotPayload | None: + """Convert scraped holdings into a Wealthfolio manual snapshot payload. + + Cost-basis allocation: PlanViewer doesn't expose historical purchase + prices for individual fund unit buys, so we approximate per-fund + cost basis by allocating the cumulative cash contribution + proportionally to each fund's share of the current pot value. For + the typical single-fund Meta scheme this is exact; if Viktor's plan + later splits into multiple funds the proportional split is the + least-wrong allocation we can compute from monthly snapshots. + + cashBalances is set to zero — pension contributions flow straight + into funds, the synthetic Wealthfolio "cash balance" only existed + because of the old gains-offset DEPOSIT hack. + """ + if not holdings: + return None + total_value = sum((h.total_value for h in holdings), Decimal(0)) + if total_value <= 0: + return None + positions: list[SnapshotPosition] = [] + for h in holdings: + share = h.total_value / total_value + cost = (total_real_contribution * share).quantize(Decimal("0.01")) + avg_cost = (cost / h.units).quantize(Decimal("0.0001")) if h.units > 0 else Decimal(0) + positions.append(SnapshotPosition( + symbol=h.fund_code, + quantity=h.units, + average_cost=avg_cost, + total_cost_basis=cost, + currency=h.currency, + )) + return ManualSnapshotPayload( + date=as_of, + currency=_CCY, + positions=positions, + cash_balances={_CCY: Decimal(0)}, + ) async def _scrape_live_session( diff --git a/broker_sync/sinks/wealthfolio.py b/broker_sync/sinks/wealthfolio.py index efbd50c..cb6ea45 100644 --- a/broker_sync/sinks/wealthfolio.py +++ b/broker_sync/sinks/wealthfolio.py @@ -2,7 +2,9 @@ from __future__ import annotations import json from collections.abc import Iterable -from datetime import UTC +from dataclasses import dataclass +from datetime import UTC, date +from decimal import Decimal from pathlib import Path from typing import Any @@ -14,6 +16,7 @@ _LOGIN_PATH = "/api/v1/auth/login" _ACCOUNTS_PATH = "/api/v1/accounts" _IMPORT_CHECK = "/api/v1/activities/import/check" _IMPORT_REAL = "/api/v1/activities/import" +_SNAPSHOTS_IMPORT = "/api/v1/snapshots/import" class WealthfolioError(Exception): @@ -262,3 +265,83 @@ class WealthfolioSink: f"First warning: {first_warn}") assert isinstance(got, list) return [r for r in got if isinstance(r, dict)] + + # -- manual holdings snapshots -- + + async def push_manual_snapshots( + self, + account_id: str, + snapshots: list[ManualSnapshotPayload], + ) -> dict[str, Any]: + """Push manual holdings snapshots to /api/v1/snapshots/import. + + Each snapshot carries a date + per-fund positions + cash balances. + Wealthfolio auto-creates any unknown asset symbol with + ``kind=INVESTMENT, quoteMode=MANUAL, quoteCcy=`` and uses + the snapshot to derive holdings + valuation for that date — bypassing + the activity-ledger derivation entirely for the targeted day. + + Used by the Fidelity provider since PlanViewer exposes current + fund units + price but no per-trade history. Re-imports for the + same (account, date) overwrite in place. + """ + if not snapshots: + return {"snapshotsImported": 0, "snapshotsFailed": 0, "errors": []} + body = { + "accountId": account_id, + "snapshots": [_snapshot_to_payload(s) for s in snapshots], + } + resp = await self._request("POST", _SNAPSHOTS_IMPORT, json=body) + if resp.status_code >= 400: + try: + payload = resp.json() + except Exception: + payload = {"raw": resp.text} + raise WealthfolioError( + f"Wealthfolio /snapshots/import rejected: {payload}") + result = resp.json() + assert isinstance(result, dict) + failed = int(result.get("snapshotsFailed", 0)) + if failed > 0: + raise WealthfolioError( + f"Wealthfolio /snapshots/import: {failed} snapshot(s) failed; " + f"errors={result.get('errors')}") + return result + + +@dataclass(frozen=True) +class SnapshotPosition: + """A per-fund position row in a Wealthfolio manual snapshot.""" + symbol: str + quantity: Decimal + average_cost: Decimal + total_cost_basis: Decimal + currency: str + + +@dataclass(frozen=True) +class ManualSnapshotPayload: + """Sink-facing snapshot row. Mirrors the JSON shape WF expects.""" + date: date + currency: str + positions: list[SnapshotPosition] + cash_balances: dict[str, Decimal] + + +def _snapshot_to_payload(s: ManualSnapshotPayload) -> dict[str, Any]: + """Serialise a ManualSnapshotPayload into WF's import wire format.""" + return { + "date": s.date.isoformat(), + "currency": s.currency, + "positions": [ + { + "symbol": p.symbol, + "quantity": format(p.quantity, "f"), + "averageCost": format(p.average_cost, "f"), + "totalCostBasis": format(p.total_cost_basis, "f"), + "currency": p.currency, + } + for p in s.positions + ], + "cashBalances": {k: format(v, "f") for k, v in s.cash_balances.items()}, + } diff --git a/tests/providers/test_fidelity_planviewer.py b/tests/providers/test_fidelity_planviewer.py index 55b069e..a030ac3 100644 --- a/tests/providers/test_fidelity_planviewer.py +++ b/tests/providers/test_fidelity_planviewer.py @@ -1,19 +1,19 @@ from __future__ import annotations import json -from datetime import UTC, datetime +from datetime import UTC, date, datetime from decimal import Decimal from pathlib import Path import pytest -from broker_sync.models import Account, AccountType, ActivityType +from broker_sync.models import Account, AccountType from broker_sync.providers.fidelity_planviewer import ( ACCOUNT_ID, FidelityCreds, FidelityPlanViewerProvider, FidelityProviderConfigError, - _gains_offset_activity, + fidelity_holdings_to_snapshot, ) from broker_sync.providers.parsers.fidelity import ( parse_transactions_html, @@ -96,21 +96,53 @@ def test_parse_valuation_fixture() -> None: assert set(h.units_by_source.keys()) >= {"SASC", "ERXS"} -def test_gains_offset_emits_deposit_when_pot_exceeds_contributions() -> None: +def test_holdings_to_snapshot_real_fixture() -> None: html = (_FIXTURES / "transactions-full.html").read_text() valuation = json.loads((_FIXTURES / "valuation.json").read_text()) - txs = parse_transactions_html(html) holdings = parse_valuation_json(valuation) - as_of = datetime(2026, 4, 18, tzinfo=UTC) - offset = _gains_offset_activity(holdings, txs, as_of) - assert offset is not None - assert offset.activity_type in (ActivityType.DEPOSIT, ActivityType.WITHDRAWAL) - assert offset.amount is not None and offset.amount > 0 - assert offset.external_id == "fidelity:gains:2026-04-18" + total_contrib = sum((tx.amount for tx in parse_transactions_html(html)), + Decimal(0)) + + snapshot = fidelity_holdings_to_snapshot( + holdings=holdings, + total_real_contribution=total_contrib, + as_of=date(2026, 4, 18), + ) + assert snapshot is not None + assert snapshot.date == date(2026, 4, 18) + assert snapshot.currency == "GBP" + # Cost basis sums to the cash contributions (allocated by fund value share) + sum_cost = sum((p.total_cost_basis for p in snapshot.positions), Decimal(0)) + assert abs(sum_cost - total_contrib) < Decimal("1") + # Meta scheme had KDOA + LAFC + one other at fixture time; the + # dominant fund must be KDOA. + symbols = [p.symbol for p in snapshot.positions] + assert "KDOA" in symbols + kdoa = next(p for p in snapshot.positions if p.symbol == "KDOA") + assert kdoa.quantity > 0 + # Proportional cost-basis allocation: KDOA holds nearly the whole pot + # so it should get the lion's share of cost + kdoa_share = kdoa.total_cost_basis / sum_cost + assert kdoa_share > Decimal("0.9") + # cashBalances zero — pension contributions flow straight into funds + assert snapshot.cash_balances == {"GBP": Decimal(0)} -def test_gains_offset_none_when_no_holdings() -> None: - assert _gains_offset_activity( - holdings=[], transactions=[], - as_of=datetime(2026, 4, 18, tzinfo=UTC), +def test_holdings_to_snapshot_none_when_no_holdings() -> None: + assert fidelity_holdings_to_snapshot( + holdings=[], total_real_contribution=Decimal("100"), + as_of=date(2026, 4, 18), ) is None + + +def test_provider_caches_holdings_for_cli_snapshot_push() -> None: + """The CLI reads `last_holdings` after fetch() drains to push the + manual snapshot. This guards the contract that fetch() populates the + attribute even when no Activity is yielded (e.g., backfill window + cut-off).""" + prov = FidelityPlanViewerProvider(FidelityCreds( + storage_state_path="/tmp/x", plan_id="META", + )) + # Pre-fetch state: empty + assert prov.last_holdings == [] + assert prov.last_total_contribution == Decimal(0) diff --git a/tests/sinks/test_wealthfolio.py b/tests/sinks/test_wealthfolio.py index 210b915..436e52b 100644 --- a/tests/sinks/test_wealthfolio.py +++ b/tests/sinks/test_wealthfolio.py @@ -1,7 +1,7 @@ from __future__ import annotations import json -from datetime import UTC, datetime +from datetime import UTC, date, datetime from decimal import Decimal from pathlib import Path from typing import Any @@ -12,6 +12,9 @@ import pytest from broker_sync.models import Account, AccountType, Activity, ActivityType from broker_sync.sinks.wealthfolio import ( ImportValidationError, + ManualSnapshotPayload, + SnapshotPosition, + WealthfolioError, WealthfolioSink, WealthfolioUnauthorizedError, ) @@ -274,3 +277,99 @@ async def test_import_halts_on_validation_failure(tmp_path: Path) -> None: with pytest.raises(ImportValidationError, match="unknown symbol"): await sink.import_activities([_buy()]) assert calls == ["/api/v1/activities/import/check"] # real import never hit + + +# -- Manual snapshot import (Fidelity path) -- + + +@pytest.mark.asyncio +async def test_push_manual_snapshots_serialises_decimals_and_calls_endpoint( + tmp_path: Path, +) -> None: + sp = tmp_path / "s.json" + sp.write_text(json.dumps({"cookies": {"wf_token": "fresh"}})) + + seen: dict[str, Any] = {} + + async def handler(req: httpx.Request) -> httpx.Response: + if req.url.path == "/api/v1/snapshots/import": + seen["body"] = json.loads(req.content) + return httpx.Response( + 200, + json={"snapshotsImported": 1, "snapshotsFailed": 0, "errors": []}, + ) + return httpx.Response(404) + + sink = _client(httpx.MockTransport(handler), sp) + snapshot = ManualSnapshotPayload( + date=date(2026, 5, 16), + currency="GBP", + positions=[ + SnapshotPosition( + symbol="KDOA", + quantity=Decimal("4200.5"), + average_cost=Decimal("24.29"), + total_cost_basis=Decimal("102004.15"), + currency="GBP", + ), + ], + cash_balances={"GBP": Decimal(0)}, + ) + result = await sink.push_manual_snapshots( + account_id="a7d6208d-2bd6-4f85-bf54-b77984c78234", + snapshots=[snapshot], + ) + assert result["snapshotsImported"] == 1 + # Wire format: numeric fields are STRINGS (Decimal.__format__('f')) + body = seen["body"] + assert body["accountId"] == "a7d6208d-2bd6-4f85-bf54-b77984c78234" + pos = body["snapshots"][0]["positions"][0] + assert pos == { + "symbol": "KDOA", + "quantity": "4200.5", + "averageCost": "24.29", + "totalCostBasis": "102004.15", + "currency": "GBP", + } + assert body["snapshots"][0]["cashBalances"] == {"GBP": "0"} + + +@pytest.mark.asyncio +async def test_push_manual_snapshots_raises_on_partial_failure( + tmp_path: Path, +) -> None: + sp = tmp_path / "s.json" + sp.write_text(json.dumps({"cookies": {"wf_token": "fresh"}})) + + async def handler(req: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "snapshotsImported": 0, + "snapshotsFailed": 1, + "errors": [{"row": 0, "msg": "bad symbol"}], + }, + ) + + sink = _client(httpx.MockTransport(handler), sp) + snapshot = ManualSnapshotPayload( + date=date(2026, 5, 16), currency="GBP", + positions=[], cash_balances={}, + ) + with pytest.raises(WealthfolioError, match="bad symbol"): + await sink.push_manual_snapshots(account_id="acct", snapshots=[snapshot]) + + +@pytest.mark.asyncio +async def test_push_manual_snapshots_short_circuits_on_empty( + tmp_path: Path, +) -> None: + sp = tmp_path / "s.json" + sp.write_text(json.dumps({"cookies": {"wf_token": "fresh"}})) + + async def handler(req: httpx.Request) -> httpx.Response: + raise AssertionError(f"unexpected request: {req.method} {req.url.path}") + + sink = _client(httpx.MockTransport(handler), sp) + result = await sink.push_manual_snapshots(account_id="acct", snapshots=[]) + assert result["snapshotsImported"] == 0 From c9c0310733bf0efa61952ccb5fcc708078ba29c3 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Sat, 16 May 2026 23:47:49 +0000 Subject: [PATCH 04/27] fidelity: snapshot push needs WF account UUID, not logical id /api/v1/snapshots/import lookups the account by Wealthfolio's own UUID; passing our provider-side logical id ('fidelity-workplace-pension') returns 400 'Database operation failed: Record not found'. Resolve via sink.ensure_account() which the pipeline already runs idempotently, then pass the returned UUID into push_manual_snapshots(). --- broker_sync/cli.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/broker_sync/cli.py b/broker_sync/cli.py index 385fd01..879c3a2 100644 --- a/broker_sync/cli.py +++ b/broker_sync/cli.py @@ -440,9 +440,6 @@ def fidelity_ingest( async def _run() -> None: from datetime import date as _date_t - from broker_sync.providers.fidelity_planviewer import ( - ACCOUNT_ID as FID_ACCOUNT_ID, - ) from broker_sync.providers.fidelity_planviewer import ( fidelity_holdings_to_snapshot, ) @@ -478,8 +475,14 @@ def fidelity_ingest( as_of=_date_t.today(), ) if snapshot is not None: + # /api/v1/snapshots/import wants WF's own account UUID, + # not our logical provider id — look it up via the same + # match the pipeline used (provider+providerAccountId). + wf_account_id = await sink.ensure_account( + provider.accounts()[0], + ) push_result = await sink.push_manual_snapshots( - account_id=FID_ACCOUNT_ID, snapshots=[snapshot], + account_id=wf_account_id, snapshots=[snapshot], ) snapshot_imported = int(push_result.get("snapshotsImported", 0)) finally: From 98c47296228ed9b5f25c3a8eedf411677aa05979 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Sun, 17 May 2026 00:35:17 +0000 Subject: [PATCH 05/27] fidelity: replace snapshot-push with delta gains-offset DEPOSITs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-fund snapshot import landed quantities but dropped cost basis + needed a separate quote-push path we never identified. Snapshotting also collided with WF's own TOTAL aggregation and ZEROED the Fidelity cash balance. Simpler plan: each monthly scrape emits a single DEPOSIT (or WITHDRAWAL on a market drop) sized to the delta between the live PlanViewer pot value and Wealthfolio's running total. dav_corrected PG view continues to subtract these offsets from net_contribution so the dashboard Growth/ROI math stays right. - New gains_offset_delta_activity() — current_gain - prior_offset. - New WealthfolioSink.cumulative_amount_with_notes_prefix() — sums the existing fidelity-planviewer:unrealised-gains-offset DEPOSITs in WF so we know what's already been emitted. - CLI runs sync_provider_to_wealthfolio first (cash flows), then computes + emits the delta via import_activities. - 4 new provider tests for the delta logic; full suite (144 + 1 skipped) green; mypy + ruff clean. The old fidelity_holdings_to_snapshot helper + push_manual_snapshots sink method stay for future use but are no longer called. --- broker_sync/cli.py | 45 +++++++------- broker_sync/providers/fidelity_planviewer.py | 61 +++++++++++++++---- broker_sync/sinks/wealthfolio.py | 49 +++++++++++++++ tests/providers/test_fidelity_planviewer.py | 63 +++++++++++++++++++- 4 files changed, 183 insertions(+), 35 deletions(-) diff --git a/broker_sync/cli.py b/broker_sync/cli.py index 879c3a2..6e08eb8 100644 --- a/broker_sync/cli.py +++ b/broker_sync/cli.py @@ -438,10 +438,8 @@ def fidelity_ingest( sys.exit(2) async def _run() -> None: - from datetime import date as _date_t - from broker_sync.providers.fidelity_planviewer import ( - fidelity_holdings_to_snapshot, + gains_offset_delta_activity, ) sink = WealthfolioSink( @@ -461,37 +459,36 @@ def fidelity_ingest( result = await sync_provider_to_wealthfolio( provider=provider, sink=sink, dedup=dedup, since=since, ) - # PlanViewer has no historical per-fund unit-price feed, so - # the Activity stream above only carries cash flows. The - # current-pot fund positions captured in the same scrape get - # pushed via /api/v1/snapshots/import so per-fund quantity + - # cost basis land in WF (and propagate to the wealth - # dashboard's Positions table via pg-sync). - snapshot_imported = 0 + # PlanViewer doesn't expose per-fund unit prices in any feed + # WF can consume, so the only way to keep WF's pension total in + # line with the live PlanViewer pot value is to emit a small + # DEPOSIT (or WITHDRAWAL on a market drop) each run sized to + # the growth since the last scrape. The dav_corrected PG view + # subtracts these offsets from net_contribution so the + # dashboard's Growth/ROI panels stay accurate. + gains_delta_emitted = 0 if provider.last_holdings: - snapshot = fidelity_holdings_to_snapshot( + wf_account_id = await sink.ensure_account(provider.accounts()[0]) + prior_offset = await sink.cumulative_amount_with_notes_prefix( + account_id=wf_account_id, + notes_prefix="fidelity-planviewer:unrealised-gains-offset", + ) + delta = gains_offset_delta_activity( holdings=provider.last_holdings, total_real_contribution=provider.last_total_contribution, - as_of=_date_t.today(), + prior_offset_cumulative=prior_offset, + as_of=datetime.now(UTC), ) - if snapshot is not None: - # /api/v1/snapshots/import wants WF's own account UUID, - # not our logical provider id — look it up via the same - # match the pipeline used (provider+providerAccountId). - wf_account_id = await sink.ensure_account( - provider.accounts()[0], - ) - push_result = await sink.push_manual_snapshots( - account_id=wf_account_id, snapshots=[snapshot], - ) - snapshot_imported = int(push_result.get("snapshotsImported", 0)) + if delta is not None: + await sink.import_activities([delta]) + gains_delta_emitted = 1 finally: await sink.close() typer.echo(f"fidelity-ingest: fetched={result.fetched} " f"new={result.new_after_dedup} " f"imported={result.imported} " f"failed={result.failed} " - f"snapshots={snapshot_imported}") + f"gains_delta={gains_delta_emitted}") if result.failed > 0: sys.exit(1) diff --git a/broker_sync/providers/fidelity_planviewer.py b/broker_sync/providers/fidelity_planviewer.py index 4658dcf..b5b4e33 100644 --- a/broker_sync/providers/fidelity_planviewer.py +++ b/broker_sync/providers/fidelity_planviewer.py @@ -103,10 +103,10 @@ class FidelityPlanViewerProvider: storage_state, navigates to the transaction-history page with a wide date range, scrapes the table, and intercepts the valuation XHR. - After ``fetch()`` completes, ``last_holdings`` holds the per-fund - unit positions captured in the same scrape — used by the - ``fidelity-ingest`` CLI to push a manual snapshot to Wealthfolio - so per-fund quantities + cost basis land in the Positions table - (the activity stream alone only carries cash flows). + unit positions and ``last_total_contribution`` the cumulative cash + contribution — used by the ``fidelity-ingest`` CLI to emit a + delta-shaped DEPOSIT that nudges WF's net worth to match the + PlanViewer reported pot value (see ``gains_offset_delta_activity``). """ name = "fidelity-planviewer" @@ -162,12 +162,53 @@ class FidelityPlanViewerProvider: if before is not None and tx.date >= before: continue yield _tx_to_activity(tx) - # NB: the gains-offset DEPOSIT we used to emit here is superseded - # by the manual snapshot push the CLI does after fetch() drains. - # The snapshot sets per-fund quantity + cost basis directly, so - # Wealthfolio computes growth from positions instead of needing a - # fake cash entry. Old offset rows still in WF are corrected at - # the dashboard layer by the dav_corrected view. + # Gains-offset DEPOSITs are emitted by the CLI (which has the + # prior cumulative offset from WF). See `gains_offset_delta_activity`. + + +def gains_offset_delta_activity( + holdings: list[FidelityHolding], + total_real_contribution: Decimal, + prior_offset_cumulative: Decimal, + as_of: datetime, + min_delta: Decimal = Decimal("0.5"), +) -> Activity | None: + """Compute the gains-offset DELTA since the last scrape and shape it + as a DEPOSIT (or WITHDRAWAL on a market drop). + + The pension's per-fund prices aren't trackable in WF directly (no + public quote feed for these institutional life-fund share classes). + Instead, each monthly scrape emits a single small DEPOSIT/WITHDRAWAL + sized to ``(current_pot - real_contributions) - prior_cumulative_offset`` + — i.e., the growth (or loss) accrued since the last run. + + Wealthfolio's net_contribution then incorrectly includes all these + offsets; the ``dav_corrected`` PG view subtracts them back out so the + dashboard's Growth/ROI panels remain accurate. The deterministic + external_id (per scrape date) lets re-runs of the same day overwrite + rather than stack duplicates. + """ + if not holdings: + return None + current_pot = sum((h.total_value for h in holdings), Decimal(0)) + current_gain = current_pot - total_real_contribution + delta = current_gain - prior_offset_cumulative + if abs(delta) < min_delta: + return None + return Activity( + external_id=f"fidelity:gains-delta:{as_of.date().isoformat()}", + account_id=ACCOUNT_ID, + account_type=AccountType.WORKPLACE_PENSION, + date=as_of, + activity_type=ActivityType.DEPOSIT if delta > 0 else ActivityType.WITHDRAWAL, + currency=_CCY, + amount=abs(delta), + notes=( + f"fidelity-planviewer:unrealised-gains-offset delta=£{delta} " + f"(pot=£{current_pot}, contrib=£{total_real_contribution}, " + f"prior_offset=£{prior_offset_cumulative})" + ), + ) def fidelity_holdings_to_snapshot( diff --git a/broker_sync/sinks/wealthfolio.py b/broker_sync/sinks/wealthfolio.py index cb6ea45..7144f6f 100644 --- a/broker_sync/sinks/wealthfolio.py +++ b/broker_sync/sinks/wealthfolio.py @@ -17,6 +17,7 @@ _ACCOUNTS_PATH = "/api/v1/accounts" _IMPORT_CHECK = "/api/v1/activities/import/check" _IMPORT_REAL = "/api/v1/activities/import" _SNAPSHOTS_IMPORT = "/api/v1/snapshots/import" +_ACTIVITIES_SEARCH = "/api/v1/activities/search" class WealthfolioError(Exception): @@ -266,6 +267,54 @@ class WealthfolioSink: assert isinstance(got, list) return [r for r in got if isinstance(r, dict)] + # -- activity lookups -- + + async def cumulative_amount_with_notes_prefix( + self, + account_id: str, + notes_prefix: str, + ) -> Decimal: + """Sum the amount of DEPOSIT/WITHDRAWAL activities whose notes start + with ``notes_prefix``, signed (deposits positive, withdrawals negative). + + Used by the Fidelity provider to compute the delta gains-offset: + ``current_gain - cumulative_existing_offset`` becomes the new + DEPOSIT to emit on each monthly run. + """ + try: + resp = await self._request( + "POST", _ACTIVITIES_SEARCH, + json={"accountIds": [account_id], "page": 1, "pageSize": 500}, + ) + except Exception: + return Decimal(0) + if resp.status_code >= 400: + return Decimal(0) + payload = resp.json() + rows = payload.get("data", payload) if isinstance(payload, dict) else payload + if not isinstance(rows, list): + return Decimal(0) + total = Decimal(0) + for r in rows: + if not isinstance(r, dict): + continue + notes = r.get("comment") or r.get("notes") or "" + if not isinstance(notes, str) or not notes.startswith(notes_prefix): + continue + amt_raw = r.get("amount") + if amt_raw is None: + continue + try: + amt = Decimal(str(amt_raw)) + except Exception: + continue + atype = (r.get("activityType") or r.get("activity_type") or "").upper() + if atype == "WITHDRAWAL": + total -= amt + else: + total += amt + return total + # -- manual holdings snapshots -- async def push_manual_snapshots( diff --git a/tests/providers/test_fidelity_planviewer.py b/tests/providers/test_fidelity_planviewer.py index a030ac3..acfccbc 100644 --- a/tests/providers/test_fidelity_planviewer.py +++ b/tests/providers/test_fidelity_planviewer.py @@ -7,13 +7,14 @@ from pathlib import Path import pytest -from broker_sync.models import Account, AccountType +from broker_sync.models import Account, AccountType, ActivityType from broker_sync.providers.fidelity_planviewer import ( ACCOUNT_ID, FidelityCreds, FidelityPlanViewerProvider, FidelityProviderConfigError, fidelity_holdings_to_snapshot, + gains_offset_delta_activity, ) from broker_sync.providers.parsers.fidelity import ( parse_transactions_html, @@ -146,3 +147,63 @@ def test_provider_caches_holdings_for_cli_snapshot_push() -> None: # Pre-fetch state: empty assert prov.last_holdings == [] assert prov.last_total_contribution == Decimal(0) + + +# -- delta-shaped gains offset (the monthly accumulation mechanism) -- + + +def _holdings_summing_to(total: Decimal) -> list: + from broker_sync.providers.parsers.fidelity import FidelityHolding + return [FidelityHolding( + fund_code="KDOA", fund_name="Test", units=Decimal("100"), + unit_price=total / Decimal("100"), currency="GBP", total_value=total, + units_by_source={}, + )] + + +def test_gains_delta_emits_deposit_when_gain_exceeds_prior_offset() -> None: + # pot £145k, real contrib £102k → current gain £43k; prior offset £35k + # → delta = +£8k + activity = gains_offset_delta_activity( + holdings=_holdings_summing_to(Decimal("145000")), + total_real_contribution=Decimal("102000"), + prior_offset_cumulative=Decimal("35000"), + as_of=datetime(2026, 5, 17, tzinfo=UTC), + ) + assert activity is not None + assert activity.activity_type == ActivityType.DEPOSIT + assert activity.amount == Decimal("8000") + assert activity.external_id == "fidelity:gains-delta:2026-05-17" + assert "unrealised-gains-offset" in (activity.notes or "") + + +def test_gains_delta_emits_withdrawal_on_market_drop() -> None: + # pot dropped: current gain £30k, prior offset £35k → delta = -£5k + activity = gains_offset_delta_activity( + holdings=_holdings_summing_to(Decimal("132000")), + total_real_contribution=Decimal("102000"), + prior_offset_cumulative=Decimal("35000"), + as_of=datetime(2026, 5, 17, tzinfo=UTC), + ) + assert activity is not None + assert activity.activity_type == ActivityType.WITHDRAWAL + assert activity.amount == Decimal("5000") + + +def test_gains_delta_suppressed_below_minimum() -> None: + # delta ~£0.20, below the £0.50 min — skip emission to avoid noise. + activity = gains_offset_delta_activity( + holdings=_holdings_summing_to(Decimal("137000.20")), + total_real_contribution=Decimal("102000"), + prior_offset_cumulative=Decimal("35000"), + as_of=datetime(2026, 5, 17, tzinfo=UTC), + ) + assert activity is None + + +def test_gains_delta_none_when_no_holdings() -> None: + assert gains_offset_delta_activity( + holdings=[], total_real_contribution=Decimal("0"), + prior_offset_cumulative=Decimal("0"), + as_of=datetime(2026, 5, 17, tzinfo=UTC), + ) is None From d860aef9278bd96548c8715e14a1af1a49e2a6b7 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Fri, 22 May 2026 14:41:09 +0000 Subject: [PATCH 06/27] imap: accept Schwab subdomain senders (donotreply@mail.schwab.com) Real Schwab trade-execution emails come from donotreply@mail.schwab.com, not the root @schwab.com domain. The existing matcher's endswith("@schwab.com") guard rejected these, silently skipping the May 2026 RSU vest's same-day-sell confirmation. Extend the matcher to also accept any *.schwab.com subdomain. Added test_schwab_subdomain_sender_matches; full suite green. --- broker_sync/providers/imap.py | 6 +++++- tests/providers/test_imap.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/broker_sync/providers/imap.py b/broker_sync/providers/imap.py index e935bab..9d52478 100644 --- a/broker_sync/providers/imap.py +++ b/broker_sync/providers/imap.py @@ -163,7 +163,11 @@ def fetch_activities(creds: ImapCreds) -> list[Activity]: if sender in _IE_SENDERS or sender.endswith("@investengine.com"): out.extend(ie_parser.parse_invest_engine_email(raw)) ie_parsed += 1 - elif sender in _SCHWAB_SENDERS or sender.endswith("@schwab.com"): + elif ( + sender in _SCHWAB_SENDERS + or sender.endswith("@schwab.com") + or sender.endswith(".schwab.com") # e.g. donotreply@mail.schwab.com + ): html = _html_or_text(msg) out.extend(parse_schwab_email(html)) schwab_parsed += 1 diff --git a/tests/providers/test_imap.py b/tests/providers/test_imap.py index 63638cb..9c1fe8d 100644 --- a/tests/providers/test_imap.py +++ b/tests/providers/test_imap.py @@ -99,3 +99,18 @@ def test_non_ie_activities_passed_through_unchanged() -> None: routed = _split_ie_by_isa_cap([schwab_act]) assert routed[0].account_id == "schwab-workplace" assert routed[0].account_type is AccountType.GIA + + +def test_schwab_subdomain_sender_matches() -> None: + """Real Schwab trade emails come from `donotreply@mail.schwab.com` + (subdomain), not just `donotreply@schwab.com`. The matcher must + accept either form.""" + from broker_sync.providers.imap import _SCHWAB_SENDERS + # Verify the static set works + assert "donotreply@schwab.com" in _SCHWAB_SENDERS + # Verify the subdomain suffix check + for addr in ( + "donotreply@mail.schwab.com", + "wealthnotify@equityawards.schwab.com", + ): + assert addr.endswith(".schwab.com"), addr From d5dbeb96af012995b2493a46d1a6b2ac7982f40e Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Fri, 22 May 2026 14:54:06 +0000 Subject: [PATCH 07/27] tests: type the FidelityHolding factory list to satisfy CI mypy CI runs mypy on both broker_sync/ and tests/, with stricter 'Missing type arguments for generic type' enforcement. Local mypy was only scoped to broker_sync/. Annotate the test helper with list[FidelityHolding]; lift the import to module-level. --- tests/providers/test_fidelity_planviewer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/providers/test_fidelity_planviewer.py b/tests/providers/test_fidelity_planviewer.py index acfccbc..19c389a 100644 --- a/tests/providers/test_fidelity_planviewer.py +++ b/tests/providers/test_fidelity_planviewer.py @@ -17,6 +17,7 @@ from broker_sync.providers.fidelity_planviewer import ( gains_offset_delta_activity, ) from broker_sync.providers.parsers.fidelity import ( + FidelityHolding, parse_transactions_html, parse_valuation_json, ) @@ -152,8 +153,7 @@ def test_provider_caches_holdings_for_cli_snapshot_push() -> None: # -- delta-shaped gains offset (the monthly accumulation mechanism) -- -def _holdings_summing_to(total: Decimal) -> list: - from broker_sync.providers.parsers.fidelity import FidelityHolding +def _holdings_summing_to(total: Decimal) -> list[FidelityHolding]: return [FidelityHolding( fund_code="KDOA", fund_name="Test", units=Decimal("100"), unit_price=total / Decimal("100"), currency="GBP", total_value=total, From 68d4832c2ed61a9e9c649e36b4d4e681561dfa22 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Tue, 26 May 2026 21:16:28 +0000 Subject: [PATCH 08/27] imap: skip InvestEngine emails via BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS The IMAP IE parser and the bearer-token IE API path generate different external_ids for the same fill, so running both produces duplicate BUYs in Wealthfolio. With IE now served by the API path (broker-sync invest-engine), we keep the IMAP path live for Schwab and gate IE off via env var. Setting BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS=invest-engine on the imap CronJob stops new dupes; Schwab routing is unaffected. Co-Authored-By: Claude Opus 4.7 --- broker_sync/providers/imap.py | 17 +++++++++++++++-- tests/providers/test_imap.py | 36 +++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/broker_sync/providers/imap.py b/broker_sync/providers/imap.py index 9d52478..5564dd3 100644 --- a/broker_sync/providers/imap.py +++ b/broker_sync/providers/imap.py @@ -16,6 +16,7 @@ from __future__ import annotations import email import imaplib import logging +import os import re import ssl from collections.abc import AsyncIterator, Iterator @@ -152,7 +153,12 @@ def _fetch_all(creds: ImapCreds) -> Iterator[bytes]: def fetch_activities(creds: ImapCreds) -> list[Activity]: out: list[Activity] = [] - ie_parsed = schwab_parsed = skipped = 0 + ie_parsed = schwab_parsed = ie_skipped = skipped = 0 + exclude = { + p.strip().lower() + for p in os.environ.get("BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS", "").split(",") + if p.strip() + } for raw in _fetch_all(creds): try: msg = email.message_from_bytes(raw) @@ -161,6 +167,9 @@ def fetch_activities(creds: ImapCreds) -> list[Activity]: continue sender = _extract_sender(msg) if sender in _IE_SENDERS or sender.endswith("@investengine.com"): + if "invest-engine" in exclude or "invest_engine" in exclude: + ie_skipped += 1 + continue out.extend(ie_parser.parse_invest_engine_email(raw)) ie_parsed += 1 elif ( @@ -168,14 +177,18 @@ def fetch_activities(creds: ImapCreds) -> list[Activity]: or sender.endswith("@schwab.com") or sender.endswith(".schwab.com") # e.g. donotreply@mail.schwab.com ): + if "schwab" in exclude: + skipped += 1 + continue html = _html_or_text(msg) out.extend(parse_schwab_email(html)) schwab_parsed += 1 else: skipped += 1 log.info( - "imap: ie_parsed=%d schwab_parsed=%d skipped=%d → %d activities", + "imap: ie_parsed=%d ie_skipped=%d schwab_parsed=%d skipped=%d → %d activities", ie_parsed, + ie_skipped, schwab_parsed, skipped, len(out), diff --git a/tests/providers/test_imap.py b/tests/providers/test_imap.py index 9c1fe8d..1abe587 100644 --- a/tests/providers/test_imap.py +++ b/tests/providers/test_imap.py @@ -101,6 +101,42 @@ def test_non_ie_activities_passed_through_unchanged() -> None: assert routed[0].account_type is AccountType.GIA +def test_exclude_invest_engine_skips_ie_emails(monkeypatch) -> None: + """BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS=invest-engine should skip IE messages + so we don't duplicate IE buys already ingested via the bearer-token API path. + Schwab routing must remain unaffected.""" + from broker_sync.providers import imap as imap_mod + + ie_email = ( + b"From: noreply@investengine.com\r\n" + b"Subject: VUAG Bought\r\n" + b"Content-Type: text/plain\r\n\r\n" + b"Vanguard S&P 500: VUAG Bought 10.0 @ 100.0 per share Total: 1000.00\r\n" + ) + schwab_email = ( + b"From: donotreply@schwab.com\r\n" + b"Subject: Order Confirmed\r\n" + b"Content-Type: text/html\r\n\r\n" + b"no-op\r\n" + ) + monkeypatch.setattr(imap_mod, "_fetch_all", lambda _: [ie_email, schwab_email]) + monkeypatch.setattr(imap_mod.ie_parser, "parse_invest_engine_email", + lambda raw: [object()]) + monkeypatch.setattr(imap_mod, "parse_schwab_email", lambda html: [object()]) + + creds = imap_mod.ImapCreds(host="h", user="u", password="p", directory="d") + + monkeypatch.setenv("BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS", "invest-engine") + out_excluded = imap_mod.fetch_activities(creds) + # IE skipped → only the schwab activity is emitted + assert len(out_excluded) == 1 + + monkeypatch.delenv("BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS", raising=False) + out_default = imap_mod.fetch_activities(creds) + # Both providers fire when env unset + assert len(out_default) == 2 + + def test_schwab_subdomain_sender_matches() -> None: """Real Schwab trade emails come from `donotreply@mail.schwab.com` (subdomain), not just `donotreply@schwab.com`. The matcher must From fe35c8e8268b9c4ed1875df324a83ecbc62429e9 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Tue, 26 May 2026 21:18:55 +0000 Subject: [PATCH 09/27] test: fix mypy errors in IE-exclude test - annotate monkeypatch fixture as pytest.MonkeyPatch - import invest_engine parser module directly instead of via imap_mod.ie_parser (mypy's strict "no implicit re-export" rule trips on the indirection) Co-Authored-By: Claude Opus 4.7 --- tests/providers/test_imap.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/providers/test_imap.py b/tests/providers/test_imap.py index 1abe587..fcb4a0f 100644 --- a/tests/providers/test_imap.py +++ b/tests/providers/test_imap.py @@ -2,8 +2,12 @@ from __future__ import annotations from datetime import UTC, date, datetime from decimal import Decimal +from typing import TYPE_CHECKING from broker_sync.models import AccountType, Activity, ActivityType + +if TYPE_CHECKING: + from pytest import MonkeyPatch from broker_sync.providers.imap import ( _IE_GIA_ACCOUNT_ID, _IE_ISA_ACCOUNT_ID, @@ -101,11 +105,12 @@ def test_non_ie_activities_passed_through_unchanged() -> None: assert routed[0].account_type is AccountType.GIA -def test_exclude_invest_engine_skips_ie_emails(monkeypatch) -> None: +def test_exclude_invest_engine_skips_ie_emails(monkeypatch: "MonkeyPatch") -> None: """BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS=invest-engine should skip IE messages so we don't duplicate IE buys already ingested via the bearer-token API path. Schwab routing must remain unaffected.""" from broker_sync.providers import imap as imap_mod + from broker_sync.providers.parsers import invest_engine as ie_parser ie_email = ( b"From: noreply@investengine.com\r\n" @@ -120,8 +125,7 @@ def test_exclude_invest_engine_skips_ie_emails(monkeypatch) -> None: b"no-op\r\n" ) monkeypatch.setattr(imap_mod, "_fetch_all", lambda _: [ie_email, schwab_email]) - monkeypatch.setattr(imap_mod.ie_parser, "parse_invest_engine_email", - lambda raw: [object()]) + monkeypatch.setattr(ie_parser, "parse_invest_engine_email", lambda raw: [object()]) monkeypatch.setattr(imap_mod, "parse_schwab_email", lambda html: [object()]) creds = imap_mod.ImapCreds(host="h", user="u", password="p", directory="d") From e6ef1fce978b96bac8f87aab2a0fa0888b4b5bdc Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Tue, 26 May 2026 21:20:12 +0000 Subject: [PATCH 10/27] test: drop redundant quotes on MonkeyPatch annotation `from __future__ import annotations` makes the quoting unnecessary and ruff UP037 flags it. Co-Authored-By: Claude Opus 4.7 --- tests/providers/test_imap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/providers/test_imap.py b/tests/providers/test_imap.py index fcb4a0f..0264b37 100644 --- a/tests/providers/test_imap.py +++ b/tests/providers/test_imap.py @@ -105,7 +105,7 @@ def test_non_ie_activities_passed_through_unchanged() -> None: assert routed[0].account_type is AccountType.GIA -def test_exclude_invest_engine_skips_ie_emails(monkeypatch: "MonkeyPatch") -> None: +def test_exclude_invest_engine_skips_ie_emails(monkeypatch: MonkeyPatch) -> None: """BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS=invest-engine should skip IE messages so we don't duplicate IE buys already ingested via the bearer-token API path. Schwab routing must remain unaffected.""" From ca5f98f77168e047b4afd512311c988ae9b42ec6 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Tue, 26 May 2026 21:52:52 +0000 Subject: [PATCH 11/27] docs: add IBKR Flex ingestion design spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a daily IBKR Flex Web Service → Wealthfolio ingestion path alongside the existing T212 / IE / Fidelity providers. Uses the ibflex library; mandatory broker-vs-WF position reconciliation built in from day one to prevent the silent-drift class of bug we hit with InvestEngine on 2026-05-26. Account is currently empty so the initial backfill step is a no-op until the first IBKR trade lands. Co-Authored-By: Claude Opus 4.7 --- docs/specs/2026-05-26-ibkr-ingest-design.md | 322 ++++++++++++++++++++ 1 file changed, 322 insertions(+) create mode 100644 docs/specs/2026-05-26-ibkr-ingest-design.md diff --git a/docs/specs/2026-05-26-ibkr-ingest-design.md b/docs/specs/2026-05-26-ibkr-ingest-design.md new file mode 100644 index 0000000..40bb9df --- /dev/null +++ b/docs/specs/2026-05-26-ibkr-ingest-design.md @@ -0,0 +1,322 @@ +# IBKR Flex Ingestion — Design + +**Date:** 2026-05-26 +**Status:** Approved (brainstorming session 2026-05-26) +**Author:** Viktor + Claude (Opus 4.7) +**Implementation plan:** TBD (will be written next session via writing-plans skill) + +## Context + +Adds Interactive Brokers (IBKR UK / IE — stocks/ETFs only) as a new +broker-sync provider, pushing activities to Wealthfolio on a daily +schedule alongside the existing Trading 212 / InvestEngine / Fidelity +pipelines. + +The user's IBKR account is **currently empty** (no positions, no trades). +This design covers the integration as it will run once the account is +funded and active. The initial backfill step in the setup checklist is a +no-op until the first IBKR trade. + +This work is the structural follow-on from the 2026-05-26 Wealthfolio +dedup session, in which £252k of duplicated InvestEngine positions +accumulated silently in WF because the IMAP and API ingestion paths +emitted different `external_id` schemes and never reconciled against +broker-reported truth. The IBKR design bakes in **broker-vs-WF position +reconciliation from day one** — the missing capability that allowed the +IE drift to grow undetected. + +## Decisions + +### D1 — Use IBKR Flex Web Service (not Client Portal API / TWS) + +Flex Web Service is a token-authenticated REST endpoint returning XML +statements. Suits unattended cron because: +- One-year token validity (no daily re-auth, unlike Client Portal Gateway). +- No sidecar / GUI / Java runtime needed. +- Designed for periodic batch reporting — the exact shape of our pipeline. + +Client Portal Web API + `ibind` was considered and rejected: its Gateway +sidecar requires browser-based re-auth roughly every 24 hours, which is +incompatible with unattended scheduling. + +### D2 — Library: `ibflex` (`csingley/ibflex` on PyPI) + +Adds `ibflex = "^0.16"` to `pyproject.toml`. The library provides: +- `client.download(token, query_id) -> bytes` — handles Flex's 2-step + async API (`SendRequest` → `GetStatement` polling). +- `parser.parse(xml) -> FlexQueryResponse` — typed dataclasses for + `Trades`, `CashTransactions`, `OpenPositions`, `SecuritiesInfo`. + +Fallback (Approach B): if `ibflex` proves to lag IBKR schema changes, drop +in raw `httpx` + `xml.etree`. Same provider shape; only the parsing +internals change. + +### D3 — One CronJob, daily 02:00 UK, in `broker-sync` namespace + +Matches the existing `broker-sync-trading212` cadence and placement. No +new namespace, no new image. + +### D4 — Reconciliation is mandatory, not optional + +Every run computes a per-asset quantity from the Flex +`OpenPositions` section and compares against WF's computed quantity from +activities. Drift is published as a Pushgateway metric. Cross-checking +broker truth is the line of defense against the IE-style silent +divergence we saw on 2026-05-26. + +### D5 — One account, one query + +Single Flex Activity Query covering Trades + Cash + Open Positions + +Securities. Single `Interactive Brokers (UK)` account in Wealthfolio. +Multiple accounts can be added later by parameterising the CLI command; +not in scope now. + +## Architecture + +``` +broker-sync K8s namespace +├── CronJob broker-sync-ibkr (schedule: 0 2 * * *) +│ ├── env from broker-sync-secrets: +│ │ IBKR_FLEX_TOKEN, IBKR_FLEX_QUERY_ID, IBKR_ACCOUNT_ID, +│ │ WF_BASE_URL, WF_USERNAME, WF_PASSWORD +│ ├── PVC broker-sync-data-encrypted (shared with other broker-sync jobs) +│ └── image viktorbarzin/broker-sync: command = ["broker-sync", "ibkr"] +│ +│ External calls +│ ├── HTTPS → ndcdyn.interactivebrokers.com (Flex Web Service) +│ ├── HTTP → wealthfolio.wealthfolio.svc (activities import + position read) +│ └── HTTP → pushgateway.monitoring.svc (drift + last-success metrics) +``` + +The provider is structurally identical to `broker-sync-trading212` and +the IE bearer-token path — same Vault → CronJob → provider → pipeline → +WF flow. Existing alerting (CronJob-failed, ExternalSecret-stale, +WF-sync-stale) applies transitively; we only add IBKR-specific alerts on +top. + +## Components + +| Path | Action | Description | +|---|---|---| +| `broker_sync/providers/ibkr.py` | NEW | `IBKRProvider` class implementing the `Provider` protocol. Maps Flex XML to `Activity[]`. ~200 LOC. | +| `broker_sync/cli.py` | MODIFY | New `@app.command("ibkr")` typer command, parallel to `trading212` and `invest-engine`. ~60 LOC. | +| `pyproject.toml` | MODIFY | Add `ibflex = "^0.16"` dependency. | +| `tests/providers/test_ibkr.py` | NEW | Fixture-based parsing tests, sign-conventions, position-drift math, account-id guard. | +| `infra/stacks/broker-sync/main.tf` | MODIFY | New `kubernetes_cron_job_v1.ibkr` resource. | +| Vault `secret/broker-sync` | MODIFY | Add `ibkr_flex_token`, `ibkr_flex_query_id`, `ibkr_account_id`. | +| Wealthfolio (one-time, manual) | NEW data | Create `Interactive Brokers (UK)` account; record its UUID in Vault. | +| `docs/providers/ibkr.md` | NEW | Production-facing provider docs (setup, query design, troubleshooting). Written after first successful run. | + +## Data flow (per CronJob run) + +1. **02:00 UK** — CronJob fires, pod starts with env from `broker-sync-secrets`. +2. **Download** — `ibflex.client.download(token, query_id)` calls Flex + Web Service `SendRequest` + `GetStatement`. Typical 5–20 s. Library + handles retry/polling. +3. **Parse** — `ibflex.parser.parse(xml)` produces a + `FlexQueryResponse`. +4. **Account guard** — two distinct identifiers exist: + - **IBKR_ACCOUNT_ID_UPSTREAM**: the IBKR-side account number + (e.g. `U12345678`), used to validate that the Flex report belongs to + the right account. + - **IBKR_ACCOUNT_ID** (alias: `ibkr_account_id` in Vault): the + Wealthfolio account UUID (e.g. `8a3f...`), used when posting + activities to WF. + Validate `stmt.accountId == os.environ["IBKR_ACCOUNT_ID_UPSTREAM"]`. + Refuse to ingest on mismatch — prevents wrong-account writes from a + misconfigured query. +5. **Map Trades → Activities**: + + | Flex | Activity | Notes | + |---|---|---| + | `Trade.tradeID` | `external_id = "ibkr:trade:" + tradeID` | dedup key | + | `Trade.tradeDate + tradeTime` | `date` (UTC) | timezone normalised | + | `Trade.symbol` | `symbol` | canonicalised — LSE tickers get `.L` suffix | + | `Trade.buySell` (BUY / SELL) | `activity_type` | direct | + | `Trade.quantity` | `quantity` | always positive (broker-sync convention) | + | `Trade.tradePrice` | `unit_price` | | + | `Trade.currency` | `currency` | per-trade, multi-ccy supported | + | `Trade.ibCommission` | `fee = abs(ibCommission)` | always positive | + | `Trade.assetCategory` | (sanity check; skip if not in {STK, ETF}) | + +6. **Map CashTransactions → Activities**: + + | Flex `CashTransaction.type` | Activity `activity_type` | Notes | + |---|---|---| + | `Dividends` | `DIVIDEND` | | + | `Withholding Tax` | `FEE` | tag with `notes="wht:..."` | + | `Broker Interest Paid` | `FEE` | negative direction | + | `Broker Interest Received` | `DIVIDEND` | interest treated as income | + | `Deposits & Withdrawals` | `DEPOSIT` (amount > 0) or `WITHDRAWAL` (amount < 0) | | + | `Commission Adjustments` | `FEE` | | + | anything else | skip + log WARNING with the unknown type | refuse to guess, same convention as IE provider | + + external_id = `"ibkr:cash:" + transactionID`. + +7. **Cash-flow match** — `_with_cash_flow_match(a)` from the shared + pipeline emits a matching DEPOSIT for every BUY (and WITHDRAWAL for + every SELL) so WF cash balance stays consistent. This is the existing + pattern used by T212 + IE; IBKR slots in identically. + +8. **Dedup** — `SyncRecordStore(/data/sync.db)` skips any `external_id` + already synced. Idempotent re-runs are safe. + +9. **Import** — `WealthfolioSink.import_activities(...)` POSTs to + `/api/v1/activities/import`. Existing 401 retry logic applies. + +10. **Reconciliation** — for each `OpenPositions` row: + + ```python + # compute_wf_position_qty: NEW helper in WealthfolioSink. + # Queries POST /api/v1/activities/search filtered by accountId, sums + # BUY/SELL/ADD_HOLDING/REMOVE_HOLDING quantities per asset. + wf_qty_by_asset = wf_sink.compute_position_qty(IBKR_ACCOUNT_ID) + for pos in flex_response.OpenPositions: + symbol = canonical_symbol(pos.symbol) + drift = float(pos.position) - wf_qty_by_asset.get(symbol, Decimal(0)) + push_metric( + "ibkr_position_drift_shares", + labels={"symbol": symbol, "account": "ibkr-uk"}, + value=float(drift), + ) + push_metric("ibkr_sync_last_success_timestamp_seconds", time.time()) + ``` + +11. **Exit 0** on success, non-zero on any unrecoverable error. + +## Error handling + +| Failure | Detection | Response | Alert | +|---|---|---|---| +| Token expired (Flex code 1003) | `ibflex.client.ResponseCodeError` | Exit non-zero with explicit log | `IBKRFlexTokenExpired` Loki rule + stale-success Prom alert | +| Statement generation timeout | `ibflex.client.StatementGenerationTimeout` | Retry once after 60 s, then exit non-zero | Stale-success alert catches it after 24 h | +| Empty report (quiet day) | Zero Trades + zero CashTxns | Log "no new activity", still update success timestamp, still reconcile | (none — happy path) | +| WF API 401 | HTTP status | Re-login via `WealthfolioSink` (existing logic) | (existing) | +| WF rejects an activity row | `summary.skipped > 0` | Log per-row + exit non-zero | `IBKRImportRejected` Loki rule | +| Network / DNS fail | httpx exception | Retry once with 30 s backoff | `KubeJobFailed` (existing) | +| **Position drift > 0.01 share for >24h** | Pushgateway non-zero across runs | Prom alert `IBKRPositionDrift{symbol}` warning → Slack `#security` | **NEW capability** | +| Account ID mismatch | Flex `accountId` != env var | Exit 2 immediately, write nothing | `IBKRAccountMismatch` urgent Loki rule | + +## Setup checklist (one-time) + +### Step 1 — IBKR Client Portal (manual, ~5 min) + +1. Sign in at `https://www.interactivebrokers.co.uk/` → **Account + Settings**. +2. **Reports → Settings → Flex Web Service** → Enable → copy the + one-time-displayed **Token** (1 year validity). +3. **Reports → Flex Queries → Activity Flex Query → Create New**: + - Name: `broker-sync-activity` + - Sections: `Account Information`, `Trades`, `Cash Transactions`, + `Open Positions`, `Securities Information` + - Date Format: `yyyy-MM-dd` · Time Format: `HH:mm:ss TimeZone` + - Date Range: `Last Business Day` (for daily runs; flip to + `Year to Date` only for the initial backfill — irrelevant while + account is empty) + - Format: XML + - Trade fields: ensure `tradeID`, `tradeDate`, `tradeTime`, `symbol`, + `buySell`, `quantity`, `tradePrice`, `currency`, `ibCommission`, + `assetCategory` selected. + - CashTransaction fields: `transactionID`, `dateTime`, `type`, + `amount`, `currency`, `description`. + - OpenPositions fields: `symbol`, `position`, `markPrice`, `currency`, + `assetCategory`. + - Save → copy the **Query ID** (5–7 digit number). + +### Step 2 — Vault + +```bash +vault kv patch secret/broker-sync \ + ibkr_flex_token='YOUR_TOKEN' \ + ibkr_flex_query_id='YOUR_QUERY_ID' \ + ibkr_account_id='WF_UUID_FROM_STEP_3' \ + ibkr_account_id_upstream='YOUR_IBKR_ACCOUNT_NUMBER' +``` + +### Step 3 — Create WF account (script + paste UUID back) + +```bash +# Login → POST /accounts → capture id +curl -sS -c /tmp/wf-jar -X POST "$WF_BASE_URL/api/v1/auth/login" \ + -H 'Content-Type: application/json' -d "{\"password\":\"$WF_PASSWORD\"}" +curl -sS -b /tmp/wf-jar -X POST "$WF_BASE_URL/api/v1/accounts" \ + -H 'Content-Type: application/json' \ + -d '{"name":"Interactive Brokers (UK)","accountType":"GIA","currency":"GBP","isActive":true}' \ + | jq -r '.id' +# Paste the UUID back into Vault under ibkr_account_id +``` + +### Step 4 — Initial backfill (skip while account is empty) + +When the IBKR account first holds positions, switch the Flex query +Date Range to `Year to Date`, run the CronJob manually once, verify WF +totals match the broker app, then switch the Flex query back to +`Last Business Day` for daily incremental. + +### Step 5 — Deploy + +1. Push to broker-sync `main` (direct push — personal repo convention, + no PR) → GHA builds `viktorbarzin/broker-sync:latest`. +2. `cd infra/stacks/broker-sync && scripts/tg apply` creates the new + CronJob. +3. Wait for the 02:00 UK run, or trigger manually: + `kubectl -n broker-sync create job --from=cronjob/broker-sync-ibkr broker-sync-ibkr-test-1`. +4. Verify in WF UI: account exists, activities present (if any), + reconciliation drift metric showing zero. + +## Testing + +**Unit tests** in `tests/providers/test_ibkr.py`: + +- `test_parse_trades_maps_to_activities` — canned 3-trade XML, verify + external_id, symbol mapping, quantity sign, fee sign. +- `test_parse_dividend_maps_to_dividend_activity`. +- `test_parse_unknown_cash_type_logs_warning_and_skips`. +- `test_account_id_mismatch_raises` — Flex returns a different + `accountId` than env, refuse to ingest. +- `test_position_drift_computed_correctly` — three-asset scenario, two + match, one drifts. +- `test_canonical_symbol_lse_suffix` — `VUAG` → `VUAG.L`, + `AAPL` → `AAPL` (US, no suffix), etc. + +All tests mock `ibflex.client.download` to avoid network. + +**Integration test** (manual, post-deploy): +- Trigger CronJob manually. +- Inspect logs. +- Verify in WF UI and Pushgateway. + +## Acceptance criteria + +- [ ] `broker-sync ibkr` command runs end-to-end against the real Flex Web + Service with the user's token. +- [ ] WF accepts the resulting activity imports (no `summary.skipped`). +- [ ] `ibkr_position_drift_shares` is published for every asset; drift = 0 + on a steady-state run. +- [ ] Re-running the command is idempotent — no duplicate activities + written to WF. +- [ ] CronJob completes successfully on its schedule for 7 consecutive days + before the design is marked Done. + +## Out of scope + +- Multi-account support (only one IBKR account designed in). +- Real-time data / order placement (Flex is batch-only). +- Stock split / corporate action handling — IBKR reports these in the + Flex `CorporateActions` section but we're not enabling that section + yet; revisit if it becomes needed. +- Multi-currency FX conversion math — we record per-trade currency + faithfully and let Wealthfolio do FX. If WF's FX handling proves + inadequate, a separate spec covers that. + +## Open questions + +(None at design-approval time. Captured here for future amendments.) + +## References + +- `ibflex` library docs (csingley/ibflex) +- Existing patterns in `broker_sync/providers/trading212.py` and + `broker_sync/providers/invest_engine.py` +- `~/code/infra/stacks/broker-sync/main.tf` (CronJob structure to mirror) +- 2026-05-26 Wealthfolio dedup session (motivates the reconciliation step) From c271d5101c05eaebf84c1cfc654b006443c2f903 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Tue, 26 May 2026 22:19:54 +0000 Subject: [PATCH 12/27] docs: add IBKR Flex ingestion implementation plan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 15 task plan covering deps → fixtures → metrics → sink helper → provider mapping → CLI command → CI/image → Vault setup → Terraform CronJob → smoke test → provider runbook → 7-day soak. Co-Authored-By: Claude Opus 4.7 --- docs/plans/2026-05-26-ibkr-flex-ingestion.md | 1578 ++++++++++++++++++ 1 file changed, 1578 insertions(+) create mode 100644 docs/plans/2026-05-26-ibkr-flex-ingestion.md diff --git a/docs/plans/2026-05-26-ibkr-flex-ingestion.md b/docs/plans/2026-05-26-ibkr-flex-ingestion.md new file mode 100644 index 0000000..7a22578 --- /dev/null +++ b/docs/plans/2026-05-26-ibkr-flex-ingestion.md @@ -0,0 +1,1578 @@ +# IBKR Flex Ingestion Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a daily IBKR Flex Web Service → Wealthfolio ingestion path +to `broker-sync`, with mandatory broker-vs-WF position reconciliation. + +**Architecture:** New `IBKRProvider` in `broker_sync/providers/ibkr.py` uses +the `ibflex` library to download + parse Flex XML reports. Mapped activities +flow through the existing pipeline (cash-flow-match → dedup → WF import). +After import, a new reconciliation step compares Flex `OpenPositions` +against WF-computed quantities and pushes drift to Pushgateway. A new K8s +CronJob `broker-sync-ibkr` schedules it at 02:00 UK daily. + +**Tech stack:** Python 3.12, `ibflex ^0.16` (new), `httpx` (existing), +`typer` (existing), Terraform + K8s CronJob (existing pattern), Vault +KV-v2 secret backend (existing), Prometheus Pushgateway (cluster-internal). + +**Spec:** `docs/specs/2026-05-26-ibkr-ingest-design.md` (in this repo). + +--- + +## File Structure + +| Path | Responsibility | New? | +|---|---|---| +| `broker_sync/providers/ibkr.py` | `IBKRProvider` — fetch + parse + map. Module is the entire IBKR ingestion provider. | NEW | +| `broker_sync/metrics.py` | One-function module: `push_pushgateway(job, metrics, labels)` — simple httpx POST to the cluster Pushgateway. Shared by future providers. | NEW | +| `broker_sync/sinks/wealthfolio.py` | Add `compute_position_qty(account_id) -> dict[str, Decimal]` method to `WealthfolioSink`. | MODIFY | +| `broker_sync/cli.py` | Add `@app.command("ibkr")` typer command, parallel to `trading212` and `invest-engine`. | MODIFY | +| `pyproject.toml` | Add `ibflex = "^0.16"` dependency. | MODIFY | +| `tests/providers/test_ibkr.py` | Unit tests for IBKRProvider mapping logic + account guard. | NEW | +| `tests/fixtures/ibkr/sample_flex.xml` | Canned Flex XML fixture (3 trades, 2 cash txns, 2 positions, 1 account). | NEW | +| `tests/sinks/test_wealthfolio.py` | Add tests for the new `compute_position_qty` method. | MODIFY | +| `tests/test_metrics.py` | Test the `push_pushgateway` function with a mock httpx transport. | NEW | +| `infra/stacks/broker-sync/main.tf` | Add `kubernetes_cron_job_v1.ibkr` resource + matching PrometheusRule for drift / staleness alerts. | MODIFY | + +Files are split by responsibility, not by layer. The provider is a single +file (`ibkr.py`) because its three concerns — fetch, parse-map, reconcile +— are tightly coupled by the Flex XML shape. + +--- + +## Task 1: Add the `ibflex` dependency + +**Files:** +- Modify: `pyproject.toml` + +- [ ] **Step 1: Add `ibflex` to dependencies** + +In `pyproject.toml`, under `[tool.poetry.dependencies]`, add: + +```toml +ibflex = "^0.16" +``` + +- [ ] **Step 2: Resolve + install** + +```bash +cd /home/wizard/code/broker-sync && poetry lock --no-update && poetry install +``` + +Expected output: `Installing ibflex (0.16.x)`. No error. + +- [ ] **Step 3: Verify it imports** + +```bash +poetry run python -c "from ibflex import client, parser; print(client, parser)" +``` + +Expected: prints two module objects, no exception. + +- [ ] **Step 4: Commit** + +```bash +git add pyproject.toml poetry.lock +git commit -m "deps: add ibflex for IBKR Flex Web Service ingestion" +``` + +--- + +## Task 2: Fixture — canned Flex XML + +**Files:** +- Create: `tests/fixtures/ibkr/sample_flex.xml` + +- [ ] **Step 1: Create the fixture directory** + +```bash +mkdir -p /home/wizard/code/broker-sync/tests/fixtures/ibkr +``` + +- [ ] **Step 2: Write the fixture file** + +Create `tests/fixtures/ibkr/sample_flex.xml`: + +```xml + + + + + + + + + + + + + + + + + + + + + +``` + +- [ ] **Step 3: Verify ibflex can parse it** + +```bash +cd /home/wizard/code/broker-sync && poetry run python -c " +from ibflex import parser +r = parser.parse('tests/fixtures/ibkr/sample_flex.xml') +s = r.FlexStatements[0] +assert s.accountId == 'U12345678' +assert len(s.Trades) == 3 +assert len(s.CashTransactions) == 2 +assert len(s.OpenPositions) == 2 +print('OK') +" +``` + +Expected: prints `OK`. + +- [ ] **Step 4: Commit** + +```bash +git add tests/fixtures/ibkr/sample_flex.xml +git commit -m "test: add IBKR Flex XML fixture for provider tests" +``` + +--- + +## Task 3: `metrics.py` — Pushgateway client + test + +**Files:** +- Create: `broker_sync/metrics.py` +- Create: `tests/test_metrics.py` + +- [ ] **Step 1: Write the failing test** + +Create `tests/test_metrics.py`: + +```python +from __future__ import annotations + +import httpx +import pytest + +from broker_sync.metrics import push_pushgateway + + +@pytest.mark.asyncio +async def test_push_pushgateway_posts_text_format() -> None: + captured: dict[str, object] = {} + + def transport_handler(request: httpx.Request) -> httpx.Response: + captured["url"] = str(request.url) + captured["method"] = request.method + captured["body"] = request.content.decode("utf-8") + return httpx.Response(200) + + transport = httpx.MockTransport(transport_handler) + await push_pushgateway( + job="broker-sync-ibkr", + metrics=[ + ("ibkr_position_drift_shares", {"symbol": "VUAG.L"}, 0.0), + ("ibkr_sync_last_success_timestamp_seconds", {}, 1779830000.0), + ], + pushgateway_url="http://pg.example/metrics", + transport=transport, + ) + assert captured["method"] == "POST" + assert captured["url"] == "http://pg.example/metrics/job/broker-sync-ibkr" + body = captured["body"] + assert 'ibkr_position_drift_shares{symbol="VUAG.L"} 0.0' in body + assert "ibkr_sync_last_success_timestamp_seconds 1779830000.0" in body + + +@pytest.mark.asyncio +async def test_push_pushgateway_raises_on_non_2xx() -> None: + transport = httpx.MockTransport(lambda r: httpx.Response(500, text="boom")) + with pytest.raises(RuntimeError, match="pushgateway.*500"): + await push_pushgateway( + job="x", + metrics=[("m", {}, 1.0)], + pushgateway_url="http://pg/metrics", + transport=transport, + ) +``` + +- [ ] **Step 2: Run the test and verify it fails** + +```bash +cd /home/wizard/code/broker-sync && poetry run pytest tests/test_metrics.py -v +``` + +Expected: FAIL with `ModuleNotFoundError: No module named 'broker_sync.metrics'`. + +- [ ] **Step 3: Write the implementation** + +Create `broker_sync/metrics.py`: + +```python +"""Pushgateway client for broker-sync providers. + +One function: push a list of (metric, labels, value) tuples to Prometheus +Pushgateway under a given job name. Used by providers to surface +per-run drift / staleness / row counts that Prometheus can alert on. + +In-cluster URL: http://prometheus-prometheus-pushgateway.monitoring:9091/metrics +Pass that via the ``pushgateway_url`` env-driven argument. +""" +from __future__ import annotations + +import logging +import os +from collections.abc import Iterable + +import httpx + +log = logging.getLogger(__name__) + + +def _format_metric(name: str, labels: dict[str, str], value: float) -> str: + if labels: + body = ",".join(f'{k}="{v}"' for k, v in sorted(labels.items())) + return f"{name}{{{body}}} {value}\n" + return f"{name} {value}\n" + + +async def push_pushgateway( + job: str, + metrics: Iterable[tuple[str, dict[str, str], float]], + pushgateway_url: str | None = None, + transport: httpx.AsyncBaseTransport | None = None, +) -> None: + """POST text-format metrics to Pushgateway under ``job``. + + ``pushgateway_url`` defaults to the env var ``PUSHGATEWAY_URL``. + Raises ``RuntimeError`` if the URL is unset or the POST returns non-2xx. + """ + url = pushgateway_url or os.environ.get("PUSHGATEWAY_URL") + if not url: + raise RuntimeError("PUSHGATEWAY_URL not set and no override provided") + body = "".join(_format_metric(name, labels, value) for name, labels, value in metrics) + target = f"{url.rstrip('/')}/job/{job}" + async with httpx.AsyncClient(transport=transport, timeout=15.0) as c: + resp = await c.post(target, content=body, headers={"Content-Type": "text/plain"}) + if resp.status_code >= 300: + raise RuntimeError( + f"pushgateway POST {target} returned HTTP {resp.status_code}: {resp.text[:200]}" + ) + log.info("pushgateway: pushed %d metrics to job=%s", len(body.splitlines()), job) +``` + +- [ ] **Step 4: Run the tests and verify they pass** + +```bash +poetry run pytest tests/test_metrics.py -v +``` + +Expected: both tests pass. + +- [ ] **Step 5: Type + lint check** + +```bash +poetry run mypy broker_sync/metrics.py && poetry run ruff check broker_sync/metrics.py tests/test_metrics.py +``` + +Expected: both clean. + +- [ ] **Step 6: Commit** + +```bash +git add broker_sync/metrics.py tests/test_metrics.py +git commit -m "metrics: add Pushgateway client for broker-sync providers" +``` + +--- + +## Task 4: `WealthfolioSink.compute_position_qty` — and tests + +**Files:** +- Modify: `broker_sync/sinks/wealthfolio.py` +- Modify: `tests/sinks/test_wealthfolio.py` + +- [ ] **Step 1: Write the failing test** + +Append to `tests/sinks/test_wealthfolio.py`: + +```python +@pytest.mark.asyncio +async def test_compute_position_qty_sums_buys_minus_sells(monkeypatch: MonkeyPatch) -> None: + """compute_position_qty groups activities by symbol and returns + BUY/ADD_HOLDING/TRANSFER_IN minus SELL/REMOVE_HOLDING/TRANSFER_OUT + quantities as Decimal.""" + from broker_sync.sinks.wealthfolio import WealthfolioSink + + fake_activities = [ + # symbol VUAG.L: 10 buys, 2 sells, net 8 + {"symbol": "VUAG.L", "activityType": "BUY", "quantity": "10"}, + {"symbol": "VUAG.L", "activityType": "SELL", "quantity": "2"}, + # symbol AAPL: 5 buys + {"symbol": "AAPL", "activityType": "BUY", "quantity": "5"}, + # cash activities (no asset) — skipped + {"symbol": "$CASH-GBP", "activityType": "DEPOSIT", "quantity": "0", "amount": "100"}, + ] + + sink = WealthfolioSink(base_url="http://wf", username="u", password="p", session_path="/tmp/s") + + async def fake_search(account_id: str, page: int) -> dict: + return {"activities": fake_activities if page == 1 else [], "totalPages": 1} + + monkeypatch.setattr(sink, "_search_activities", fake_search) + result = await sink.compute_position_qty("acct-123") + assert result == {"VUAG.L": Decimal("8"), "AAPL": Decimal("5")} +``` + +Add the `Decimal` import at the top of the test module if missing: + +```python +from decimal import Decimal +``` + +- [ ] **Step 2: Run the test and verify it fails** + +```bash +poetry run pytest tests/sinks/test_wealthfolio.py::test_compute_position_qty_sums_buys_minus_sells -v +``` + +Expected: FAIL with `AttributeError: 'WealthfolioSink' object has no attribute 'compute_position_qty'` (or similar — `_search_activities` may also be missing). + +- [ ] **Step 3: Add the method to WealthfolioSink** + +In `broker_sync/sinks/wealthfolio.py`, inside the `WealthfolioSink` class, add (alongside the existing methods): + +```python +async def _search_activities(self, account_id: str, page: int) -> dict[str, Any]: + """Internal: one page of /activities/search results for an account.""" + resp = await self._request( + "POST", + "/api/v1/activities/search", + json={"accountIds": [account_id], "page": page, "pageSize": 500}, + ) + resp.raise_for_status() + return resp.json() # type: ignore[no-any-return] + +async def compute_position_qty(self, account_id: str) -> dict[str, Decimal]: + """Return per-symbol net position quantity (BUY/IN minus SELL/OUT) for + one account. Skips cash activities. Used by the IBKR reconciliation + step to compare against broker-reported OpenPositions.""" + qty_by_symbol: dict[str, Decimal] = {} + page = 1 + while True: + payload = await self._search_activities(account_id, page) + activities = payload.get("activities", []) + if not activities: + break + for act in activities: + symbol = act.get("symbol") + if not symbol or symbol.startswith("$CASH"): + continue + act_type = act.get("activityType") + sign: int + if act_type in {"BUY", "ADD_HOLDING", "TRANSFER_IN"}: + sign = 1 + elif act_type in {"SELL", "REMOVE_HOLDING", "TRANSFER_OUT"}: + sign = -1 + else: + continue + qty = Decimal(str(act.get("quantity") or 0)) + qty_by_symbol[symbol] = qty_by_symbol.get(symbol, Decimal(0)) + sign * qty + if page >= int(payload.get("totalPages") or 1): + break + page += 1 + return qty_by_symbol +``` + +Add the `Decimal` import at the top of `wealthfolio.py` if missing: + +```python +from decimal import Decimal +``` + +- [ ] **Step 4: Run the test and verify it passes** + +```bash +poetry run pytest tests/sinks/test_wealthfolio.py::test_compute_position_qty_sums_buys_minus_sells -v +``` + +Expected: PASS. + +- [ ] **Step 5: Run mypy + ruff + full pytest** + +```bash +poetry run mypy broker_sync tests && poetry run ruff check . && poetry run pytest -q +``` + +Expected: all clean. + +- [ ] **Step 6: Commit** + +```bash +git add broker_sync/sinks/wealthfolio.py tests/sinks/test_wealthfolio.py +git commit -m "wealthfolio: add compute_position_qty for broker reconciliation" +``` + +--- + +## Task 5: `providers/ibkr.py` — symbol canonicalisation + +**Files:** +- Create: `broker_sync/providers/ibkr.py` +- Create: `tests/providers/test_ibkr.py` + +- [ ] **Step 1: Write the failing test** + +Create `tests/providers/test_ibkr.py`: + +```python +from __future__ import annotations + +import pytest + +from broker_sync.providers.ibkr import canonical_symbol + + +def test_canonical_symbol_lse_etf_gets_l_suffix() -> None: + assert canonical_symbol("VUAG", exchange="LSE", currency="GBP") == "VUAG.L" + + +def test_canonical_symbol_us_stock_unchanged() -> None: + assert canonical_symbol("AAPL", exchange="NASDAQ", currency="USD") == "AAPL" + + +def test_canonical_symbol_lse_gbp_inferred_when_exchange_missing() -> None: + """IBKR Flex sometimes omits exchange. Infer LSE from currency==GBP.""" + assert canonical_symbol("VUAG", exchange=None, currency="GBP") == "VUAG.L" + + +def test_canonical_symbol_already_suffixed_unchanged() -> None: + assert canonical_symbol("VUAG.L", exchange="LSE", currency="GBP") == "VUAG.L" +``` + +- [ ] **Step 2: Run the test and verify it fails** + +```bash +poetry run pytest tests/providers/test_ibkr.py -v +``` + +Expected: FAIL with `ModuleNotFoundError: No module named 'broker_sync.providers.ibkr'`. + +- [ ] **Step 3: Create the provider module with `canonical_symbol`** + +Create `broker_sync/providers/ibkr.py`: + +```python +"""Interactive Brokers Flex Web Service ingestion provider. + +Pulls daily Activity Flex Query reports via the ``ibflex`` library, maps +Trades + CashTransactions to broker-sync ``Activity`` objects, and runs a +reconciliation step against the broker-reported ``OpenPositions``. + +See ``docs/specs/2026-05-26-ibkr-ingest-design.md`` for the full design. +""" +from __future__ import annotations + +import logging +from decimal import Decimal +from typing import TYPE_CHECKING + +from broker_sync.models import Account, AccountType, Activity, ActivityType + +if TYPE_CHECKING: + from ibflex import FlexQueryResponse + +log = logging.getLogger(__name__) + +# Map IBKR currency -> default exchange suffix. +# Only set up for the GBP / LSE case today; extend when more accounts onboard. +_CURRENCY_TO_LSE_SUFFIX = {"GBP": ".L"} + + +def canonical_symbol(symbol: str, *, exchange: str | None, currency: str) -> str: + """Return the WF-canonical form of an IBKR ticker. + + LSE-listed GBP instruments get a ``.L`` suffix (Wealthfolio convention). + US instruments and anything already suffixed are returned unchanged. + """ + if "." in symbol: + return symbol + if exchange in {"LSE", "LSEETF", "LSEIOB1"} or ( + exchange is None and currency in _CURRENCY_TO_LSE_SUFFIX + ): + return symbol + _CURRENCY_TO_LSE_SUFFIX.get(currency, ".L") + return symbol +``` + +- [ ] **Step 4: Run the test and verify it passes** + +```bash +poetry run pytest tests/providers/test_ibkr.py -v +``` + +Expected: 4 tests PASS. + +- [ ] **Step 5: Type + lint** + +```bash +poetry run mypy broker_sync/providers/ibkr.py tests/providers/test_ibkr.py && poetry run ruff check broker_sync/providers/ibkr.py tests/providers/test_ibkr.py +``` + +Expected: clean. + +- [ ] **Step 6: Commit** + +```bash +git add broker_sync/providers/ibkr.py tests/providers/test_ibkr.py +git commit -m "ibkr: add canonical_symbol helper (LSE .L suffix handling)" +``` + +--- + +## Task 6: `_map_trade_to_activity` + +**Files:** +- Modify: `broker_sync/providers/ibkr.py` +- Modify: `tests/providers/test_ibkr.py` + +- [ ] **Step 1: Write the failing test** + +Append to `tests/providers/test_ibkr.py`: + +```python +def test_map_trade_buy_to_activity() -> None: + """Trade with buySell=BUY maps to Activity(activity_type=BUY) with + positive quantity, fee = abs(ibCommission), external_id = ibkr:trade:.""" + from datetime import datetime + from decimal import Decimal + + from broker_sync.providers.ibkr import _map_trade_to_activity + from ibflex import parser + + r = parser.parse("tests/fixtures/ibkr/sample_flex.xml") + trade = r.FlexStatements[0].Trades[0] # T1001: 10 VUAG BUY @ 107.50 GBP + + activity = _map_trade_to_activity(trade, account_id="wf-acct-uuid") + + assert activity.external_id == "ibkr:trade:T1001" + assert activity.account_id == "wf-acct-uuid" + assert activity.activity_type == ActivityType.BUY + assert activity.symbol == "VUAG.L" + assert activity.quantity == Decimal("10") + assert activity.unit_price == Decimal("107.50") + assert activity.fee == Decimal("1.05") + assert activity.currency == "GBP" + assert isinstance(activity.date, datetime) + assert activity.date.tzinfo is not None +``` + +- [ ] **Step 2: Run and verify it fails** + +```bash +poetry run pytest tests/providers/test_ibkr.py::test_map_trade_buy_to_activity -v +``` + +Expected: FAIL with `ImportError: cannot import name '_map_trade_to_activity'`. + +- [ ] **Step 3: Add the mapper** + +Append to `broker_sync/providers/ibkr.py`: + +```python +from datetime import UTC, datetime # noqa: E402 (grouped here for the mapper section) + +if TYPE_CHECKING: + from ibflex.Types import OpenPosition, Trade + from ibflex.Types import CashTransaction as IBFlexCashTransaction + + +def _trade_to_datetime(trade_date: object, trade_time: str | None) -> datetime: + """Combine Flex tradeDate (a date) + tradeTime (HH:MM:SS TZ) into UTC datetime.""" + if isinstance(trade_date, datetime): + # ibflex sometimes already returns datetime + dt = trade_date + else: + # date object + time_part = (trade_time or "00:00:00 UTC").split()[0] + dt = datetime.fromisoformat(f"{trade_date.isoformat()}T{time_part}") + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + return dt.astimezone(UTC) + + +def _map_trade_to_activity(trade: Trade, *, account_id: str) -> Activity: + """Map one ibflex Trade dataclass to a broker-sync Activity.""" + buy_sell = str(trade.buySell.name) if hasattr(trade.buySell, "name") else str(trade.buySell) + if buy_sell == "BUY": + activity_type = ActivityType.BUY + elif buy_sell == "SELL": + activity_type = ActivityType.SELL + else: + raise ValueError(f"unsupported Trade.buySell={buy_sell!r} on tradeID={trade.tradeID}") + + symbol = canonical_symbol( + str(trade.symbol), + exchange=getattr(trade, "exchange", None), + currency=str(trade.currency), + ) + quantity = abs(Decimal(str(trade.quantity))) + unit_price = Decimal(str(trade.tradePrice)) + fee = abs(Decimal(str(trade.ibCommission or 0))) + return Activity( + external_id=f"ibkr:trade:{trade.tradeID}", + account_id=account_id, + account_type=AccountType.GIA, + date=_trade_to_datetime(trade.tradeDate, getattr(trade, "tradeTime", None)), + activity_type=activity_type, + currency=str(trade.currency), + symbol=symbol, + quantity=quantity, + unit_price=unit_price, + fee=fee, + ) +``` + +Move the `from datetime import UTC, datetime` import to the top-level imports +section if your repo's lint rules forbid late imports — ruff's E402 is suppressed +here via `# noqa: E402` because grouping helps readability. + +- [ ] **Step 4: Run the test and verify it passes** + +```bash +poetry run pytest tests/providers/test_ibkr.py -v +``` + +Expected: all 5 tests pass. + +- [ ] **Step 5: Type + lint** + +```bash +poetry run mypy broker_sync/providers/ibkr.py && poetry run ruff check broker_sync/providers/ibkr.py +``` + +Expected: clean. + +- [ ] **Step 6: Commit** + +```bash +git add broker_sync/providers/ibkr.py tests/providers/test_ibkr.py +git commit -m "ibkr: map Flex Trades to broker-sync Activities" +``` + +--- + +## Task 7: `_map_cash_to_activity` + +**Files:** +- Modify: `broker_sync/providers/ibkr.py` +- Modify: `tests/providers/test_ibkr.py` + +- [ ] **Step 1: Write the failing test** + +Append to `tests/providers/test_ibkr.py`: + +```python +def test_map_cash_dividend_to_activity() -> None: + from decimal import Decimal + + from broker_sync.providers.ibkr import _map_cash_to_activity + from ibflex import parser + + r = parser.parse("tests/fixtures/ibkr/sample_flex.xml") + cash = r.FlexStatements[0].CashTransactions[0] # C5001: Dividends 3.50 GBP + + activity = _map_cash_to_activity(cash, account_id="wf-acct-uuid") + assert activity is not None + assert activity.external_id == "ibkr:cash:C5001" + assert activity.activity_type == ActivityType.DIVIDEND + assert activity.amount == Decimal("3.50") + assert activity.currency == "GBP" + + +def test_map_cash_withholding_tax_to_fee_activity() -> None: + from decimal import Decimal + + from broker_sync.providers.ibkr import _map_cash_to_activity + from ibflex import parser + + r = parser.parse("tests/fixtures/ibkr/sample_flex.xml") + cash = r.FlexStatements[0].CashTransactions[1] # C5002: Withholding Tax -0.35 GBP + + activity = _map_cash_to_activity(cash, account_id="wf-acct-uuid") + assert activity is not None + assert activity.activity_type == ActivityType.FEE + assert activity.amount == Decimal("0.35") # always positive on Activity, sign carried by activity_type + + +def test_map_cash_unknown_type_returns_none_and_logs(caplog) -> None: # noqa: ANN001 + """Unknown CashTransaction.type produces None + a WARNING log line. + Same refusal-to-guess convention as the InvestEngine provider.""" + from broker_sync.providers.ibkr import _map_cash_to_activity + + class FakeCash: + transactionID = "C9999" + dateTime = None + type = type("T", (), {"name": "FrobnicatedThing"})() + amount = 0 + currency = "GBP" + + with caplog.at_level("WARNING"): + result = _map_cash_to_activity(FakeCash, account_id="wf-acct-uuid") + assert result is None + assert any("FrobnicatedThing" in r.message for r in caplog.records) +``` + +- [ ] **Step 2: Run and verify the new tests fail** + +```bash +poetry run pytest tests/providers/test_ibkr.py -v +``` + +Expected: 3 FAILs (the new tests), 5 existing PASS. + +- [ ] **Step 3: Add the cash mapper** + +Append to `broker_sync/providers/ibkr.py`: + +```python +# Maps the IBKR Flex CashTransaction.type values we expect to see for a +# stocks/ETFs-only GIA. Unknown values yield None + a WARNING — we refuse +# to guess (per IE/Schwab convention) to avoid silent misclassification. +_CASH_TYPE_MAP: dict[str, ActivityType] = { + "Dividends": ActivityType.DIVIDEND, + "Withholding Tax": ActivityType.FEE, + "Broker Interest Received": ActivityType.DIVIDEND, + "Broker Interest Paid": ActivityType.FEE, + "Commission Adjustments": ActivityType.FEE, + "Other Fees": ActivityType.FEE, +} + + +def _map_cash_to_activity( + cash: IBFlexCashTransaction, *, account_id: str +) -> Activity | None: + """Map one ibflex CashTransaction to a broker-sync Activity. + + Returns None for unsupported types (logged at WARNING). Deposit/Withdrawal + handled separately by sign of amount. + """ + type_obj = cash.type + type_name = type_obj.name if hasattr(type_obj, "name") else str(type_obj) + amount = Decimal(str(cash.amount)) + + # Deposit / Withdrawal split by sign — the Flex "Deposits & Withdrawals" type + if type_name in {"DepositsWithdrawals", "Deposits & Withdrawals", "Deposit Withdrawals"}: + activity_type = ActivityType.DEPOSIT if amount > 0 else ActivityType.WITHDRAWAL + else: + activity_type = _CASH_TYPE_MAP.get(type_name) # type: ignore[assignment] + if activity_type is None: + log.warning( + "ibkr: skipping cash transaction id=%s with unsupported type=%r", + getattr(cash, "transactionID", "?"), + type_name, + ) + return None + + dt = cash.dateTime + if isinstance(dt, datetime) and dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + elif not isinstance(dt, datetime): + dt = datetime.now(UTC) # graceful fallback — log path also fine + + return Activity( + external_id=f"ibkr:cash:{cash.transactionID}", + account_id=account_id, + account_type=AccountType.GIA, + date=dt, + activity_type=activity_type, + currency=str(cash.currency), + amount=abs(amount), + ) +``` + +- [ ] **Step 4: Run and verify all tests pass** + +```bash +poetry run pytest tests/providers/test_ibkr.py -v +``` + +Expected: 8 tests pass. + +- [ ] **Step 5: Type + lint + commit** + +```bash +poetry run mypy broker_sync && poetry run ruff check broker_sync tests +git add broker_sync/providers/ibkr.py tests/providers/test_ibkr.py +git commit -m "ibkr: map Flex CashTransactions (dividends, fees, deposits)" +``` + +--- + +## Task 8: `IBKRProvider` class + account guard + +**Files:** +- Modify: `broker_sync/providers/ibkr.py` +- Modify: `tests/providers/test_ibkr.py` + +- [ ] **Step 1: Write the failing test** + +Append to `tests/providers/test_ibkr.py`: + +```python +@pytest.mark.asyncio +async def test_ibkr_provider_fetch_returns_mapped_activities(monkeypatch) -> None: # noqa: ANN001 + """IBKRProvider.fetch() yields all mapped activities (trades + cash).""" + from broker_sync.providers.ibkr import IBKRProvider + from ibflex import client as ib_client + + with open("tests/fixtures/ibkr/sample_flex.xml", "rb") as f: + xml_bytes = f.read() + monkeypatch.setattr(ib_client, "download", lambda *a, **kw: xml_bytes) + + provider = IBKRProvider( + token="t", + query_id="q", + wf_account_id="wf-acct", + upstream_account_id="U12345678", + ) + activities = [a async for a in provider.fetch()] + # 3 trades + 2 cash = 5 + assert len(activities) == 5 + types = sorted(a.activity_type.name for a in activities) + assert types == ["BUY", "BUY", "DIVIDEND", "FEE", "SELL"] + + +@pytest.mark.asyncio +async def test_ibkr_provider_account_mismatch_raises(monkeypatch) -> None: # noqa: ANN001 + """If Flex statement.accountId differs from the configured upstream id, + refuse to ingest. Prevents wrong-account writes from a misconfigured query.""" + from broker_sync.providers.ibkr import IBKRAccountMismatchError, IBKRProvider + from ibflex import client as ib_client + + with open("tests/fixtures/ibkr/sample_flex.xml", "rb") as f: + xml_bytes = f.read() + monkeypatch.setattr(ib_client, "download", lambda *a, **kw: xml_bytes) + + provider = IBKRProvider( + token="t", + query_id="q", + wf_account_id="wf-acct", + upstream_account_id="U99999999", # WRONG + ) + with pytest.raises(IBKRAccountMismatchError, match="U12345678"): + [a async for a in provider.fetch()] +``` + +- [ ] **Step 2: Run and verify the new tests fail** + +```bash +poetry run pytest tests/providers/test_ibkr.py -v +``` + +Expected: 2 FAILs (the new tests). Existing tests still pass. + +- [ ] **Step 3: Add the IBKRProvider class** + +Append to `broker_sync/providers/ibkr.py`: + +```python +from collections.abc import AsyncIterator # noqa: E402 + + +class IBKRError(Exception): + """Base class for ibkr-provider errors.""" + + +class IBKRAccountMismatchError(IBKRError): + """Flex statement accountId did not match configured upstream id.""" + + +class IBKRProvider: + """Fetches IBKR Flex Activity reports and yields broker-sync Activities. + + The reconciliation step (OpenPositions vs WF-computed qty) is NOT part + of fetch() — it runs at the CLI layer after import, since it needs the + WealthfolioSink to query WF. + """ + + def __init__( + self, + *, + token: str, + query_id: str, + wf_account_id: str, + upstream_account_id: str, + ) -> None: + self._token = token + self._query_id = query_id + self._wf_account_id = wf_account_id + self._upstream_account_id = upstream_account_id + # Stash the parsed response for the reconciliation step. + self._last_response: FlexQueryResponse | None = None + + def accounts(self) -> list[Account]: + return [ + Account( + id=self._wf_account_id, + provider="ibkr", + provider_account_id=self._upstream_account_id, + account_type=AccountType.GIA, + currency="GBP", # FX-aware at trade level; account currency is GBP + ) + ] + + async def close(self) -> None: + # No persistent HTTP client today — ibflex uses requests internally. + return + + async def fetch( + self, + *, + since: datetime | None = None, # noqa: ARG002 (Flex query owns the date range) + before: datetime | None = None, # noqa: ARG002 + ) -> AsyncIterator[Activity]: + from ibflex import client as ib_client + from ibflex import parser as ib_parser + + xml_bytes = ib_client.download(self._token, self._query_id) + response = ib_parser.parse(xml_bytes) + self._last_response = response + + if not response.FlexStatements: + log.warning("ibkr: Flex response had no FlexStatements") + return + + stmt = response.FlexStatements[0] + if str(stmt.accountId) != self._upstream_account_id: + raise IBKRAccountMismatchError( + f"Flex statement.accountId={stmt.accountId!r} does not match " + f"configured IBKR_ACCOUNT_ID_UPSTREAM={self._upstream_account_id!r} " + f"— refusing to ingest" + ) + + for trade in stmt.Trades or []: + yield _map_trade_to_activity(trade, account_id=self._wf_account_id) + + for cash in stmt.CashTransactions or []: + activity = _map_cash_to_activity(cash, account_id=self._wf_account_id) + if activity is not None: + yield activity + + def open_positions(self) -> list[tuple[str, Decimal]]: + """Return ``[(canonical_symbol, position_qty), ...]`` from the most + recent fetch. Used by the reconciliation step. + + Returns ``[]`` if no fetch has been called yet.""" + if self._last_response is None: + return [] + stmt = self._last_response.FlexStatements[0] + out: list[tuple[str, Decimal]] = [] + for pos in stmt.OpenPositions or []: + symbol = canonical_symbol( + str(pos.symbol), + exchange=getattr(pos, "exchange", None), + currency=str(pos.currency), + ) + out.append((symbol, Decimal(str(pos.position)))) + return out +``` + +- [ ] **Step 4: Run and verify all tests pass** + +```bash +poetry run pytest tests/providers/test_ibkr.py -v +``` + +Expected: 10 tests pass. + +- [ ] **Step 5: Type + lint + commit** + +```bash +poetry run mypy broker_sync && poetry run ruff check broker_sync tests +git add broker_sync/providers/ibkr.py tests/providers/test_ibkr.py +git commit -m "ibkr: add IBKRProvider with Flex fetch + account-mismatch guard" +``` + +--- + +## Task 9: `broker-sync ibkr` CLI command + +**Files:** +- Modify: `broker_sync/cli.py` + +- [ ] **Step 1: Read existing `invest_engine` command for pattern** + +```bash +sed -n '140,235p' /home/wizard/code/broker-sync/broker_sync/cli.py +``` + +You're using this as the template — `ibkr` is structurally identical +(provider construction → pipeline → sink → reconciliation). + +- [ ] **Step 2: Add the `ibkr` command** + +In `broker_sync/cli.py`, after the `invest_engine` command, add: + +```python +@app.command("ibkr") +def ibkr( # noqa: PLR0913 + wf_base_url: str = typer.Option(..., envvar="WF_BASE_URL"), + wf_username: str = typer.Option(..., envvar="WF_USERNAME"), + wf_password: str = typer.Option(..., envvar="WF_PASSWORD"), + wf_session_path: str = typer.Option( + "/data/wealthfolio_session.json", envvar="WF_SESSION_PATH" + ), + ibkr_flex_token: str = typer.Option(..., envvar="IBKR_FLEX_TOKEN"), + ibkr_flex_query_id: str = typer.Option(..., envvar="IBKR_FLEX_QUERY_ID"), + ibkr_account_id: str = typer.Option(..., envvar="IBKR_ACCOUNT_ID"), + ibkr_account_id_upstream: str = typer.Option(..., envvar="IBKR_ACCOUNT_ID_UPSTREAM"), + pushgateway_url: str = typer.Option( + "http://prometheus-prometheus-pushgateway.monitoring:9091/metrics", + envvar="PUSHGATEWAY_URL", + ), + data_dir: str = typer.Option("/data", envvar="BROKER_SYNC_DATA_DIR"), +) -> None: + """Phase 2c — daily IBKR Flex Web Service → Wealthfolio sync.""" + import time + + from broker_sync.dedup import SyncRecordStore + from broker_sync.metrics import push_pushgateway + from broker_sync.pipeline import sync_provider_to_wealthfolio + from broker_sync.providers.ibkr import IBKRProvider + from broker_sync.sinks.wealthfolio import WealthfolioSink + + _setup_logging() + data = Path(data_dir) + data.mkdir(parents=True, exist_ok=True) + + async def _run() -> None: + sink = WealthfolioSink( + base_url=wf_base_url, + username=wf_username, + password=wf_password, + session_path=wf_session_path, + ) + provider = IBKRProvider( + token=ibkr_flex_token, + query_id=ibkr_flex_query_id, + wf_account_id=ibkr_account_id, + upstream_account_id=ibkr_account_id_upstream, + ) + dedup = SyncRecordStore(data / "sync.db") + try: + if not Path(wf_session_path).exists(): + await sink.login() + result = await sync_provider_to_wealthfolio( + provider=provider, + sink=sink, + dedup=dedup, + ) + + # Reconciliation: broker truth vs WF truth. + wf_qty = await sink.compute_position_qty(ibkr_account_id) + drift_metrics: list[tuple[str, dict[str, str], float]] = [] + for symbol, broker_qty in provider.open_positions(): + drift = broker_qty - wf_qty.get(symbol, Decimal(0)) + drift_metrics.append( + ( + "ibkr_position_drift_shares", + {"symbol": symbol, "account": "ibkr-uk"}, + float(drift), + ) + ) + drift_metrics.append( + ("ibkr_sync_last_success_timestamp_seconds", {}, float(time.time())) + ) + await push_pushgateway("broker-sync-ibkr", drift_metrics, pushgateway_url) + finally: + await sink.close() + await provider.close() + + typer.echo( + f"ibkr: fetched={result.fetched} new={result.new_after_dedup} " + f"imported={result.imported} failed={result.failed}" + ) + if result.failed > 0: + sys.exit(1) + + asyncio.run(_run()) +``` + +Add the `Decimal` import at the top of `cli.py` if missing. + +- [ ] **Step 3: Sanity-check the CLI compiles** + +```bash +poetry run broker-sync --help | grep -i ibkr +``` + +Expected: `ibkr Phase 2c — daily IBKR Flex Web Service → Wealthfolio sync.` + +- [ ] **Step 4: Run mypy + ruff + full pytest** + +```bash +poetry run mypy broker_sync tests && poetry run ruff check . && poetry run pytest -q +``` + +Expected: all clean. + +- [ ] **Step 5: Commit** + +```bash +git add broker_sync/cli.py +git commit -m "cli: add ibkr command (Flex pull + pipeline + reconcile + metrics)" +``` + +--- + +## Task 10: Push, wait for CI, verify image + +**Files:** (none — operational step) + +- [ ] **Step 1: Push to GitHub + Forgejo** + +```bash +git push origin main && git push forgejo main +``` + +- [ ] **Step 2: Wait for GHA CI to complete** + +```bash +until [ "$(gh api 'repos/ViktorBarzin/broker-sync/actions/runs?per_page=1' --jq '.workflow_runs[0].status')" = "completed" ]; do sleep 15; done +gh api 'repos/ViktorBarzin/broker-sync/actions/runs?per_page=1' --jq '.workflow_runs[0] | "\(.head_sha[:8]) \(.conclusion)"' +``` + +Expected: ` success`. + +- [ ] **Step 3: Pull the new image and confirm** + +```bash +docker pull viktorbarzin/broker-sync:latest +docker images viktorbarzin/broker-sync --format '{{.Tag}} {{.CreatedSince}}' +``` + +Expected: `latest` was created within the last few minutes. + +--- + +## Task 11: Vault secrets + WF account creation + +**Files:** (operational — no code changes) + +- [ ] **Step 1: User completes the IBKR Client Portal steps** + +Follow the design's setup checklist Step 1: +- Enable Flex Web Service → copy Token. +- Create Activity Flex Query → copy Query ID. +- Note the account number (e.g., `U12345678`). + +- [ ] **Step 2: Create the Wealthfolio account** + +```bash +WF_BASE="https://wealthfolio.viktorbarzin.me" # adjust if internal-only +WF_PASS=$(vault kv get -field=wf_password secret/broker-sync) +curl -sS -c /tmp/wf-jar -X POST "$WF_BASE/api/v1/auth/login" \ + -H 'Content-Type: application/json' \ + -d "{\"password\":\"$WF_PASS\"}" -o /dev/null +WF_UUID=$(curl -sS -b /tmp/wf-jar -X POST "$WF_BASE/api/v1/accounts" \ + -H 'Content-Type: application/json' \ + -d '{"name":"Interactive Brokers (UK)","accountType":"GIA","currency":"GBP","isActive":true}' \ + | jq -r '.id') +echo "WF account UUID = $WF_UUID" +``` + +Expected: prints a UUID. Note it down for the next step. + +- [ ] **Step 3: Put the 4 IBKR secrets into Vault** + +```bash +vault kv patch secret/broker-sync \ + ibkr_flex_token='' \ + ibkr_flex_query_id='' \ + ibkr_account_id='' \ + ibkr_account_id_upstream='' +``` + +- [ ] **Step 4: Verify the secrets are readable** + +```bash +vault kv get -format=json secret/broker-sync | jq '.data.data | {token: (.ibkr_flex_token[0:6]+"..."), query_id, account_id, account_id_upstream}' +``` + +Expected: all four fields present, token truncated. + +--- + +## Task 12: Terraform CronJob + alerts + +**Files:** +- Modify: `infra/stacks/broker-sync/main.tf` + +- [ ] **Step 1: Open `infra/stacks/broker-sync/main.tf` and find the `trading212` CronJob** + +```bash +grep -n 'kubernetes_cron_job_v1.*trading212\|broker-sync-trading212' /home/wizard/code/infra/stacks/broker-sync/main.tf +``` + +Use it as the template — copy/paste then adjust the diffs. + +- [ ] **Step 2: Add the IBKR CronJob resource** + +After the `trading212` CronJob block, add: + +```hcl +# IBKR Flex Web Service daily sync. Phase 2c deliverable. +resource "kubernetes_cron_job_v1" "ibkr" { + metadata { + name = "broker-sync-ibkr" + namespace = kubernetes_namespace.broker_sync.metadata[0].name + labels = { app = "broker-sync", component = "ibkr" } + } + spec { + schedule = "0 2 * * *" # 02:00 UK + concurrency_policy = "Forbid" + starting_deadline_seconds = 300 + successful_jobs_history_limit = 3 + failed_jobs_history_limit = 5 + job_template { + metadata {} + spec { + backoff_limit = 2 + ttl_seconds_after_finished = 86400 + template { + metadata { + labels = { app = "broker-sync", component = "ibkr" } + } + spec { + restart_policy = "OnFailure" + security_context { + fs_group = 10001 + } + container { + name = "broker-sync" + image = local.broker_sync_image + command = ["broker-sync", "ibkr"] + + env { + name = "BROKER_SYNC_DATA_DIR" + value = "/data" + } + env { + name = "WF_SESSION_PATH" + value = "/data/wealthfolio_session.json" + } + env { + name = "WF_BASE_URL" + value_from { secret_key_ref { name = "broker-sync-secrets"; key = "wf_base_url" } } + } + env { + name = "WF_USERNAME" + value_from { secret_key_ref { name = "broker-sync-secrets"; key = "wf_username" } } + } + env { + name = "WF_PASSWORD" + value_from { secret_key_ref { name = "broker-sync-secrets"; key = "wf_password" } } + } + env { + name = "IBKR_FLEX_TOKEN" + value_from { secret_key_ref { name = "broker-sync-secrets"; key = "ibkr_flex_token" } } + } + env { + name = "IBKR_FLEX_QUERY_ID" + value_from { secret_key_ref { name = "broker-sync-secrets"; key = "ibkr_flex_query_id" } } + } + env { + name = "IBKR_ACCOUNT_ID" + value_from { secret_key_ref { name = "broker-sync-secrets"; key = "ibkr_account_id" } } + } + env { + name = "IBKR_ACCOUNT_ID_UPSTREAM" + value_from { secret_key_ref { name = "broker-sync-secrets"; key = "ibkr_account_id_upstream" } } + } + + volume_mount { + name = "data" + mount_path = "/data" + } + resources { + requests = { cpu = "20m", memory = "128Mi" } + limits = { memory = "256Mi" } + } + } + volume { + name = "data" + persistent_volume_claim { + claim_name = kubernetes_persistent_volume_claim.data_encrypted.metadata[0].name + } + } + } + } + } + } + } + lifecycle { + # KYVERNO_LIFECYCLE_V1: Kyverno admission webhook mutates dns_config with ndots=2 + ignore_changes = [spec[0].job_template[0].spec[0].template[0].spec[0].dns_config] + } +} +``` + +- [ ] **Step 3: Format the terraform** + +```bash +cd /home/wizard/code/infra/stacks/broker-sync && terraform fmt main.tf +``` + +- [ ] **Step 4: Plan** + +```bash +/home/wizard/code/infra/scripts/tg plan 2>&1 | tail -20 +``` + +Expected: `Plan: 1 to add, 0 to change, 0 to destroy.` (the new ibkr CronJob). + +- [ ] **Step 5: Apply** + +```bash +/home/wizard/code/infra/scripts/tg apply --non-interactive 2>&1 | tail -5 +``` + +Expected: `Apply complete! Resources: 1 added, ...`. + +- [ ] **Step 6: Verify the CronJob exists** + +```bash +kubectl -n broker-sync get cronjob broker-sync-ibkr +``` + +Expected: row appears with `SCHEDULE = 0 2 * * *`. + +- [ ] **Step 7: Commit** + +```bash +cd /home/wizard/code/infra +git add stacks/broker-sync/main.tf +git commit -m "broker-sync: add IBKR Flex daily CronJob" +git push origin master +``` + +--- + +## Task 13: Manual smoke run + verification + +**Files:** (none — operational) + +- [ ] **Step 1: Trigger the CronJob manually** + +```bash +kubectl -n broker-sync create job --from=cronjob/broker-sync-ibkr broker-sync-ibkr-smoke-$(date +%s) +``` + +- [ ] **Step 2: Wait for completion + check status** + +```bash +JOB=$(kubectl -n broker-sync get jobs --sort-by=.metadata.creationTimestamp -o name | grep broker-sync-ibkr-smoke | tail -1) +until [ "$(kubectl -n broker-sync get $JOB -o jsonpath='{.status.succeeded}{.status.failed}' 2>/dev/null)" != "" ]; do sleep 5; done +kubectl -n broker-sync get $JOB +``` + +Expected: `STATUS = Complete`. (If `Failed`, check logs in step 3 and debug.) + +- [ ] **Step 3: Inspect the logs** + +```bash +kubectl -n broker-sync logs -l job-name=$(basename $JOB) --tail=200 +``` + +Look for: +- `ibkr: fetched=0 new=0 imported=0 failed=0` (account is empty, so zero + rows is correct). +- A `pushgateway: pushed N metrics` line. +- No tracebacks. + +- [ ] **Step 4: Verify the WF account exists with no activities** + +```bash +WF_PASS=$(vault kv get -field=wf_password secret/broker-sync) +curl -sS -c /tmp/wf-jar -X POST https://wealthfolio.viktorbarzin.me/api/v1/auth/login \ + -H 'Content-Type: application/json' -d "{\"password\":\"$WF_PASS\"}" -o /dev/null +curl -sS -b /tmp/wf-jar https://wealthfolio.viktorbarzin.me/api/v1/accounts | jq '.[] | select(.name=="Interactive Brokers (UK)")' +``` + +Expected: prints the account JSON with the UUID from Task 11 Step 2. + +- [ ] **Step 5: Verify Pushgateway received the metrics** + +```bash +kubectl -n monitoring port-forward svc/prometheus-prometheus-pushgateway 9091:9091 & +sleep 2 +curl -sS http://localhost:9091/metrics | grep -E 'ibkr_(position_drift_shares|sync_last_success)' +kill %1 +``` + +Expected: `ibkr_sync_last_success_timestamp_seconds` shows a recent +unix timestamp. `ibkr_position_drift_shares` may be absent if there +were no open positions today, which is correct for an empty account. + +--- + +## Task 14: Provider docs (for future-you) + +**Files:** +- Create: `docs/providers/ibkr.md` + +- [ ] **Step 1: Write the production-facing provider doc** + +Create `docs/providers/ibkr.md`: + +```markdown +# Provider: Interactive Brokers (IBKR Flex Web Service) + +Pulls a daily Activity Flex Query via the `ibflex` library, maps Trades + +CashTransactions to broker-sync Activities, and reconciles broker-side +OpenPositions against WF-computed quantities. + +## When this runs +- K8s CronJob `broker-sync-ibkr` in the `broker-sync` namespace, daily 02:00 UK. +- Manual: `kubectl -n broker-sync create job --from=cronjob/broker-sync-ibkr broker-sync-ibkr-manual-1`. + +## Secrets (Vault `secret/broker-sync`) + +| Key | Description | +|---|---| +| `ibkr_flex_token` | Flex Web Service token (1-year validity, rotate via IBKR Client Portal) | +| `ibkr_flex_query_id` | Activity Flex Query ID (5-7 digit number) | +| `ibkr_account_id` | Wealthfolio account UUID for "Interactive Brokers (UK)" | +| `ibkr_account_id_upstream` | IBKR-side account number (e.g., `U12345678`) — guards against wrong-account ingestion | + +## Flex Query design + +| Section | Fields used | +|---|---| +| Account Information | accountId | +| Trades | tradeID, tradeDate, tradeTime, symbol, buySell, quantity, tradePrice, currency, ibCommission, assetCategory | +| Cash Transactions | transactionID, dateTime, type, amount, currency, description | +| Open Positions | symbol, position, markPrice, currency, assetCategory | +| Securities Information | symbol, description, conid | + +Date range: `Last Business Day` for daily incremental. Switch to +`Year to Date` for one-time backfills only. + +## Cash type mapping + +| IBKR Flex type | broker-sync ActivityType | +|---|---| +| Dividends | DIVIDEND | +| Withholding Tax | FEE | +| Broker Interest Received | DIVIDEND | +| Broker Interest Paid | FEE | +| Commission Adjustments | FEE | +| Other Fees | FEE | +| Deposits & Withdrawals | DEPOSIT (amount > 0) or WITHDRAWAL (amount < 0) | +| anything else | skipped + WARNING logged (refusal-to-guess) | + +## External IDs (dedup keys) +- Trades: `ibkr:trade:` +- Cash: `ibkr:cash:` + +Both are stable across re-runs — the `dedup.SyncRecordStore` rejects +already-seen IDs. + +## Symbol canonicalisation +LSE-listed GBP instruments get a `.L` suffix (Wealthfolio convention). +US instruments and anything already suffixed pass through unchanged. + +## Position reconciliation +Each run pushes to Pushgateway: +- `ibkr_position_drift_shares{symbol, account}` — broker_qty − wf_qty per asset +- `ibkr_sync_last_success_timestamp_seconds` — unix timestamp + +Alerts (defined in monitoring stack — TBD until first non-zero drift): +- `IBKRPositionDrift{symbol}` — `|drift| > 0.01` for >24h, Slack `#security`. +- `IBKRSyncStale` — timestamp > 36h old. +- `IBKRFlexTokenExpired` — Loki rule on the "code 1003" log line. + +## Token rotation +Flex tokens expire after 1 year. When the cron starts failing with +`ResponseCodeError(code=1003)`: +1. Sign in to IBKR Client Portal → Reports → Settings → Flex Web Service → regenerate token. +2. `vault kv patch secret/broker-sync ibkr_flex_token=''`. +3. ExternalSecrets controller picks it up within 15 min; no manual restart needed. + +## Spec / plan +Design: `docs/specs/2026-05-26-ibkr-ingest-design.md` +Implementation plan: `docs/plans/2026-05-26-ibkr-flex-ingestion.md` +``` + +- [ ] **Step 2: Commit** + +```bash +cd /home/wizard/code/broker-sync +git add docs/providers/ibkr.md +git commit -m "docs: add IBKR provider runbook" +git push origin main && git push forgejo main +``` + +--- + +## Task 15: Acceptance — 7-day soak + +**Files:** (none — observational) + +- [ ] **Step 1: Set a 7-day calendar reminder to re-check** + +Set a reminder for `2026-06-02` (today + 7 days). + +- [ ] **Step 2: On 2026-06-02, run the acceptance check** + +```bash +# Last 7 days of CronJob outcomes +kubectl -n broker-sync get jobs --sort-by=.metadata.creationTimestamp -o wide \ + | grep broker-sync-ibkr-2 + +# Pushgateway should have a recent success timestamp +kubectl -n monitoring port-forward svc/prometheus-prometheus-pushgateway 9091:9091 & +sleep 2 +curl -sS http://localhost:9091/metrics | grep ibkr_sync_last_success +kill %1 + +# Pushgateway drift should be zero on all symbols (account still empty, or +# else broker matches WF) +curl -sS http://localhost:9091/metrics | grep ibkr_position_drift_shares +``` + +Expected: +- ≥6 of the 7 nightly runs `Complete`. +- `ibkr_sync_last_success_timestamp_seconds` within the last 36 hours. +- `ibkr_position_drift_shares` all zero. + +- [ ] **Step 3: If all green, close the implementation plan** + +Mark this plan file as `Status: Done` at the top and commit. + +If not green, file beads tasks for the specific issues and revisit. + +--- + +## Self-review notes + +- **Spec coverage**: every section of `docs/specs/2026-05-26-ibkr-ingest-design.md` + maps to one or more tasks (deps→1, fixtures→2, metrics→3, sink helper→4, + symbol canon→5, trade map→6, cash map→7, provider→8, CLI→9, image→10, + setup→11, CronJob→12, smoke→13, docs→14, soak→15). +- **Placeholder scan**: no `TBD` in the plan body. The doc file + `docs/providers/ibkr.md` includes one explicit TBD about + PrometheusRule definitions — that's intentional, deferred to the + monitoring stack work (out-of-scope here; first non-zero drift event + will prompt the alert PR). +- **Type consistency**: `IBKRProvider.fetch` is `AsyncIterator[Activity]` + throughout. `compute_position_qty` returns `dict[str, Decimal]` in + both the sink and the CLI consumer. External_id schemes + (`ibkr:trade:` and `ibkr:cash:`) match between the mapper, the + provider, and the documentation. From 7cba540c37f7b02211f33232ab03a9703ec9ac42 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Tue, 26 May 2026 22:21:54 +0000 Subject: [PATCH 13/27] deps: add ibflex with web extras for IBKR Flex Web Service ingestion --- poetry.lock | 199 ++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 3 + 2 files changed, 201 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index f4abb62..56df0e2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -73,6 +73,145 @@ files = [ {file = "certifi-2026.2.25.tar.gz", hash = "sha256:e887ab5cee78ea814d3472169153c2d12cd43b14bd03329a39a9c6e2e80bfba7"}, ] +[[package]] +name = "charset-normalizer" +version = "3.4.7" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "charset_normalizer-3.4.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-win32.whl", hash = "sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-win_amd64.whl", hash = "sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943"}, + {file = "charset_normalizer-3.4.7-cp310-cp310-win_arm64.whl", hash = "sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-win32.whl", hash = "sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-win_amd64.whl", hash = "sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00"}, + {file = "charset_normalizer-3.4.7-cp311-cp311-win_arm64.whl", hash = "sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-win32.whl", hash = "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", hash = "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6"}, + {file = "charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", hash = "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-win32.whl", hash = "sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-win_amd64.whl", hash = "sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110"}, + {file = "charset_normalizer-3.4.7-cp313-cp313-win_arm64.whl", hash = "sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-win32.whl", hash = "sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", hash = "sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f"}, + {file = "charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", hash = "sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-win32.whl", hash = "sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", hash = "sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c"}, + {file = "charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", hash = "sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e5f4d355f0a2b1a31bc3edec6795b46324349c9cb25eed068049e4f472fb4259"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:16d971e29578a5e97d7117866d15889a4a07befe0e87e703ed63cd90cb348c01"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:dca4bbc466a95ba9c0234ef56d7dd9509f63da22274589ebd4ed7f1f4d4c54e3"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:e80c8378d8f3d83cd3164da1ad2df9e37a666cdde7b1cb2298ed0b558064be30"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36836d6ff945a00b88ba1e4572d721e60b5b8c98c155d465f56ad19d68f23734"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-manylinux_2_31_armv7l.whl", hash = "sha256:bd9b23791fe793e4968dba0c447e12f78e425c59fc0e3b97f6450f4781f3ee60"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aef65cd602a6d0e0ff6f9930fcb1c8fec60dd2cfcb6facaf4bdb0e5873042db0"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:82b271f5137d07749f7bf32f70b17ab6eaabedd297e75dce75081a24f76eb545"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:1efde3cae86c8c273f1eb3b287be7d8499420cf2fe7585c41d370d3e790054a5"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:c593052c465475e64bbfe5dbd81680f64a67fdc752c56d7a0ae205dc8aeefe0f"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-musllinux_1_2_riscv64.whl", hash = "sha256:af21eb4409a119e365397b2adbaca4c9ccab56543a65d5dbd9f920d6ac29f686"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:84c018e49c3bf790f9c2771c45e9313a08c2c2a6342b162cd650258b57817706"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:dd915403e231e6b1809fe9b6d9fc55cf8fb5e02765ac625d9cd623342a7905d7"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-win32.whl", hash = "sha256:320ade88cfb846b8cd6b4ddf5ee9e80ee0c1f52401f2456b84ae1ae6a1a5f207"}, + {file = "charset_normalizer-3.4.7-cp38-cp38-win_amd64.whl", hash = "sha256:1dc8b0ea451d6e69735094606991f32867807881400f808a106ee1d963c46a83"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:177a0ba5f0211d488e295aaf82707237e331c24788d8d76c96c5a41594723217"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e0d51f618228538a3e8f46bd246f87a6cd030565e015803691603f55e12afb5"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:14265bfe1f09498b9d8ec91e9ec9fa52775edf90fcbde092b25f4a33d444fea9"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:87fad7d9ba98c86bcb41b2dc8dbb326619be2562af1f8ff50776a39e55721c5a"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f22dec1690b584cea26fade98b2435c132c1b5f68e39f5a0b7627cd7ae31f1dc"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:d61f00a0869d77422d9b2aba989e2d24afa6ffd552af442e0e58de4f35ea6d00"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6370e8686f662e6a3941ee48ed4742317cafbe5707e36406e9df792cdb535776"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:a6c5863edfbe888d9eff9c8b8087354e27618d9da76425c119293f11712a6319"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:ed065083d0898c9d5b4bbec7b026fd755ff7454e6e8b73a67f8c744b13986e24"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:2cd4a60d0e2fb04537162c62bbbb4182f53541fe0ede35cdf270a1c1e723cc42"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:813c0e0132266c08eb87469a642cb30aaff57c5f426255419572aaeceeaa7bf4"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:07d9e39b01743c3717745f4c530a6349eadbfa043c7577eef86c502c15df2c67"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c0f081d69a6e58272819b70288d3221a6ee64b98df852631c80f293514d3b274"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-win32.whl", hash = "sha256:8751d2787c9131302398b11e6c8068053dcb55d5a8964e114b6e196cf16cb366"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-win_amd64.whl", hash = "sha256:12a6fff75f6bc66711b73a2f0addfc4c8c15a20e805146a02d147a318962c444"}, + {file = "charset_normalizer-3.4.7-cp39-cp39-win_arm64.whl", hash = "sha256:bb8cc7534f51d9a017b93e3e85b260924f909601c3df002bcdb58ddb4dc41a5c"}, + {file = "charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d"}, + {file = "charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5"}, +] + [[package]] name = "click" version = "8.1.8" @@ -234,6 +373,24 @@ http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "ibflex" +version = "1.1" +description = "Parse Interactive Brokers Flex XML reports and convert to Python types" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "ibflex-1.1-py3-none-any.whl", hash = "sha256:c84e02dafcd17f70587777c2e2f00e3cc1e949e045790bf4fe562fb03dbef434"}, + {file = "ibflex-1.1.tar.gz", hash = "sha256:3e5cac02cadcbd22ea46ae4ca306d67c274b7166f40119f5d7d7103a130d032a"}, +] + +[package.dependencies] +requests = {version = "*", optional = true, markers = "extra == \"web\""} + +[package.extras] +web = ["requests"] + [[package]] name = "idna" version = "3.11" @@ -663,6 +820,28 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "requests" +version = "2.34.2" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0"}, + {file = "requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed"}, +] + +[package.dependencies] +certifi = ">=2023.5.7" +charset_normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.26,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<8)"] + [[package]] name = "rich" version = "15.0.0" @@ -800,6 +979,24 @@ files = [ {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, ] +[[package]] +name = "urllib3" +version = "2.7.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897"}, + {file = "urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c"}, +] + +[package.extras] +brotli = ["brotli (>=1.2.0) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=1.2.0.0) ; platform_python_implementation != \"CPython\""] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] + [[package]] name = "yapf" version = "0.43.0" @@ -818,4 +1015,4 @@ platformdirs = ">=3.5.1" [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.13" -content-hash = "b3896b2258a425cce9498be9ada5bd48a06d5f2bd7c53ead044ad27c53086bd7" +content-hash = "8a704e79729d5bd3cbe78a7e35c51e9da724880915c0152788273b94bd00610d" diff --git a/pyproject.toml b/pyproject.toml index e5860d5..e6281b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,9 @@ aiomysql = "^0.3.2" # long-lived session alive (storage_state + device-trust cookie); actual data # is fetched via httpx against the SPA's private JSON backend. playwright = "^1.47" +# IBKR Flex Web Service: pulls Activity Flex Query XML reports (token-auth) +# and parses to typed dataclasses. No Gateway / daily re-auth needed. +ibflex = { version = "^1.1", extras = ["web"] } [tool.poetry.group.dev.dependencies] pytest = "^8.3" From 82797908b774ac52935d3cbb976eb835e121306f Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Tue, 26 May 2026 22:23:20 +0000 Subject: [PATCH 14/27] test: add IBKR Flex XML fixture (3 trades, 2 cash txns, 2 positions) --- tests/fixtures/ibkr/sample_flex.xml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 tests/fixtures/ibkr/sample_flex.xml diff --git a/tests/fixtures/ibkr/sample_flex.xml b/tests/fixtures/ibkr/sample_flex.xml new file mode 100644 index 0000000..0d82fcf --- /dev/null +++ b/tests/fixtures/ibkr/sample_flex.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + From 975c3b4bf7e13bc545fce8f9511d1006fb123fc5 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Tue, 26 May 2026 22:24:55 +0000 Subject: [PATCH 15/27] metrics: add Pushgateway client for broker-sync providers --- broker_sync/metrics.py | 51 ++++++++++++++++++++++++++++++++ tests/test_metrics.py | 66 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 broker_sync/metrics.py create mode 100644 tests/test_metrics.py diff --git a/broker_sync/metrics.py b/broker_sync/metrics.py new file mode 100644 index 0000000..41566d8 --- /dev/null +++ b/broker_sync/metrics.py @@ -0,0 +1,51 @@ +"""Pushgateway client for broker-sync providers. + +One function: push a list of (metric, labels, value) tuples to Prometheus +Pushgateway under a given job name. Used by providers to surface per-run +drift / staleness / row counts that Prometheus can alert on. + +In-cluster URL: http://prometheus-prometheus-pushgateway.monitoring:9091/metrics +Pass via the ``pushgateway_url`` argument or the ``PUSHGATEWAY_URL`` env var. +""" +from __future__ import annotations + +import logging +import os +from collections.abc import Iterable + +import httpx + +log = logging.getLogger(__name__) + + +def _format_metric(name: str, labels: dict[str, str], value: float) -> str: + if labels: + body = ",".join(f'{k}="{v}"' for k, v in sorted(labels.items())) + return f"{name}{{{body}}} {value}\n" + return f"{name} {value}\n" + + +async def push_pushgateway( + job: str, + metrics: Iterable[tuple[str, dict[str, str], float]], + pushgateway_url: str | None = None, + transport: httpx.AsyncBaseTransport | None = None, +) -> None: + """POST text-format metrics to Pushgateway under ``job``. + + ``pushgateway_url`` falls back to the env var ``PUSHGATEWAY_URL``. + Raises ``RuntimeError`` if the URL is unset or POST returns non-2xx. + """ + url = pushgateway_url or os.environ.get("PUSHGATEWAY_URL") + if not url: + raise RuntimeError("PUSHGATEWAY_URL not set and no override provided") + body = "".join(_format_metric(n, lbls, v) for n, lbls, v in metrics) + target = f"{url.rstrip('/')}/job/{job}" + async with httpx.AsyncClient(transport=transport, timeout=15.0) as c: + resp = await c.post(target, content=body, headers={"Content-Type": "text/plain"}) + if resp.status_code >= 300: + raise RuntimeError( + f"pushgateway POST {target} returned HTTP {resp.status_code}: " + f"{resp.text[:200]}" + ) + log.info("pushgateway: pushed %d metrics to job=%s", len(body.splitlines()), job) diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 0000000..6a82012 --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import httpx +import pytest + +from broker_sync.metrics import push_pushgateway + + +async def test_push_pushgateway_posts_text_format() -> None: + captured: dict[str, str] = {} + + def transport_handler(request: httpx.Request) -> httpx.Response: + captured["url"] = str(request.url) + captured["method"] = request.method + captured["body"] = request.content.decode("utf-8") + return httpx.Response(200) + + transport = httpx.MockTransport(transport_handler) + await push_pushgateway( + job="broker-sync-ibkr", + metrics=[ + ("ibkr_position_drift_shares", {"symbol": "VUAG.L"}, 0.0), + ("ibkr_sync_last_success_timestamp_seconds", {}, 1779830000.0), + ], + pushgateway_url="http://pg.example/metrics", + transport=transport, + ) + assert captured["method"] == "POST" + assert captured["url"] == "http://pg.example/metrics/job/broker-sync-ibkr" + body = captured["body"] + assert 'ibkr_position_drift_shares{symbol="VUAG.L"} 0.0' in body + assert "ibkr_sync_last_success_timestamp_seconds 1779830000.0" in body + + +async def test_push_pushgateway_raises_on_non_2xx() -> None: + transport = httpx.MockTransport(lambda r: httpx.Response(500, text="boom")) + with pytest.raises(RuntimeError, match="pushgateway.*500"): + await push_pushgateway( + job="x", + metrics=[("m", {}, 1.0)], + pushgateway_url="http://pg/metrics", + transport=transport, + ) + + +async def test_push_pushgateway_uses_env_var(monkeypatch: pytest.MonkeyPatch) -> None: + captured: dict[str, str] = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["url"] = str(request.url) + return httpx.Response(200) + + transport = httpx.MockTransport(handler) + monkeypatch.setenv("PUSHGATEWAY_URL", "http://from-env/metrics") + await push_pushgateway( + job="j", + metrics=[("m", {}, 1.0)], + transport=transport, + ) + assert captured["url"] == "http://from-env/metrics/job/j" + + +async def test_push_pushgateway_raises_when_url_missing(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("PUSHGATEWAY_URL", raising=False) + with pytest.raises(RuntimeError, match="PUSHGATEWAY_URL not set"): + await push_pushgateway(job="j", metrics=[("m", {}, 1.0)]) From 882415464e3bee2d23d4d491dab0d8c9d82d430b Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Tue, 26 May 2026 22:26:24 +0000 Subject: [PATCH 16/27] wealthfolio: add compute_position_qty for broker reconciliation --- broker_sync/sinks/wealthfolio.py | 44 ++++++++++++++++++++++++ tests/sinks/test_wealthfolio.py | 59 ++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/broker_sync/sinks/wealthfolio.py b/broker_sync/sinks/wealthfolio.py index 7144f6f..51a2d41 100644 --- a/broker_sync/sinks/wealthfolio.py +++ b/broker_sync/sinks/wealthfolio.py @@ -315,6 +315,50 @@ class WealthfolioSink: total += amt return total + async def compute_position_qty(self, account_id: str) -> dict[str, Decimal]: + """Return per-symbol net position quantity (BUY/IN minus SELL/OUT) for + one account. Skips cash activities and unknown activity types. + + Used by the IBKR reconciliation step to compare against broker-reported + OpenPositions. + """ + qty_by_symbol: dict[str, Decimal] = {} + page = 1 + while True: + resp = await self._request( + "POST", _ACTIVITIES_SEARCH, + json={"accountIds": [account_id], "page": page, "pageSize": 500}, + ) + resp.raise_for_status() + payload = resp.json() + activities = payload.get("activities", []) if isinstance(payload, dict) else [] + if not activities: + break + for act in activities: + if not isinstance(act, dict): + continue + symbol = act.get("symbol") or "" + if not symbol or symbol.startswith("$CASH"): + continue + act_type = act.get("activityType") or "" + sign: int + if act_type in {"BUY", "ADD_HOLDING", "TRANSFER_IN"}: + sign = 1 + elif act_type in {"SELL", "REMOVE_HOLDING", "TRANSFER_OUT"}: + sign = -1 + else: + continue + try: + qty = Decimal(str(act.get("quantity") or 0)) + except Exception: + continue + qty_by_symbol[symbol] = qty_by_symbol.get(symbol, Decimal(0)) + sign * qty + total_pages = int(payload.get("totalPages") or 1) if isinstance(payload, dict) else 1 + if page >= total_pages: + break + page += 1 + return qty_by_symbol + # -- manual holdings snapshots -- async def push_manual_snapshots( diff --git a/tests/sinks/test_wealthfolio.py b/tests/sinks/test_wealthfolio.py index 436e52b..2b43681 100644 --- a/tests/sinks/test_wealthfolio.py +++ b/tests/sinks/test_wealthfolio.py @@ -373,3 +373,62 @@ async def test_push_manual_snapshots_short_circuits_on_empty( sink = _client(httpx.MockTransport(handler), sp) result = await sink.push_manual_snapshots(account_id="acct", snapshots=[]) assert result["snapshotsImported"] == 0 + + +# -- compute_position_qty (used by IBKR reconciliation) -- + + +@pytest.mark.asyncio +async def test_compute_position_qty_sums_buys_minus_sells(tmp_path: Path) -> None: + """Sums BUY/ADD_HOLDING/TRANSFER_IN minus SELL/REMOVE_HOLDING/TRANSFER_OUT + quantities per symbol, skipping cash activities.""" + sp = tmp_path / "s.json" + sp.write_text(json.dumps({"cookies": {"wf_token": "fresh"}})) + + page_1: dict[str, Any] = { + "activities": [ + {"symbol": "VUAG.L", "activityType": "BUY", "quantity": "10"}, + {"symbol": "VUAG.L", "activityType": "SELL", "quantity": "2"}, + {"symbol": "AAPL", "activityType": "BUY", "quantity": "5"}, + {"symbol": "$CASH-GBP", "activityType": "DEPOSIT", "quantity": "0", + "amount": "100"}, + # Unknown activity type — must be skipped, not crash. + {"symbol": "VUAG.L", "activityType": "DIVIDEND", "quantity": "0", + "amount": "0.5"}, + ], + "totalPages": 1, + } + + async def handler(req: httpx.Request) -> httpx.Response: + if req.url.path == "/api/v1/activities/search": + return httpx.Response(200, json=page_1) + raise AssertionError(f"unexpected request: {req.method} {req.url.path}") + + sink = _client(httpx.MockTransport(handler), sp) + result = await sink.compute_position_qty("acct-123") + assert result == {"VUAG.L": Decimal("8"), "AAPL": Decimal("5")} + + +@pytest.mark.asyncio +async def test_compute_position_qty_paginates(tmp_path: Path) -> None: + """Walks all pages until totalPages reached.""" + sp = tmp_path / "s.json" + sp.write_text(json.dumps({"cookies": {"wf_token": "fresh"}})) + + pages: dict[int, dict[str, Any]] = { + 1: {"activities": [{"symbol": "VUAG.L", "activityType": "BUY", + "quantity": "3"}], "totalPages": 2}, + 2: {"activities": [{"symbol": "VUAG.L", "activityType": "BUY", + "quantity": "4"}], "totalPages": 2}, + } + seen_pages: list[int] = [] + + async def handler(req: httpx.Request) -> httpx.Response: + body = json.loads(req.content) + seen_pages.append(body["page"]) + return httpx.Response(200, json=pages[body["page"]]) + + sink = _client(httpx.MockTransport(handler), sp) + result = await sink.compute_position_qty("acct-x") + assert sorted(seen_pages) == [1, 2] + assert result == {"VUAG.L": Decimal("7")} From e83c5a0a8fe72515fc9f568a6f35ce100ac1899d Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Tue, 26 May 2026 22:28:35 +0000 Subject: [PATCH 17/27] =?UTF-8?q?ibkr:=20add=20Flex=20provider=20=E2=80=94?= =?UTF-8?q?=20Trade/Cash=20mapping=20+=20OpenPositions=20snapshot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Maps Trades (BUY/SELL) and CashTransactions (DIVIDEND, TAX, INTEREST, FEE, DEPOSIT, WITHDRAWAL) from an IBKR Flex Activity Query to broker-sync Activity objects. Adds canonical_symbol helper (LSE → .L suffix when exchange=LSE* or currency=GBP). Exposes OpenPositions for the reconciliation step that runs at the CLI layer. Guards against wrong-account writes by checking stmt.accountId == IBKR_ACCOUNT_ID_UPSTREAM before yielding any activities. 13 unit tests cover all the mappings + the mismatch guard. Co-Authored-By: Claude Opus 4.7 --- broker_sync/providers/ibkr.py | 255 ++++++++++++++++++++++++++++++++++ tests/providers/test_ibkr.py | 199 ++++++++++++++++++++++++++ 2 files changed, 454 insertions(+) create mode 100644 broker_sync/providers/ibkr.py create mode 100644 tests/providers/test_ibkr.py diff --git a/broker_sync/providers/ibkr.py b/broker_sync/providers/ibkr.py new file mode 100644 index 0000000..f156a3f --- /dev/null +++ b/broker_sync/providers/ibkr.py @@ -0,0 +1,255 @@ +"""Interactive Brokers Flex Web Service ingestion provider. + +Pulls daily Activity Flex Query reports via the ``ibflex`` library, maps +Trades + CashTransactions to broker-sync ``Activity`` objects, and runs a +reconciliation step against the broker-reported ``OpenPositions``. + +See ``docs/specs/2026-05-26-ibkr-ingest-design.md`` for the full design. +""" +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator +from datetime import UTC, date, datetime +from decimal import Decimal +from typing import Any + +from broker_sync.models import Account, AccountType, Activity, ActivityType + +log = logging.getLogger(__name__) + +# Map IBKR currency → default exchange suffix. +# Today: GBP → LSE (.L). Extend when more accounts onboard. +_LSE_EXCHANGES = {"LSE", "LSEETF", "LSEIOB1"} +_GBP_SUFFIX = ".L" + + +def canonical_symbol(symbol: str, *, exchange: str | None, currency: str) -> str: + """Return the WF-canonical form of an IBKR ticker. + + LSE-listed GBP instruments get a ``.L`` suffix (Wealthfolio convention). + US instruments and anything already suffixed are returned unchanged. + """ + if "." in symbol: + return symbol + if exchange in _LSE_EXCHANGES or (exchange is None and currency == "GBP"): + return symbol + _GBP_SUFFIX + return symbol + + +def _to_utc_datetime(value: Any, time_value: Any = None) -> datetime: + """Combine a date (with optional time) into a UTC datetime.""" + if isinstance(value, datetime): + dt = value + elif isinstance(value, date): + if isinstance(time_value, str): + dt = datetime.fromisoformat(f"{value.isoformat()}T{time_value}") + elif hasattr(time_value, "isoformat"): + dt = datetime.fromisoformat(f"{value.isoformat()}T{time_value.isoformat()}") + else: + dt = datetime.fromisoformat(f"{value.isoformat()}T00:00:00") + else: + # Last-resort: ISO string + dt = datetime.fromisoformat(str(value)) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + return dt.astimezone(UTC) + + +def _map_trade_to_activity(trade: Any, *, account_id: str) -> Activity: + """Map one ibflex Trade dataclass to a broker-sync Activity.""" + buy_sell_obj = trade.buySell + buy_sell = buy_sell_obj.name if hasattr(buy_sell_obj, "name") else str(buy_sell_obj) + if buy_sell == "BUY": + activity_type = ActivityType.BUY + elif buy_sell == "SELL": + activity_type = ActivityType.SELL + else: + raise ValueError( + f"unsupported Trade.buySell={buy_sell!r} on tradeID={trade.tradeID}" + ) + + exchange = getattr(trade, "exchange", None) + symbol = canonical_symbol( + str(trade.symbol), + exchange=str(exchange) if exchange is not None else None, + currency=str(trade.currency), + ) + quantity = abs(Decimal(str(trade.quantity))) + unit_price = Decimal(str(trade.tradePrice)) + commission = trade.ibCommission if trade.ibCommission is not None else Decimal(0) + fee = abs(Decimal(str(commission))) + return Activity( + external_id=f"ibkr:trade:{trade.tradeID}", + account_id=account_id, + account_type=AccountType.GIA, + date=_to_utc_datetime(trade.tradeDate, getattr(trade, "tradeTime", None)), + activity_type=activity_type, + currency=str(trade.currency), + symbol=symbol, + quantity=quantity, + unit_price=unit_price, + fee=fee, + ) + + +# Map known IBKR Flex CashTransaction.type values to broker-sync ActivityType. +# Unknown values yield None + a WARNING — we refuse to guess. +_CASH_TYPE_MAP: dict[str, ActivityType] = { + "DIVIDEND": ActivityType.DIVIDEND, + "DIVIDENDS": ActivityType.DIVIDEND, + "PAYMENT_IN_LIEU_OF_DIVIDENDS": ActivityType.DIVIDEND, + "WITHHOLDING_TAX": ActivityType.TAX, + "WHTAX": ActivityType.TAX, + "BROKER_INTEREST_RECEIVED": ActivityType.INTEREST, + "BROKER_INTEREST_PAID": ActivityType.FEE, + "COMMISSION_ADJUSTMENTS": ActivityType.FEE, + "OTHER_FEES": ActivityType.FEE, +} + +_DEPOSIT_WITHDRAWAL_TYPES = { + "DEPOSITS_WITHDRAWALS", + "DEPOSIT_WITHDRAWALS", + "DEPOSITWITHDRAW", +} + + +def _normalise_cash_type(type_obj: Any) -> str: + """Canonicalise the IBKR Flex CashTransaction.type enum to an UPPER_SNAKE name.""" + if hasattr(type_obj, "name"): + return str(type_obj.name).upper() + return str(type_obj).strip().upper().replace(" ", "_").replace("&", "AND") + + +def _map_cash_to_activity(cash: Any, *, account_id: str) -> Activity | None: + """Map one ibflex CashTransaction to a broker-sync Activity. + + Returns None for unsupported types (logged at WARNING). + """ + type_name = _normalise_cash_type(cash.type) + amount = Decimal(str(cash.amount)) + + if type_name in _DEPOSIT_WITHDRAWAL_TYPES: + activity_type = ActivityType.DEPOSIT if amount > 0 else ActivityType.WITHDRAWAL + else: + mapped = _CASH_TYPE_MAP.get(type_name) + if mapped is None: + log.warning( + "ibkr: skipping cash transaction id=%s with unsupported type=%r", + getattr(cash, "transactionID", "?"), + type_name, + ) + return None + activity_type = mapped + + dt_raw = cash.dateTime + dt = _to_utc_datetime(dt_raw) if dt_raw is not None else datetime.now(UTC) + + return Activity( + external_id=f"ibkr:cash:{cash.transactionID}", + account_id=account_id, + account_type=AccountType.GIA, + date=dt, + activity_type=activity_type, + currency=str(cash.currency), + amount=abs(amount), + ) + + +class IBKRError(Exception): + """Base class for ibkr-provider errors.""" + + +class IBKRAccountMismatchError(IBKRError): + """Flex statement accountId did not match configured upstream id.""" + + +class IBKRProvider: + """Fetches IBKR Flex Activity reports and yields broker-sync Activities. + + Reconciliation (OpenPositions vs WF-computed qty) is NOT part of + ``fetch()`` — it runs at the CLI layer after import, where the + WealthfolioSink is available to query WF. + """ + + def __init__( + self, + *, + token: str, + query_id: str, + wf_account_id: str, + upstream_account_id: str, + ) -> None: + self._token = token + self._query_id = query_id + self._wf_account_id = wf_account_id + self._upstream_account_id = upstream_account_id + # Stashed for the reconciliation step after fetch() drains. + self._last_response: Any = None + + def accounts(self) -> list[Account]: + return [ + Account( + id=self._wf_account_id, + name="Interactive Brokers (UK)", + account_type=AccountType.GIA, + currency="GBP", # FX-aware per-trade; account ccy is GBP + provider="ibkr", + ) + ] + + async def close(self) -> None: + # ibflex.client uses synchronous `requests` under the hood; no resources to close. + return + + async def fetch( + self, + *, + since: datetime | None = None, # Flex query owns the date range + before: datetime | None = None, + ) -> AsyncIterator[Activity]: + from ibflex import client as ib_client + from ibflex import parser as ib_parser + + del since, before # unused; Flex query defines the period + + xml_bytes = ib_client.download(self._token, self._query_id) + response = ib_parser.parse(xml_bytes) + self._last_response = response + + if not response.FlexStatements: + log.warning("ibkr: Flex response had no FlexStatements") + return + + stmt = response.FlexStatements[0] + if str(stmt.accountId) != self._upstream_account_id: + raise IBKRAccountMismatchError( + f"Flex statement.accountId={stmt.accountId!r} does not match " + f"configured IBKR_ACCOUNT_ID_UPSTREAM={self._upstream_account_id!r} " + f"— refusing to ingest" + ) + + for trade in stmt.Trades or []: + yield _map_trade_to_activity(trade, account_id=self._wf_account_id) + + for cash in stmt.CashTransactions or []: + activity = _map_cash_to_activity(cash, account_id=self._wf_account_id) + if activity is not None: + yield activity + + def open_positions(self) -> list[tuple[str, Decimal]]: + """Return ``[(canonical_symbol, position_qty), ...]`` from the most + recent fetch. Empty list before the first ``fetch()`` call.""" + if self._last_response is None: + return [] + stmt = self._last_response.FlexStatements[0] + out: list[tuple[str, Decimal]] = [] + for pos in stmt.OpenPositions or []: + exchange = getattr(pos, "exchange", None) + symbol = canonical_symbol( + str(pos.symbol), + exchange=str(exchange) if exchange is not None else None, + currency=str(pos.currency), + ) + out.append((symbol, Decimal(str(pos.position)))) + return out diff --git a/tests/providers/test_ibkr.py b/tests/providers/test_ibkr.py new file mode 100644 index 0000000..ea83e26 --- /dev/null +++ b/tests/providers/test_ibkr.py @@ -0,0 +1,199 @@ +from __future__ import annotations + +from datetime import datetime +from decimal import Decimal + +import pytest + +from broker_sync.models import ActivityType +from broker_sync.providers.ibkr import ( + IBKRAccountMismatchError, + IBKRProvider, + _map_cash_to_activity, + _map_trade_to_activity, + canonical_symbol, +) + +# -- canonical_symbol -- + + +def test_canonical_symbol_lse_etf_gets_l_suffix() -> None: + assert canonical_symbol("VUAG", exchange="LSEETF", currency="GBP") == "VUAG.L" + + +def test_canonical_symbol_us_stock_unchanged() -> None: + assert canonical_symbol("AAPL", exchange="NASDAQ", currency="USD") == "AAPL" + + +def test_canonical_symbol_lse_gbp_inferred_when_exchange_missing() -> None: + """IBKR Flex sometimes omits exchange — infer LSE from currency==GBP.""" + assert canonical_symbol("VUAG", exchange=None, currency="GBP") == "VUAG.L" + + +def test_canonical_symbol_already_suffixed_unchanged() -> None: + assert canonical_symbol("VUAG.L", exchange="LSEETF", currency="GBP") == "VUAG.L" + + +# -- Trade mapping -- + + +def test_map_trade_buy_to_activity() -> None: + from ibflex import parser + + r = parser.parse("tests/fixtures/ibkr/sample_flex.xml") + trade = r.FlexStatements[0].Trades[0] # T1001: 10 VUAG BUY @ 107.50 GBP, comm -1.05 + + activity = _map_trade_to_activity(trade, account_id="wf-acct-uuid") + + assert activity.external_id == "ibkr:trade:T1001" + assert activity.account_id == "wf-acct-uuid" + assert activity.activity_type == ActivityType.BUY + assert activity.symbol == "VUAG.L" + assert activity.quantity == Decimal("10") + assert activity.unit_price == Decimal("107.50") + assert activity.fee == Decimal("1.05") + assert activity.currency == "GBP" + assert isinstance(activity.date, datetime) + assert activity.date.tzinfo is not None + + +def test_map_trade_sell_to_activity() -> None: + from ibflex import parser + + r = parser.parse("tests/fixtures/ibkr/sample_flex.xml") + trade = r.FlexStatements[0].Trades[2] # T1003: 2 VUAG SELL @ 108.00 GBP + + activity = _map_trade_to_activity(trade, account_id="wf-acct") + assert activity.activity_type == ActivityType.SELL + assert activity.symbol == "VUAG.L" + assert activity.quantity == Decimal("2") + assert activity.unit_price == Decimal("108.00") + + +def test_map_trade_us_stock_keeps_usd_currency_and_no_suffix() -> None: + from ibflex import parser + + r = parser.parse("tests/fixtures/ibkr/sample_flex.xml") + trade = r.FlexStatements[0].Trades[1] # T1002: AAPL BUY USD + + activity = _map_trade_to_activity(trade, account_id="wf-acct") + assert activity.symbol == "AAPL" + assert activity.currency == "USD" + + +# -- Cash mapping -- + + +def test_map_cash_dividend_to_activity() -> None: + from ibflex import parser + + r = parser.parse("tests/fixtures/ibkr/sample_flex.xml") + cash = r.FlexStatements[0].CashTransactions[0] # C5001: Dividends 3.50 GBP + + activity = _map_cash_to_activity(cash, account_id="wf-acct") + assert activity is not None + assert activity.external_id == "ibkr:cash:C5001" + assert activity.activity_type == ActivityType.DIVIDEND + assert activity.amount == Decimal("3.50") + assert activity.currency == "GBP" + + +def test_map_cash_withholding_tax_to_tax_activity() -> None: + from ibflex import parser + + r = parser.parse("tests/fixtures/ibkr/sample_flex.xml") + cash = r.FlexStatements[0].CashTransactions[1] # C5002: Withholding Tax -0.35 GBP + + activity = _map_cash_to_activity(cash, account_id="wf-acct") + assert activity is not None + assert activity.activity_type == ActivityType.TAX + assert activity.amount == Decimal("0.35") # always positive on Activity + + +def test_map_cash_unknown_type_returns_none_and_logs(caplog: pytest.LogCaptureFixture) -> None: + """Unknown CashTransaction.type produces None + a WARNING log line.""" + + class FakeType: + name = "FrobnicatedThing" + + class FakeCash: + transactionID = "C9999" + dateTime = None + type = FakeType() + amount = Decimal("0") + currency = "GBP" + + with caplog.at_level("WARNING"): + result = _map_cash_to_activity(FakeCash, account_id="wf-acct") + assert result is None + assert any("FROBNICATEDTHING" in r.message for r in caplog.records) + + +# -- IBKRProvider end-to-end -- + + +async def test_ibkr_provider_fetch_returns_mapped_activities( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """IBKRProvider.fetch() yields all mapped activities (trades + cash).""" + from ibflex import client as ib_client + + with open("tests/fixtures/ibkr/sample_flex.xml", "rb") as f: + xml_bytes = f.read() + monkeypatch.setattr(ib_client, "download", lambda *a, **kw: xml_bytes) + + provider = IBKRProvider( + token="t", + query_id="q", + wf_account_id="wf-acct", + upstream_account_id="U12345678", + ) + activities = [a async for a in provider.fetch()] + # 3 trades + 2 cash = 5 + assert len(activities) == 5 + types = sorted(a.activity_type.name for a in activities) + assert types == ["BUY", "BUY", "DIVIDEND", "SELL", "TAX"] + + +async def test_ibkr_provider_account_mismatch_raises( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Mismatched accountId raises and writes nothing.""" + from ibflex import client as ib_client + + with open("tests/fixtures/ibkr/sample_flex.xml", "rb") as f: + xml_bytes = f.read() + monkeypatch.setattr(ib_client, "download", lambda *a, **kw: xml_bytes) + + provider = IBKRProvider( + token="t", + query_id="q", + wf_account_id="wf-acct", + upstream_account_id="U99999999", # WRONG + ) + with pytest.raises(IBKRAccountMismatchError, match="U12345678"): + _ = [a async for a in provider.fetch()] + + +async def test_ibkr_provider_open_positions_after_fetch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """open_positions() returns canonicalised symbol + qty after fetch drained.""" + from ibflex import client as ib_client + + with open("tests/fixtures/ibkr/sample_flex.xml", "rb") as f: + xml_bytes = f.read() + monkeypatch.setattr(ib_client, "download", lambda *a, **kw: xml_bytes) + + provider = IBKRProvider( + token="t", + query_id="q", + wf_account_id="wf-acct", + upstream_account_id="U12345678", + ) + # drain the iterator before reading positions + [a async for a in provider.fetch()] + + positions = provider.open_positions() + # VUAG → VUAG.L (LSE inferred from GBP); AAPL unchanged (USD) + assert dict(positions) == {"VUAG.L": Decimal("8"), "AAPL": Decimal("5")} From a4dab03bc5b0b8cb9500e39e2cb660858214a4c6 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Tue, 26 May 2026 22:29:44 +0000 Subject: [PATCH 18/27] cli: add 'broker-sync ibkr' command (Flex pull + import + reconcile + metrics) --- broker_sync/cli.py | 94 +++++++++++++++++++++++++++++++++++ broker_sync/providers/ibkr.py | 2 + 2 files changed, 96 insertions(+) diff --git a/broker_sync/cli.py b/broker_sync/cli.py index 6e08eb8..cef7526 100644 --- a/broker_sync/cli.py +++ b/broker_sync/cli.py @@ -230,6 +230,100 @@ def invest_engine( asyncio.run(_run()) +@app.command("ibkr") +def ibkr( + wf_base_url: str = typer.Option(..., envvar="WF_BASE_URL"), + wf_username: str = typer.Option(..., envvar="WF_USERNAME"), + wf_password: str = typer.Option(..., envvar="WF_PASSWORD"), + wf_session_path: str = typer.Option( + "/data/wealthfolio_session.json", envvar="WF_SESSION_PATH" + ), + ibkr_flex_token: str = typer.Option(..., envvar="IBKR_FLEX_TOKEN"), + ibkr_flex_query_id: str = typer.Option(..., envvar="IBKR_FLEX_QUERY_ID"), + ibkr_account_id: str = typer.Option(..., envvar="IBKR_ACCOUNT_ID"), + ibkr_account_id_upstream: str = typer.Option(..., envvar="IBKR_ACCOUNT_ID_UPSTREAM"), + pushgateway_url: str = typer.Option( + "http://prometheus-prometheus-pushgateway.monitoring:9091/metrics", + envvar="PUSHGATEWAY_URL", + ), + data_dir: str = typer.Option("/data", envvar="BROKER_SYNC_DATA_DIR"), +) -> None: + """Phase 2c — daily IBKR Flex Web Service → Wealthfolio sync. + + Pulls an Activity Flex Query (Trades + Cash + OpenPositions), maps to + broker-sync Activities, pushes through the shared pipeline, then + reconciles broker-reported OpenPositions against WF-computed quantities + and publishes a Pushgateway drift metric. + """ + import time + from decimal import Decimal + + from broker_sync.dedup import SyncRecordStore + from broker_sync.metrics import push_pushgateway + from broker_sync.pipeline import sync_provider_to_wealthfolio + from broker_sync.providers.ibkr import IBKRAccountMismatchError, IBKRProvider + from broker_sync.sinks.wealthfolio import WealthfolioSink + + _setup_logging() + data = Path(data_dir) + data.mkdir(parents=True, exist_ok=True) + + async def _run() -> None: + sink = WealthfolioSink( + base_url=wf_base_url, + username=wf_username, + password=wf_password, + session_path=wf_session_path, + ) + provider = IBKRProvider( + token=ibkr_flex_token, + query_id=ibkr_flex_query_id, + wf_account_id=ibkr_account_id, + upstream_account_id=ibkr_account_id_upstream, + ) + dedup = SyncRecordStore(data / "sync.db") + try: + if not Path(wf_session_path).exists(): + await sink.login() + result = await sync_provider_to_wealthfolio( + provider=provider, + sink=sink, + dedup=dedup, + ) + + # Reconciliation: broker truth vs WF truth. + wf_qty = await sink.compute_position_qty(ibkr_account_id) + drift_metrics: list[tuple[str, dict[str, str], float]] = [] + for symbol, broker_qty in provider.open_positions(): + drift = broker_qty - wf_qty.get(symbol, Decimal(0)) + drift_metrics.append( + ( + "ibkr_position_drift_shares", + {"symbol": symbol, "account": "ibkr-uk"}, + float(drift), + ) + ) + drift_metrics.append( + ("ibkr_sync_last_success_timestamp_seconds", {}, float(time.time())) + ) + await push_pushgateway("broker-sync-ibkr", drift_metrics, pushgateway_url) + except IBKRAccountMismatchError as e: + typer.echo(f"IBKR: {e}", err=True) + sys.exit(2) + finally: + await provider.close() + await sink.close() + + typer.echo( + f"ibkr: fetched={result.fetched} new={result.new_after_dedup} " + f"imported={result.imported} failed={result.failed}" + ) + if result.failed > 0: + sys.exit(1) + + asyncio.run(_run()) + + @app.command("finance-mysql-import") def finance_mysql_import( wf_base_url: str = typer.Option(..., envvar="WF_BASE_URL"), diff --git a/broker_sync/providers/ibkr.py b/broker_sync/providers/ibkr.py index f156a3f..741c79a 100644 --- a/broker_sync/providers/ibkr.py +++ b/broker_sync/providers/ibkr.py @@ -172,6 +172,8 @@ class IBKRProvider: WealthfolioSink is available to query WF. """ + name = "ibkr" + def __init__( self, *, From 2fb1fbbdd8b38e9fe01ff4121786b9580052efb1 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Tue, 26 May 2026 22:34:46 +0000 Subject: [PATCH 19/27] docs: add IBKR provider runbook --- docs/providers/ibkr.md | 124 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 docs/providers/ibkr.md diff --git a/docs/providers/ibkr.md b/docs/providers/ibkr.md new file mode 100644 index 0000000..a21df5b --- /dev/null +++ b/docs/providers/ibkr.md @@ -0,0 +1,124 @@ +# Provider: Interactive Brokers (IBKR Flex Web Service) + +Pulls a daily Activity Flex Query via the [`ibflex`](https://github.com/csingley/ibflex) +library, maps Trades + CashTransactions to broker-sync Activities, and +reconciles broker-side OpenPositions against WF-computed quantities. + +## When this runs + +- K8s CronJob `broker-sync-ibkr` in the `broker-sync` namespace, daily 02:00 UK. +- Manual trigger: + ```bash + kubectl -n broker-sync create job --from=cronjob/broker-sync-ibkr broker-sync-ibkr-manual-$(date +%s) + ``` + +## Vault secrets — `secret/broker-sync` + +| Key | Description | +|---|---| +| `ibkr_flex_token` | Flex Web Service token (1-year validity, rotate via IBKR Client Portal). | +| `ibkr_flex_query_id` | Activity Flex Query ID (5–7 digit number). | +| `ibkr_account_id` | Wealthfolio account UUID for "Interactive Brokers (UK)". | +| `ibkr_account_id_upstream` | IBKR-side account number (e.g. `U12345678`) — guards against wrong-account ingestion. | + +ExternalSecret `broker-sync-secrets` syncs all keys from `secret/broker-sync` +to a K8s secret of the same name. New keys take ~15 min to propagate. + +## IBKR Flex Query design + +In IBKR Client Portal → Reports → Flex Queries → Activity Flex Query, create +a new query named `broker-sync-activity` with: + +| Section | Required fields | +|---|---| +| Account Information | accountId | +| Trades | tradeID, tradeDate, tradeTime, symbol, buySell, quantity, tradePrice, currency, ibCommission, assetCategory, exchange | +| Cash Transactions | transactionID, dateTime, type, amount, currency, description | +| Open Positions | symbol, position, markPrice, currency, assetCategory, exchange | +| Securities Information | symbol, description, conid | + +**Date Format:** `yyyy-MM-dd`. **Time Format:** `HH:mm:ss` (no timezone +suffix — ibflex 1.1 rejects timezone abbreviations in the time field). +**Date Range:** `Last Business Day` for daily incremental. Switch to +`Year to Date` only for one-off backfills. + +## Cash type mapping + +| IBKR Flex `CashTransaction.type` | broker-sync `ActivityType` | +|---|---| +| Dividends | DIVIDEND | +| Withholding Tax | TAX | +| Broker Interest Received | INTEREST | +| Broker Interest Paid | FEE | +| Commission Adjustments | FEE | +| Other Fees | FEE | +| Deposits & Withdrawals | DEPOSIT (amount > 0) / WITHDRAWAL (amount < 0) | +| anything else | skipped + WARNING logged (refuse to guess) | + +## Dedup keys + +- Trades: `external_id = "ibkr:trade:" + tradeID` +- Cash: `external_id = "ibkr:cash:" + transactionID` + +Both are stable across re-runs; `dedup.SyncRecordStore` rejects already- +synced IDs. + +## Symbol canonicalisation + +LSE-listed GBP instruments get a `.L` suffix (Wealthfolio convention). +US instruments and anything already suffixed pass through unchanged. + +The heuristic: `exchange in {LSE, LSEETF, LSEIOB1}` OR +`(exchange is None AND currency == GBP)` → suffix with `.L`. Edge cases +not yet covered (Euronext, XETRA) — extend `canonical_symbol` when those +holdings exist. + +## Position reconciliation + +Each run pushes to Pushgateway under job `broker-sync-ibkr`: +- `ibkr_position_drift_shares{symbol, account="ibkr-uk"}` — + broker_qty − wf_qty per asset. +- `ibkr_sync_last_success_timestamp_seconds` — unix timestamp. + +Alerts (TODO, will be added to the monitoring stack on first +non-zero drift): +- `IBKRPositionDrift{symbol}` — `|drift| > 0.01` for >24h, Slack `#security`. +- `IBKRSyncStale` — timestamp > 36h old. +- `IBKRFlexTokenExpired` — Loki rule on the "code 1003" log line. + +## Account guard + +Before yielding any activities, the provider checks +`flex.accountId == IBKR_ACCOUNT_ID_UPSTREAM`. Mismatch → raises +`IBKRAccountMismatchError` and writes nothing. Prevents wrong-account +ingestion from a misconfigured query (e.g., someone replaced the token +with another user's by mistake). + +## Token rotation + +Flex tokens expire after 1 year. When the cron starts failing with +`ResponseCodeError(code=1003)`: + +1. Sign in to IBKR Client Portal → Reports → Settings → Flex Web Service + → regenerate token. +2. `vault kv patch secret/broker-sync ibkr_flex_token=''` +3. ExternalSecrets controller picks up the new value within ~15 min; no + manual pod restart needed. + +## Troubleshooting + +| Symptom | Likely cause | Fix | +|---|---|---| +| `IBKR_FLEX_TOKEN not provided` exit 2 | Vault has placeholder value or key missing | `vault kv patch secret/broker-sync ibkr_flex_token=''` | +| `IBKRAccountMismatchError` | `ibkr_account_id_upstream` doesn't match the account in the Flex query | Re-check IBKR account number; fix the Vault value | +| `ResponseCodeError(code=1003)` | Flex token expired | See "Token rotation" above | +| `StatementGenerationTimeout` | IBKR side slow | Single retry built in; if it persists, try a smaller date range | +| `Can't convert '... TZ' to time` parser error | Flex query has Time Format with timezone suffix | Switch to `HH:mm:ss` (no TZ) in Flex query settings | +| `'ETF' is not a valid AssetClass` | ETF set in fixture not in ibflex enum | Use `STK` in fixtures (IBKR Flex categorises ETFs under STK) | + +## References + +- Spec: [`docs/specs/2026-05-26-ibkr-ingest-design.md`](../specs/2026-05-26-ibkr-ingest-design.md) +- Plan: [`docs/plans/2026-05-26-ibkr-flex-ingestion.md`](../plans/2026-05-26-ibkr-flex-ingestion.md) +- Library: +- IBKR Flex Web Service docs: From 0ab069349f8b2968f3f265100a74f8c3dba2bd9e Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Tue, 26 May 2026 22:52:11 +0000 Subject: [PATCH 20/27] sinks/wealthfolio: treat duplicates as success in import-summary check The IMAP cronjob re-processes the full mailbox window on every run, so on steady-state runs all activities come back tagged duplicate=N. The existing logic raises ImportValidationError whenever imported_n < total_n, which makes the cron exit 1 (and the Job is reported FAILED) even though the data path is healthy. Treat (imported + duplicates) as "accounted for". Only raise when rows go missing entirely (silently dropped / validation rejected). Co-Authored-By: Claude Opus 4.7 --- broker_sync/sinks/wealthfolio.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/broker_sync/sinks/wealthfolio.py b/broker_sync/sinks/wealthfolio.py index 51a2d41..f459952 100644 --- a/broker_sync/sinks/wealthfolio.py +++ b/broker_sync/sinks/wealthfolio.py @@ -247,10 +247,14 @@ class WealthfolioSink: if summary is not None: imported_n = int(summary.get("imported", 0)) total_n = int(summary.get("total", len(valid_rows))) - if imported_n < total_n: + dupes = int(summary.get("duplicates", 0)) + skipped = int(summary.get("skipped", 0)) + # Duplicates are expected on every re-run (the cron re-processes the + # full IMAP window each night) — treat (imported + duplicates) as + # accounted-for. Only fail if something was genuinely lost. + accounted = imported_n + dupes + if accounted < total_n: err_msg = summary.get("errorMessage") or "no errorMessage" - skipped = int(summary.get("skipped", 0)) - dupes = int(summary.get("duplicates", 0)) raise ImportValidationError(f"Wealthfolio /import persisted {imported_n}/{total_n} " f"(skipped={skipped} duplicates={dupes}). " f"errorMessage: {err_msg}") From 30af5fe2c96ab0fc91dcf4ee155815d52b4af278 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Wed, 27 May 2026 09:11:57 +0000 Subject: [PATCH 21/27] =?UTF-8?q?docs(ibkr):=20change=20Flex=20date=20rang?= =?UTF-8?q?e=20from=20Last=20Business=20Day=20=E2=86=92=20Last=2090=20Days?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Trailing window backed by SyncRecordStore dedup is strictly better than a single-day window — a single missed cron run with Last Business Day loses that day's activity permanently. SyncRecordStore is keyed by ibkr:trade: / ibkr:cash:, so overlapping pulls are no-ops. Caught during the brainstorming review. Co-Authored-By: Claude Opus 4.7 --- docs/plans/2026-05-26-ibkr-flex-ingestion.md | 6 ++++-- docs/providers/ibkr.md | 7 +++++-- docs/specs/2026-05-26-ibkr-ingest-design.md | 20 +++++++++++++------- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/docs/plans/2026-05-26-ibkr-flex-ingestion.md b/docs/plans/2026-05-26-ibkr-flex-ingestion.md index 7a22578..1fa57bb 100644 --- a/docs/plans/2026-05-26-ibkr-flex-ingestion.md +++ b/docs/plans/2026-05-26-ibkr-flex-ingestion.md @@ -1461,8 +1461,10 @@ OpenPositions against WF-computed quantities. | Open Positions | symbol, position, markPrice, currency, assetCategory | | Securities Information | symbol, description, conid | -Date range: `Last Business Day` for daily incremental. Switch to -`Year to Date` for one-time backfills only. +Date range: `Last 90 Days` — trailing window so a missed cron run +doesn't lose data. SyncRecordStore makes overlapping pulls idempotent. +Switch to `Year to Date` or `Custom Date Range` only for one-time +historical backfills. ## Cash type mapping diff --git a/docs/providers/ibkr.md b/docs/providers/ibkr.md index a21df5b..501c641 100644 --- a/docs/providers/ibkr.md +++ b/docs/providers/ibkr.md @@ -39,8 +39,11 @@ a new query named `broker-sync-activity` with: **Date Format:** `yyyy-MM-dd`. **Time Format:** `HH:mm:ss` (no timezone suffix — ibflex 1.1 rejects timezone abbreviations in the time field). -**Date Range:** `Last Business Day` for daily incremental. Switch to -`Year to Date` only for one-off backfills. +**Date Range:** `Last 90 Days` — trailing window so a missed cron run +doesn't lose data. SyncRecordStore (keyed by `external_id`) makes +overlapping pulls idempotent. For a one-off historical backfill, widen +temporarily to `Year to Date` or `Custom Date Range`, run once, then +switch back. ## Cash type mapping diff --git a/docs/specs/2026-05-26-ibkr-ingest-design.md b/docs/specs/2026-05-26-ibkr-ingest-design.md index 40bb9df..21aaf12 100644 --- a/docs/specs/2026-05-26-ibkr-ingest-design.md +++ b/docs/specs/2026-05-26-ibkr-ingest-design.md @@ -210,9 +210,12 @@ top. - Sections: `Account Information`, `Trades`, `Cash Transactions`, `Open Positions`, `Securities Information` - Date Format: `yyyy-MM-dd` · Time Format: `HH:mm:ss TimeZone` - - Date Range: `Last Business Day` (for daily runs; flip to - `Year to Date` only for the initial backfill — irrelevant while - account is empty) + - Date Range: `Last 90 Days` — trailing window so a missed cron run + (failed pod, outage, vacation) doesn't lose data. SyncRecordStore + keys on `ibkr:trade:` / `ibkr:cash:`, so + overlapping pulls are no-ops. `Last Business Day` was the original + choice but creates a "single missed run = permanent data loss" + failure mode — rejected in favour of dedup-backed resync window. - Format: XML - Trade fields: ensure `tradeID`, `tradeDate`, `tradeTime`, `symbol`, `buySell`, `quantity`, `tradePrice`, `currency`, `ibCommission`, @@ -248,10 +251,13 @@ curl -sS -b /tmp/wf-jar -X POST "$WF_BASE_URL/api/v1/accounts" \ ### Step 4 — Initial backfill (skip while account is empty) -When the IBKR account first holds positions, switch the Flex query -Date Range to `Year to Date`, run the CronJob manually once, verify WF -totals match the broker app, then switch the Flex query back to -`Last Business Day` for daily incremental. +When the IBKR account first holds positions, the daily CronJob will +backfill automatically up to the 90-day trailing window. For older +history, temporarily switch the Flex query Date Range to +`Year to Date` (or `Custom Date Range` with a 1-year window), run the +CronJob manually once, verify WF totals match the broker app, then +switch the Flex query back to `Last 90 Days` for daily incremental. +Dedup makes the temporary widening safe — already-synced rows are no-ops. ### Step 5 — Deploy From ceb652b62363cce2ea71147a0af07e035d72e5e4 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Wed, 27 May 2026 09:18:42 +0000 Subject: [PATCH 22/27] ibkr: use IBKR account number as the canonical Account.id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: provider passed the WF UUID as Account.id. ensure_account looks up existing accounts by (provider, providerAccountId=Account.id), so the WF-UUID-as-providerAccountId would never match the manually-created account (which has providerAccountId=U13279690), causing the pipeline to create a duplicate WF account on every cron run. Fix: Account.id is now the IBKR account number (U13279690) throughout. The pipeline's _ensure_accounts() resolves it to the WF UUID via the canonical (provider, providerAccountId) lookup; activities are remapped before import. CLI no longer takes the WF UUID — derives it post-import via a cheap idempotent ensure_account call for the reconciliation step. Co-Authored-By: Claude Opus 4.7 --- broker_sync/cli.py | 13 ++++++++++--- broker_sync/providers/ibkr.py | 12 +++++++----- tests/providers/test_ibkr.py | 3 --- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/broker_sync/cli.py b/broker_sync/cli.py index cef7526..7f855f5 100644 --- a/broker_sync/cli.py +++ b/broker_sync/cli.py @@ -240,7 +240,6 @@ def ibkr( ), ibkr_flex_token: str = typer.Option(..., envvar="IBKR_FLEX_TOKEN"), ibkr_flex_query_id: str = typer.Option(..., envvar="IBKR_FLEX_QUERY_ID"), - ibkr_account_id: str = typer.Option(..., envvar="IBKR_ACCOUNT_ID"), ibkr_account_id_upstream: str = typer.Option(..., envvar="IBKR_ACCOUNT_ID_UPSTREAM"), pushgateway_url: str = typer.Option( "http://prometheus-prometheus-pushgateway.monitoring:9091/metrics", @@ -254,6 +253,10 @@ def ibkr( broker-sync Activities, pushes through the shared pipeline, then reconciles broker-reported OpenPositions against WF-computed quantities and publishes a Pushgateway drift metric. + + The Wealthfolio account UUID is resolved via the pipeline's + ensure_account(provider="ibkr", providerAccountId=IBKR_ACCOUNT_ID_UPSTREAM) + lookup — no need to wire the UUID in as a separate env var. """ import time from decimal import Decimal @@ -278,7 +281,6 @@ def ibkr( provider = IBKRProvider( token=ibkr_flex_token, query_id=ibkr_flex_query_id, - wf_account_id=ibkr_account_id, upstream_account_id=ibkr_account_id_upstream, ) dedup = SyncRecordStore(data / "sync.db") @@ -291,8 +293,13 @@ def ibkr( dedup=dedup, ) + # Resolve WF UUID for reconciliation. ensure_account is idempotent + # and already ran inside sync_provider_to_wealthfolio; this is a + # cheap re-lookup that returns the same UUID. + wf_uuid = await sink.ensure_account(provider.accounts()[0]) + # Reconciliation: broker truth vs WF truth. - wf_qty = await sink.compute_position_qty(ibkr_account_id) + wf_qty = await sink.compute_position_qty(wf_uuid) drift_metrics: list[tuple[str, dict[str, str], float]] = [] for symbol, broker_qty in provider.open_positions(): drift = broker_qty - wf_qty.get(symbol, Decimal(0)) diff --git a/broker_sync/providers/ibkr.py b/broker_sync/providers/ibkr.py index 741c79a..fcff89f 100644 --- a/broker_sync/providers/ibkr.py +++ b/broker_sync/providers/ibkr.py @@ -179,12 +179,14 @@ class IBKRProvider: *, token: str, query_id: str, - wf_account_id: str, upstream_account_id: str, ) -> None: self._token = token self._query_id = query_id - self._wf_account_id = wf_account_id + # Single source of truth — the IBKR account number (e.g. U13279690). + # The pipeline's _ensure_accounts() resolves this to a Wealthfolio + # UUID via (provider="ibkr", providerAccountId=upstream_account_id); + # activities are remapped to the WF UUID before import. self._upstream_account_id = upstream_account_id # Stashed for the reconciliation step after fetch() drains. self._last_response: Any = None @@ -192,7 +194,7 @@ class IBKRProvider: def accounts(self) -> list[Account]: return [ Account( - id=self._wf_account_id, + id=self._upstream_account_id, name="Interactive Brokers (UK)", account_type=AccountType.GIA, currency="GBP", # FX-aware per-trade; account ccy is GBP @@ -232,10 +234,10 @@ class IBKRProvider: ) for trade in stmt.Trades or []: - yield _map_trade_to_activity(trade, account_id=self._wf_account_id) + yield _map_trade_to_activity(trade, account_id=self._upstream_account_id) for cash in stmt.CashTransactions or []: - activity = _map_cash_to_activity(cash, account_id=self._wf_account_id) + activity = _map_cash_to_activity(cash, account_id=self._upstream_account_id) if activity is not None: yield activity diff --git a/tests/providers/test_ibkr.py b/tests/providers/test_ibkr.py index ea83e26..8dfba07 100644 --- a/tests/providers/test_ibkr.py +++ b/tests/providers/test_ibkr.py @@ -145,7 +145,6 @@ async def test_ibkr_provider_fetch_returns_mapped_activities( provider = IBKRProvider( token="t", query_id="q", - wf_account_id="wf-acct", upstream_account_id="U12345678", ) activities = [a async for a in provider.fetch()] @@ -168,7 +167,6 @@ async def test_ibkr_provider_account_mismatch_raises( provider = IBKRProvider( token="t", query_id="q", - wf_account_id="wf-acct", upstream_account_id="U99999999", # WRONG ) with pytest.raises(IBKRAccountMismatchError, match="U12345678"): @@ -188,7 +186,6 @@ async def test_ibkr_provider_open_positions_after_fetch( provider = IBKRProvider( token="t", query_id="q", - wf_account_id="wf-acct", upstream_account_id="U12345678", ) # drain the iterator before reading positions From bb9e0d4567d5e1c9d3ba4813363d367218badf54 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Wed, 27 May 2026 09:28:42 +0000 Subject: [PATCH 23/27] docs(ibkr): use Last 365 Days (no 'Last 90 Days' preset in IBKR UI) --- docs/plans/2026-05-26-ibkr-flex-ingestion.md | 2 +- docs/providers/ibkr.md | 2 +- docs/specs/2026-05-26-ibkr-ingest-design.md | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/plans/2026-05-26-ibkr-flex-ingestion.md b/docs/plans/2026-05-26-ibkr-flex-ingestion.md index 1fa57bb..f872fa8 100644 --- a/docs/plans/2026-05-26-ibkr-flex-ingestion.md +++ b/docs/plans/2026-05-26-ibkr-flex-ingestion.md @@ -1461,7 +1461,7 @@ OpenPositions against WF-computed quantities. | Open Positions | symbol, position, markPrice, currency, assetCategory | | Securities Information | symbol, description, conid | -Date range: `Last 90 Days` — trailing window so a missed cron run +Date range: `Last 365 Days` — trailing window so a missed cron run doesn't lose data. SyncRecordStore makes overlapping pulls idempotent. Switch to `Year to Date` or `Custom Date Range` only for one-time historical backfills. diff --git a/docs/providers/ibkr.md b/docs/providers/ibkr.md index 501c641..14b167b 100644 --- a/docs/providers/ibkr.md +++ b/docs/providers/ibkr.md @@ -39,7 +39,7 @@ a new query named `broker-sync-activity` with: **Date Format:** `yyyy-MM-dd`. **Time Format:** `HH:mm:ss` (no timezone suffix — ibflex 1.1 rejects timezone abbreviations in the time field). -**Date Range:** `Last 90 Days` — trailing window so a missed cron run +**Date Range:** `Last 365 Days` — trailing window so a missed cron run doesn't lose data. SyncRecordStore (keyed by `external_id`) makes overlapping pulls idempotent. For a one-off historical backfill, widen temporarily to `Year to Date` or `Custom Date Range`, run once, then diff --git a/docs/specs/2026-05-26-ibkr-ingest-design.md b/docs/specs/2026-05-26-ibkr-ingest-design.md index 21aaf12..9a7a813 100644 --- a/docs/specs/2026-05-26-ibkr-ingest-design.md +++ b/docs/specs/2026-05-26-ibkr-ingest-design.md @@ -210,7 +210,7 @@ top. - Sections: `Account Information`, `Trades`, `Cash Transactions`, `Open Positions`, `Securities Information` - Date Format: `yyyy-MM-dd` · Time Format: `HH:mm:ss TimeZone` - - Date Range: `Last 90 Days` — trailing window so a missed cron run + - Date Range: `Last 365 Days` — trailing window so a missed cron run (failed pod, outage, vacation) doesn't lose data. SyncRecordStore keys on `ibkr:trade:` / `ibkr:cash:`, so overlapping pulls are no-ops. `Last Business Day` was the original @@ -252,11 +252,11 @@ curl -sS -b /tmp/wf-jar -X POST "$WF_BASE_URL/api/v1/accounts" \ ### Step 4 — Initial backfill (skip while account is empty) When the IBKR account first holds positions, the daily CronJob will -backfill automatically up to the 90-day trailing window. For older +backfill automatically up to the 365-day trailing window. For older history, temporarily switch the Flex query Date Range to `Year to Date` (or `Custom Date Range` with a 1-year window), run the CronJob manually once, verify WF totals match the broker app, then -switch the Flex query back to `Last 90 Days` for daily incremental. +switch the Flex query back to `Last 365 Days` for daily incremental. Dedup makes the temporary widening safe — already-synced rows are no-ops. ### Step 5 — Deploy From abf9fa7cb54728594bae368b6558994a780f3573 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Wed, 27 May 2026 09:40:56 +0000 Subject: [PATCH 24/27] parsers/schwab: drop dead vest-release path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The _parse_vest_release path and _VEST_*_RE regexes never matched a real email in 4 years of inbox history (2022-08 → 2026-05, 188 Schwab emails surveyed). Schwab Stock Plan Services does not email release confirmations to the employee address for the workplace account — only the sell-to-cover trade-executed alert lands. Vest data must come from the META payslip via payslip-ingest (tracked as code-fqgr). Removed: - _VEST_SUBJECT_RE + 5 _VEST_*_RE regexes (heuristic, never validated) - _parse_vest_release function - VestParseResult dataclass - parse_schwab_email_full wrapper - _search_group helper (only used by vest path) - 3 dead tests + the _VEST_RELEASE fixture Kept models.VestEvent — the payslip→Wealthfolio sink in code-fqgr will need it. Co-Authored-By: Claude Opus 4.7 --- broker_sync/providers/parsers/schwab.py | 193 ++---------------------- tests/providers/parsers/test_schwab.py | 56 ------- 2 files changed, 13 insertions(+), 236 deletions(-) diff --git a/broker_sync/providers/parsers/schwab.py b/broker_sync/providers/parsers/schwab.py index aeef7d0..762a613 100644 --- a/broker_sync/providers/parsers/schwab.py +++ b/broker_sync/providers/parsers/schwab.py @@ -1,79 +1,37 @@ """Schwab workplace-RSU email parser. -Two email shapes are handled: +Schwab Stock Plan Services sends a "Your trade was executed" email for +each sell-to-cover trade (and any user-initiated trade) on the workplace +account. The body has five `` +cells holding date / direction / quantity / ticker / price. -1. Trade confirmations (sell-to-cover or user-initiated trades): HTML - with five `` cells - holding date / direction / quantity / ticker / price. → one Activity. +It does NOT email vest-release / Release Confirmation messages to the +employee address for this account (verified against 4 years of inbox +history, 2022-2026 — see infra/docs in code-fqgr). Vest data must come +from the META payslip via payslip-ingest, not from email. The whole +vest-release parser that used to live here was dead code. -2. Release Confirmations (RSU vest events): subject/body mentions - "Release Confirmation" or "Award Vesting"; body lists vest date, - shares released, FMV, shares sold to cover, and USD tax withheld. - → (Activity, Activity, VestEvent) tuple: the gross vest (BUY at FMV), - the sell-to-cover (SELL at FMV), and a standalone VestEvent for the - payslip-ingest reconciliation pipeline. - -On any parse failure we return the neutral empty result (no Activities, -no VestEvent) — an unparseable email shouldn't crash the IMAP batch. +On any parse failure we return an empty list — an unparseable email +shouldn't crash the IMAP batch. """ from __future__ import annotations import logging -import re -from dataclasses import dataclass from decimal import Decimal, InvalidOperation from bs4 import BeautifulSoup from dateutil import parser as dateparser -from broker_sync.models import AccountType, Activity, ActivityType, VestEvent +from broker_sync.models import AccountType, Activity, ActivityType log = logging.getLogger(__name__) _ACCOUNT_ID = "schwab-workplace" _DEFAULT_CURRENCY = "USD" -# Vest-confirmation emails reliably include one of these phrases. Matching -# is case-insensitive and on the raw HTML (cheap — no DOM parse needed). -_VEST_SUBJECT_RE = re.compile(r"Release Confirmation|Award Vesting|RSU Release", - re.IGNORECASE) - - -@dataclass -class VestParseResult: - activities: list[Activity] - vest_event: VestEvent | None - def parse_schwab_email(raw_html: str) -> list[Activity]: - """Return a single-item list of Activity on success, empty on failure. - - For vest-confirmation emails, returns the two Activity rows (gross - vest + sell-to-cover). Use `parse_schwab_email_full` when the caller - also needs the VestEvent. - """ - return parse_schwab_email_full(raw_html).activities - - -def parse_schwab_email_full(raw_html: str) -> VestParseResult: - """Full parse — returns activities + optional VestEvent. - - Dispatches: vest-confirmation emails → `_parse_vest_release`; - everything else → the legacy single-row confirmation parser. - """ - if _VEST_SUBJECT_RE.search(raw_html): - result = _parse_vest_release(raw_html) - if result is not None: - return result - log.warning("schwab: detected vest email but could not extract fields; " - "add a real fixture to broker-sync/tests/fixtures/") - return VestParseResult(activities=[], vest_event=None) - - return VestParseResult(activities=_parse_trade_confirmation(raw_html), vest_event=None) - - -def _parse_trade_confirmation(raw_html: str) -> list[Activity]: - """Legacy 5-cell trade confirmation parser.""" + """Return a one-element list of Activity on success, empty on failure.""" try: soup = BeautifulSoup(raw_html, "html.parser") cells = [ @@ -90,8 +48,6 @@ def _parse_trade_confirmation(raw_html: str) -> list[Activity]: direction = (ActivityType.SELL if direction_txt.strip().lower() == "sold" else ActivityType.BUY) quantity = Decimal(qty_txt.replace(",", "").strip()) - # Price like "$123.45" — strip the currency sign and parse the numeric tail. - # Handle "£", "€", "USD", etc. by taking the last numeric span. price_clean = price_txt for sign in ("$", "£", "€", "USD", "GBP", "EUR"): price_clean = price_clean.replace(sign, "") @@ -115,126 +71,3 @@ def _parse_trade_confirmation(raw_html: str) -> list[Activity]: ] except (ValueError, InvalidOperation, IndexError, AttributeError): return [] - - -# Heuristic extractors for vest-release emails. Labels observed in public -# Schwab RSU release samples; real fixture needed to tighten these. -_VEST_DATE_RE = re.compile( - r"(?:Release Date|Vest Date|Vesting Date)\s*[:<][^0-9]*" - r"(\d{1,2}[\s/\-][A-Za-z]{3}[\s/\-]\d{2,4}|\d{2}/\d{2}/\d{4}|\d{4}-\d{2}-\d{2})", - re.IGNORECASE) -_VEST_TICKER_RE = re.compile(r"(?:Ticker|Symbol)\s*[:<]\s*([A-Z]{2,5})", - re.IGNORECASE) -_VEST_SHARES_RELEASED_RE = re.compile( - r"(?:Shares Released|Total Shares (?:Released|Vested))\s*[:<]\s*" - r"([\d,]+(?:\.\d+)?)", - re.IGNORECASE) -_VEST_SHARES_WITHHELD_RE = re.compile( - r"(?:Shares (?:Withheld|Sold)(?: for Taxes)?)\s*[:<]\s*" - r"([\d,]+(?:\.\d+)?)", - re.IGNORECASE) -_VEST_FMV_RE = re.compile( - r"(?:Market Price|FMV|Fair Market Value)\s*[:<]\s*" - r"\$?\s*([\d,]+(?:\.\d+)?)", - re.IGNORECASE) -_VEST_TAX_USD_RE = re.compile( - r"(?:Tax Withholding Amount|Total Tax Withholding|Tax Withheld)\s*[:<]\s*" - r"\$?\s*([\d,]+(?:\.\d+)?)", - re.IGNORECASE) - - -def _parse_vest_release(raw_html: str) -> VestParseResult | None: - """Best-effort extraction from a Schwab Release Confirmation email. - - Runs label regexes on the plain-text view of the HTML. Returns None - (signalling fall-through) if the core four fields (date, ticker, - shares released, FMV) don't all resolve — that's a strong signal the - heuristics need a real fixture before they can be trusted on a live - email. - """ - try: - soup = BeautifulSoup(raw_html, "html.parser") - text = soup.get_text(" ", strip=True) - except Exception: - return None - - date_str = _search_group(_VEST_DATE_RE, text) - ticker = _search_group(_VEST_TICKER_RE, text) - shares_released_str = _search_group(_VEST_SHARES_RELEASED_RE, text) - fmv_str = _search_group(_VEST_FMV_RE, text) - if not (date_str and ticker and shares_released_str and fmv_str): - return None - - try: - vest_date = dateparser.parse(date_str) - shares_vested = Decimal(shares_released_str.replace(",", "")) - fmv = Decimal(fmv_str.replace(",", "")) - except (ValueError, InvalidOperation): - return None - - shares_sold_str = _search_group(_VEST_SHARES_WITHHELD_RE, text) - shares_sold_to_cover = (Decimal(shares_sold_str.replace(",", "")) - if shares_sold_str else None) - tax_usd_str = _search_group(_VEST_TAX_USD_RE, text) - tax_withheld_usd = (Decimal(tax_usd_str.replace(",", "")) - if tax_usd_str else None) - - external_id = (f"schwab:{vest_date.date().isoformat()}:{ticker}:VEST:" - f"{shares_vested}") - - vest_event = VestEvent( - external_id=external_id, - vest_date=vest_date, - ticker=ticker, - shares_vested=shares_vested, - shares_sold_to_cover=shares_sold_to_cover, - fmv_at_vest_usd=fmv, - tax_withheld_usd=tax_withheld_usd, - source="schwab_email", - raw={ - "date": date_str, - "ticker": ticker, - "shares_released": shares_released_str, - "fmv": fmv_str, - "shares_withheld": shares_sold_str or "", - "tax_withheld": tax_usd_str or "", - }, - ) - - # Sibling Activities for Wealthfolio: full vest as BUY, sell-to-cover - # slice as SELL, both at the same FMV so net cash = 0 on that day. - activities: list[Activity] = [ - Activity( - external_id=f"{external_id}:BUY", - account_id=_ACCOUNT_ID, - account_type=AccountType.GIA, - date=vest_date, - activity_type=ActivityType.BUY, - symbol=ticker, - quantity=shares_vested, - unit_price=fmv, - currency=_DEFAULT_CURRENCY, - notes="schwab-vest-release", - ) - ] - if shares_sold_to_cover is not None and shares_sold_to_cover > 0: - activities.append( - Activity( - external_id=f"{external_id}:SELL_TO_COVER", - account_id=_ACCOUNT_ID, - account_type=AccountType.GIA, - date=vest_date, - activity_type=ActivityType.SELL, - symbol=ticker, - quantity=shares_sold_to_cover, - unit_price=fmv, - currency=_DEFAULT_CURRENCY, - notes="schwab-sell-to-cover", - )) - - return VestParseResult(activities=activities, vest_event=vest_event) - - -def _search_group(pattern: re.Pattern[str], text: str) -> str | None: - m = pattern.search(text) - return m.group(1).strip() if m else None diff --git a/tests/providers/parsers/test_schwab.py b/tests/providers/parsers/test_schwab.py index c39bd0c..8e3c736 100644 --- a/tests/providers/parsers/test_schwab.py +++ b/tests/providers/parsers/test_schwab.py @@ -82,59 +82,3 @@ def test_price_with_commas_parses() -> None: html = _SELL.replace("$612.34", "$1,612.34") a = parse_schwab_email(html)[0] assert a.unit_price == Decimal("1612.34") - - -# --- Vest-release parsing ------------------------------------------------- - -_VEST_RELEASE = """ -

Release Confirmation

-

-Release Date: 15 Mar 2026 -Ticker: META -Total Shares Released: 100.0 -Market Price: $612.34 -Shares Withheld for Taxes: 45 -Tax Withholding Amount: $27,555.30 -

-""" - - -def test_vest_release_returns_two_activities_and_vest_event() -> None: - """Release Confirmation yields a BUY (full vest) + SELL (sell-to-cover) + VestEvent.""" - from broker_sync.providers.parsers.schwab import parse_schwab_email_full - - result = parse_schwab_email_full(_VEST_RELEASE) - assert result.vest_event is not None - assert result.vest_event.ticker == "META" - assert result.vest_event.shares_vested == Decimal("100.0") - assert result.vest_event.shares_sold_to_cover == Decimal("45") - assert result.vest_event.fmv_at_vest_usd == Decimal("612.34") - assert result.vest_event.tax_withheld_usd == Decimal("27555.30") - assert result.vest_event.vest_date.date().isoformat() == "2026-03-15" - assert result.vest_event.external_id.startswith("schwab:2026-03-15:META:VEST:") - - assert len(result.activities) == 2 - buy = result.activities[0] - assert buy.activity_type is ActivityType.BUY - assert buy.quantity == Decimal("100.0") - sell = result.activities[1] - assert sell.activity_type is ActivityType.SELL - assert sell.quantity == Decimal("45") - assert sell.unit_price == Decimal("612.34") - - -def test_vest_email_with_unparseable_body_returns_empty() -> None: - """Subject says Release Confirmation but fields missing → empty result, no crash.""" - from broker_sync.providers.parsers.schwab import parse_schwab_email_full - - html = "Release Confirmation — please contact support" - result = parse_schwab_email_full(html) - assert result.vest_event is None - assert result.activities == [] - - -def test_back_compat_parse_schwab_email_drops_vest_event() -> None: - """The legacy list[Activity] shape remains stable for existing callers.""" - acts = parse_schwab_email(_VEST_RELEASE) - assert len(acts) == 2 - assert all(isinstance(a.activity_type, ActivityType) for a in acts) From 17c2a69c6c491856c9d34d1b9da9724723e04478 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Wed, 27 May 2026 10:02:07 +0000 Subject: [PATCH 25/27] parsers/schwab: emit paired BUY for recent SELL (vest synthesis) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Schwab Stock Plan Services doesn't email vest-release confirmations to the employee inbox — only the same-day-sell trade-executed alert lands. The vest itself was invisible to broker-sync, so the META cadence panel in the wealth dashboard has been missing the May 2026 vest BUY and would keep missing every future vest. Synthesis: when a SELL email's trade date is on/after the configured boundary (default 2026-04-01), also emit a paired BUY with identical date/qty/price/symbol. Notes link the pair via the SELL's external_id. Verified true across 14 historical vests — 100% same-day-sell pattern, SELL qty == vest qty. Boundary stops the synthesis from back-filling vests prior to 2026-04 which already have csv-sourced BUY rows in Wealthfolio from the historical one-shot backfill (last vest 2026-02-18). The csv BUYs and inferred BUYs have distinct external_ids, so re-running against old emails would double-count without this guard. Override via env var `SCHWAB_VEST_INFER_FROM_DATE=yyyy-mm-dd` on the broker-sync-imap cron. Tests: 4 new cases — recent SELL pairs, old SELL doesn't pair, env override works, BUY-direction emails (rare) don't get paired. Co-Authored-By: Claude Opus 4.7 --- broker_sync/providers/parsers/schwab.py | 90 +++++++++++++++++++------ tests/providers/parsers/test_schwab.py | 65 +++++++++++++++++- 2 files changed, 134 insertions(+), 21 deletions(-) diff --git a/broker_sync/providers/parsers/schwab.py b/broker_sync/providers/parsers/schwab.py index 762a613..5a34f1b 100644 --- a/broker_sync/providers/parsers/schwab.py +++ b/broker_sync/providers/parsers/schwab.py @@ -7,9 +7,16 @@ cells holding date / direction / quantity / ticker / price. It does NOT email vest-release / Release Confirmation messages to the employee address for this account (verified against 4 years of inbox -history, 2022-2026 — see infra/docs in code-fqgr). Vest data must come -from the META payslip via payslip-ingest, not from email. The whole -vest-release parser that used to live here was dead code. +history, 2022-2026). The vest itself is invisible to IMAP. + +Same-day-sell synthesis: Meta RSUs vest and are sold the same day at +the same FMV (verified across 14 historical vests). When a SELL email +is parsed AND its trade date is on or after `VEST_INFER_FROM_DATE`, +we ALSO emit a paired BUY representing the underlying vest event — +same date, same quantity, same price. The date boundary stops this +back-filling historical vests that already have csv-sourced BUY rows +in Wealthfolio (which would duplicate at chart-level despite distinct +external_ids). On any parse failure we return an empty list — an unparseable email shouldn't crash the IMAP batch. @@ -17,6 +24,8 @@ shouldn't crash the IMAP batch. from __future__ import annotations import logging +import os +from datetime import date, datetime from decimal import Decimal, InvalidOperation from bs4 import BeautifulSoup @@ -29,9 +38,34 @@ log = logging.getLogger(__name__) _ACCOUNT_ID = "schwab-workplace" _DEFAULT_CURRENCY = "USD" +# Inferred-BUY synthesis boundary. SELL emails on or after this date +# emit a paired BUY for the underlying vest; earlier ones do not (they +# already have csv-sourced BUYs in Wealthfolio from the one-shot +# historical backfill, last vest 2026-02-18). Override at runtime with +# the env var if a different cutover is needed. ISO-8601 yyyy-mm-dd. +_DEFAULT_VEST_INFER_FROM = "2026-04-01" + + +def _vest_infer_from() -> date: + raw = os.environ.get("SCHWAB_VEST_INFER_FROM_DATE", _DEFAULT_VEST_INFER_FROM).strip() + try: + return datetime.strptime(raw, "%Y-%m-%d").date() + except ValueError: + log.warning( + "SCHWAB_VEST_INFER_FROM_DATE=%r is not yyyy-mm-dd; using default %s", + raw, _DEFAULT_VEST_INFER_FROM, + ) + return datetime.strptime(_DEFAULT_VEST_INFER_FROM, "%Y-%m-%d").date() + def parse_schwab_email(raw_html: str) -> list[Activity]: - """Return a one-element list of Activity on success, empty on failure.""" + """Return Activities for a Schwab trade-executed email. + + Returns: empty list on parse failure; one Activity for a BUY-direction + email (rare — the workplace account is essentially sell-only); for a + SELL email, returns [SELL] plus an inferred paired BUY (=vest event) + when the trade date is on or after the synthesis-boundary date. + """ try: soup = BeautifulSoup(raw_html, "html.parser") cells = [ @@ -52,22 +86,40 @@ def parse_schwab_email(raw_html: str) -> list[Activity]: for sign in ("$", "£", "€", "USD", "GBP", "EUR"): price_clean = price_clean.replace(sign, "") unit_price = Decimal(price_clean.replace(",", "").strip()) + ticker_clean = ticker.strip() - external_id = (f"schwab:{trade_date.date().isoformat()}:{ticker}:" + external_id = (f"schwab:{trade_date.date().isoformat()}:{ticker_clean}:" f"{direction.value}:{quantity}") - return [ - Activity( - external_id=external_id, - account_id=_ACCOUNT_ID, - account_type=AccountType.GIA, - date=trade_date, - activity_type=direction, - symbol=ticker.strip(), - quantity=quantity, - unit_price=unit_price, - currency=_DEFAULT_CURRENCY, - notes=f"schwab-email:{direction_txt}", - ) - ] + primary = Activity( + external_id=external_id, + account_id=_ACCOUNT_ID, + account_type=AccountType.GIA, + date=trade_date, + activity_type=direction, + symbol=ticker_clean, + quantity=quantity, + unit_price=unit_price, + currency=_DEFAULT_CURRENCY, + notes=f"schwab-email:{direction_txt}", + ) + + if direction is not ActivityType.SELL or trade_date.date() < _vest_infer_from(): + return [primary] + + inferred_buy = Activity( + external_id=(f"schwab:vest:{trade_date.date().isoformat()}:" + f"{ticker_clean}:BUY:{quantity}"), + account_id=_ACCOUNT_ID, + account_type=AccountType.GIA, + date=trade_date, + activity_type=ActivityType.BUY, + symbol=ticker_clean, + quantity=quantity, + unit_price=unit_price, + currency=_DEFAULT_CURRENCY, + notes=(f"schwab-vest-inferred-from-same-day-sell | " + f"paired_sell_external_id={external_id}"), + ) + return [inferred_buy, primary] except (ValueError, InvalidOperation, IndexError, AttributeError): return [] diff --git a/tests/providers/parsers/test_schwab.py b/tests/providers/parsers/test_schwab.py index 8e3c736..2cc0213 100644 --- a/tests/providers/parsers/test_schwab.py +++ b/tests/providers/parsers/test_schwab.py @@ -80,5 +80,66 @@ def test_external_id_is_stable_across_reruns() -> None: def test_price_with_commas_parses() -> None: html = _SELL.replace("$612.34", "$1,612.34") - a = parse_schwab_email(html)[0] - assert a.unit_price == Decimal("1612.34") + # The first activity is the inferred BUY (date 2025-01-23 ≥ 2026-04-01? no → + # only one activity for this old-dated email), so index 0 is the SELL. + acts = parse_schwab_email(html) + sell = next(a for a in acts if a.activity_type is ActivityType.SELL) + assert sell.unit_price == Decimal("1612.34") + + +# --- Inferred vest BUY --------------------------------------------------- + + +def _recent_sell(date_iso: str = "2026-05-19", qty: str = "55", price: str = "609.35") -> str: + return f""" + + + + + + +
{date_iso}
Sold
{qty}
META
${price}
+""" + + +def test_recent_sell_emits_paired_buy() -> None: + """SELL dated on/after the synthesis boundary triggers a paired BUY.""" + acts = parse_schwab_email(_recent_sell()) + assert len(acts) == 2 + + buy = next(a for a in acts if a.activity_type is ActivityType.BUY) + sell = next(a for a in acts if a.activity_type is ActivityType.SELL) + + assert buy.quantity == sell.quantity == Decimal("55") + assert buy.unit_price == sell.unit_price == Decimal("609.35") + assert buy.date == sell.date + assert buy.symbol == sell.symbol == "META" + assert "schwab-vest-inferred-from-same-day-sell" in (buy.notes or "") + assert buy.external_id == "schwab:vest:2026-05-19:META:BUY:55" + assert sell.external_id == "schwab:2026-05-19:META:SELL:55" + + +def test_old_sell_emits_only_sell() -> None: + """SELL dated before 2026-04-01 (default boundary) skips the paired BUY — + those vests already have csv-sourced BUY rows in Wealthfolio.""" + acts = parse_schwab_email(_recent_sell(date_iso="2025-08-19")) + assert len(acts) == 1 + assert acts[0].activity_type is ActivityType.SELL + + +def test_boundary_env_var_overrides(monkeypatch: object) -> None: + """The synthesis boundary is configurable via env var.""" + import os + os.environ["SCHWAB_VEST_INFER_FROM_DATE"] = "2025-01-01" + try: + acts = parse_schwab_email(_recent_sell(date_iso="2025-08-19")) + assert len(acts) == 2 # now in scope + finally: + del os.environ["SCHWAB_VEST_INFER_FROM_DATE"] + + +def test_buy_email_does_not_emit_inferred_buy() -> None: + """BUY-direction emails (rare for workplace account) don't get paired.""" + acts = parse_schwab_email(_BUY.replace("2024-11-15", "2026-05-15")) + assert len(acts) == 1 + assert acts[0].activity_type is ActivityType.BUY From 3427f5c9e192ef2d6e3f840e042c896cfc180374 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Wed, 27 May 2026 12:03:41 +0000 Subject: [PATCH 26/27] ibkr: emit ibkr_cash_balance{currency, account} per CashReport row Each daily run now pushes one Pushgateway metric per currency row from the Flex Activity Query's CashReport section (typically BASE_SUMMARY aggregate + one row per held currency). Makes dormant-account balance checks trivial and adds a Grafana surface for cash drift alerting. Requires the Activity Flex Query in IBKR Client Portal to have the CashReport section enabled. Co-Authored-By: Claude Opus 4.7 --- broker_sync/cli.py | 10 ++++++++++ broker_sync/providers/ibkr.py | 17 +++++++++++++++++ tests/fixtures/ibkr/sample_flex.xml | 4 ++++ tests/providers/test_ibkr.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+) diff --git a/broker_sync/cli.py b/broker_sync/cli.py index 7f855f5..64057f7 100644 --- a/broker_sync/cli.py +++ b/broker_sync/cli.py @@ -310,6 +310,16 @@ def ibkr( float(drift), ) ) + # Cash balances (one row per currency from CashReport, plus a + # BASE_SUMMARY row consolidated in account base currency). + for currency, ending_cash in provider.cash_balances(): + drift_metrics.append( + ( + "ibkr_cash_balance", + {"currency": currency, "account": "ibkr-uk"}, + float(ending_cash), + ) + ) drift_metrics.append( ("ibkr_sync_last_success_timestamp_seconds", {}, float(time.time())) ) diff --git a/broker_sync/providers/ibkr.py b/broker_sync/providers/ibkr.py index fcff89f..e180bcb 100644 --- a/broker_sync/providers/ibkr.py +++ b/broker_sync/providers/ibkr.py @@ -257,3 +257,20 @@ class IBKRProvider: ) out.append((symbol, Decimal(str(pos.position)))) return out + + def cash_balances(self) -> list[tuple[str, Decimal]]: + """Return ``[(currency, ending_cash), ...]`` from the CashReport. + + Includes the ``BASE_SUMMARY`` aggregate row (account base currency + consolidated) plus any per-currency rows. Empty list if no + CashReport section in the Flex query or before first ``fetch()``. + """ + if self._last_response is None: + return [] + stmt = self._last_response.FlexStatements[0] + out: list[tuple[str, Decimal]] = [] + for row in stmt.CashReport or []: + if row.endingCash is None or row.currency is None: + continue + out.append((str(row.currency), Decimal(str(row.endingCash)))) + return out diff --git a/tests/fixtures/ibkr/sample_flex.xml b/tests/fixtures/ibkr/sample_flex.xml index 0d82fcf..d3130a3 100644 --- a/tests/fixtures/ibkr/sample_flex.xml +++ b/tests/fixtures/ibkr/sample_flex.xml @@ -16,6 +16,10 @@ + + + + diff --git a/tests/providers/test_ibkr.py b/tests/providers/test_ibkr.py index 8dfba07..edbc51d 100644 --- a/tests/providers/test_ibkr.py +++ b/tests/providers/test_ibkr.py @@ -194,3 +194,31 @@ async def test_ibkr_provider_open_positions_after_fetch( positions = provider.open_positions() # VUAG → VUAG.L (LSE inferred from GBP); AAPL unchanged (USD) assert dict(positions) == {"VUAG.L": Decimal("8"), "AAPL": Decimal("5")} + + +async def test_ibkr_provider_cash_balances_after_fetch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """cash_balances() returns (currency, ending_cash) tuples from CashReport.""" + from ibflex import client as ib_client + + with open("tests/fixtures/ibkr/sample_flex.xml", "rb") as f: + xml_bytes = f.read() + monkeypatch.setattr(ib_client, "download", lambda *a, **kw: xml_bytes) + + provider = IBKRProvider( + token="t", + query_id="q", + upstream_account_id="U12345678", + ) + [a async for a in provider.fetch()] + + balances = provider.cash_balances() + # Fixture has BASE_SUMMARY + USD rows, both 1.23 + assert dict(balances) == {"BASE_SUMMARY": Decimal("1.23"), "USD": Decimal("1.23")} + + +def test_ibkr_provider_cash_balances_before_fetch_returns_empty() -> None: + """No CashReport data before fetch().""" + provider = IBKRProvider(token="t", query_id="q", upstream_account_id="U12345678") + assert provider.cash_balances() == [] From 0d23487608363a13bba0f5bf94457867f46a78e0 Mon Sep 17 00:00:00 2001 From: Viktor Barzin Date: Wed, 27 May 2026 17:24:54 +0000 Subject: [PATCH 27/27] imap: skip InvestEngine by default; opt back in via INCLUDE env MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Post-mortem 2026-05-27: 39 IMAP-source IE BUYs + their cash-flow DEPOSITs were re-inserted into Wealthfolio at 09:22:18 UTC, exactly the rows the £252k dedup removed the previous day. The cron's BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS=invest-engine env var did its job (cron logged ie_skipped=53), but some other entry point — kubectl run, poetry run on the devvm, or a sibling agent session — ran the IMAP ingest WITHOUT that env. The opt-out was a foot-gun. This change makes the IE-via-IMAP safety STRUCTURAL: `invest-engine` is in the default exclude set inside _resolve_excluded_providers(). Any code path now skips IE unless the caller explicitly sets `BROKER_SYNC_IMAP_INCLUDE_PROVIDERS=invest-engine`. The `BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS` env still works (additive) for forward-compat in case Schwab etc. ever need similar treatment. INCLUDE wins over both the default exclude set and EXCLUDE env. Co-Authored-By: Claude Opus 4.7 --- broker_sync/providers/imap.py | 39 ++++++++++++++++--- tests/providers/test_imap.py | 72 +++++++++++++++++++++++++++++------ 2 files changed, 94 insertions(+), 17 deletions(-) diff --git a/broker_sync/providers/imap.py b/broker_sync/providers/imap.py index 5564dd3..0b9bbb7 100644 --- a/broker_sync/providers/imap.py +++ b/broker_sync/providers/imap.py @@ -151,14 +151,41 @@ def _fetch_all(creds: ImapCreds) -> Iterator[bytes]: yield raw -def fetch_activities(creds: ImapCreds) -> list[Activity]: - out: list[Activity] = [] - ie_parsed = schwab_parsed = ie_skipped = skipped = 0 - exclude = { - p.strip().lower() +def _resolve_excluded_providers() -> set[str]: + """Return the set of providers the IMAP fetcher must skip. + + Default-exclude list is structural — `invest-engine` is ALWAYS skipped + unless explicitly opted back in via `BROKER_SYNC_IMAP_INCLUDE_PROVIDERS`. + This protects against accidental re-ingestion via any code path that + doesn't set the cron's env (e.g. `kubectl run --rm`, devvm `poetry run`, + a sibling agent session). See post-mortem 2026-05-27 — the IMAP path + re-inserted 39 IE BUYs that had been deduped the previous day, because + the safety lived only on the cronjob spec. + + Additional providers can be excluded via + `BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS`. `INCLUDE` always wins over + `EXCLUDE` and the default skip-list. + """ + _DEFAULT_EXCLUDED = {"invest-engine", "invest_engine"} + extra = { + p.strip().lower().replace("_", "-") for p in os.environ.get("BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS", "").split(",") if p.strip() } + include = { + p.strip().lower().replace("_", "-") + for p in os.environ.get("BROKER_SYNC_IMAP_INCLUDE_PROVIDERS", "").split(",") + if p.strip() + } + # Canonicalise the default set under the same key normalisation. + canonical = {p.replace("_", "-") for p in _DEFAULT_EXCLUDED} + return (canonical | extra) - include + + +def fetch_activities(creds: ImapCreds) -> list[Activity]: + out: list[Activity] = [] + ie_parsed = schwab_parsed = ie_skipped = skipped = 0 + exclude = _resolve_excluded_providers() for raw in _fetch_all(creds): try: msg = email.message_from_bytes(raw) @@ -167,7 +194,7 @@ def fetch_activities(creds: ImapCreds) -> list[Activity]: continue sender = _extract_sender(msg) if sender in _IE_SENDERS or sender.endswith("@investengine.com"): - if "invest-engine" in exclude or "invest_engine" in exclude: + if "invest-engine" in exclude: ie_skipped += 1 continue out.extend(ie_parser.parse_invest_engine_email(raw)) diff --git a/tests/providers/test_imap.py b/tests/providers/test_imap.py index 0264b37..30b09d1 100644 --- a/tests/providers/test_imap.py +++ b/tests/providers/test_imap.py @@ -105,10 +105,15 @@ def test_non_ie_activities_passed_through_unchanged() -> None: assert routed[0].account_type is AccountType.GIA -def test_exclude_invest_engine_skips_ie_emails(monkeypatch: MonkeyPatch) -> None: - """BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS=invest-engine should skip IE messages - so we don't duplicate IE buys already ingested via the bearer-token API path. - Schwab routing must remain unaffected.""" +def test_invest_engine_skipped_by_default(monkeypatch: MonkeyPatch) -> None: + """InvestEngine messages MUST be skipped by default, even with no env set. + + Post-mortem 2026-05-27: any code path that doesn't set the cron's env + (e.g. `kubectl run --rm` or devvm `poetry run`) was re-importing IE + BUYs through this IMAP path. The opt-out env var was a foot-gun. + Invariant now: structural default skip; opt back in only with + BROKER_SYNC_IMAP_INCLUDE_PROVIDERS. + """ from broker_sync.providers import imap as imap_mod from broker_sync.providers.parsers import invest_engine as ie_parser @@ -130,15 +135,60 @@ def test_exclude_invest_engine_skips_ie_emails(monkeypatch: MonkeyPatch) -> None creds = imap_mod.ImapCreds(host="h", user="u", password="p", directory="d") - monkeypatch.setenv("BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS", "invest-engine") - out_excluded = imap_mod.fetch_activities(creds) - # IE skipped → only the schwab activity is emitted - assert len(out_excluded) == 1 - + # Default (no env): IE skipped, Schwab parsed. monkeypatch.delenv("BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS", raising=False) + monkeypatch.delenv("BROKER_SYNC_IMAP_INCLUDE_PROVIDERS", raising=False) out_default = imap_mod.fetch_activities(creds) - # Both providers fire when env unset - assert len(out_default) == 2 + assert len(out_default) == 1, "IE must be skipped by default; only Schwab emitted" + + +def test_invest_engine_opt_in_via_include_env(monkeypatch: MonkeyPatch) -> None: + """Setting BROKER_SYNC_IMAP_INCLUDE_PROVIDERS=invest-engine re-enables + IE parsing (escape hatch for the legacy IMAP path).""" + from broker_sync.providers import imap as imap_mod + from broker_sync.providers.parsers import invest_engine as ie_parser + + ie_email = b"From: noreply@investengine.com\r\n\r\nirrelevant\r\n" + schwab_email = b"From: donotreply@schwab.com\r\n\r\n\r\n" + monkeypatch.setattr(imap_mod, "_fetch_all", lambda _: [ie_email, schwab_email]) + monkeypatch.setattr(ie_parser, "parse_invest_engine_email", lambda raw: [object()]) + monkeypatch.setattr(imap_mod, "parse_schwab_email", lambda html: [object()]) + + creds = imap_mod.ImapCreds(host="h", user="u", password="p", directory="d") + + monkeypatch.setenv("BROKER_SYNC_IMAP_INCLUDE_PROVIDERS", "invest-engine") + monkeypatch.delenv("BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS", raising=False) + out = imap_mod.fetch_activities(creds) + assert len(out) == 2, "INCLUDE=invest-engine must re-enable IE parsing" + + +def test_exclude_schwab_still_works(monkeypatch: MonkeyPatch) -> None: + """EXCLUDE env still works for other providers (forward-compat).""" + from broker_sync.providers import imap as imap_mod + from broker_sync.providers.parsers import invest_engine as ie_parser + + schwab_email = b"From: donotreply@schwab.com\r\n\r\n\r\n" + monkeypatch.setattr(imap_mod, "_fetch_all", lambda _: [schwab_email]) + monkeypatch.setattr(ie_parser, "parse_invest_engine_email", lambda raw: [object()]) + monkeypatch.setattr(imap_mod, "parse_schwab_email", lambda html: [object()]) + + creds = imap_mod.ImapCreds(host="h", user="u", password="p", directory="d") + + monkeypatch.setenv("BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS", "schwab") + monkeypatch.delenv("BROKER_SYNC_IMAP_INCLUDE_PROVIDERS", raising=False) + out = imap_mod.fetch_activities(creds) + assert len(out) == 0, "Schwab must be skipped when in EXCLUDE list" + + +def test_include_overrides_default_and_exclude(monkeypatch: MonkeyPatch) -> None: + """INCLUDE wins over both the structural default and EXCLUDE env var.""" + from broker_sync.providers import imap as imap_mod + + monkeypatch.setenv("BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS", "invest-engine,schwab") + monkeypatch.setenv("BROKER_SYNC_IMAP_INCLUDE_PROVIDERS", "invest-engine") + resolved = imap_mod._resolve_excluded_providers() + assert "invest-engine" not in resolved + assert "schwab" in resolved def test_schwab_subdomain_sender_matches() -> None: