"""Fidelity UK PlanViewer provider — workplace pension backfill + monthly sync. PlanViewer has no public individual-member API. The SPA (at ``pv.planviewer.fidelity.co.uk``) and the legacy HTML app (at ``www.planviewer.fidelity.co.uk``) share session cookies via PingFederate OAuth at ``id.fidelity.co.uk``. We keep a Playwright-maintained session via ``storage_state.json``: 1. **One-off seed** (``broker-sync fidelity-seed``): Viktor runs a headed Chromium, logs in (password + memorable word + SMS MFA), clicks "Remember device". The storage_state is persisted to Vault. 2. **Monthly cron**: loads storage_state, boots headless Chromium, navigates to the transaction-history page with a wide date range, parses the HTML table, and intercepts the ``DisplayValuation`` XHR for the current fund holdings. On 401/idle-timeout we raise :class:`FidelitySessionError` so Prometheus alerts Viktor to re-seed. ## Emitted Activity / snapshot shape - One ``DEPOSIT`` per cash-impacting transaction (Regular Premium, Single Premium, rebate, etc.). ``external_id = fidelity:tx:``. - Bulk Switches / Fund Switches are skipped (no cash movement). - After the activity stream drains, the ``fidelity-ingest`` CLI calls ``WealthfolioSink.push_manual_snapshots`` with one ``ManualSnapshotPayload`` per fund holding (today's date, units + cost basis allocated proportionally to fund value share). This sets per-fund quantity and cost basis in WF so the dashboard Positions table shows the pension funds alongside the brokerage assets. - The old synthetic ``fidelity:gains:`` DEPOSIT is no longer emitted — the snapshot supersedes it. Old offset rows that landed before this change are corrected at the dashboard layer by the ``dav_corrected`` PG view (``infra/stacks/wealthfolio/main.tf``). """ from __future__ import annotations import contextlib import logging from collections.abc import AsyncIterator from datetime import date, datetime from decimal import Decimal from pathlib import Path from typing import Any, NamedTuple from broker_sync.models import Account, AccountType, Activity, ActivityType from broker_sync.providers.parsers.fidelity import ( FidelityCashTx, FidelityHolding, parse_transactions_html, parse_valuation_json, ) from broker_sync.sinks.wealthfolio import ManualSnapshotPayload, SnapshotPosition log = logging.getLogger(__name__) ACCOUNT_ID = "fidelity-workplace-pension" _CCY = "GBP" _PV_BASE = "https://www.planviewer.fidelity.co.uk" _PV_TX_PATH = "/planviewer/DisplayMyPlanMemberTransHist.action" _PV_VALUATION_PATH = "/planviewer/DisplayValuation.action" _PV_LANDING = "https://www.planviewer.fidelity.co.uk/" # A wide backfill cap; scheme can't predate 1990. _BACKFILL_START = "01 Jan 1990" class FidelityCreds(NamedTuple): """Paths needed to run the provider.""" storage_state_path: str plan_id: str headless: bool = True class FidelitySessionError(Exception): """Raised when PlanViewer rejects the saved session — re-seed required.""" class FidelityProviderConfigError(Exception): """Raised when provider config is missing or obviously wrong.""" def _tx_to_activity(tx: FidelityCashTx) -> Activity: """Map a Fidelity cash transaction to a canonical DEPOSIT.""" return Activity( external_id=tx.external_id, account_id=ACCOUNT_ID, account_type=AccountType.WORKPLACE_PENSION, date=tx.date, activity_type=ActivityType.DEPOSIT, currency=_CCY, amount=tx.amount, notes=f"fidelity-planviewer:{tx.tx_type}", ) class FidelityPlanViewerProvider: """Read-only provider against Fidelity UK PlanViewer. Lifecycle: - ``accounts()`` advertises the single WF workplace-pension account. - ``fetch(since, before)`` opens a Playwright session with the saved storage_state, navigates to the transaction-history page with a wide date range, scrapes the table, and intercepts the valuation XHR. - After ``fetch()`` completes, ``last_holdings`` holds the per-fund unit positions and ``last_total_contribution`` the cumulative cash contribution — used by the ``fidelity-ingest`` CLI to emit a delta-shaped DEPOSIT that nudges WF's net worth to match the PlanViewer reported pot value (see ``gains_offset_delta_activity``). """ name = "fidelity-planviewer" def __init__(self, creds: FidelityCreds) -> None: self._creds = creds self.last_holdings: list[FidelityHolding] = [] self.last_total_contribution: Decimal = Decimal(0) def accounts(self) -> list[Account]: return [ Account( id=ACCOUNT_ID, name="Fidelity UK Pension", account_type=AccountType.WORKPLACE_PENSION, currency=_CCY, provider=self.name, ), ] async def fetch( self, *, since: datetime | None = None, before: datetime | None = None, ) -> AsyncIterator[Activity]: state_path = self._creds.storage_state_path if not Path(state_path).exists(): raise FidelityProviderConfigError( f"storage_state not found at {state_path} — " "run `broker-sync fidelity-seed` first") tx_html, valuation_json = await _scrape_live_session( state_path=state_path, headless=self._creds.headless, ) transactions = parse_transactions_html(tx_html) holdings = parse_valuation_json(valuation_json) log.info("fidelity: parsed %d transactions, %d holdings", len(transactions), len(holdings)) # Snapshot the per-fund holdings for the CLI to push as a manual # holdings_snapshot after this generator drains. Wealthfolio's # activity model can't represent pension fund unit purchases (no # per-purchase price feed from PlanViewer), so we record current # state via /api/v1/snapshots/import instead. self.last_holdings = holdings self.last_total_contribution = sum( (t.amount for t in transactions), Decimal(0) ) for tx in transactions: if since is not None and tx.date < since: continue if before is not None and tx.date >= before: continue yield _tx_to_activity(tx) # Gains-offset DEPOSITs are emitted by the CLI (which has the # prior cumulative offset from WF). See `gains_offset_delta_activity`. def gains_offset_delta_activity( holdings: list[FidelityHolding], total_real_contribution: Decimal, prior_offset_cumulative: Decimal, as_of: datetime, min_delta: Decimal = Decimal("0.5"), ) -> Activity | None: """Compute the gains-offset DELTA since the last scrape and shape it as a DEPOSIT (or WITHDRAWAL on a market drop). The pension's per-fund prices aren't trackable in WF directly (no public quote feed for these institutional life-fund share classes). Instead, each monthly scrape emits a single small DEPOSIT/WITHDRAWAL sized to ``(current_pot - real_contributions) - prior_cumulative_offset`` — i.e., the growth (or loss) accrued since the last run. Wealthfolio's net_contribution then incorrectly includes all these offsets; the ``dav_corrected`` PG view subtracts them back out so the dashboard's Growth/ROI panels remain accurate. The deterministic external_id (per scrape date) lets re-runs of the same day overwrite rather than stack duplicates. """ if not holdings: return None current_pot = sum((h.total_value for h in holdings), Decimal(0)) current_gain = current_pot - total_real_contribution delta = current_gain - prior_offset_cumulative if abs(delta) < min_delta: return None return Activity( external_id=f"fidelity:gains-delta:{as_of.date().isoformat()}", account_id=ACCOUNT_ID, account_type=AccountType.WORKPLACE_PENSION, date=as_of, activity_type=ActivityType.DEPOSIT if delta > 0 else ActivityType.WITHDRAWAL, currency=_CCY, amount=abs(delta), notes=( f"fidelity-planviewer:unrealised-gains-offset delta=£{delta} " f"(pot=£{current_pot}, contrib=£{total_real_contribution}, " f"prior_offset=£{prior_offset_cumulative})" ), ) def fidelity_holdings_to_snapshot( holdings: list[FidelityHolding], total_real_contribution: Decimal, as_of: date, ) -> ManualSnapshotPayload | None: """Convert scraped holdings into a Wealthfolio manual snapshot payload. Cost-basis allocation: PlanViewer doesn't expose historical purchase prices for individual fund unit buys, so we approximate per-fund cost basis by allocating the cumulative cash contribution proportionally to each fund's share of the current pot value. For the typical single-fund Meta scheme this is exact; if Viktor's plan later splits into multiple funds the proportional split is the least-wrong allocation we can compute from monthly snapshots. cashBalances is set to zero — pension contributions flow straight into funds, the synthetic Wealthfolio "cash balance" only existed because of the old gains-offset DEPOSIT hack. """ if not holdings: return None total_value = sum((h.total_value for h in holdings), Decimal(0)) if total_value <= 0: return None positions: list[SnapshotPosition] = [] for h in holdings: share = h.total_value / total_value cost = (total_real_contribution * share).quantize(Decimal("0.01")) avg_cost = (cost / h.units).quantize(Decimal("0.0001")) if h.units > 0 else Decimal(0) positions.append(SnapshotPosition( symbol=h.fund_code, quantity=h.units, average_cost=avg_cost, total_cost_basis=cost, currency=h.currency, )) return ManualSnapshotPayload( date=as_of, currency=_CCY, positions=positions, cash_balances={_CCY: Decimal(0)}, ) async def _scrape_live_session( *, state_path: str, headless: bool, ) -> tuple[str, dict[str, Any]]: """Load storage_state, navigate the transaction + valuation pages, return (transactions HTML, valuation JSON payload). Raises :class:`FidelitySessionError` if the session is dead (15-min idle, cookie expiry, etc.) — Viktor must re-seed. """ from playwright.async_api import async_playwright captured_valuation: dict[str, dict[str, Any]] = {} async with async_playwright() as pw: browser = await pw.chromium.launch(headless=headless) try: ctx = await browser.new_context( storage_state=state_path, user_agent=("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/147.0.0.0 Safari/537.36"), viewport={"width": 1280, "height": 900}, ) page = await ctx.new_page() async def on_response(resp: Any) -> None: if _PV_VALUATION_PATH in resp.url and resp.status < 400: with contextlib.suppress(Exception): captured_valuation["payload"] = await resp.json() page.on("response", on_response) # Trigger session + capture valuation by navigating through landing # → main page. The SPA fires DisplayValuation on the main page. await page.goto(_PV_LANDING, wait_until="networkidle", timeout=30000) await page.wait_for_timeout(2000) main_url = f"{_PV_BASE}/planviewer/DisplayMainPage.action" await page.goto(main_url, wait_until="networkidle", timeout=30000) await page.wait_for_timeout(3000) if "idle for more than 15 minutes" in (await page.content()) \ or "id.fidelity.co.uk" in page.url: raise FidelitySessionError( "PlanViewer session stale — run `broker-sync fidelity-seed`") # Now pull the transactions page with a wide date range. await page.goto(f"{_PV_BASE}{_PV_TX_PATH}", wait_until="networkidle", timeout=30000) await page.wait_for_timeout(1500) await page.fill('input[name="startDate"]', _BACKFILL_START) today = await page.evaluate( "new Date().toLocaleDateString('en-GB'," "{day:'2-digit',month:'short',year:'numeric'}).replace(/,/g,'')") await page.fill('input[name="endDate"]', today) await page.focus('input[name="endDate"]') await page.keyboard.press("Enter") with contextlib.suppress(Exception): await page.wait_for_load_state("networkidle", timeout=15000) await page.wait_for_timeout(2000) tx_html = await page.content() # If valuation wasn't picked up on the main page, request directly. if "payload" not in captured_valuation: r = await page.request.get(f"{_PV_BASE}{_PV_VALUATION_PATH}") if r.ok: with contextlib.suppress(Exception): captured_valuation["payload"] = await r.json() # Roll the storage_state so the next run benefits from any refresh. await ctx.storage_state(path=state_path) finally: await browser.close() valuation: dict[str, Any] = captured_valuation.get("payload") or {} return tx_html, valuation