diff --git a/broker_sync/providers/fidelity_planviewer.py b/broker_sync/providers/fidelity_planviewer.py index 6031bc2..e201ac8 100644 --- a/broker_sync/providers/fidelity_planviewer.py +++ b/broker_sync/providers/fidelity_planviewer.py @@ -1,90 +1,130 @@ """Fidelity UK PlanViewer provider — workplace pension backfill + monthly sync. -PlanViewer has no public individual-member API; Fidelity International's -developer portal only catalogues B2B scheme/HR endpoints. The SPA (at -``pv.planviewer.fidelity.co.uk``) does call a private JSON backend at -``prd.wiciam.fidelity.co.uk/cvmfe/api/*`` — we reverse-engineer that and feed -it through a Playwright-maintained session. +PlanViewer has no public individual-member API. The SPA (at +``pv.planviewer.fidelity.co.uk``) and the legacy HTML app (at +``www.planviewer.fidelity.co.uk``) share session cookies via PingFederate +OAuth at ``id.fidelity.co.uk``. -## Session lifecycle +We keep a Playwright-maintained session via ``storage_state.json``: 1. **One-off seed** (``broker-sync fidelity-seed``): Viktor runs a headed - Chromium, logs in (password + memorable word + MFA), clicks "Remember - device". Playwright dumps the resulting ``storage_state.json`` (cookies + - localStorage) which we stash in Vault. - + Chromium, logs in (password + memorable word + SMS MFA), clicks + "Remember device". The storage_state is persisted to Vault. 2. **Monthly cron**: loads storage_state, boots headless Chromium, navigates - to the SPA once to let it refresh rolling session tokens, intercepts the - first outbound XHR to capture the ``sid``/``fid``/``tbid``/``rid`` headers, - then closes the browser and continues with plain httpx. + to the transaction-history page with a wide date range, parses the HTML + table, and intercepts the ``DisplayValuation`` XHR for the current + fund holdings. On 401/idle-timeout we raise + :class:`FidelitySessionError` so Prometheus alerts Viktor to re-seed. -3. **Re-seed trigger**: on any 401 from the JSON API we raise - :class:`FidelitySessionError`; the CronJob fails loudly and Prometheus - alerts Viktor to run the seed command again. +## Emitted Activity shape -Remember-device typically survives 30-90 days on Fidelity, so we expect the -re-seed to be a quarterly manual step — not monthly. - -## Data model - -Salary-sacrifice scheme with two contribution streams (employee + employer), -both pre-tax. Each contribution buys units across one or more funds. We emit: - -- ``DEPOSIT`` per employee-or-employer cash inflow (external_id carries - ``fidelity::``). -- ``BUY`` per fund-unit purchase (``symbol`` = fund ISIN or Fidelity code, - ``quantity`` = units, ``unit_price`` = GBp or GBP per unit). - -All currency is GBP. The single WF account is ``AccountType.WORKPLACE_PENSION``. +- One ``DEPOSIT`` per cash-impacting transaction (Regular Premium, Single + Premium, rebate, etc.). ``external_id = fidelity:tx:``. +- One synthetic ``DEPOSIT`` for unrealised gains so WF's Net Worth matches + the Fidelity dashboard. ``external_id = + fidelity:gains:``. +- Bulk Switches / Fund Switches are skipped (no cash movement). """ from __future__ import annotations +import contextlib import logging from collections.abc import AsyncIterator -from datetime import datetime -from typing import NamedTuple +from datetime import UTC, datetime +from decimal import Decimal +from pathlib import Path +from typing import Any, NamedTuple -from broker_sync.models import Account, AccountType, Activity +from broker_sync.models import Account, AccountType, Activity, ActivityType +from broker_sync.providers.parsers.fidelity import ( + FidelityCashTx, + FidelityHolding, + parse_transactions_html, + parse_valuation_json, +) log = logging.getLogger(__name__) ACCOUNT_ID = "fidelity-workplace-pension" _CCY = "GBP" -# PlanViewer's private JSON backend. Endpoint paths are reverse-engineered from -# Viktor's DevTools cURLs and validated by the unit tests' fixtures. -_API_BASE = "https://prd.wiciam.fidelity.co.uk" +_PV_BASE = "https://www.planviewer.fidelity.co.uk" +_PV_TX_PATH = "/planviewer/DisplayMyPlanMemberTransHist.action" +_PV_VALUATION_PATH = "/planviewer/DisplayValuation.action" +_PV_LANDING = "https://www.planviewer.fidelity.co.uk/" + +# A wide backfill cap; scheme can't predate 1990. +_BACKFILL_START = "01 Jan 1990" class FidelityCreds(NamedTuple): - """Credentials + session state required to hit the PlanViewer backend.""" + """Paths needed to run the provider.""" storage_state_path: str plan_id: str headless: bool = True class FidelitySessionError(Exception): - """Raised when PlanViewer returns 401/403 — storage_state is stale. - - Recovery: run ``broker-sync fidelity-seed`` in a browser to refresh the - storage_state blob in Vault, then re-run the CronJob. - """ + """Raised when PlanViewer rejects the saved session — re-seed required.""" class FidelityProviderConfigError(Exception): - """Raised when the provider is asked to run but required config (plan id, - storage_state path) is missing or obviously wrong.""" + """Raised when provider config is missing or obviously wrong.""" + + +def _tx_to_activity(tx: FidelityCashTx) -> Activity: + """Map a Fidelity cash transaction to a canonical DEPOSIT.""" + return Activity( + external_id=tx.external_id, + account_id=ACCOUNT_ID, + account_type=AccountType.WORKPLACE_PENSION, + date=tx.date, + activity_type=ActivityType.DEPOSIT, + currency=_CCY, + amount=tx.amount, + notes=f"fidelity-planviewer:{tx.tx_type}", + ) + + +def _gains_offset_activity( + holdings: list[FidelityHolding], + transactions: list[FidelityCashTx], + as_of: datetime, +) -> Activity | None: + """Create a synthetic DEPOSIT/WITHDRAWAL so WF Net Worth matches the + Fidelity dashboard's reported pot value. + + The offset carries a date-derived external_id so monthly runs refresh + the same synthetic entry rather than stacking duplicates. + """ + if not holdings: + return None + total_value = sum((h.total_value for h in holdings), Decimal(0)) + total_contrib = sum((t.amount for t in transactions), Decimal(0)) + gains = total_value - total_contrib + if gains == 0: + return None + return Activity( + external_id=f"fidelity:gains:{as_of.date().isoformat()}", + account_id=ACCOUNT_ID, + account_type=AccountType.WORKPLACE_PENSION, + date=as_of, + activity_type=ActivityType.DEPOSIT if gains > 0 else ActivityType.WITHDRAWAL, + currency=_CCY, + amount=abs(gains), + notes=(f"fidelity-planviewer:unrealised-gains-offset " + f"(pot=£{total_value}, contrib=£{total_contrib})"), + ) class FidelityPlanViewerProvider: """Read-only provider against Fidelity UK PlanViewer. - Per the Provider protocol consumed by ``broker_sync.pipeline``: - - - ``.accounts()`` advertises the single workplace-pension WF account we - write into. - - ``.fetch(since, before)`` is an async generator that yields canonical - ``Activity`` objects. + Lifecycle: + - ``accounts()`` advertises the single WF workplace-pension account. + - ``fetch(since, before)`` opens a Playwright session with the saved + storage_state, navigates to the transaction-history page with a wide + date range, scrapes the table, and intercepts the valuation XHR. """ name = "fidelity-planviewer" @@ -108,21 +148,106 @@ class FidelityPlanViewerProvider: since: datetime | None = None, before: datetime | None = None, ) -> AsyncIterator[Activity]: - """Yield Activity records. + state_path = self._creds.storage_state_path + if not Path(state_path).exists(): + raise FidelityProviderConfigError( + f"storage_state not found at {state_path} — " + "run `broker-sync fidelity-seed` first") - Implementation blocked on captured endpoint shapes. Viktor will paste - the transactions + holdings POST cURLs from DevTools, then we wire the - parsers and this method lights up. - """ - # Guard against accidentally running before endpoint reverse-engineering - # is done — makes the CronJob fail loudly with an actionable message - # rather than silently importing nothing. - raise FidelityProviderConfigError( - "Fidelity ingest not yet enabled — PlanViewer endpoint paths have " - "not been captured. Paste the POST cURLs from DevTools for the " - "transactions + holdings views and re-apply the provider update." + tx_html, valuation_json = await _scrape_live_session( + state_path=state_path, headless=self._creds.headless, ) - # Unreachable yield — keeps the return type AsyncIterator[Activity] - # once the raise above is removed. - if False: # pragma: no cover - yield + transactions = parse_transactions_html(tx_html) + holdings = parse_valuation_json(valuation_json) + log.info("fidelity: parsed %d transactions, %d holdings", + len(transactions), len(holdings)) + + for tx in transactions: + if since is not None and tx.date < since: + continue + if before is not None and tx.date >= before: + continue + yield _tx_to_activity(tx) + + # The gains offset is always "as of now" so it reflects today's pot. + # Only emit when the caller isn't windowing (full state). + if since is None and before is None: + offset = _gains_offset_activity(holdings, transactions, datetime.now(UTC)) + if offset is not None: + yield offset + + +async def _scrape_live_session( + *, + state_path: str, + headless: bool, +) -> tuple[str, dict[str, Any]]: + """Load storage_state, navigate the transaction + valuation pages, + return (transactions HTML, valuation JSON payload). + + Raises :class:`FidelitySessionError` if the session is dead (15-min idle, + cookie expiry, etc.) — Viktor must re-seed. + """ + from playwright.async_api import async_playwright + + captured_valuation: dict[str, dict[str, Any]] = {} + async with async_playwright() as pw: + browser = await pw.chromium.launch(headless=headless) + try: + ctx = await browser.new_context( + storage_state=state_path, + user_agent=("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/147.0.0.0 Safari/537.36"), + viewport={"width": 1280, "height": 900}, + ) + page = await ctx.new_page() + + async def on_response(resp: Any) -> None: + if _PV_VALUATION_PATH in resp.url and resp.status < 400: + with contextlib.suppress(Exception): + captured_valuation["payload"] = await resp.json() + page.on("response", on_response) + + # Trigger session + capture valuation by navigating through landing + # → main page. The SPA fires DisplayValuation on the main page. + await page.goto(_PV_LANDING, wait_until="networkidle", timeout=30000) + await page.wait_for_timeout(2000) + main_url = f"{_PV_BASE}/planviewer/DisplayMainPage.action" + await page.goto(main_url, wait_until="networkidle", timeout=30000) + await page.wait_for_timeout(3000) + if "idle for more than 15 minutes" in (await page.content()) \ + or "id.fidelity.co.uk" in page.url: + raise FidelitySessionError( + "PlanViewer session stale — run `broker-sync fidelity-seed`") + + # Now pull the transactions page with a wide date range. + await page.goto(f"{_PV_BASE}{_PV_TX_PATH}", + wait_until="networkidle", timeout=30000) + await page.wait_for_timeout(1500) + await page.fill('input[name="startDate"]', _BACKFILL_START) + today = await page.evaluate( + "new Date().toLocaleDateString('en-GB'," + "{day:'2-digit',month:'short',year:'numeric'}).replace(/,/g,'')") + await page.fill('input[name="endDate"]', today) + await page.focus('input[name="endDate"]') + await page.keyboard.press("Enter") + with contextlib.suppress(Exception): + await page.wait_for_load_state("networkidle", timeout=15000) + await page.wait_for_timeout(2000) + tx_html = await page.content() + + # If valuation wasn't picked up on the main page, request directly. + if "payload" not in captured_valuation: + r = await page.request.get(f"{_PV_BASE}{_PV_VALUATION_PATH}") + if r.ok: + with contextlib.suppress(Exception): + captured_valuation["payload"] = await r.json() + + # Roll the storage_state so the next run benefits from any refresh. + await ctx.storage_state(path=state_path) + finally: + await browser.close() + + valuation: dict[str, Any] = captured_valuation.get("payload") or {} + return tx_html, valuation diff --git a/broker_sync/providers/parsers/fidelity.py b/broker_sync/providers/parsers/fidelity.py new file mode 100644 index 0000000..b53875c --- /dev/null +++ b/broker_sync/providers/parsers/fidelity.py @@ -0,0 +1,129 @@ +"""Parsers for Fidelity UK PlanViewer scraped data. + +Two inputs: + +- **Transactions HTML** from ``/planviewer/DisplayMyPlanMemberTransHist.action`` + rendered with a wide date range. The relevant has + ``id="myplan_member_transhist_support"``. +- **Valuation JSON** from the XHR ``/planviewer/DisplayValuation.action`` — + the SPA calls this to render the my-investments dashboard. Contains + current unit holdings + price + breakdown by contribution type. +""" +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass +from datetime import UTC, datetime +from decimal import Decimal +from typing import Any + +from bs4 import BeautifulSoup + +_AMOUNT_RE = re.compile(r"\u00a3([\d,]+(?:\.\d+)?)") + +# Fidelity transaction type strings we care about +_TX_DEPOSIT_TYPES = { + "regular premium", + "single premium", + "investment management rebate", +} +_TX_IGNORE_TYPES = { + "bulk switch", # pure reallocation, no cash impact + "fund switch", +} + + +@dataclass(frozen=True) +class FidelityCashTx: + """A single cash-impacting transaction from the transaction history page.""" + date: datetime + tx_type: str # raw Fidelity label ("Regular Premium", "Single Premium", …) + amount: Decimal + external_id: str + + +@dataclass(frozen=True) +class FidelityHolding: + """A current fund-unit holding from DisplayValuation.action.""" + fund_code: str + fund_name: str + units: Decimal + unit_price: Decimal + currency: str + total_value: Decimal + # Contribution-type breakdown ({"SASC": Decimal(...), "ERXS": Decimal(...)}) + units_by_source: dict[str, Decimal] + + +def parse_transactions_html(html: str) -> list[FidelityCashTx]: + """Extract cash-impacting transactions from the transaction history page. + + Skips bulk switches (no cash movement) and header/total rows. Deterministic + external_id so re-runs dedup against the same rows. + """ + soup = BeautifulSoup(html, "html.parser") + out: list[FidelityCashTx] = [] + for tr in soup.select("table#myplan_member_transhist_support tr"): + cells = [td.get_text(" ", strip=True) for td in tr.find_all("td")] + if len(cells) != 7: + continue + date_str, tx_type, _f, _c, _u, _p, amount_str = cells + m_date = re.match(r"(\d{2})/(\d{2})/(\d{4})", date_str) + if not m_date: + continue + tx_lower = tx_type.lower() + if tx_lower in _TX_IGNORE_TYPES or tx_type in ("-",): + continue + m_amt = _AMOUNT_RE.search(amount_str) + if not m_amt: + continue + amount = Decimal(m_amt.group(1).replace(",", "")) + if amount == 0: + continue + dd, mm, yyyy = m_date.groups() + dt = datetime(int(yyyy), int(mm), int(dd), tzinfo=UTC) + fp = hashlib.sha256( + f"{dt.isoformat()}|{tx_type}|{amount}".encode() + ).hexdigest()[:16] + out.append(FidelityCashTx( + date=dt, + tx_type=tx_type, + amount=amount, + external_id=f"fidelity:tx:{fp}", + )) + return out + + +def parse_valuation_json(payload: Any) -> list[FidelityHolding]: + """Extract current fund holdings from DisplayValuation.action JSON.""" + out: list[FidelityHolding] = [] + for v in payload.get("valuations", []): + asset = v.get("asset") or {} + fund_code = next( + (a.get("value") for a in asset.get("assetId", []) if a.get("type") == "FUND_CODE"), + None, + ) + if not fund_code: + continue + fund_name = asset.get("name") or fund_code + units = Decimal(str((v.get("units") or {}).get("total") or 0)) + price = (v.get("price") or {}) + unit_price = Decimal(str(price.get("value") or 0)) + currency = price.get("currency") or "GBP" + total = Decimal(str((v.get("valuation") or {}).get("total") or 0)) + groups = (v.get("units") or {}).get("group", []) or [] + by_src = {} + for g in groups: + if g.get("type") == "CONTRIBUTION_TYPE" and g.get("groupId"): + by_src[g["groupId"]] = Decimal(str(g.get("unit", {}).get("total") or 0)) + out.append(FidelityHolding( + fund_code=fund_code, + fund_name=fund_name, + units=units, + unit_price=unit_price, + currency=currency, + total_value=total, + units_by_source=by_src, + )) + return out diff --git a/tests/fixtures/fidelity/transactions-full.html b/tests/fixtures/fidelity/transactions-full.html new file mode 100644 index 0000000..1b71f80 --- /dev/null +++ b/tests/fixtures/fidelity/transactions-full.html @@ -0,0 +1,1707 @@ + + + + + Fidelity's PlanViewer + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Fidelity uses cookies to provide you with the best possible online experience. If you continue without changing your settings, we'll assume that you are happy to receive all cookies on our site. However, you can change the cookie settings and view our cookie policy at any time.

Continue
+ + + + + + + + + + +
+ +
+ + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + +
+ + +
+ + + + + Contact us + | + Help + + + + + + + Log out + + + + + +
+
+ + + +
+
+ +
+ +
+ + + + + + + + + + +
+ + + + + + + + +
+ + + + + +
+
+ + + Meta UK Retirement Plan + +
+
+
+
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + +
Meta UK Retirement Plan
+
CIMP
+ + + + + + + + + + + + + + +

+ Transaction history +

+ +

+ Recent transactions are shown by default but you can refine the date range using the filters. PlanViewer uses the most recent data prior to the date requested. +

+ + + + + +
+ + + + +
+ + + +
+ + + + + + + + + + + +
+
View by: +
+
+ +
+
+ +
+
+
+ + + +
+
+ +
+ +
+
+
+ + + +
+
+ +
+ +
+
+ + + + + +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Date Transaction Type Funds Contribution Types Units/shares Price Transaction Amount
-454--£102,004.15
+ Transactions by fund + +  |  + + Transactions by contribution type +
Open transaction details for the member 16/04/2026Regular Premium 1 2 £1,546.02
Open transaction details for the member 16/03/2026Regular Premium 1 2 £1,500.50
Open transaction details for the member 16/02/2026Regular Premium 1 2 £1,500.50
Open transaction details for the member 16/01/2026Regular Premium 1 2 £1,500.50
Open transaction details for the member 16/12/2025Regular Premium 1 2 £1,500.50
Open transaction details for the member 17/11/2025Regular Premium 1 2 £1,500.50
Open transaction details for the member 16/10/2025Regular Premium 1 2 £1,500.50
Open transaction details for the member 16/09/2025Regular Premium 1 2 £1,500.50
Open transaction details for the member 18/08/2025Regular Premium 1 2 £1,500.50
Open transaction details for the member 15/07/2025Regular Premium 1 2 £1,500.50
Open transaction details for the member 16/06/2025Regular Premium 1 2 £1,500.50
Open transaction details for the member 16/05/2025Regular Premium 1 2 £1,500.50
Open transaction details for the member 11/04/2025Regular Premium 1 2 £1,500.50
Open transaction details for the member 11/04/2025Single Premium 1 1 £26,969.00
Open transaction details for the member 17/03/2025Regular Premium 1 2 £1,448.52
Open transaction details for the member 17/02/2025Regular Premium 1 2 £1,448.52
Open transaction details for the member 16/01/2025Regular Premium 1 2 £1,448.52
Open transaction details for the member 16/12/2024Regular Premium 1 2 £1,448.52
Open transaction details for the member 15/11/2024Regular Premium 1 2 £1,448.52
Open transaction details for the member 05/11/2024Bulk Switch 2 3 £0.00
Open transaction details for the member 15/10/2024Regular Premium 1 2 £1,448.52
Open transaction details for the member 13/09/2024Regular Premium 1 2 £1,448.52
Open transaction details for the member 16/08/2024Regular Premium 1 2 £1,448.52
Open transaction details for the member 12/07/2024Regular Premium 1 2 £1,448.52
Open transaction details for the member 14/06/2024Regular Premium 1 2 £1,448.52
Open transaction details for the member 16/05/2024Regular Premium 1 2 £1,448.52
Open transaction details for the member 16/04/2024Regular Premium 1 2 £1,448.52
Open transaction details for the member 18/03/2024Regular Premium 1 2 £1,387.50
Open transaction details for the member 16/02/2024Regular Premium 1 2 £1,387.50
Open transaction details for the member 16/01/2024Regular Premium 1 2 £1,387.50
Open transaction details for the member 28/12/2023Regular Premium 1 2 £1,387.50
Open transaction details for the member 17/11/2023Regular Premium 1 2 £1,387.50
Open transaction details for the member 16/10/2023Regular Premium 1 2 £1,387.50
Open transaction details for the member 11/10/2023Bulk Switch 2 2 £0.00
Open transaction details for the member 15/09/2023Regular Premium 1 2 £1,387.50
Open transaction details for the member 16/08/2023Regular Premium 1 2 £1,387.50
Open transaction details for the member 17/07/2023Regular Premium 1 2 £1,387.50
Open transaction details for the member 14/06/2023Regular Premium 1 2 £1,387.50
Open transaction details for the member 17/05/2023Regular Premium 1 2 £1,387.50
Open transaction details for the member 17/04/2023Regular Premium 1 2 £1,387.50
Open transaction details for the member 15/03/2023Regular Premium 1 2 £1,347.50
Open transaction details for the member 20/02/2023Regular Premium 1 2 £1,347.50
Open transaction details for the member 17/01/2023Regular Premium 1 2 £1,347.50
Open transaction details for the member 13/12/2022Regular Premium 1 2 £1,347.50
Open transaction details for the member 17/11/2022Regular Premium 1 2 £1,347.50
Open transaction details for the member 17/10/2022Regular Premium 1 2 £1,347.50
Open transaction details for the member 20/09/2022Regular Premium 1 2 £1,099.60
Open transaction details for the member 22/08/2022Regular Premium 1 2 £1,099.60
Open transaction details for the member 19/07/2022Regular Premium 1 2 £1,099.60
Open transaction details for the member 15/07/2022Investment Management Rebate 1 1 £6.68
Open transaction details for the member 20/06/2022Regular Premium 1 2 £1,099.60
Open transaction details for the member 17/06/2022Single Premium 1 1 £8,301.05
Open transaction details for the member 16/05/2022Regular Premium 2 2 £659.76
+ +
+Fidelity International +18 Apr 2026 +
+ + + + + + +

+ *Any exchange rates used to show account values in different currencies are indicative only and updated daily. +

+ + +
+ Want to change your + contributions? +

+ Depending on the rules of your retirement plan, you may have the option to make extra payments into your plan savings. If you decide to increase your monthly contributions, you may even find that your employer will increase their contributions too. +

+ + + + + + + +
+

+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+

+ + Issued in the UK by FIL Pensions Management (FPM) authorised and regulated by the Financial Conduct Authority, FIL Life Insurance Limited (FIL Life) authorised by the Prudential Regulation Authority and regulated by the Financial Conduct Authority and the Prudential Regulation Authority and in Ireland by FIL Life Insurance (Ireland) Limited (FIL Life Ireland), authorised and regulated by the Central Bank of Ireland. + +

+
+
+ + +
+
+ © FIL Pensions Management +
+ + + Important legal information + + + + | + + + + Terms and conditions + + | + + + + + + + + Cookie policy + + + + + + + | + + Contact us + + | + + Online security + + + + + + + + +
+
+
+ + + + + + + + + + + + + +
\ No newline at end of file diff --git a/tests/fixtures/fidelity/valuation.json b/tests/fixtures/fidelity/valuation.json new file mode 100644 index 0000000..5ad66e3 --- /dev/null +++ b/tests/fixtures/fidelity/valuation.json @@ -0,0 +1,2 @@ +{"valuations":[{"asset":{"assetId":[{"type":"FUND_CODE","value":"KDOA"}],"name":"Passive Global Equity Fund - Class 9"},"units":{"total":44920.21,"available":null,"crystallised":null,"uncrystallised":null,"group":[{"groupId":"BONW","type":"CONTRIBUTION_TYPE","name":"Bonus Waiver","unit":{"total":11490.84,"available":null,"crystallised":null,"uncrystallised":null}},{"groupId":"ERXS","type":"CONTRIBUTION_TYPE","name":"Company","unit":{"total":17148.27,"available":null,"crystallised":null,"uncrystallised":null}},{"groupId":"SASC","type":"CONTRIBUTION_TYPE","name":"Salary Sacrifice","unit":{"total":11432.20,"available":null,"crystallised":null,"uncrystallised":null}},{"groupId":"TREX","type":"CONTRIBUTION_TYPE","name":"Transfer In","unit":{"total":4848.90,"available":null,"crystallised":null,"uncrystallised":null}}]},"price":{"value":3.066,"datetime":"2026-04-17","currency":"GBP"},"valuation":{"total":137725.35,"available":null,"crystallised":null,"uncrystallised":null,"group":[{"groupId":"BONW","type":"CONTRIBUTION_TYPE","name":"Bonus Waiver","valuation":{"total":35230.91,"available":null,"crystallised":null,"uncrystallised":null}},{"groupId":"ERXS","type":"CONTRIBUTION_TYPE","name":"Company","valuation":{"total":52576.60,"available":null,"crystallised":null,"uncrystallised":null}},{"groupId":"SASC","type":"CONTRIBUTION_TYPE","name":"Salary Sacrifice","valuation":{"total":35051.12,"available":null,"crystallised":null,"uncrystallised":null}},{"groupId":"TREX","type":"CONTRIBUTION_TYPE","name":"Transfer In","valuation":{"total":14866.72,"available":null,"crystallised":null,"uncrystallised":null}}],"valuationType":"Value"},"currency":"GBP"},{"asset":{"assetId":[{"type":"FUND_CODE","value":"KCVT"}],"name":"FutureWise Target 2065 - Class 10"},"units":{"total":230.02,"available":null,"crystallised":null,"uncrystallised":null,"group":[{"groupId":"ERXS","type":"CONTRIBUTION_TYPE","name":"Company","unit":{"total":153.35,"available":null,"crystallised":null,"uncrystallised":null}},{"groupId":"SASC","type":"CONTRIBUTION_TYPE","name":"Salary Sacrifice","unit":{"total":76.67,"available":null,"crystallised":null,"uncrystallised":null}}]},"price":{"value":3.254,"datetime":"2026-04-17","currency":"GBP"},"valuation":{"total":748.48,"available":null,"crystallised":null,"uncrystallised":null,"group":[{"groupId":"ERXS","type":"CONTRIBUTION_TYPE","name":"Company","valuation":{"total":498.99,"available":null,"crystallised":null,"uncrystallised":null}},{"groupId":"SASC","type":"CONTRIBUTION_TYPE","name":"Salary Sacrifice","valuation":{"total":249.49,"available":null,"crystallised":null,"uncrystallised":null}}],"valuationType":"Value"},"currency":"GBP"},{"asset":{"assetId":[{"type":"FUND_CODE","value":"LAFC"}],"name":"Volatility Managed Multi Asset Fund"},"units":{"total":106.64,"available":null,"crystallised":null,"uncrystallised":null,"group":[{"groupId":"ERXS","type":"CONTRIBUTION_TYPE","name":"Company","unit":{"total":71.09,"available":null,"crystallised":null,"uncrystallised":null}},{"groupId":"SASC","type":"CONTRIBUTION_TYPE","name":"Salary Sacrifice","unit":{"total":35.55,"available":null,"crystallised":null,"uncrystallised":null}}]},"price":{"value":252.9000,"datetime":"2026-04-17","currency":"GBP"},"valuation":{"total":269.70,"available":null,"crystallised":null,"uncrystallised":null,"group":[{"groupId":"ERXS","type":"CONTRIBUTION_TYPE","name":"Company","valuation":{"total":179.80,"available":null,"crystallised":null,"uncrystallised":null}},{"groupId":"SASC","type":"CONTRIBUTION_TYPE","name":"Salary Sacrifice","valuation":{"total":89.90,"available":null,"crystallised":null,"uncrystallised":null}}],"valuationType":"Value"},"currency":"GBP"}],"valuationSum":{"total":138743.53,"available":0.0,"crystallised":null,"uncrystallised":null,"currency":"GBP"},"asOfDateTime":"2026-04-17T12:00:00+01:00"} + diff --git a/tests/providers/test_fidelity_planviewer.py b/tests/providers/test_fidelity_planviewer.py index 838d2b8..fe4feca 100644 --- a/tests/providers/test_fidelity_planviewer.py +++ b/tests/providers/test_fidelity_planviewer.py @@ -1,22 +1,33 @@ from __future__ import annotations +import json +from datetime import UTC, datetime +from decimal import Decimal +from pathlib import Path + import pytest -from broker_sync.models import Account, AccountType +from broker_sync.models import Account, AccountType, ActivityType from broker_sync.providers.fidelity_planviewer import ( ACCOUNT_ID, FidelityCreds, FidelityPlanViewerProvider, FidelityProviderConfigError, + _gains_offset_activity, ) +from broker_sync.providers.parsers.fidelity import ( + parse_transactions_html, + parse_valuation_json, +) + +_FIXTURES = Path(__file__).parent.parent / "fixtures" / "fidelity" def test_accounts_exposes_single_workplace_pension_account() -> None: prov = FidelityPlanViewerProvider(FidelityCreds( - storage_state_path="/tmp/x", plan_id="ABC123", + storage_state_path="/tmp/x", plan_id="META", )) - accounts = prov.accounts() - assert accounts == [ + assert prov.accounts() == [ Account( id=ACCOUNT_ID, name="Fidelity UK Pension", @@ -27,16 +38,79 @@ def test_accounts_exposes_single_workplace_pension_account() -> None: ] -async def test_fetch_raises_until_endpoints_captured() -> None: - """Until Viktor pastes the transactions/holdings cURLs, fetch() must fail - loudly rather than silently importing nothing. - - Swap this test for real parser tests once the API shapes are known and - `FidelityPlanViewerProvider.fetch` is wired up against fixtures. - """ +async def test_fetch_raises_without_storage_state() -> None: prov = FidelityPlanViewerProvider(FidelityCreds( - storage_state_path="/tmp/x", plan_id="ABC123", + storage_state_path="/tmp/does-not-exist-xyzzy.json", plan_id="META", )) - with pytest.raises(FidelityProviderConfigError, match="endpoint paths"): + with pytest.raises(FidelityProviderConfigError, match="storage_state"): async for _ in prov.fetch(): - pytest.fail("fetch should not yield before endpoints are configured") + pytest.fail("should have raised before yielding") + + +# -- parser tests against real (captured) fixture -- + + +def test_parse_transactions_real_fixture() -> None: + html = (_FIXTURES / "transactions-full.html").read_text() + txs = parse_transactions_html(html) + # Scheme has ~48 months + a couple of single premiums + 1 rebate; + # Bulk Switches must be filtered out (zero-amount rows). + assert 40 <= len(txs) <= 100 + # All dates are within the scheme's lifetime (2022-03 to today-ish). + assert all(tx.date >= datetime(2022, 1, 1, tzinfo=UTC) for tx in txs) + # Sum should match the header total on the page (£102,004.15 at + # fixture time). Allow a £5 tolerance in case the page summary row + # changes in future captures — the unit test primarily guards parsing + # correctness, not drift in the fixture. + total = sum((tx.amount for tx in txs), Decimal(0)) + assert abs(total - Decimal("102004.15")) < Decimal("5") + + +def test_parse_transactions_skips_bulk_switch() -> None: + html = (_FIXTURES / "transactions-full.html").read_text() + txs = parse_transactions_html(html) + assert not any("bulk switch" in tx.tx_type.lower() for tx in txs) + + +def test_parse_transactions_external_id_deterministic() -> None: + html = (_FIXTURES / "transactions-full.html").read_text() + a = parse_transactions_html(html) + b = parse_transactions_html(html) + assert [tx.external_id for tx in a] == [tx.external_id for tx in b] + assert all(tx.external_id.startswith("fidelity:tx:") for tx in a) + + +def test_parse_valuation_fixture() -> None: + payload = json.loads((_FIXTURES / "valuation.json").read_text()) + holdings = parse_valuation_json(payload) + assert len(holdings) >= 1 + h = holdings[0] + assert h.fund_code == "KDOA" + assert "Passive Global Equity" in h.fund_name + assert h.currency == "GBP" + assert h.units > 0 + assert h.unit_price > 0 + # Value ≈ units * price + assert abs(h.total_value - h.units * h.unit_price) < Decimal("1") + # Contribution-type breakdown must parse + assert set(h.units_by_source.keys()) >= {"SASC", "ERXS"} + + +def test_gains_offset_emits_deposit_when_pot_exceeds_contributions() -> None: + html = (_FIXTURES / "transactions-full.html").read_text() + valuation = json.loads((_FIXTURES / "valuation.json").read_text()) + txs = parse_transactions_html(html) + holdings = parse_valuation_json(valuation) + as_of = datetime(2026, 4, 18, tzinfo=UTC) + offset = _gains_offset_activity(holdings, txs, as_of) + assert offset is not None + assert offset.activity_type in (ActivityType.DEPOSIT, ActivityType.WITHDRAWAL) + assert offset.amount > 0 + assert offset.external_id == "fidelity:gains:2026-04-18" + + +def test_gains_offset_none_when_no_holdings() -> None: + assert _gains_offset_activity( + holdings=[], transactions=[], + as_of=datetime(2026, 4, 18, tzinfo=UTC), + ) is None