fidelity: push per-fund manual snapshot instead of gains-offset DEPOSIT
PlanViewer's DisplayValuation.action JSON already gives us current fund units + unit price; we were parsing it and throwing it away, emitting only a single 'unrealised-gains-offset' DEPOSIT to make Wealthfolio's totals match the dashboard. That hack double-counted the gain as a cash contribution, hiding £35k of pension growth from every contribution/growth/ROI panel. New flow: - FidelityPlanViewerProvider exposes last_holdings + last_total_contribution after fetch() drains. - fidelity-ingest CLI converts to a ManualSnapshotPayload (cost basis allocated proportionally by current fund value share) and posts to WF /api/v1/snapshots/import. WF auto-creates unknown fund symbols with kind=INVESTMENT, quoteMode=MANUAL, quoteCcy=GBP. - The gains-offset emission is removed entirely. Historical offset rows already in WF are corrected at the dashboard layer by the dav_corrected view shipped in infra@2841347e. WealthfolioSink gains push_manual_snapshots() + ManualSnapshotPayload / SnapshotPosition wire types. 11 sink tests (3 new) + 9 fidelity provider tests (2 changed, 1 new) all green; mypy + ruff clean.
This commit is contained in:
parent
5adc4a7ba4
commit
cb159e17d9
5 changed files with 339 additions and 60 deletions
|
|
@ -16,21 +16,28 @@ We keep a Playwright-maintained session via ``storage_state.json``:
|
|||
fund holdings. On 401/idle-timeout we raise
|
||||
:class:`FidelitySessionError` so Prometheus alerts Viktor to re-seed.
|
||||
|
||||
## Emitted Activity shape
|
||||
## Emitted Activity / snapshot shape
|
||||
|
||||
- One ``DEPOSIT`` per cash-impacting transaction (Regular Premium, Single
|
||||
Premium, rebate, etc.). ``external_id = fidelity:tx:<sha256[:16]>``.
|
||||
- One synthetic ``DEPOSIT`` for unrealised gains so WF's Net Worth matches
|
||||
the Fidelity dashboard. ``external_id =
|
||||
fidelity:gains:<YYYY-MM-DD>``.
|
||||
- Bulk Switches / Fund Switches are skipped (no cash movement).
|
||||
- After the activity stream drains, the ``fidelity-ingest`` CLI calls
|
||||
``WealthfolioSink.push_manual_snapshots`` with one ``ManualSnapshotPayload``
|
||||
per fund holding (today's date, units + cost basis allocated
|
||||
proportionally to fund value share). This sets per-fund quantity and
|
||||
cost basis in WF so the dashboard Positions table shows the pension
|
||||
funds alongside the brokerage assets.
|
||||
- The old synthetic ``fidelity:gains:<date>`` DEPOSIT is no longer
|
||||
emitted — the snapshot supersedes it. Old offset rows that landed
|
||||
before this change are corrected at the dashboard layer by the
|
||||
``dav_corrected`` PG view (``infra/stacks/wealthfolio/main.tf``).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from typing import Any, NamedTuple
|
||||
|
|
@ -42,6 +49,7 @@ from broker_sync.providers.parsers.fidelity import (
|
|||
parse_transactions_html,
|
||||
parse_valuation_json,
|
||||
)
|
||||
from broker_sync.sinks.wealthfolio import ManualSnapshotPayload, SnapshotPosition
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -86,37 +94,6 @@ def _tx_to_activity(tx: FidelityCashTx) -> Activity:
|
|||
)
|
||||
|
||||
|
||||
def _gains_offset_activity(
|
||||
holdings: list[FidelityHolding],
|
||||
transactions: list[FidelityCashTx],
|
||||
as_of: datetime,
|
||||
) -> Activity | None:
|
||||
"""Create a synthetic DEPOSIT/WITHDRAWAL so WF Net Worth matches the
|
||||
Fidelity dashboard's reported pot value.
|
||||
|
||||
The offset carries a date-derived external_id so monthly runs refresh
|
||||
the same synthetic entry rather than stacking duplicates.
|
||||
"""
|
||||
if not holdings:
|
||||
return None
|
||||
total_value = sum((h.total_value for h in holdings), Decimal(0))
|
||||
total_contrib = sum((t.amount for t in transactions), Decimal(0))
|
||||
gains = total_value - total_contrib
|
||||
if gains == 0:
|
||||
return None
|
||||
return Activity(
|
||||
external_id=f"fidelity:gains:{as_of.date().isoformat()}",
|
||||
account_id=ACCOUNT_ID,
|
||||
account_type=AccountType.WORKPLACE_PENSION,
|
||||
date=as_of,
|
||||
activity_type=ActivityType.DEPOSIT if gains > 0 else ActivityType.WITHDRAWAL,
|
||||
currency=_CCY,
|
||||
amount=abs(gains),
|
||||
notes=(f"fidelity-planviewer:unrealised-gains-offset "
|
||||
f"(pot=£{total_value}, contrib=£{total_contrib})"),
|
||||
)
|
||||
|
||||
|
||||
class FidelityPlanViewerProvider:
|
||||
"""Read-only provider against Fidelity UK PlanViewer.
|
||||
|
||||
|
|
@ -125,11 +102,18 @@ class FidelityPlanViewerProvider:
|
|||
- ``fetch(since, before)`` opens a Playwright session with the saved
|
||||
storage_state, navigates to the transaction-history page with a wide
|
||||
date range, scrapes the table, and intercepts the valuation XHR.
|
||||
- After ``fetch()`` completes, ``last_holdings`` holds the per-fund
|
||||
unit positions captured in the same scrape — used by the
|
||||
``fidelity-ingest`` CLI to push a manual snapshot to Wealthfolio
|
||||
so per-fund quantities + cost basis land in the Positions table
|
||||
(the activity stream alone only carries cash flows).
|
||||
"""
|
||||
name = "fidelity-planviewer"
|
||||
|
||||
def __init__(self, creds: FidelityCreds) -> None:
|
||||
self._creds = creds
|
||||
self.last_holdings: list[FidelityHolding] = []
|
||||
self.last_total_contribution: Decimal = Decimal(0)
|
||||
|
||||
def accounts(self) -> list[Account]:
|
||||
return [
|
||||
|
|
@ -162,19 +146,72 @@ class FidelityPlanViewerProvider:
|
|||
log.info("fidelity: parsed %d transactions, %d holdings",
|
||||
len(transactions), len(holdings))
|
||||
|
||||
# Snapshot the per-fund holdings for the CLI to push as a manual
|
||||
# holdings_snapshot after this generator drains. Wealthfolio's
|
||||
# activity model can't represent pension fund unit purchases (no
|
||||
# per-purchase price feed from PlanViewer), so we record current
|
||||
# state via /api/v1/snapshots/import instead.
|
||||
self.last_holdings = holdings
|
||||
self.last_total_contribution = sum(
|
||||
(t.amount for t in transactions), Decimal(0)
|
||||
)
|
||||
|
||||
for tx in transactions:
|
||||
if since is not None and tx.date < since:
|
||||
continue
|
||||
if before is not None and tx.date >= before:
|
||||
continue
|
||||
yield _tx_to_activity(tx)
|
||||
# NB: the gains-offset DEPOSIT we used to emit here is superseded
|
||||
# by the manual snapshot push the CLI does after fetch() drains.
|
||||
# The snapshot sets per-fund quantity + cost basis directly, so
|
||||
# Wealthfolio computes growth from positions instead of needing a
|
||||
# fake cash entry. Old offset rows still in WF are corrected at
|
||||
# the dashboard layer by the dav_corrected view.
|
||||
|
||||
# The gains offset is always "as of now" so it reflects today's pot.
|
||||
# Only emit when the caller isn't windowing (full state).
|
||||
if since is None and before is None:
|
||||
offset = _gains_offset_activity(holdings, transactions, datetime.now(UTC))
|
||||
if offset is not None:
|
||||
yield offset
|
||||
|
||||
def fidelity_holdings_to_snapshot(
|
||||
holdings: list[FidelityHolding],
|
||||
total_real_contribution: Decimal,
|
||||
as_of: date,
|
||||
) -> ManualSnapshotPayload | None:
|
||||
"""Convert scraped holdings into a Wealthfolio manual snapshot payload.
|
||||
|
||||
Cost-basis allocation: PlanViewer doesn't expose historical purchase
|
||||
prices for individual fund unit buys, so we approximate per-fund
|
||||
cost basis by allocating the cumulative cash contribution
|
||||
proportionally to each fund's share of the current pot value. For
|
||||
the typical single-fund Meta scheme this is exact; if Viktor's plan
|
||||
later splits into multiple funds the proportional split is the
|
||||
least-wrong allocation we can compute from monthly snapshots.
|
||||
|
||||
cashBalances is set to zero — pension contributions flow straight
|
||||
into funds, the synthetic Wealthfolio "cash balance" only existed
|
||||
because of the old gains-offset DEPOSIT hack.
|
||||
"""
|
||||
if not holdings:
|
||||
return None
|
||||
total_value = sum((h.total_value for h in holdings), Decimal(0))
|
||||
if total_value <= 0:
|
||||
return None
|
||||
positions: list[SnapshotPosition] = []
|
||||
for h in holdings:
|
||||
share = h.total_value / total_value
|
||||
cost = (total_real_contribution * share).quantize(Decimal("0.01"))
|
||||
avg_cost = (cost / h.units).quantize(Decimal("0.0001")) if h.units > 0 else Decimal(0)
|
||||
positions.append(SnapshotPosition(
|
||||
symbol=h.fund_code,
|
||||
quantity=h.units,
|
||||
average_cost=avg_cost,
|
||||
total_cost_basis=cost,
|
||||
currency=h.currency,
|
||||
))
|
||||
return ManualSnapshotPayload(
|
||||
date=as_of,
|
||||
currency=_CCY,
|
||||
positions=positions,
|
||||
cash_balances={_CCY: Decimal(0)},
|
||||
)
|
||||
|
||||
|
||||
async def _scrape_live_session(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue