fidelity: push per-fund manual snapshot instead of gains-offset DEPOSIT
PlanViewer's DisplayValuation.action JSON already gives us current fund units + unit price; we were parsing it and throwing it away, emitting only a single 'unrealised-gains-offset' DEPOSIT to make Wealthfolio's totals match the dashboard. That hack double-counted the gain as a cash contribution, hiding £35k of pension growth from every contribution/growth/ROI panel. New flow: - FidelityPlanViewerProvider exposes last_holdings + last_total_contribution after fetch() drains. - fidelity-ingest CLI converts to a ManualSnapshotPayload (cost basis allocated proportionally by current fund value share) and posts to WF /api/v1/snapshots/import. WF auto-creates unknown fund symbols with kind=INVESTMENT, quoteMode=MANUAL, quoteCcy=GBP. - The gains-offset emission is removed entirely. Historical offset rows already in WF are corrected at the dashboard layer by the dav_corrected view shipped in infra@2841347e. WealthfolioSink gains push_manual_snapshots() + ManualSnapshotPayload / SnapshotPosition wire types. 11 sink tests (3 new) + 9 fidelity provider tests (2 changed, 1 new) all green; mypy + ruff clean.
This commit is contained in:
parent
5adc4a7ba4
commit
cb159e17d9
5 changed files with 339 additions and 60 deletions
|
|
@ -438,6 +438,15 @@ def fidelity_ingest(
|
|||
sys.exit(2)
|
||||
|
||||
async def _run() -> None:
|
||||
from datetime import date as _date_t
|
||||
|
||||
from broker_sync.providers.fidelity_planviewer import (
|
||||
ACCOUNT_ID as FID_ACCOUNT_ID,
|
||||
)
|
||||
from broker_sync.providers.fidelity_planviewer import (
|
||||
fidelity_holdings_to_snapshot,
|
||||
)
|
||||
|
||||
sink = WealthfolioSink(
|
||||
base_url=wf_base_url,
|
||||
username=wf_username,
|
||||
|
|
@ -455,12 +464,31 @@ def fidelity_ingest(
|
|||
result = await sync_provider_to_wealthfolio(
|
||||
provider=provider, sink=sink, dedup=dedup, since=since,
|
||||
)
|
||||
# PlanViewer has no historical per-fund unit-price feed, so
|
||||
# the Activity stream above only carries cash flows. The
|
||||
# current-pot fund positions captured in the same scrape get
|
||||
# pushed via /api/v1/snapshots/import so per-fund quantity +
|
||||
# cost basis land in WF (and propagate to the wealth
|
||||
# dashboard's Positions table via pg-sync).
|
||||
snapshot_imported = 0
|
||||
if provider.last_holdings:
|
||||
snapshot = fidelity_holdings_to_snapshot(
|
||||
holdings=provider.last_holdings,
|
||||
total_real_contribution=provider.last_total_contribution,
|
||||
as_of=_date_t.today(),
|
||||
)
|
||||
if snapshot is not None:
|
||||
push_result = await sink.push_manual_snapshots(
|
||||
account_id=FID_ACCOUNT_ID, snapshots=[snapshot],
|
||||
)
|
||||
snapshot_imported = int(push_result.get("snapshotsImported", 0))
|
||||
finally:
|
||||
await sink.close()
|
||||
typer.echo(f"fidelity-ingest: fetched={result.fetched} "
|
||||
f"new={result.new_after_dedup} "
|
||||
f"imported={result.imported} "
|
||||
f"failed={result.failed}")
|
||||
f"failed={result.failed} "
|
||||
f"snapshots={snapshot_imported}")
|
||||
if result.failed > 0:
|
||||
sys.exit(1)
|
||||
|
||||
|
|
|
|||
|
|
@ -16,21 +16,28 @@ We keep a Playwright-maintained session via ``storage_state.json``:
|
|||
fund holdings. On 401/idle-timeout we raise
|
||||
:class:`FidelitySessionError` so Prometheus alerts Viktor to re-seed.
|
||||
|
||||
## Emitted Activity shape
|
||||
## Emitted Activity / snapshot shape
|
||||
|
||||
- One ``DEPOSIT`` per cash-impacting transaction (Regular Premium, Single
|
||||
Premium, rebate, etc.). ``external_id = fidelity:tx:<sha256[:16]>``.
|
||||
- One synthetic ``DEPOSIT`` for unrealised gains so WF's Net Worth matches
|
||||
the Fidelity dashboard. ``external_id =
|
||||
fidelity:gains:<YYYY-MM-DD>``.
|
||||
- Bulk Switches / Fund Switches are skipped (no cash movement).
|
||||
- After the activity stream drains, the ``fidelity-ingest`` CLI calls
|
||||
``WealthfolioSink.push_manual_snapshots`` with one ``ManualSnapshotPayload``
|
||||
per fund holding (today's date, units + cost basis allocated
|
||||
proportionally to fund value share). This sets per-fund quantity and
|
||||
cost basis in WF so the dashboard Positions table shows the pension
|
||||
funds alongside the brokerage assets.
|
||||
- The old synthetic ``fidelity:gains:<date>`` DEPOSIT is no longer
|
||||
emitted — the snapshot supersedes it. Old offset rows that landed
|
||||
before this change are corrected at the dashboard layer by the
|
||||
``dav_corrected`` PG view (``infra/stacks/wealthfolio/main.tf``).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import UTC, datetime
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from typing import Any, NamedTuple
|
||||
|
|
@ -42,6 +49,7 @@ from broker_sync.providers.parsers.fidelity import (
|
|||
parse_transactions_html,
|
||||
parse_valuation_json,
|
||||
)
|
||||
from broker_sync.sinks.wealthfolio import ManualSnapshotPayload, SnapshotPosition
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -86,37 +94,6 @@ def _tx_to_activity(tx: FidelityCashTx) -> Activity:
|
|||
)
|
||||
|
||||
|
||||
def _gains_offset_activity(
|
||||
holdings: list[FidelityHolding],
|
||||
transactions: list[FidelityCashTx],
|
||||
as_of: datetime,
|
||||
) -> Activity | None:
|
||||
"""Create a synthetic DEPOSIT/WITHDRAWAL so WF Net Worth matches the
|
||||
Fidelity dashboard's reported pot value.
|
||||
|
||||
The offset carries a date-derived external_id so monthly runs refresh
|
||||
the same synthetic entry rather than stacking duplicates.
|
||||
"""
|
||||
if not holdings:
|
||||
return None
|
||||
total_value = sum((h.total_value for h in holdings), Decimal(0))
|
||||
total_contrib = sum((t.amount for t in transactions), Decimal(0))
|
||||
gains = total_value - total_contrib
|
||||
if gains == 0:
|
||||
return None
|
||||
return Activity(
|
||||
external_id=f"fidelity:gains:{as_of.date().isoformat()}",
|
||||
account_id=ACCOUNT_ID,
|
||||
account_type=AccountType.WORKPLACE_PENSION,
|
||||
date=as_of,
|
||||
activity_type=ActivityType.DEPOSIT if gains > 0 else ActivityType.WITHDRAWAL,
|
||||
currency=_CCY,
|
||||
amount=abs(gains),
|
||||
notes=(f"fidelity-planviewer:unrealised-gains-offset "
|
||||
f"(pot=£{total_value}, contrib=£{total_contrib})"),
|
||||
)
|
||||
|
||||
|
||||
class FidelityPlanViewerProvider:
|
||||
"""Read-only provider against Fidelity UK PlanViewer.
|
||||
|
||||
|
|
@ -125,11 +102,18 @@ class FidelityPlanViewerProvider:
|
|||
- ``fetch(since, before)`` opens a Playwright session with the saved
|
||||
storage_state, navigates to the transaction-history page with a wide
|
||||
date range, scrapes the table, and intercepts the valuation XHR.
|
||||
- After ``fetch()`` completes, ``last_holdings`` holds the per-fund
|
||||
unit positions captured in the same scrape — used by the
|
||||
``fidelity-ingest`` CLI to push a manual snapshot to Wealthfolio
|
||||
so per-fund quantities + cost basis land in the Positions table
|
||||
(the activity stream alone only carries cash flows).
|
||||
"""
|
||||
name = "fidelity-planviewer"
|
||||
|
||||
def __init__(self, creds: FidelityCreds) -> None:
|
||||
self._creds = creds
|
||||
self.last_holdings: list[FidelityHolding] = []
|
||||
self.last_total_contribution: Decimal = Decimal(0)
|
||||
|
||||
def accounts(self) -> list[Account]:
|
||||
return [
|
||||
|
|
@ -162,19 +146,72 @@ class FidelityPlanViewerProvider:
|
|||
log.info("fidelity: parsed %d transactions, %d holdings",
|
||||
len(transactions), len(holdings))
|
||||
|
||||
# Snapshot the per-fund holdings for the CLI to push as a manual
|
||||
# holdings_snapshot after this generator drains. Wealthfolio's
|
||||
# activity model can't represent pension fund unit purchases (no
|
||||
# per-purchase price feed from PlanViewer), so we record current
|
||||
# state via /api/v1/snapshots/import instead.
|
||||
self.last_holdings = holdings
|
||||
self.last_total_contribution = sum(
|
||||
(t.amount for t in transactions), Decimal(0)
|
||||
)
|
||||
|
||||
for tx in transactions:
|
||||
if since is not None and tx.date < since:
|
||||
continue
|
||||
if before is not None and tx.date >= before:
|
||||
continue
|
||||
yield _tx_to_activity(tx)
|
||||
# NB: the gains-offset DEPOSIT we used to emit here is superseded
|
||||
# by the manual snapshot push the CLI does after fetch() drains.
|
||||
# The snapshot sets per-fund quantity + cost basis directly, so
|
||||
# Wealthfolio computes growth from positions instead of needing a
|
||||
# fake cash entry. Old offset rows still in WF are corrected at
|
||||
# the dashboard layer by the dav_corrected view.
|
||||
|
||||
# The gains offset is always "as of now" so it reflects today's pot.
|
||||
# Only emit when the caller isn't windowing (full state).
|
||||
if since is None and before is None:
|
||||
offset = _gains_offset_activity(holdings, transactions, datetime.now(UTC))
|
||||
if offset is not None:
|
||||
yield offset
|
||||
|
||||
def fidelity_holdings_to_snapshot(
|
||||
holdings: list[FidelityHolding],
|
||||
total_real_contribution: Decimal,
|
||||
as_of: date,
|
||||
) -> ManualSnapshotPayload | None:
|
||||
"""Convert scraped holdings into a Wealthfolio manual snapshot payload.
|
||||
|
||||
Cost-basis allocation: PlanViewer doesn't expose historical purchase
|
||||
prices for individual fund unit buys, so we approximate per-fund
|
||||
cost basis by allocating the cumulative cash contribution
|
||||
proportionally to each fund's share of the current pot value. For
|
||||
the typical single-fund Meta scheme this is exact; if Viktor's plan
|
||||
later splits into multiple funds the proportional split is the
|
||||
least-wrong allocation we can compute from monthly snapshots.
|
||||
|
||||
cashBalances is set to zero — pension contributions flow straight
|
||||
into funds, the synthetic Wealthfolio "cash balance" only existed
|
||||
because of the old gains-offset DEPOSIT hack.
|
||||
"""
|
||||
if not holdings:
|
||||
return None
|
||||
total_value = sum((h.total_value for h in holdings), Decimal(0))
|
||||
if total_value <= 0:
|
||||
return None
|
||||
positions: list[SnapshotPosition] = []
|
||||
for h in holdings:
|
||||
share = h.total_value / total_value
|
||||
cost = (total_real_contribution * share).quantize(Decimal("0.01"))
|
||||
avg_cost = (cost / h.units).quantize(Decimal("0.0001")) if h.units > 0 else Decimal(0)
|
||||
positions.append(SnapshotPosition(
|
||||
symbol=h.fund_code,
|
||||
quantity=h.units,
|
||||
average_cost=avg_cost,
|
||||
total_cost_basis=cost,
|
||||
currency=h.currency,
|
||||
))
|
||||
return ManualSnapshotPayload(
|
||||
date=as_of,
|
||||
currency=_CCY,
|
||||
positions=positions,
|
||||
cash_balances={_CCY: Decimal(0)},
|
||||
)
|
||||
|
||||
|
||||
async def _scrape_live_session(
|
||||
|
|
|
|||
|
|
@ -2,7 +2,9 @@ from __future__ import annotations
|
|||
|
||||
import json
|
||||
from collections.abc import Iterable
|
||||
from datetime import UTC
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, date
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
|
@ -14,6 +16,7 @@ _LOGIN_PATH = "/api/v1/auth/login"
|
|||
_ACCOUNTS_PATH = "/api/v1/accounts"
|
||||
_IMPORT_CHECK = "/api/v1/activities/import/check"
|
||||
_IMPORT_REAL = "/api/v1/activities/import"
|
||||
_SNAPSHOTS_IMPORT = "/api/v1/snapshots/import"
|
||||
|
||||
|
||||
class WealthfolioError(Exception):
|
||||
|
|
@ -262,3 +265,83 @@ class WealthfolioSink:
|
|||
f"First warning: {first_warn}")
|
||||
assert isinstance(got, list)
|
||||
return [r for r in got if isinstance(r, dict)]
|
||||
|
||||
# -- manual holdings snapshots --
|
||||
|
||||
async def push_manual_snapshots(
|
||||
self,
|
||||
account_id: str,
|
||||
snapshots: list[ManualSnapshotPayload],
|
||||
) -> dict[str, Any]:
|
||||
"""Push manual holdings snapshots to /api/v1/snapshots/import.
|
||||
|
||||
Each snapshot carries a date + per-fund positions + cash balances.
|
||||
Wealthfolio auto-creates any unknown asset symbol with
|
||||
``kind=INVESTMENT, quoteMode=MANUAL, quoteCcy=<currency>`` and uses
|
||||
the snapshot to derive holdings + valuation for that date — bypassing
|
||||
the activity-ledger derivation entirely for the targeted day.
|
||||
|
||||
Used by the Fidelity provider since PlanViewer exposes current
|
||||
fund units + price but no per-trade history. Re-imports for the
|
||||
same (account, date) overwrite in place.
|
||||
"""
|
||||
if not snapshots:
|
||||
return {"snapshotsImported": 0, "snapshotsFailed": 0, "errors": []}
|
||||
body = {
|
||||
"accountId": account_id,
|
||||
"snapshots": [_snapshot_to_payload(s) for s in snapshots],
|
||||
}
|
||||
resp = await self._request("POST", _SNAPSHOTS_IMPORT, json=body)
|
||||
if resp.status_code >= 400:
|
||||
try:
|
||||
payload = resp.json()
|
||||
except Exception:
|
||||
payload = {"raw": resp.text}
|
||||
raise WealthfolioError(
|
||||
f"Wealthfolio /snapshots/import rejected: {payload}")
|
||||
result = resp.json()
|
||||
assert isinstance(result, dict)
|
||||
failed = int(result.get("snapshotsFailed", 0))
|
||||
if failed > 0:
|
||||
raise WealthfolioError(
|
||||
f"Wealthfolio /snapshots/import: {failed} snapshot(s) failed; "
|
||||
f"errors={result.get('errors')}")
|
||||
return result
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SnapshotPosition:
|
||||
"""A per-fund position row in a Wealthfolio manual snapshot."""
|
||||
symbol: str
|
||||
quantity: Decimal
|
||||
average_cost: Decimal
|
||||
total_cost_basis: Decimal
|
||||
currency: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ManualSnapshotPayload:
|
||||
"""Sink-facing snapshot row. Mirrors the JSON shape WF expects."""
|
||||
date: date
|
||||
currency: str
|
||||
positions: list[SnapshotPosition]
|
||||
cash_balances: dict[str, Decimal]
|
||||
|
||||
|
||||
def _snapshot_to_payload(s: ManualSnapshotPayload) -> dict[str, Any]:
|
||||
"""Serialise a ManualSnapshotPayload into WF's import wire format."""
|
||||
return {
|
||||
"date": s.date.isoformat(),
|
||||
"currency": s.currency,
|
||||
"positions": [
|
||||
{
|
||||
"symbol": p.symbol,
|
||||
"quantity": format(p.quantity, "f"),
|
||||
"averageCost": format(p.average_cost, "f"),
|
||||
"totalCostBasis": format(p.total_cost_basis, "f"),
|
||||
"currency": p.currency,
|
||||
}
|
||||
for p in s.positions
|
||||
],
|
||||
"cashBalances": {k: format(v, "f") for k, v in s.cash_balances.items()},
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue