4da58fe56e
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
The account_snapshot cache fed /networth, /networth/history, and
/scenarios/{id}/progress. No CronJob populated it, so the cache had
drifted ~18 days behind the wealthfolio_sync mirror (last refresh
2026-05-09 via manual kubectl exec; Grafana reads wf_sync directly
and stayed fresh).
Switch to lazy refresh on read: each request to those endpoints now
checks MAX(account_snapshot.snapshot_date) — if it's older than
NETWORTH_CACHE_TTL_DAYS (default 1), pull fresh rows from wf_sync via
read_account_snapshots_from_pg and upsert. Idempotent under
concurrency (existing ON CONFLICT DO UPDATE).
Plumbing:
- Add get_wf_sync_session dependency that yields None when the wf_sync
factory isn't wired (keeps existing tests' behaviour: no refresh
attempted, they continue to seed account_snapshot directly).
- Wire wf_sync engine + session_factory in app.lifespan when
WEALTHFOLIO_SYNC_DB_CONNECTION_STRING is set.
- Centralise the staleness check in refresh_account_snapshots_if_stale.
Tests:
- 271 existing tests still green.
- Three new tests in test_api_networth_refresh.py covering: empty cache
triggers refresh, stale cache triggers refresh, fresh cache skips
refresh (asserts the wf_sync value is NOT served).
107 lines
3.9 KiB
Python
107 lines
3.9 KiB
Python
"""Upsert helper + lazy-refresh for the wealthfolio account snapshot cache.
|
|
|
|
`account_snapshot` is a disk cache of the live wealthfolio_sync mirror
|
|
(`daily_account_valuation` JOIN `accounts`). It's populated on demand by
|
|
`refresh_account_snapshots_if_stale` — invoked from the `/networth`,
|
|
`/networth/history`, and `/scenarios/{id}/progress` endpoints on every
|
|
request. If the cache is fresher than the TTL, the call is a no-op; if
|
|
not, we read from wf_sync, upsert, and serve the fresh data.
|
|
|
|
Prior to 2026-05-27 the cache was populated by a manual CLI ingest with
|
|
no automation, so it drifted up to 18 days behind reality (see
|
|
post-mortem of that day). Lazy refresh-on-read removes the manual step.
|
|
|
|
rows = await read_account_snapshots_from_pg(wf_session)
|
|
await upsert_snapshots(session, rows)
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
from datetime import date
|
|
from typing import Any
|
|
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from fire_planner.db import AccountSnapshot
|
|
from fire_planner.ingest.wealthfolio_pg import read_account_snapshots_from_pg
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
_DEFAULT_TTL_DAYS = 1
|
|
|
|
|
|
def _dialect_insert(session: AsyncSession) -> Any:
|
|
bind = session.get_bind()
|
|
if bind.dialect.name == "sqlite":
|
|
return sqlite_insert
|
|
return pg_insert
|
|
|
|
|
|
async def upsert_snapshots(session: AsyncSession, rows: list[dict[str, Any]]) -> int:
|
|
"""Idempotent upsert on `external_id` (one row per account per day)."""
|
|
if not rows:
|
|
return 0
|
|
insert_ = _dialect_insert(session)
|
|
stmt = insert_(AccountSnapshot).values(rows)
|
|
update_cols = {
|
|
"market_value": stmt.excluded.market_value,
|
|
"market_value_gbp": stmt.excluded.market_value_gbp,
|
|
"snapshot_date": stmt.excluded.snapshot_date,
|
|
"account_name": stmt.excluded.account_name,
|
|
"account_type": stmt.excluded.account_type,
|
|
"currency": stmt.excluded.currency,
|
|
"cost_basis_gbp": stmt.excluded.cost_basis_gbp,
|
|
}
|
|
stmt = stmt.on_conflict_do_update(index_elements=["external_id"], set_=update_cols)
|
|
await session.execute(stmt)
|
|
return len(rows)
|
|
|
|
|
|
def _ttl_days() -> int:
|
|
raw = os.environ.get("NETWORTH_CACHE_TTL_DAYS", "")
|
|
if not raw.strip():
|
|
return _DEFAULT_TTL_DAYS
|
|
try:
|
|
return max(0, int(raw))
|
|
except ValueError:
|
|
log.warning("NETWORTH_CACHE_TTL_DAYS=%r is not an int; using default %d",
|
|
raw, _DEFAULT_TTL_DAYS)
|
|
return _DEFAULT_TTL_DAYS
|
|
|
|
|
|
async def refresh_account_snapshots_if_stale(
|
|
session: AsyncSession,
|
|
wf_sync: AsyncSession | None,
|
|
*,
|
|
ttl_days: int | None = None,
|
|
now: date | None = None,
|
|
) -> bool:
|
|
"""Refresh `account_snapshot` from wf_sync if the cache is older than TTL.
|
|
|
|
Returns True if a refresh ran (rows were upserted), False if the cache
|
|
was already fresh or `wf_sync` was None (e.g. unconfigured environment).
|
|
|
|
The check uses `MAX(snapshot_date)` — if no rows exist yet the cache is
|
|
considered stale and a refresh fires. The upsert is idempotent so
|
|
concurrent requests racing on the same stale window don't conflict.
|
|
"""
|
|
if wf_sync is None:
|
|
return False
|
|
ttl = _ttl_days() if ttl_days is None else ttl_days
|
|
today = now or date.today()
|
|
latest = (await session.execute(select(func.max(AccountSnapshot.snapshot_date)))).scalar()
|
|
if latest is not None and (today - latest).days < ttl:
|
|
return False
|
|
rows = await read_account_snapshots_from_pg(wf_sync)
|
|
if not rows:
|
|
log.warning("refresh: wf_sync returned no rows; cache left as-is "
|
|
"(latest cached date=%s)", latest)
|
|
return False
|
|
n = await upsert_snapshots(session, rows)
|
|
await session.commit()
|
|
log.info("refresh: upserted %d snapshot row(s) from wf_sync (was latest=%s)", n, latest)
|
|
return True
|