ingest: switch wealthfolio to pg-sync mirror reads
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
The previous SQLite-direct reader queried `holdings_snapshot` (singular) and `accounts.type` — both wrong against the live wealthfolio schema (plural `holdings_snapshots`, column `account_type`). It silently returned [] via the OperationalError fallback, leaving fire-planner with stale account snapshots. Switch to reading from the wealthfolio_sync PG mirror. The pg-sync sidecar (defined in infra/stacks/wealthfolio) hourly mirrors SQLite to Postgres with a clean schema. We read from `daily_account_valuation` which already has total_value, cost_basis, and explicit fx_rate_to_base per row — no JSON-decoding of position blobs. CLI ingest no longer takes --db-path (no kubectl-exec gymnastics); reads WEALTHFOLIO_SYNC_DB_CONNECTION_STRING from env. Falls back to DB_CONNECTION_STRING for single-DB local dev. 13 new tests covering: latest-per-account, multi-currency FX, explicit as-of, empty mirror, null cost_basis, full pipeline through upsert. 140 tests pass; mypy strict + ruff clean. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
parent
8880bd377f
commit
23d11bdf6d
5 changed files with 432 additions and 203 deletions
|
|
@ -1,97 +0,0 @@
|
|||
"""Wealthfolio ingest reads a real-shape sqlite and upserts cleanly."""
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from fire_planner.db import AccountSnapshot
|
||||
from fire_planner.ingest.wealthfolio import read_account_snapshots, upsert_snapshots
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def wealthfolio_db(tmp_path: Path) -> Path:
|
||||
"""Create a minimal sqlite mimicking Wealthfolio's schema."""
|
||||
db_path = tmp_path / "wealthfolio.db"
|
||||
conn = sqlite3.connect(db_path)
|
||||
cur = conn.cursor()
|
||||
cur.executescript("""
|
||||
CREATE TABLE accounts (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT,
|
||||
type TEXT,
|
||||
currency TEXT
|
||||
);
|
||||
CREATE TABLE holdings_snapshot (
|
||||
account_id TEXT,
|
||||
snapshot_date TEXT,
|
||||
symbol TEXT,
|
||||
market_value REAL,
|
||||
market_value_gbp REAL
|
||||
);
|
||||
INSERT INTO accounts VALUES ('acc-isa', 'ISA', 'ISA', 'GBP');
|
||||
INSERT INTO accounts VALUES ('acc-schwab', 'Schwab', 'BROKERAGE', 'USD');
|
||||
INSERT INTO holdings_snapshot VALUES ('acc-isa', '2026-04-25', 'VWRL', 200000, 200000);
|
||||
INSERT INTO holdings_snapshot VALUES ('acc-isa', '2026-04-25', 'BND', 100000, 100000);
|
||||
INSERT INTO holdings_snapshot VALUES ('acc-schwab', '2026-04-25', 'META', 800000, 640000);
|
||||
""")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return db_path
|
||||
|
||||
|
||||
def test_read_groups_holdings_per_account(wealthfolio_db: Path) -> None:
|
||||
rows = read_account_snapshots(wealthfolio_db)
|
||||
assert len(rows) == 2
|
||||
by_id = {r["account_id"]: r for r in rows}
|
||||
assert by_id["acc-isa"]["market_value_gbp"] == Decimal("300000")
|
||||
assert by_id["acc-schwab"]["market_value_gbp"] == Decimal("640000")
|
||||
assert by_id["acc-isa"]["snapshot_date"] == date(2026, 4, 25)
|
||||
|
||||
|
||||
def test_read_returns_empty_on_unknown_schema(tmp_path: Path) -> None:
|
||||
"""If the sqlite has a totally different shape, return [] rather
|
||||
than blow up — let the operator surface the warning."""
|
||||
db = tmp_path / "weird.db"
|
||||
conn = sqlite3.connect(db)
|
||||
conn.execute("CREATE TABLE foo (x INTEGER)")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
assert read_account_snapshots(db) == []
|
||||
|
||||
|
||||
def test_read_missing_file_raises(tmp_path: Path) -> None:
|
||||
with pytest.raises(FileNotFoundError):
|
||||
read_account_snapshots(tmp_path / "nope.db")
|
||||
|
||||
|
||||
async def test_upsert_inserts_new_rows(session: AsyncSession, wealthfolio_db: Path) -> None:
|
||||
rows = read_account_snapshots(wealthfolio_db)
|
||||
n = await upsert_snapshots(session, rows)
|
||||
await session.commit()
|
||||
assert n == 2
|
||||
persisted = (await session.execute(select(AccountSnapshot))).scalars().all()
|
||||
assert len(persisted) == 2
|
||||
by_id = {p.account_id: p for p in persisted}
|
||||
assert by_id["acc-isa"].market_value_gbp == Decimal("300000")
|
||||
|
||||
|
||||
async def test_upsert_is_idempotent(session: AsyncSession, wealthfolio_db: Path) -> None:
|
||||
rows = read_account_snapshots(wealthfolio_db)
|
||||
await upsert_snapshots(session, rows)
|
||||
await session.commit()
|
||||
# Run again — should still be 2 rows, not 4
|
||||
await upsert_snapshots(session, rows)
|
||||
await session.commit()
|
||||
persisted = (await session.execute(select(AccountSnapshot))).scalars().all()
|
||||
assert len(persisted) == 2
|
||||
|
||||
|
||||
async def test_upsert_zero_rows_is_noop(session: AsyncSession) -> None:
|
||||
n = await upsert_snapshots(session, [])
|
||||
assert n == 0
|
||||
Loading…
Add table
Add a link
Reference in a new issue