Add finance_mysql provider + CLI for historical backfill

finance.position (171 rows, 2020-06-07 to 2025-12-19) is the only source
of InvestEngine + Schwab trade history pre-dating the broker-sync project.
This provider reads it once and pushes every row into the correct WF
account (.L tickers → IE ISA, others → Schwab).

Dedup: external_id = 'finance-mysql:position:<PK>' — idempotent on re-run.
Auth: aiomysql as MySQL root (user-authorized) against the standalone
mysql:8.4 in-cluster service.

New CLI: broker-sync finance-mysql-import
New tests: 5 unit tests covering route, symbol normalise, BUY/SELL
detection.

poetry run pytest -q   →  114 passed, 1 skipped
poetry run mypy        →  clean (aiomysql shielded with type: ignore)
poetry run ruff check  →  clean
This commit is contained in:
Viktor Barzin 2026-04-17 22:38:21 +00:00
parent 74b2179c83
commit a190875f63
6 changed files with 318 additions and 9 deletions

View file

@ -230,6 +230,71 @@ def invest_engine(
asyncio.run(_run())
@app.command("finance-mysql-import")
def finance_mysql_import(
wf_base_url: str = typer.Option(..., envvar="WF_BASE_URL"),
wf_username: str = typer.Option(..., envvar="WF_USERNAME"),
wf_password: str = typer.Option(..., envvar="WF_PASSWORD"),
wf_session_path: str = typer.Option("/data/wealthfolio_session.json",
envvar="WF_SESSION_PATH"),
db_host: str = typer.Option(..., envvar="FINANCE_DB_HOST"),
db_port: int = typer.Option(3306, envvar="FINANCE_DB_PORT"),
db_user: str = typer.Option(..., envvar="FINANCE_DB_USER"),
db_password: str = typer.Option(..., envvar="FINANCE_DB_PASSWORD"),
db_name: str = typer.Option("finance", envvar="FINANCE_DB_NAME"),
data_dir: str = typer.Option("/data", envvar="BROKER_SYNC_DATA_DIR"),
) -> None:
"""One-shot backfill: read the retired finance app's MySQL position table
and push every row into the correct Wealthfolio account (IE for .L
tickers, Schwab for US tickers). Idempotent via dedup."""
from broker_sync.dedup import SyncRecordStore
from broker_sync.pipeline import sync_provider_to_wealthfolio
from broker_sync.providers.finance_mysql import (
FinanceMySQLCreds,
FinanceMySQLProvider,
)
from broker_sync.sinks.wealthfolio import WealthfolioSink
_setup_logging()
data = Path(data_dir)
data.mkdir(parents=True, exist_ok=True)
async def _run() -> None:
sink = WealthfolioSink(
base_url=wf_base_url,
username=wf_username,
password=wf_password,
session_path=wf_session_path,
)
provider = FinanceMySQLProvider(
FinanceMySQLCreds(
host=db_host,
port=db_port,
user=db_user,
password=db_password,
database=db_name,
))
dedup = SyncRecordStore(data / "sync.db")
try:
if not Path(wf_session_path).exists():
await sink.login()
result = await sync_provider_to_wealthfolio(
provider=provider,
sink=sink,
dedup=dedup,
)
finally:
await sink.close()
typer.echo(f"finance-mysql: fetched={result.fetched} "
f"new={result.new_after_dedup} "
f"imported={result.imported} "
f"failed={result.failed}")
if result.failed > 0:
sys.exit(1)
asyncio.run(_run())
@app.command("imap-ingest")
def imap_ingest(
wf_base_url: str = typer.Option(..., envvar="WF_BASE_URL"),

View file

@ -0,0 +1,144 @@
"""Backfill-from-finance provider.
The retired `finance` app's MySQL has a `position` table with 5+ years of
InvestEngine + Schwab trade history (2020 onwards) that the broker-sync
pipeline otherwise can't reconstruct (IE's emails only go back to when
Viktor started receiving them; Schwab emails are sparse). This provider
reads that table once and emits canonical Activities so a full-history
backfill into Wealthfolio is possible.
Ticker routing to Wealthfolio accounts:
*.L (VUAG.L, VUSA.L, etc.) -> InvestEngine ISA (GBP)
everything else (META, *_US_EQ) -> Schwab (US workplace, USD)
Deduplication: the finance.position PK (a giant numeric string) goes into
external_id verbatim, so re-runs are idempotent against the sync_record
store.
"""
from __future__ import annotations
import logging
from collections.abc import AsyncIterator
from datetime import UTC, datetime
from decimal import Decimal
from typing import NamedTuple
import aiomysql # type: ignore[import-untyped]
from broker_sync.models import Account, AccountType, Activity, ActivityType
log = logging.getLogger(__name__)
IE_ACCOUNT_ID = "invest-engine-primary"
SCHWAB_ACCOUNT_ID = "schwab-workplace"
class FinanceMySQLCreds(NamedTuple):
host: str
port: int
user: str
password: str
database: str
def _route(ticker: str) -> tuple[str, AccountType, str]:
"""Return (account_id, account_type, currency) for a raw ticker."""
if ticker.endswith(".L"):
return IE_ACCOUNT_ID, AccountType.ISA, "GBP"
return SCHWAB_ACCOUNT_ID, AccountType.GIA, "USD"
def _normalise_symbol(ticker: str) -> str:
"""Strip finance-app quirks so the output symbol matches T212/Wealthfolio."""
# VUAG.L -> VUAG (LSE handled by Wealthfolio's exchange_mic resolution)
if ticker.endswith(".L"):
return ticker[:-2]
# FLME_US_EQ -> FLME (Trading212-style suffix leaked into the old finance DB)
if ticker.endswith("_US_EQ"):
return ticker[:-6]
if ticker.endswith("_EQ"):
return ticker[:-3]
return ticker
def _row_to_activity(row: dict[str, object]) -> Activity:
ticker = str(row["ticker"])
account_id, account_type, default_ccy = _route(ticker)
raw_qty = Decimal(str(row["num_shares"]))
activity_type = ActivityType.BUY if raw_qty > 0 else ActivityType.SELL
# buy_date from MySQL comes back as datetime (aiomysql converts)
dt = row["buy_date"]
if isinstance(dt, datetime):
date = dt if dt.tzinfo else dt.replace(tzinfo=UTC)
else:
date = datetime.fromisoformat(str(dt)).replace(tzinfo=UTC)
currency_raw = row.get("currency")
currency = str(currency_raw) if currency_raw else default_ccy
return Activity(
external_id=f"finance-mysql:position:{row['id']}",
account_id=account_id,
account_type=account_type,
date=date,
activity_type=activity_type,
symbol=_normalise_symbol(ticker),
quantity=abs(raw_qty),
unit_price=Decimal(str(row["buy_price"])),
currency=currency,
notes=f"finance-mysql:{ticker}",
)
class FinanceMySQLProvider:
"""Read-only backfill from the retired finance MySQL `position` table."""
name = "finance-mysql"
def __init__(self, creds: FinanceMySQLCreds) -> None:
self._creds = creds
def accounts(self) -> list[Account]:
return [
Account(
id=IE_ACCOUNT_ID,
name="InvestEngine ISA",
account_type=AccountType.ISA,
currency="GBP",
provider="invest-engine",
),
Account(
id=SCHWAB_ACCOUNT_ID,
name="Schwab (US workplace)",
account_type=AccountType.GIA,
currency="USD",
provider="schwab",
),
]
async def fetch(
self,
*,
since: datetime | None = None,
before: datetime | None = None,
) -> AsyncIterator[Activity]:
conn = await aiomysql.connect(
host=self._creds.host,
port=self._creds.port,
user=self._creds.user,
password=self._creds.password,
db=self._creds.database,
autocommit=True,
)
try:
async with conn.cursor(aiomysql.DictCursor) as cur:
await cur.execute("SELECT id, ticker, buy_price, num_shares, currency, buy_date, "
"account_id FROM position ORDER BY buy_date ASC")
rows = await cur.fetchall()
log.info("finance-mysql: %d position rows", len(rows))
for row in rows:
activity = _row_to_activity(row)
if since is not None and activity.date < since:
continue
if before is not None and activity.date >= before:
continue
yield activity
finally:
conn.close()

View file

@ -243,11 +243,9 @@ class WealthfolioSink:
err_msg = summary.get("errorMessage") or "no errorMessage"
skipped = int(summary.get("skipped", 0))
dupes = int(summary.get("duplicates", 0))
raise ImportValidationError(
f"Wealthfolio /import persisted {imported_n}/{total_n} "
f"(skipped={skipped} duplicates={dupes}). "
f"errorMessage: {err_msg}"
)
raise ImportValidationError(f"Wealthfolio /import persisted {imported_n}/{total_n} "
f"(skipped={skipped} duplicates={dupes}). "
f"errorMessage: {err_msg}")
# Legacy silent-drop guard for no-summary responses.
elif valid_rows and not got:
first_warn = next(
@ -257,6 +255,6 @@ class WealthfolioSink:
raise ImportValidationError(
f"Wealthfolio /import silently dropped all {len(valid_rows)} rows. "
f"First checked row: {checked[0] if checked else 'none'}. "
f"First warning: {first_warn}"
)
return got
f"First warning: {first_warn}")
assert isinstance(got, list)
return [r for r in got if isinstance(r, dict)]