broker-sync/broker_sync/providers/invest_engine.py

"""InvestEngine Bearer-token provider.

InvestEngine (https://investengine.com) has no public API and requires MFA
(push-approval via the IE mobile app) on every login. We work around that
by having Viktor log in manually in a browser, copy the Bearer token out
of devtools, and paste it into Vault. This module consumes that token.

## Vault schema — `secret/broker-sync`

The following keys are expected in Vault. The caller (CLI/pipeline) reads
them and hands the values to the constructor. This module does NOT read
Vault directly — it stays testable.

- ``investengine_bearer_token`` — the Bearer string Viktor pastes from
  devtools. Expires on IE's refresh schedule (~monthly based on observed
  behaviour).
- ``investengine_token_expires_at`` — ISO-8601 timestamp Viktor sets WHEN
  HE PASTES the token. Used to alert 3 days before expiry and to fail
  fast before making a request with a known-dead token.
- ``investengine_refresh_token`` *(optional)* — if Viktor's devtools
  capture included a refresh token, we may attempt auto-refresh in a
  future iteration. Not used yet.

## Version probing

IE rolls the ``/api/v0.3X/`` version every 4-6 weeks. ``v0.29`` and
``v0.30`` were ``410 Gone`` at research time; ``v0.31`` and ``v0.32``
were live (401 without auth — the correct "auth required" signal);
``v0.33`` and later returned 404 (not yet created). The probe starts
at ``v0.32`` and walks forward on 410, stopping at the first version
that returns anything other than 410 (401/404/200/etc).
"""
from __future__ import annotations

import logging
from collections.abc import AsyncIterator
from datetime import UTC, datetime
from decimal import Decimal
from typing import Any

import httpx

from broker_sync.models import Account, AccountType, Activity, ActivityType

log = logging.getLogger(__name__)

_DEFAULT_BASE_URL = "https://investengine.com"
_USER_AGENT = "broker-sync/0.1 (+https://github.com/ViktorBarzin/broker-sync)"

# Version probe starts here. If this minor bumps past a live version,
# update the constant rather than relying on re-probe every process.
_START_VERSION_MINOR = 32
# Hard cap on how far forward we probe before giving up. IE has never
# skipped versions; this is defence against a runaway loop.
_MAX_VERSION_MINOR = 60

# One logical account — IE has only ever had a single ISA per user.
_ACCOUNT_ID = "invest-engine-primary"
_ACCOUNT = Account(
    id=_ACCOUNT_ID,
    name="InvestEngine ISA",
    account_type=AccountType.ISA,
    currency="GBP",
    provider="invest-engine",
)

# Type-string → ActivityType. Exact IE strings are UNVERIFIED — we match
# case-insensitively and fall back to substring checks for the common
# variants ("DEPOSIT", "WITHDRAWAL").
_EXACT_TYPE_MAP: dict[str, ActivityType] = {
    "BUY": ActivityType.BUY,
    "SELL": ActivityType.SELL,
    "DIVIDEND": ActivityType.DIVIDEND,
    "INTEREST": ActivityType.INTEREST,
    "DEPOSIT": ActivityType.DEPOSIT,
    "WITHDRAWAL": ActivityType.WITHDRAWAL,
    "FEE": ActivityType.FEE,
    "TAX": ActivityType.TAX,
}


class InvestEngineError(Exception):
    """Any non-retryable InvestEngine API failure."""


class InvestEngineTokenExpiredError(InvestEngineError):
    """Bearer token rejected by IE (401). Viktor must paste a new token."""


class InvestEngineVersionError(InvestEngineError):
    """Could not find a live /api/v0.3X/ version on the IE backend."""


def _version_path(minor: int) -> str:
    return f"/api/v0.{minor}/"


async def _probe_version(
    client: httpx.AsyncClient,
    *,
    start_minor: int = _START_VERSION_MINOR,
    max_minor: int = _MAX_VERSION_MINOR,
) -> int:
    """Walk forward from ``start_minor`` looking for a live API version.

    Returns the minor number of the first version that is NOT 410 Gone.
    A live version is one IE currently accepts; a 401 response is the
    expected "auth required" signal and confirms the version is serving
    traffic. Raises :class:`InvestEngineVersionError` if no live version
    is found before ``max_minor``.
    """
    for minor in range(start_minor, max_minor + 1):
        resp = await client.get(_version_path(minor))
        if resp.status_code != 410:
            return minor
    raise InvestEngineVersionError(
        f"No live /api/v0.3X/ between v0.{start_minor} and v0.{max_minor}")


def _parse_iso(ts: str) -> datetime:
    """Accept both ``...Z`` and ``...+HH:MM`` suffixes."""
    return datetime.fromisoformat(ts.replace("Z", "+00:00"))


def _opt_decimal(raw: Any) -> Decimal | None:
    if raw is None:
        return None
    return Decimal(str(raw))


def _classify_type(raw_type: str) -> ActivityType | None:
    """Map a raw IE type string to ActivityType, or None if unknown."""
    upper = raw_type.upper()
    exact = _EXACT_TYPE_MAP.get(upper)
    if exact is not None:
        return exact
    if "DEPOSIT" in upper:
        return ActivityType.DEPOSIT
    if "WITHDRAWAL" in upper or "WITHDRAW" in upper:
        return ActivityType.WITHDRAWAL
    return None


def _transaction_to_activity(raw: dict[str, Any]) -> Activity | None:
    """Turn one IE transaction dict into a canonical Activity.

    The IE response shape is UNVERIFIED — these assumptions WILL need
    review once Viktor pastes a live token:

    - ``id``: string or int (cast to str for ``external_id``)
    - ``type``: upper-case string — ``BUY``/``SELL``/``DIVIDEND``/etc.
    - ``symbol``: ticker string (may be missing on cash events)
    - ``quantity`` / ``price`` / ``amount``: numeric-in-a-string or float
    - ``currency``: ISO code; default ``GBP`` for an ISA
    - ``date``: ISO-8601 with ``Z`` or offset suffix

    Unknown type strings are logged at WARNING and skipped — silent
    misclassification would corrupt tax reporting, so we refuse to guess.
    """
    raw_type = str(raw.get("type", ""))
    activity_type = _classify_type(raw_type)
    if activity_type is None:
        log.warning(
            "invest-engine: skipping transaction id=%s with unknown type=%r",
            raw.get("id"),
            raw_type,
        )
        return None

    txn_id = raw.get("id")
    if txn_id is None:
        log.warning("invest-engine: skipping transaction with missing id: %r", raw)
        return None

    currency = str(raw.get("currency") or "GBP")
    date_str = raw.get("date") or raw.get("created_at") or raw.get("timestamp")
    if not isinstance(date_str, str):
        log.warning("invest-engine: skipping txn id=%s — no parseable date", txn_id)
        return None

    quantity = _opt_decimal(raw.get("quantity"))
    unit_price = _opt_decimal(raw.get("price") or raw.get("unit_price"))
    amount = _opt_decimal(raw.get("amount") or raw.get("value"))
    fee = _opt_decimal(raw.get("fee")) or Decimal("0")
    symbol_raw = raw.get("symbol") or raw.get("ticker")
    symbol = str(symbol_raw) if symbol_raw else None

    return Activity(
        external_id=f"invest-engine:{txn_id}",
        account_id=_ACCOUNT_ID,
        account_type=AccountType.ISA,
        date=_parse_iso(date_str),
        activity_type=activity_type,
        currency=currency,
        symbol=symbol,
        quantity=quantity,
        unit_price=unit_price,
        amount=amount,
        fee=fee,
    )


def _extract_list(page: dict[str, Any]) -> list[dict[str, Any]]:
    """Handle both ``{results: [...]}`` and ``{data: [...]}`` shapes."""
    for key in ("results", "data"):
        items = page.get(key)
        if isinstance(items, list):
            return [i for i in items if isinstance(i, dict)]
    return []


def _extract_next(page: dict[str, Any]) -> str | None:
    """Pull the next-page URL from either DRF ``next`` or JSON:API ``meta.next_page``."""
    nxt = page.get("next")
    if isinstance(nxt, str) and nxt:
        return nxt
    meta = page.get("meta")
    if isinstance(meta, dict):
        meta_nxt = meta.get("next_page") or meta.get("next")
        if isinstance(meta_nxt, str) and meta_nxt:
            return meta_nxt
    return None


class InvestEngineProvider:
    """Concrete Provider for InvestEngine.

    Only one logical account per user (one ISA, GBP). The token expiry is
    tracked at the Python layer: the CLI alerts 3 days before expiry, and
    this class fails fast if the clock says the token is already dead
    (cheaper than burning a request for the same 401).
    """

    name = "invest-engine"

    def __init__(
        self,
        *,
        bearer_token: str,
        token_expires_at: datetime,
        base_url: str = _DEFAULT_BASE_URL,
        transport: httpx.AsyncBaseTransport | None = None,
    ) -> None:
        self._token = bearer_token
        self._token_expires_at = token_expires_at
        self._client = httpx.AsyncClient(
            base_url=base_url,
            timeout=30.0,
            transport=transport,
            headers={
                "Authorization": f"Bearer {bearer_token}",
                "User-Agent": _USER_AGENT,
            },
        )
        self._version_minor: int | None = None

    def accounts(self) -> list[Account]:
        return [_ACCOUNT]

    async def close(self) -> None:
        await self._client.aclose()

    async def _active_version(self, *, force: bool = False) -> int:
        """Cache the live API minor. Set ``force=True`` after a 410 to re-probe."""
        if self._version_minor is not None and not force:
            return self._version_minor
        start = (self._version_minor + 1) if force and self._version_minor else _START_VERSION_MINOR
        minor = await _probe_version(self._client, start_minor=start)
        self._version_minor = minor
        return minor

    async def _request_json(self, path: str, params: dict[str, str] | None = None) -> Any:
        """GET ``path`` (relative), return JSON. Handles one 410 re-probe retry."""
        resp = await self._client.get(path, params=params)
        if resp.status_code == 401:
            raise InvestEngineTokenExpiredError(
                f"InvestEngine rejected Bearer token (HTTP 401 on {path}); "
                f"token_expires_at={self._token_expires_at.isoformat()}. "
                f"Viktor must paste a new token into Vault.")
        if resp.status_code == 410:
            # Version rolled mid-session. Re-probe, retarget path, retry once.
            old_minor = self._version_minor
            new_minor = await self._active_version(force=True)
            if old_minor is not None and new_minor != old_minor:
                new_path = path.replace(f"/v0.{old_minor}/", f"/v0.{new_minor}/", 1)
                retry = await self._client.get(new_path, params=params)
                if retry.status_code == 401:
                    raise InvestEngineTokenExpiredError(f"InvestEngine 401 on retry {new_path}")
                if retry.status_code != 200:
                    raise InvestEngineError(
                        f"InvestEngine {new_path} HTTP {retry.status_code} after re-probe")
                return retry.json()
            raise InvestEngineError(f"InvestEngine 410 on {path} and no newer version")
        if resp.status_code != 200:
            raise InvestEngineError(
                f"InvestEngine {path} HTTP {resp.status_code}: {resp.text[:200]}")
        return resp.json()

    async def fetch(
        self,
        *,
        since: datetime | None = None,
        before: datetime | None = None,
    ) -> AsyncIterator[Activity]:
        # Fail fast if the token is already known-dead.
        if self._token_expires_at <= datetime.now(UTC):
            raise InvestEngineTokenExpiredError(
                f"InvestEngine token expired at {self._token_expires_at.isoformat()} — "
                f"Viktor must paste a new token.")

        version = await self._active_version()
        portfolio_ids = await self._list_portfolio_ids(version)
        for pid in portfolio_ids:
            async for activity in self._fetch_portfolio(version, pid, since, before):
                yield activity

    async def _list_portfolio_ids(self, version: int) -> list[str]:
        """Walk `/portfolios/` pagination and return the list of ids."""
        ids: list[str] = []
        path: str | None = f"/api/v0.{version}/portfolios/"
        while path is not None:
            page = await self._request_json(path)
            if not isinstance(page, dict):
                log.warning("invest-engine: /portfolios/ returned non-dict %r", type(page))
                break
            for item in _extract_list(page):
                pid = item.get("id")
                if pid is None:
                    continue
                ids.append(str(pid))
            path = _next_page_path(_extract_next(page), current=path)
        return ids

    async def _fetch_portfolio(
        self,
        version: int,
        portfolio_id: str,
        since: datetime | None,
        before: datetime | None,
    ) -> AsyncIterator[Activity]:
        params: dict[str, str] = {"portfolio": portfolio_id}
        if since is not None:
            params["start"] = since.date().isoformat()
        if before is not None:
            params["end"] = before.date().isoformat()
        path: str | None = f"/api/v0.{version}/transactions/"
        while path is not None:
            page = await self._request_json(
                path, params=params if path.endswith("/transactions/") else None)
            if not isinstance(page, dict):
                break
            for raw in _extract_list(page):
                activity = _transaction_to_activity(raw)
                if activity is None:
                    continue
                if since is not None and activity.date < since:
                    continue
                if before is not None and activity.date >= before:
                    continue
                yield activity
            path = _next_page_path(_extract_next(page), current=path)


def _next_page_path(raw_next: str | None, *, current: str) -> str | None:
    """Normalise a ``next`` URL to a request path.

    IE might emit full URLs (``https://investengine.com/api/v0.32/...``)
    or relative paths. We return the path component only — the httpx
    client holds the base URL.
    """
    if raw_next is None:
        return None
    if raw_next.startswith("http://") or raw_next.startswith("https://"):
        from urllib.parse import urlparse
        parsed = urlparse(raw_next)
        path = parsed.path
        if parsed.query:
            path = f"{path}?{parsed.query}"
        return path
    return raw_next