diff --git a/broker_sync/providers/imap.py b/broker_sync/providers/imap.py index e935bab..5564dd3 100644 --- a/broker_sync/providers/imap.py +++ b/broker_sync/providers/imap.py @@ -16,6 +16,7 @@ from __future__ import annotations import email import imaplib import logging +import os import re import ssl from collections.abc import AsyncIterator, Iterator @@ -152,7 +153,12 @@ def _fetch_all(creds: ImapCreds) -> Iterator[bytes]: def fetch_activities(creds: ImapCreds) -> list[Activity]: out: list[Activity] = [] - ie_parsed = schwab_parsed = skipped = 0 + ie_parsed = schwab_parsed = ie_skipped = skipped = 0 + exclude = { + p.strip().lower() + for p in os.environ.get("BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS", "").split(",") + if p.strip() + } for raw in _fetch_all(creds): try: msg = email.message_from_bytes(raw) @@ -161,17 +167,28 @@ def fetch_activities(creds: ImapCreds) -> list[Activity]: continue sender = _extract_sender(msg) if sender in _IE_SENDERS or sender.endswith("@investengine.com"): + if "invest-engine" in exclude or "invest_engine" in exclude: + ie_skipped += 1 + continue out.extend(ie_parser.parse_invest_engine_email(raw)) ie_parsed += 1 - elif sender in _SCHWAB_SENDERS or sender.endswith("@schwab.com"): + elif ( + sender in _SCHWAB_SENDERS + or sender.endswith("@schwab.com") + or sender.endswith(".schwab.com") # e.g. donotreply@mail.schwab.com + ): + if "schwab" in exclude: + skipped += 1 + continue html = _html_or_text(msg) out.extend(parse_schwab_email(html)) schwab_parsed += 1 else: skipped += 1 log.info( - "imap: ie_parsed=%d schwab_parsed=%d skipped=%d → %d activities", + "imap: ie_parsed=%d ie_skipped=%d schwab_parsed=%d skipped=%d → %d activities", ie_parsed, + ie_skipped, schwab_parsed, skipped, len(out), diff --git a/tests/providers/test_fidelity_planviewer.py b/tests/providers/test_fidelity_planviewer.py index acfccbc..19c389a 100644 --- a/tests/providers/test_fidelity_planviewer.py +++ b/tests/providers/test_fidelity_planviewer.py @@ -17,6 +17,7 @@ from broker_sync.providers.fidelity_planviewer import ( gains_offset_delta_activity, ) from broker_sync.providers.parsers.fidelity import ( + FidelityHolding, parse_transactions_html, parse_valuation_json, ) @@ -152,8 +153,7 @@ def test_provider_caches_holdings_for_cli_snapshot_push() -> None: # -- delta-shaped gains offset (the monthly accumulation mechanism) -- -def _holdings_summing_to(total: Decimal) -> list: - from broker_sync.providers.parsers.fidelity import FidelityHolding +def _holdings_summing_to(total: Decimal) -> list[FidelityHolding]: return [FidelityHolding( fund_code="KDOA", fund_name="Test", units=Decimal("100"), unit_price=total / Decimal("100"), currency="GBP", total_value=total, diff --git a/tests/providers/test_imap.py b/tests/providers/test_imap.py index 63638cb..1abe587 100644 --- a/tests/providers/test_imap.py +++ b/tests/providers/test_imap.py @@ -99,3 +99,54 @@ def test_non_ie_activities_passed_through_unchanged() -> None: routed = _split_ie_by_isa_cap([schwab_act]) assert routed[0].account_id == "schwab-workplace" assert routed[0].account_type is AccountType.GIA + + +def test_exclude_invest_engine_skips_ie_emails(monkeypatch) -> None: + """BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS=invest-engine should skip IE messages + so we don't duplicate IE buys already ingested via the bearer-token API path. + Schwab routing must remain unaffected.""" + from broker_sync.providers import imap as imap_mod + + ie_email = ( + b"From: noreply@investengine.com\r\n" + b"Subject: VUAG Bought\r\n" + b"Content-Type: text/plain\r\n\r\n" + b"Vanguard S&P 500: VUAG Bought 10.0 @ 100.0 per share Total: 1000.00\r\n" + ) + schwab_email = ( + b"From: donotreply@schwab.com\r\n" + b"Subject: Order Confirmed\r\n" + b"Content-Type: text/html\r\n\r\n" + b"no-op\r\n" + ) + monkeypatch.setattr(imap_mod, "_fetch_all", lambda _: [ie_email, schwab_email]) + monkeypatch.setattr(imap_mod.ie_parser, "parse_invest_engine_email", + lambda raw: [object()]) + monkeypatch.setattr(imap_mod, "parse_schwab_email", lambda html: [object()]) + + creds = imap_mod.ImapCreds(host="h", user="u", password="p", directory="d") + + monkeypatch.setenv("BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS", "invest-engine") + out_excluded = imap_mod.fetch_activities(creds) + # IE skipped → only the schwab activity is emitted + assert len(out_excluded) == 1 + + monkeypatch.delenv("BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS", raising=False) + out_default = imap_mod.fetch_activities(creds) + # Both providers fire when env unset + assert len(out_default) == 2 + + +def test_schwab_subdomain_sender_matches() -> None: + """Real Schwab trade emails come from `donotreply@mail.schwab.com` + (subdomain), not just `donotreply@schwab.com`. The matcher must + accept either form.""" + from broker_sync.providers.imap import _SCHWAB_SENDERS + # Verify the static set works + assert "donotreply@schwab.com" in _SCHWAB_SENDERS + # Verify the subdomain suffix check + for addr in ( + "donotreply@mail.schwab.com", + "wealthnotify@equityawards.schwab.com", + ): + assert addr.endswith(".schwab.com"), addr