viktor/broker-sync
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

imap: skip InvestEngine emails via BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS
Some checks failed
CI / test (push) Waiting to run
Details
CI / build (push) Blocked by required conditions
Details
CI / deploy (push) Blocked by required conditions
Details
ci/woodpecker/push/build Pipeline failed
Details

Browse source 
The IMAP IE parser and the bearer-token IE API path generate different
external_ids for the same fill, so running both produces duplicate BUYs
in Wealthfolio. With IE now served by the API path (broker-sync invest-engine),
we keep the IMAP path live for Schwab and gate IE off via env var.

Setting BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS=invest-engine on the imap CronJob
stops new dupes; Schwab routing is unaffected.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Viktor Barzin 2026-05-26 21:16:28 +00:00
parent d5dbeb96af
commit 68d4832c2e
2 changed files with 51 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

17
broker_sync/providers/imap.py
View file

@ -16,6 +16,7 @@ from __future__ import annotations
import email
import imaplib
import logging
import os
import re
import ssl
from collections.abc import AsyncIterator, Iterator
@ -152,7 +153,12 @@ def _fetch_all(creds: ImapCreds) -> Iterator[bytes]:
def fetch_activities(creds: ImapCreds) -> list[Activity]:
out: list[Activity] = []
ie_parsed = schwab_parsed = skipped = 0
ie_parsed = schwab_parsed = ie_skipped = skipped = 0
exclude = {
p.strip().lower()
for p in os.environ.get("BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS", "").split(",")
if p.strip()
}
for raw in _fetch_all(creds):
try:
msg = email.message_from_bytes(raw)
@ -161,6 +167,9 @@ def fetch_activities(creds: ImapCreds) -> list[Activity]:
continue
sender = _extract_sender(msg)
if sender in _IE_SENDERS or sender.endswith("@investengine.com"):
if "invest-engine" in exclude or "invest_engine" in exclude:
ie_skipped += 1
continue
out.extend(ie_parser.parse_invest_engine_email(raw))
ie_parsed += 1
elif (
@ -168,14 +177,18 @@ def fetch_activities(creds: ImapCreds) -> list[Activity]:
or sender.endswith("@schwab.com")
or sender.endswith(".schwab.com") # e.g. donotreply@mail.schwab.com
):
if "schwab" in exclude:
skipped += 1
continue
html = _html_or_text(msg)
out.extend(parse_schwab_email(html))
schwab_parsed += 1
else:
skipped += 1
log.info(
"imap: ie_parsed=%d schwab_parsed=%d skipped=%d → %d activities",
"imap: ie_parsed=%d ie_skipped=%d schwab_parsed=%d skipped=%d → %d activities",
ie_parsed,
ie_skipped,
schwab_parsed,
skipped,
len(out),

36
tests/providers/test_imap.py
View file

@ -101,6 +101,42 @@ def test_non_ie_activities_passed_through_unchanged() -> None:
assert routed[0].account_type is AccountType.GIA
def test_exclude_invest_engine_skips_ie_emails(monkeypatch) -> None:
"""BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS=invest-engine should skip IE messages
so we don't duplicate IE buys already ingested via the bearer-token API path.
Schwab routing must remain unaffected."""
from broker_sync.providers import imap as imap_mod
ie_email = (
b"From: noreply@investengine.com\r\n"
b"Subject: VUAG Bought\r\n"
b"Content-Type: text/plain\r\n\r\n"
b"Vanguard S&P 500: VUAG Bought 10.0 @ 100.0 per share Total: 1000.00\r\n"
)
schwab_email = (
b"From: donotreply@schwab.com\r\n"
b"Subject: Order Confirmed\r\n"
b"Content-Type: text/html\r\n\r\n"
b"<html><body>no-op</body></html>\r\n"
)
monkeypatch.setattr(imap_mod, "_fetch_all", lambda _: [ie_email, schwab_email])
monkeypatch.setattr(imap_mod.ie_parser, "parse_invest_engine_email",
lambda raw: [object()])
monkeypatch.setattr(imap_mod, "parse_schwab_email", lambda html: [object()])
creds = imap_mod.ImapCreds(host="h", user="u", password="p", directory="d")
monkeypatch.setenv("BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS", "invest-engine")
out_excluded = imap_mod.fetch_activities(creds)
# IE skipped → only the schwab activity is emitted
assert len(out_excluded) == 1
monkeypatch.delenv("BROKER_SYNC_IMAP_EXCLUDE_PROVIDERS", raising=False)
out_default = imap_mod.fetch_activities(creds)
# Both providers fire when env unset
assert len(out_default) == 2
def test_schwab_subdomain_sender_matches() -> None:
"""Real Schwab trade emails come from `donotreply@mail.schwab.com`
(subdomain), not just `donotreply@schwab.com`. The matcher must