feat: real data pipeline — market data, DB persistence, portfolio sync, signal-trade linkage

Wire the trading bot to real Alpaca market data and persist pipeline
state to the database so the dashboard displays live information.

- Add market-data service fetching OHLCV bars from Alpaca, publishing
  to market:bars Redis Stream; signal generator consumes bars and
  injects current_price into signals for position sizing
- Sentiment analyzer now persists Article + ArticleSentiment rows to
  DB after scoring, with duplicate and error handling
- API gateway runs a background portfolio sync task that snapshots
  Alpaca account state into PortfolioSnapshot/Position DB tables
  during market hours
- TradeSignal carries a signal_id UUID; signal generator and trade
  executor both persist their records to DB with cross-references
- 303 unit tests pass (57 new tests added)
This commit is contained in:
Viktor Barzin 2026-02-22 19:52:45 +00:00
parent 5a6b20c8f1
commit e2a3bd456d
No known key found for this signature in database
GPG key ID: 0EB088298288D958
19 changed files with 2238 additions and 72 deletions

View file

@ -3,7 +3,8 @@
Consumes ``news:raw`` articles from Redis Streams, scores them using a
tiered approach (FinBERT first, Ollama fallback for low-confidence results),
extracts ticker mentions, and publishes ``ScoredArticle`` messages to
``news:scored``.
``news:scored``. Also persists scored articles to the database (articles +
article_sentiments tables) so the dashboard can display real data.
"""
from __future__ import annotations
@ -14,11 +15,15 @@ import signal
import time
from redis.asyncio import Redis
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import async_sessionmaker
from services.sentiment_analyzer.analyzers.finbert import FinBERTAnalyzer
from services.sentiment_analyzer.analyzers.ollama_analyzer import OllamaAnalyzer
from services.sentiment_analyzer.config import SentimentAnalyzerConfig
from services.sentiment_analyzer.ticker_extractor import extract_tickers
from shared.db import create_db
from shared.models.news import Article, ArticleSentiment
from shared.redis_streams import StreamConsumer, StreamPublisher
from shared.schemas.news import RawArticle, ScoredArticle
from shared.telemetry import setup_telemetry
@ -33,6 +38,7 @@ async def process_article(
publisher: StreamPublisher,
config: SentimentAnalyzerConfig,
counters: dict,
db_session_factory: async_sessionmaker | None = None,
) -> None:
"""Score a single article and publish one ScoredArticle per extracted ticker.
@ -50,6 +56,9 @@ async def process_article(
Service configuration (confidence threshold, etc.).
counters:
Dict of OpenTelemetry counter/histogram instruments.
db_session_factory:
Optional async session factory for persisting to the DB.
When ``None``, DB persistence is skipped (backward compatible).
"""
start = time.monotonic()
@ -103,6 +112,46 @@ async def process_article(
counters["articles_scored"].add(1)
# --- Step 5: Persist to DB ---
if db_session_factory is not None:
try:
async with db_session_factory() as session:
db_article = Article(
source=article.source,
url=article.url,
title=article.title,
published_at=article.published_at,
fetched_at=article.fetched_at,
content_hash=article.content_hash,
)
session.add(db_article)
for ticker in tickers:
sentiment = ArticleSentiment(
article_id=db_article.id,
ticker=ticker,
score=score,
confidence=confidence,
model_used=model_used,
)
session.add(sentiment)
await session.commit()
logger.debug(
"Persisted article '%s' with %d sentiments to DB",
article.title[:60],
len(tickers),
)
except IntegrityError:
logger.debug(
"Article already exists in DB (content_hash=%s), skipping",
article.content_hash,
)
except Exception:
logger.exception(
"Failed to persist article to DB: %s", article.title[:60]
)
async def run(config: SentimentAnalyzerConfig | None = None) -> None:
"""Main service loop.
@ -150,6 +199,14 @@ async def run(config: SentimentAnalyzerConfig | None = None) -> None:
)
ollama = OllamaAnalyzer(model=config.ollama_model, host=config.ollama_host)
# --- Database ---
db_session_factory = None
try:
_engine, db_session_factory = create_db(config)
logger.info("Database session factory initialised")
except Exception:
logger.exception("Failed to initialise DB — articles will NOT be persisted")
logger.info("Consuming from news:raw, publishing to news:scored")
# Graceful shutdown on SIGTERM/SIGINT
@ -165,7 +222,7 @@ async def run(config: SentimentAnalyzerConfig | None = None) -> None:
break
try:
article = RawArticle.model_validate(data)
await process_article(article, finbert, ollama, publisher, config, counters)
await process_article(article, finbert, ollama, publisher, config, counters, db_session_factory)
except Exception:
logger.exception("Error processing article: %s", data.get("title", "<unknown>"))
finally: