feat: real data pipeline — market data, DB persistence, portfolio sync, signal-trade linkage
Wire the trading bot to real Alpaca market data and persist pipeline state to the database so the dashboard displays live information. - Add market-data service fetching OHLCV bars from Alpaca, publishing to market:bars Redis Stream; signal generator consumes bars and injects current_price into signals for position sizing - Sentiment analyzer now persists Article + ArticleSentiment rows to DB after scoring, with duplicate and error handling - API gateway runs a background portfolio sync task that snapshots Alpaca account state into PortfolioSnapshot/Position DB tables during market hours - TradeSignal carries a signal_id UUID; signal generator and trade executor both persist their records to DB with cross-references - 303 unit tests pass (57 new tests added)
This commit is contained in:
parent
5a6b20c8f1
commit
e2a3bd456d
19 changed files with 2238 additions and 72 deletions
|
|
@ -3,7 +3,8 @@
|
|||
Consumes ``news:raw`` articles from Redis Streams, scores them using a
|
||||
tiered approach (FinBERT first, Ollama fallback for low-confidence results),
|
||||
extracts ticker mentions, and publishes ``ScoredArticle`` messages to
|
||||
``news:scored``.
|
||||
``news:scored``. Also persists scored articles to the database (articles +
|
||||
article_sentiments tables) so the dashboard can display real data.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -14,11 +15,15 @@ import signal
|
|||
import time
|
||||
|
||||
from redis.asyncio import Redis
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from services.sentiment_analyzer.analyzers.finbert import FinBERTAnalyzer
|
||||
from services.sentiment_analyzer.analyzers.ollama_analyzer import OllamaAnalyzer
|
||||
from services.sentiment_analyzer.config import SentimentAnalyzerConfig
|
||||
from services.sentiment_analyzer.ticker_extractor import extract_tickers
|
||||
from shared.db import create_db
|
||||
from shared.models.news import Article, ArticleSentiment
|
||||
from shared.redis_streams import StreamConsumer, StreamPublisher
|
||||
from shared.schemas.news import RawArticle, ScoredArticle
|
||||
from shared.telemetry import setup_telemetry
|
||||
|
|
@ -33,6 +38,7 @@ async def process_article(
|
|||
publisher: StreamPublisher,
|
||||
config: SentimentAnalyzerConfig,
|
||||
counters: dict,
|
||||
db_session_factory: async_sessionmaker | None = None,
|
||||
) -> None:
|
||||
"""Score a single article and publish one ScoredArticle per extracted ticker.
|
||||
|
||||
|
|
@ -50,6 +56,9 @@ async def process_article(
|
|||
Service configuration (confidence threshold, etc.).
|
||||
counters:
|
||||
Dict of OpenTelemetry counter/histogram instruments.
|
||||
db_session_factory:
|
||||
Optional async session factory for persisting to the DB.
|
||||
When ``None``, DB persistence is skipped (backward compatible).
|
||||
"""
|
||||
start = time.monotonic()
|
||||
|
||||
|
|
@ -103,6 +112,46 @@ async def process_article(
|
|||
|
||||
counters["articles_scored"].add(1)
|
||||
|
||||
# --- Step 5: Persist to DB ---
|
||||
if db_session_factory is not None:
|
||||
try:
|
||||
async with db_session_factory() as session:
|
||||
db_article = Article(
|
||||
source=article.source,
|
||||
url=article.url,
|
||||
title=article.title,
|
||||
published_at=article.published_at,
|
||||
fetched_at=article.fetched_at,
|
||||
content_hash=article.content_hash,
|
||||
)
|
||||
session.add(db_article)
|
||||
|
||||
for ticker in tickers:
|
||||
sentiment = ArticleSentiment(
|
||||
article_id=db_article.id,
|
||||
ticker=ticker,
|
||||
score=score,
|
||||
confidence=confidence,
|
||||
model_used=model_used,
|
||||
)
|
||||
session.add(sentiment)
|
||||
|
||||
await session.commit()
|
||||
logger.debug(
|
||||
"Persisted article '%s' with %d sentiments to DB",
|
||||
article.title[:60],
|
||||
len(tickers),
|
||||
)
|
||||
except IntegrityError:
|
||||
logger.debug(
|
||||
"Article already exists in DB (content_hash=%s), skipping",
|
||||
article.content_hash,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to persist article to DB: %s", article.title[:60]
|
||||
)
|
||||
|
||||
|
||||
async def run(config: SentimentAnalyzerConfig | None = None) -> None:
|
||||
"""Main service loop.
|
||||
|
|
@ -150,6 +199,14 @@ async def run(config: SentimentAnalyzerConfig | None = None) -> None:
|
|||
)
|
||||
ollama = OllamaAnalyzer(model=config.ollama_model, host=config.ollama_host)
|
||||
|
||||
# --- Database ---
|
||||
db_session_factory = None
|
||||
try:
|
||||
_engine, db_session_factory = create_db(config)
|
||||
logger.info("Database session factory initialised")
|
||||
except Exception:
|
||||
logger.exception("Failed to initialise DB — articles will NOT be persisted")
|
||||
|
||||
logger.info("Consuming from news:raw, publishing to news:scored")
|
||||
|
||||
# Graceful shutdown on SIGTERM/SIGINT
|
||||
|
|
@ -165,7 +222,7 @@ async def run(config: SentimentAnalyzerConfig | None = None) -> None:
|
|||
break
|
||||
try:
|
||||
article = RawArticle.model_validate(data)
|
||||
await process_article(article, finbert, ollama, publisher, config, counters)
|
||||
await process_article(article, finbert, ollama, publisher, config, counters, db_session_factory)
|
||||
except Exception:
|
||||
logger.exception("Error processing article: %s", data.get("title", "<unknown>"))
|
||||
finally:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue