feat: real data pipeline — market data, DB persistence, portfolio sync, signal-trade linkage
Wire the trading bot to real Alpaca market data and persist pipeline state to the database so the dashboard displays live information. - Add market-data service fetching OHLCV bars from Alpaca, publishing to market:bars Redis Stream; signal generator consumes bars and injects current_price into signals for position sizing - Sentiment analyzer now persists Article + ArticleSentiment rows to DB after scoring, with duplicate and error handling - API gateway runs a background portfolio sync task that snapshots Alpaca account state into PortfolioSnapshot/Position DB tables during market hours - TradeSignal carries a signal_id UUID; signal generator and trade executor both persist their records to DB with cross-references - 303 unit tests pass (57 new tests added)
This commit is contained in:
Viktor Barzin
parent
5a6b20c8f1
commit
e2a3bd456d
19 changed files with 2238 additions and 72 deletions
|
|
@ -1,8 +1,9 @@
|
|||
"""Signal Generator service -- main entry point.
|
||||
|
||||
Consumes ``news:scored`` articles from Redis Streams, updates sentiment
|
||||
context per ticker, runs the weighted ensemble of trading strategies, and
|
||||
publishes qualifying ``TradeSignal`` messages to ``signals:generated``.
|
||||
Consumes ``news:scored`` articles and ``market:bars`` OHLCV data from
|
||||
Redis Streams, updates sentiment context and market data per ticker,
|
||||
runs the weighted ensemble of trading strategies, and publishes
|
||||
qualifying ``TradeSignal`` messages to ``signals:generated``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -10,16 +11,21 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
import logging
|
||||
import signal
|
||||
import uuid
|
||||
from collections import defaultdict, deque
|
||||
|
||||
from redis.asyncio import Redis
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from services.signal_generator.config import SignalGeneratorConfig
|
||||
from services.signal_generator.ensemble import WeightedEnsemble
|
||||
from services.signal_generator.market_data import MarketDataManager
|
||||
from shared.db import create_db
|
||||
from shared.models.trading import Signal as SignalModel
|
||||
from shared.models.trading import SignalDirection as SignalDirectionModel
|
||||
from shared.redis_streams import StreamConsumer, StreamPublisher
|
||||
from shared.schemas.news import ScoredArticle
|
||||
from shared.schemas.trading import SentimentContext
|
||||
from shared.schemas.trading import MarketSnapshot, SentimentContext
|
||||
from shared.strategies import MeanReversionStrategy, MomentumStrategy, NewsDrivenStrategy
|
||||
from shared.telemetry import setup_telemetry
|
||||
|
||||
|
|
@ -53,12 +59,150 @@ def _build_sentiment_context(
|
|||
)
|
||||
|
||||
|
||||
async def _consume_market_bars(
|
||||
bars_consumer: StreamConsumer,
|
||||
market_data: MarketDataManager,
|
||||
shutdown_event: asyncio.Event,
|
||||
bars_received_counter,
|
||||
) -> None:
|
||||
"""Consume OHLCV bars from ``market:bars`` and feed them to the MarketDataManager.
|
||||
|
||||
Runs as a concurrent task alongside the scored-article consumer.
|
||||
"""
|
||||
logger.info("Starting market:bars consumer")
|
||||
async for _msg_id, data in bars_consumer.consume():
|
||||
if shutdown_event.is_set():
|
||||
break
|
||||
try:
|
||||
ticker = data.get("ticker")
|
||||
if not ticker:
|
||||
logger.warning("Received bar message without ticker field: %s", data)
|
||||
continue
|
||||
|
||||
# Build bar_data dict without the ticker key (OHLCVBar doesn't have it)
|
||||
bar_data = {k: v for k, v in data.items() if k != "ticker"}
|
||||
market_data.add_bar(ticker, bar_data)
|
||||
bars_received_counter.add(1)
|
||||
logger.debug("Added bar for %s: close=%s", ticker, data.get("close"))
|
||||
except Exception:
|
||||
logger.exception("Error processing market bar: %s", data)
|
||||
|
||||
|
||||
async def _consume_scored_articles(
|
||||
articles_consumer: StreamConsumer,
|
||||
market_data: MarketDataManager,
|
||||
ensemble: WeightedEnsemble,
|
||||
weights: dict[str, float],
|
||||
publisher: StreamPublisher,
|
||||
shutdown_event: asyncio.Event,
|
||||
signals_generated,
|
||||
per_strategy_signal_count,
|
||||
db_session_factory: async_sessionmaker | None = None,
|
||||
) -> None:
|
||||
"""Consume scored articles from ``news:scored``, run the ensemble, and publish signals.
|
||||
|
||||
Runs as a concurrent task alongside the market-bars consumer.
|
||||
"""
|
||||
# Per-ticker sentiment accumulators
|
||||
sentiment_scores: dict[str, deque[float]] = defaultdict(
|
||||
lambda: deque(maxlen=_MAX_SENTIMENT_SCORES)
|
||||
)
|
||||
sentiment_confidences: dict[str, deque[float]] = defaultdict(
|
||||
lambda: deque(maxlen=_MAX_SENTIMENT_SCORES)
|
||||
)
|
||||
|
||||
logger.info("Starting news:scored consumer")
|
||||
async for _msg_id, data in articles_consumer.consume():
|
||||
if shutdown_event.is_set():
|
||||
break
|
||||
try:
|
||||
article = ScoredArticle.model_validate(data)
|
||||
ticker = article.ticker
|
||||
|
||||
# Update sentiment accumulators
|
||||
sentiment_scores[ticker].append(article.sentiment_score)
|
||||
sentiment_confidences[ticker].append(article.confidence)
|
||||
|
||||
# Build sentiment context
|
||||
sentiment = _build_sentiment_context(
|
||||
ticker,
|
||||
sentiment_scores[ticker],
|
||||
sentiment_confidences[ticker],
|
||||
)
|
||||
|
||||
# Get market snapshot (may be None if no bars received yet)
|
||||
snapshot = market_data.get_snapshot(ticker)
|
||||
if snapshot is None:
|
||||
# Create a minimal snapshot from sentiment data alone
|
||||
# (the news_driven strategy does not require market indicators)
|
||||
snapshot = MarketSnapshot(
|
||||
ticker=ticker,
|
||||
current_price=0.0,
|
||||
open=0.0,
|
||||
high=0.0,
|
||||
low=0.0,
|
||||
close=0.0,
|
||||
volume=0.0,
|
||||
)
|
||||
|
||||
# Run ensemble
|
||||
signal_result = await ensemble.evaluate(ticker, snapshot, sentiment, weights)
|
||||
|
||||
if signal_result is not None:
|
||||
# Inject current price for trade executor position sizing
|
||||
if snapshot and snapshot.current_price > 0:
|
||||
if signal_result.sentiment_context is None:
|
||||
signal_result.sentiment_context = {}
|
||||
signal_result.sentiment_context["current_price"] = snapshot.current_price
|
||||
|
||||
# Persist signal to DB
|
||||
if db_session_factory is not None:
|
||||
try:
|
||||
async with db_session_factory() as session:
|
||||
direction_map = {
|
||||
"LONG": SignalDirectionModel.LONG,
|
||||
"SHORT": SignalDirectionModel.SHORT,
|
||||
"NEUTRAL": SignalDirectionModel.NEUTRAL,
|
||||
}
|
||||
db_signal = SignalModel(
|
||||
id=signal_result.signal_id,
|
||||
ticker=ticker,
|
||||
direction=direction_map[signal_result.direction.value],
|
||||
strength=signal_result.strength,
|
||||
strategy_sources=signal_result.strategy_sources,
|
||||
sentiment_score=sentiment.avg_score if sentiment else None,
|
||||
acted_on=False,
|
||||
)
|
||||
session.add(db_signal)
|
||||
await session.commit()
|
||||
except Exception:
|
||||
logger.exception("Failed to persist signal to DB")
|
||||
|
||||
await publisher.publish(signal_result.model_dump(mode="json"))
|
||||
signals_generated.add(1)
|
||||
for src in signal_result.strategy_sources:
|
||||
strategy_name = src.split(":")[0]
|
||||
per_strategy_signal_count.add(1, {"strategy": strategy_name})
|
||||
logger.info(
|
||||
"Signal generated: %s %s strength=%.4f sources=%s",
|
||||
signal_result.direction.value,
|
||||
ticker,
|
||||
signal_result.strength,
|
||||
signal_result.strategy_sources,
|
||||
)
|
||||
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Error processing scored article: %s", data.get("title", "<unknown>")
|
||||
)
|
||||
|
||||
|
||||
async def run(config: SignalGeneratorConfig | None = None) -> None:
|
||||
"""Main service loop.
|
||||
|
||||
Connects to Redis, initialises strategies and telemetry, then
|
||||
continuously consumes from ``news:scored`` and publishes qualifying
|
||||
signals to ``signals:generated``.
|
||||
continuously consumes from ``news:scored`` and ``market:bars``,
|
||||
publishing qualifying signals to ``signals:generated``.
|
||||
"""
|
||||
if config is None:
|
||||
config = SignalGeneratorConfig()
|
||||
|
|
@ -76,10 +220,19 @@ async def run(config: SignalGeneratorConfig | None = None) -> None:
|
|||
"per_strategy_signal_count",
|
||||
description="Signals emitted, broken down by strategy",
|
||||
)
|
||||
bars_received_counter = meter.create_counter(
|
||||
"bars_received",
|
||||
description="Total OHLCV bars received from market:bars stream",
|
||||
)
|
||||
|
||||
# --- Redis ---
|
||||
redis = Redis.from_url(config.redis_url, decode_responses=False)
|
||||
consumer = StreamConsumer(redis, "news:scored", "signal-generator", "worker-1")
|
||||
articles_consumer = StreamConsumer(
|
||||
redis, "news:scored", "signal-generator", "worker-1"
|
||||
)
|
||||
bars_consumer = StreamConsumer(
|
||||
redis, "market:bars", "signal-generator", "bars-worker"
|
||||
)
|
||||
publisher = StreamPublisher(redis, "signals:generated")
|
||||
|
||||
# --- Market data ---
|
||||
|
|
@ -96,11 +249,17 @@ async def run(config: SignalGeneratorConfig | None = None) -> None:
|
|||
# --- Strategy weights (default equal; could load from DB) ---
|
||||
weights = dict(_DEFAULT_WEIGHTS)
|
||||
|
||||
# --- Per-ticker sentiment accumulators ---
|
||||
sentiment_scores: dict[str, deque[float]] = defaultdict(lambda: deque(maxlen=_MAX_SENTIMENT_SCORES))
|
||||
sentiment_confidences: dict[str, deque[float]] = defaultdict(lambda: deque(maxlen=_MAX_SENTIMENT_SCORES))
|
||||
# --- Database (for persisting signals) ---
|
||||
db_session_factory = None
|
||||
try:
|
||||
_engine, db_session_factory = create_db(config)
|
||||
logger.info("Database session factory initialised for signal persistence")
|
||||
except Exception:
|
||||
logger.exception("Failed to initialise DB — signals will NOT be persisted")
|
||||
|
||||
logger.info("Consuming from news:scored, publishing to signals:generated")
|
||||
logger.info(
|
||||
"Consuming from news:scored and market:bars, publishing to signals:generated"
|
||||
)
|
||||
|
||||
# Graceful shutdown on SIGTERM/SIGINT
|
||||
shutdown_event = asyncio.Event()
|
||||
|
|
@ -108,62 +267,30 @@ async def run(config: SignalGeneratorConfig | None = None) -> None:
|
|||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
loop.add_signal_handler(sig, shutdown_event.set)
|
||||
|
||||
# --- Consume loop ---
|
||||
# --- Run both consumers concurrently ---
|
||||
try:
|
||||
async for _msg_id, data in consumer.consume():
|
||||
if shutdown_event.is_set():
|
||||
break
|
||||
try:
|
||||
article = ScoredArticle.model_validate(data)
|
||||
ticker = article.ticker
|
||||
|
||||
# Update sentiment accumulators
|
||||
sentiment_scores[ticker].append(article.sentiment_score)
|
||||
sentiment_confidences[ticker].append(article.confidence)
|
||||
|
||||
# Build sentiment context
|
||||
sentiment = _build_sentiment_context(
|
||||
ticker,
|
||||
sentiment_scores[ticker],
|
||||
sentiment_confidences[ticker],
|
||||
async with asyncio.TaskGroup() as tg:
|
||||
tg.create_task(
|
||||
_consume_scored_articles(
|
||||
articles_consumer,
|
||||
market_data,
|
||||
ensemble,
|
||||
weights,
|
||||
publisher,
|
||||
shutdown_event,
|
||||
signals_generated,
|
||||
per_strategy_signal_count,
|
||||
db_session_factory,
|
||||
)
|
||||
|
||||
# Get market snapshot (may be None if no bars received yet)
|
||||
snapshot = market_data.get_snapshot(ticker)
|
||||
if snapshot is None:
|
||||
# Create a minimal snapshot from sentiment data alone
|
||||
# (the news_driven strategy does not require market indicators)
|
||||
from shared.schemas.trading import MarketSnapshot
|
||||
|
||||
snapshot = MarketSnapshot(
|
||||
ticker=ticker,
|
||||
current_price=0.0,
|
||||
open=0.0,
|
||||
high=0.0,
|
||||
low=0.0,
|
||||
close=0.0,
|
||||
volume=0.0,
|
||||
)
|
||||
|
||||
# Run ensemble
|
||||
signal_result = await ensemble.evaluate(ticker, snapshot, sentiment, weights)
|
||||
|
||||
if signal_result is not None:
|
||||
await publisher.publish(signal_result.model_dump(mode="json"))
|
||||
signals_generated.add(1)
|
||||
for src in signal_result.strategy_sources:
|
||||
strategy_name = src.split(":")[0]
|
||||
per_strategy_signal_count.add(1, {"strategy": strategy_name})
|
||||
logger.info(
|
||||
"Signal generated: %s %s strength=%.4f sources=%s",
|
||||
signal_result.direction.value,
|
||||
ticker,
|
||||
signal_result.strength,
|
||||
signal_result.strategy_sources,
|
||||
)
|
||||
|
||||
except Exception:
|
||||
logger.exception("Error processing scored article: %s", data.get("title", "<unknown>"))
|
||||
)
|
||||
tg.create_task(
|
||||
_consume_market_bars(
|
||||
bars_consumer,
|
||||
market_data,
|
||||
shutdown_event,
|
||||
bars_received_counter,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
await redis.aclose()
|
||||
logger.info("Signal generator stopped gracefully")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue