Files
foxhunt/services/trading_service/src/prediction_generation_loop.rs
jgrusewski eae3c31e53 fix(clippy): Fix 6 unwrap_used violations in risk/data
Patterns applied:
- Pattern 2: Float comparison (2x: utils.rs, var_edge_cases_tests.rs)
- Pattern 7: Date/time construction (2x: production_streaming.rs, streaming.rs)
- Pattern 1: Duration/time ops (2x: rate limiter, semaphore)
- Pattern 4: Optional field access (1x: position_tracker.rs)

Changes:
- data/src/utils.rs: Float sort with NaN handling
- data/src/providers/benzinga/production_streaming.rs: Rate limiter + semaphore + date/time
- data/src/providers/benzinga/streaming.rs: Date/time construction
- risk/src/position_tracker.rs: Emergency fallback counter
- risk/tests/var_edge_cases_tests.rs: Test helper float sort

Test impact: 0 failures (182/182 passing)
Compilation: Clean (0 errors, 0 warnings)
Time: 25 min (44% under budget)
2025-10-23 14:58:32 +02:00

631 lines
20 KiB
Rust

//! Background ML Prediction Generation Loop
//!
//! This module implements a continuous background task that generates ML predictions
//! every 60 seconds for all configured trading symbols. Predictions are saved to the
//! ensemble_predictions table for consumption by the paper trading executor.
//!
//! ## Architecture
//!
//! ```text
//! ┌─────────────────────────────────────────────────────────┐
//! │ Prediction Generation Loop (60s interval) │
//! │ │
//! │ 1. Fetch current market data for all symbols │
//! │ 2. Extract 26 features from OHLCV data │
//! │ 3. Call EnsembleCoordinator.predict() │
//! │ 4. Save EnsembleDecision to ensemble_predictions │
//! │ 5. Sleep 60 seconds │
//! │ 6. Repeat (with graceful shutdown on SIGTERM) │
//! └─────────────────────────────────────────────────────────┘
//! ```
//!
//! ## Error Handling
//!
//! The loop is designed to be resilient:
//! - Database connection errors: Log and retry next cycle
//! - Model inference errors: Log and continue (ensemble degradation)
//! - Feature extraction errors: Log and skip symbol
//! - NO crashes: Loop always continues unless shutdown signal received
//!
//! ## Graceful Shutdown
//!
//! The task listens for SIGTERM/SIGINT and exits cleanly:
//! - Waits for current prediction cycle to complete
//! - Flushes any pending database writes
//! - Returns immediately after cleanup
use anyhow::{Context, Result};
use ml::Features;
use sqlx::PgPool;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::broadcast;
use tokio::time::interval;
use tracing::{debug, error, info, warn};
use uuid::Uuid;
use crate::ensemble_coordinator::EnsembleCoordinator;
/// Configuration for prediction generation loop
#[derive(Debug, Clone)]
pub struct PredictionLoopConfig {
/// Interval between prediction cycles (default: 60 seconds)
pub prediction_interval: Duration,
/// List of symbols to generate predictions for
pub symbols: Vec<String>,
/// Account ID for attribution
pub account_id: String,
/// Strategy ID for attribution
pub strategy_id: String,
/// Node ID for system context
pub node_id: String,
}
impl Default for PredictionLoopConfig {
fn default() -> Self {
Self {
prediction_interval: Duration::from_secs(60),
symbols: vec![
"ES.FUT".to_string(),
"NQ.FUT".to_string(),
"ZN.FUT".to_string(),
"6E.FUT".to_string(),
],
account_id: "prediction_generator_001".to_string(),
strategy_id: "ensemble_ml_v1".to_string(),
node_id: hostname::get()
.ok()
.and_then(|h| h.into_string().ok())
.unwrap_or_else(|| "unknown".to_string()),
}
}
}
impl PredictionLoopConfig {
/// Create configuration from environment variables
pub fn from_env() -> Self {
Self {
prediction_interval: std::env::var("PREDICTION_INTERVAL_SECONDS")
.ok()
.and_then(|s| s.parse().ok())
.map(Duration::from_secs)
.unwrap_or(Duration::from_secs(60)),
symbols: std::env::var("PREDICTION_SYMBOLS")
.ok()
.map(|s| s.split(',').map(|sym| sym.trim().to_string()).collect())
.unwrap_or_else(|| {
vec![
"ES.FUT".to_string(),
"NQ.FUT".to_string(),
"ZN.FUT".to_string(),
"6E.FUT".to_string(),
]
}),
account_id: std::env::var("PREDICTION_ACCOUNT_ID")
.unwrap_or_else(|_| "prediction_generator_001".to_string()),
strategy_id: std::env::var("PREDICTION_STRATEGY_ID")
.unwrap_or_else(|_| "ensemble_ml_v1".to_string()),
node_id: hostname::get()
.ok()
.and_then(|h| h.into_string().ok())
.unwrap_or_else(|| "unknown".to_string()),
}
}
}
/// Background task for continuous ML prediction generation
pub struct PredictionGenerationLoop {
coordinator: Arc<EnsembleCoordinator>,
db_pool: PgPool,
config: PredictionLoopConfig,
}
impl PredictionGenerationLoop {
/// Create new prediction generation loop
pub fn new(
coordinator: Arc<EnsembleCoordinator>,
db_pool: PgPool,
config: PredictionLoopConfig,
) -> Self {
Self {
coordinator,
db_pool,
config,
}
}
/// Start the prediction generation loop
///
/// This runs forever until a shutdown signal is received via the shutdown_rx channel.
/// The loop generates predictions for all configured symbols every `prediction_interval`.
pub async fn run(&self, mut shutdown_rx: broadcast::Receiver<()>) -> Result<()> {
info!(
"Starting prediction generation loop for symbols: {:?}, interval: {:?}",
self.config.symbols, self.config.prediction_interval
);
let mut ticker = interval(self.config.prediction_interval);
loop {
tokio::select! {
_ = ticker.tick() => {
// Generate predictions for all symbols
if let Err(e) = self.generate_predictions_for_all_symbols().await {
error!("Prediction generation cycle failed: {}", e);
// Don't crash - log and continue to next cycle
}
}
_ = shutdown_rx.recv() => {
info!("Shutdown signal received, stopping prediction generation loop");
break;
}
}
}
info!("Prediction generation loop stopped gracefully");
Ok(())
}
/// Generate predictions for all configured symbols
async fn generate_predictions_for_all_symbols(&self) -> Result<()> {
debug!(
"Starting prediction generation cycle for {} symbols",
self.config.symbols.len()
);
let start_time = std::time::Instant::now();
for symbol in &self.config.symbols {
if let Err(e) = self.generate_prediction_for_symbol(symbol).await {
warn!("Failed to generate prediction for {}: {}", symbol, e);
// Continue with other symbols even if one fails
}
}
let cycle_duration = start_time.elapsed();
info!(
"Prediction generation cycle completed in {:.2}s for {} symbols",
cycle_duration.as_secs_f64(),
self.config.symbols.len()
);
Ok(())
}
/// Generate prediction for a single symbol
async fn generate_prediction_for_symbol(&self, symbol: &str) -> Result<()> {
debug!("Generating prediction for {}", symbol);
// 1. Fetch current market data and extract features
let features = self.fetch_features_for_symbol(symbol).await?;
// 2. Generate ensemble prediction
let start_inference = std::time::Instant::now();
let decision = self
.coordinator
.predict(&features)
.await
.context("Ensemble prediction failed")?;
let inference_latency_us = start_inference.elapsed().as_micros() as i32;
// 3. Save prediction to database
self.save_prediction_to_database(symbol, &decision, inference_latency_us)
.await?;
debug!(
"Prediction saved for {}: action={:?}, confidence={:.3}, disagreement={:.3}, latency={}μs",
symbol, decision.action, decision.confidence, decision.disagreement_rate, inference_latency_us
);
Ok(())
}
/// Fetch market data and extract features for a symbol
async fn fetch_features_for_symbol(&self, symbol: &str) -> Result<Features> {
// Query recent market data from database
let market_data = self.query_recent_market_data(symbol).await?;
// Extract 26 features from OHLCV data
// For production, use real feature engineering from ml/src/features
// For now, create a simple feature vector with mock technical indicators
let features = self.extract_features_from_market_data(symbol, &market_data)?;
Ok(features)
}
/// Query recent market data from database
async fn query_recent_market_data(&self, symbol: &str) -> Result<MarketDataSnapshot> {
// Query last 100 bars of OHLCV data for technical indicators
let bars = sqlx::query_as::<_, OhlcvBar>(
r#"
SELECT
timestamp,
symbol,
open,
high,
low,
close,
volume
FROM ohlcv_bars
WHERE symbol = $1
ORDER BY timestamp DESC
LIMIT 100
"#,
)
.bind(symbol)
.fetch_all(&self.db_pool)
.await
.context("Failed to query market data")?;
if bars.is_empty() {
return Err(anyhow::anyhow!("No market data available for {}", symbol));
}
Ok(MarketDataSnapshot { bars })
}
/// Extract features from market data
fn extract_features_from_market_data(
&self,
symbol: &str,
market_data: &MarketDataSnapshot,
) -> Result<Features> {
// Use the most recent bar for current price data
let latest_bar = market_data
.bars
.first()
.ok_or_else(|| anyhow::anyhow!("No bars in market data"))?;
// Calculate basic technical indicators
// In production, use ml/src/features/technical_indicators.rs
let close_prices: Vec<f64> = market_data.bars.iter().map(|b| b.close).collect();
let volumes: Vec<f64> = market_data.bars.iter().map(|b| b.volume).collect();
// Basic features (5 OHLCV + 10 technical indicators = 15 total)
// Simplified for MVP - production should use full 26-feature extraction
let mut feature_values = vec![
latest_bar.open,
latest_bar.high,
latest_bar.low,
latest_bar.close,
latest_bar.volume,
];
// Add 10 technical indicators (simplified calculations)
feature_values.push(calculate_sma(&close_prices, 20)); // SMA_20
feature_values.push(calculate_sma(&close_prices, 50)); // SMA_50
feature_values.push(calculate_rsi(&close_prices, 14)); // RSI_14
feature_values.push(calculate_volatility(&close_prices, 20)); // Volatility_20
feature_values.push(calculate_momentum(&close_prices, 10)); // Momentum_10
feature_values.push(calculate_volume_ratio(&volumes, 20)); // Volume_Ratio
feature_values.push(latest_bar.high - latest_bar.low); // Range
feature_values.push(latest_bar.close - latest_bar.open); // Body
feature_values.push(calculate_returns(&close_prices)); // Returns_1
feature_values.push(calculate_log_returns(&close_prices)); // Log_Returns_1
let feature_names = vec![
"open",
"high",
"low",
"close",
"volume",
"sma_20",
"sma_50",
"rsi_14",
"volatility_20",
"momentum_10",
"volume_ratio_20",
"range",
"body",
"returns_1",
"log_returns_1",
]
.into_iter()
.map(|s| s.to_string())
.collect();
let mut features = Features::new(feature_values, feature_names);
features.symbol = Some(symbol.to_string());
Ok(features)
}
/// Save prediction to ensemble_predictions table
async fn save_prediction_to_database(
&self,
symbol: &str,
decision: &ml::ensemble::EnsembleDecision,
inference_latency_us: i32,
) -> Result<()> {
let prediction_id = Uuid::new_v4();
let prediction_timestamp = chrono::Utc::now();
// Extract per-model votes from decision
let (dqn_signal, dqn_confidence, dqn_weight, dqn_vote) =
extract_model_vote(decision, "DQN");
let (ppo_signal, ppo_confidence, ppo_weight, ppo_vote) =
extract_model_vote(decision, "PPO");
let (mamba2_signal, mamba2_confidence, mamba2_weight, mamba2_vote) =
extract_model_vote(decision, "MAMBA2");
let (tft_signal, tft_confidence, tft_weight, tft_vote) =
extract_model_vote(decision, "TFT");
// Convert action to string
let ensemble_action = format!("{:?}", decision.action).to_uppercase();
// Insert into database
sqlx::query(
r#"
INSERT INTO ensemble_predictions (
id,
prediction_timestamp,
symbol,
account_id,
strategy_id,
ensemble_action,
ensemble_signal,
ensemble_confidence,
disagreement_rate,
dqn_signal,
dqn_confidence,
dqn_weight,
dqn_vote,
ppo_signal,
ppo_confidence,
ppo_weight,
ppo_vote,
mamba2_signal,
mamba2_confidence,
mamba2_weight,
mamba2_vote,
tft_signal,
tft_confidence,
tft_weight,
tft_vote,
node_id,
inference_latency_us,
metadata
)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28)
"#,
)
.bind(prediction_id)
.bind(prediction_timestamp)
.bind(symbol)
.bind(&self.config.account_id)
.bind(&self.config.strategy_id)
.bind(ensemble_action)
.bind(decision.signal)
.bind(decision.confidence)
.bind(decision.disagreement_rate)
.bind(dqn_signal)
.bind(dqn_confidence)
.bind(dqn_weight)
.bind(dqn_vote)
.bind(ppo_signal)
.bind(ppo_confidence)
.bind(ppo_weight)
.bind(ppo_vote)
.bind(mamba2_signal)
.bind(mamba2_confidence)
.bind(mamba2_weight)
.bind(mamba2_vote)
.bind(tft_signal)
.bind(tft_confidence)
.bind(tft_weight)
.bind(tft_vote)
.bind(&self.config.node_id)
.bind(inference_latency_us)
.bind(serde_json::json!({"generator": "background_loop"}))
.execute(&self.db_pool)
.await
.context("Failed to insert prediction into database")?;
Ok(())
}
}
/// Extract model vote from ensemble decision
fn extract_model_vote(
decision: &ml::ensemble::EnsembleDecision,
model_id: &str,
) -> (Option<f64>, Option<f64>, Option<f64>, Option<String>) {
if let Some(vote) = decision.model_votes.get(model_id) {
let action_str = format!("{:?}", vote.action(0.0)).to_uppercase(); // threshold=0.0 for basic classification
(
Some(vote.signal),
Some(vote.confidence),
Some(vote.weight),
Some(action_str),
)
} else {
(None, None, None, None)
}
}
/// Market data snapshot for feature extraction
#[derive(Debug, Clone)]
struct MarketDataSnapshot {
bars: Vec<OhlcvBar>,
}
/// OHLCV bar from database
#[derive(Debug, Clone, sqlx::FromRow)]
struct OhlcvBar {
#[allow(dead_code)]
timestamp: chrono::DateTime<chrono::Utc>,
#[allow(dead_code)]
symbol: String,
open: f64,
high: f64,
low: f64,
close: f64,
volume: f64,
}
// ================================================================================================
// Technical Indicator Calculations (Simplified)
// For production, use ml/src/features/technical_indicators.rs
// ================================================================================================
fn calculate_sma(prices: &[f64], period: usize) -> f64 {
if prices.len() < period {
return prices.iter().sum::<f64>() / prices.len() as f64;
}
prices.iter().take(period).sum::<f64>() / period as f64
}
fn calculate_rsi(prices: &[f64], period: usize) -> f64 {
if prices.len() < period + 1 {
return 50.0; // Neutral
}
let mut gains = 0.0;
let mut losses = 0.0;
for i in 0..period {
let change = prices[i] - prices[i + 1];
if change > 0.0 {
gains += change;
} else {
losses += change.abs();
}
}
let avg_gain = gains / period as f64;
let avg_loss = losses / period as f64;
if avg_loss == 0.0 {
return 100.0;
}
let rs = avg_gain / avg_loss;
100.0 - (100.0 / (1.0 + rs))
}
fn calculate_volatility(prices: &[f64], period: usize) -> f64 {
if prices.len() < period {
return 0.0;
}
let mean = calculate_sma(prices, period);
let variance: f64 = prices
.iter()
.take(period)
.map(|p| (p - mean).powi(2))
.sum::<f64>()
/ period as f64;
variance.sqrt()
}
fn calculate_momentum(prices: &[f64], period: usize) -> f64 {
if prices.len() <= period {
return 0.0;
}
prices.first().copied().unwrap_or(0.0) - prices.get(period).copied().unwrap_or(0.0)
}
fn calculate_volume_ratio(volumes: &[f64], period: usize) -> f64 {
if volumes.is_empty() {
return 1.0;
}
let avg_volume = calculate_sma(volumes, period);
if avg_volume == 0.0 {
return 1.0;
}
volumes.first().copied().unwrap_or(0.0) / avg_volume
}
fn calculate_returns(prices: &[f64]) -> f64 {
if prices.len() < 2 {
return 0.0;
}
let price_0 = prices.first().copied().unwrap_or(0.0);
let price_1 = prices.get(1).copied().unwrap_or(1.0);
if price_1 == 0.0 {
return 0.0;
}
(price_0 - price_1) / price_1
}
fn calculate_log_returns(prices: &[f64]) -> f64 {
if prices.len() < 2 {
return 0.0;
}
let price_0 = prices.first().copied().unwrap_or(0.0);
let price_1 = prices.get(1).copied().unwrap_or(1.0);
if price_1 == 0.0 {
return 0.0;
}
(price_0 / price_1).ln()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_config_default() {
let config = PredictionLoopConfig::default();
assert_eq!(config.prediction_interval, Duration::from_secs(60));
assert_eq!(config.symbols.len(), 4);
assert!(config.symbols.contains(&"ES.FUT".to_string()));
}
#[test]
fn test_calculate_sma() {
let prices = vec![10.0, 11.0, 12.0, 13.0, 14.0];
let sma = calculate_sma(&prices, 3);
assert!((sma - 11.0).abs() < 0.01); // (10 + 11 + 12) / 3 = 11
}
#[test]
fn test_calculate_rsi() {
let prices = vec![
50.0, 51.0, 52.0, 51.5, 53.0, 52.0, 54.0, 53.5, 55.0, 54.5, 56.0, 55.5, 57.0, 56.5,
58.0,
];
let rsi = calculate_rsi(&prices, 14);
assert!(rsi >= 0.0 && rsi <= 100.0);
}
#[test]
fn test_calculate_returns() {
let prices = vec![110.0, 100.0];
let returns = calculate_returns(&prices);
assert!((returns - 0.1).abs() < 0.001); // 10% return
}
#[test]
fn test_extract_model_vote() {
use ml::ensemble::{EnsembleDecision, ModelVote, TradingAction};
use std::collections::HashMap;
let mut model_votes = HashMap::new();
model_votes.insert(
"DQN".to_string(),
ModelVote::new("DQN".to_string(), 0.8, 0.9, 0.33),
);
let decision = EnsembleDecision::new(TradingAction::Buy, 0.85, 0.75, 0.15, model_votes);
let (signal, confidence, weight, vote) = extract_model_vote(&decision, "DQN");
assert_eq!(signal, Some(0.8));
assert_eq!(confidence, Some(0.9));
assert_eq!(weight, Some(0.33));
assert_eq!(vote, Some("BUY".to_string()));
// Non-existent model
let (signal, confidence, weight, vote) = extract_model_vote(&decision, "UNKNOWN");
assert_eq!(signal, None);
assert_eq!(confidence, None);
assert_eq!(weight, None);
assert_eq!(vote, None);
}
}