diff --git a/adaptive-strategy/src/risk/mod.rs b/adaptive-strategy/src/risk/mod.rs index 1279088ae..f28947f19 100644 --- a/adaptive-strategy/src/risk/mod.rs +++ b/adaptive-strategy/src/risk/mod.rs @@ -33,25 +33,14 @@ use uuid::Uuid; use crate::config::{PositionSizingMethod, RiskConfig}; use crate::risk::kelly_position_sizer::{DrawdownTracker, VolatilityRegime}; // Import the proper MarketRegime enum from common module (has Normal variant) -pub use common::types::MarketRegime; +// DO NOT RE-EXPORT - Use explicit imports at usage sites // Enhanced Kelly Criterion implementation mod kelly_position_sizer; -pub use kelly_position_sizer::{ - ConcentrationMetrics, ConcentrationMonitor, DynamicRiskAdjuster, KellyConfig, - KellyPositionRecommendation, KellyPositionSizer, MarketData, RiskAdjustments, - VolatilityOptimizer, -}; // PPO-based position sizing implementation mod ppo_position_sizer; -pub use ppo_position_sizer::{ - ContinuousPPOConfig, ContinuousPolicyConfig, ContinuousTrajectory, - KellyComparisonMetrics, KellyIntegrationConfig, MarketData as PPOMarketData, - PPOPositionSizeRecommendation, PPOPositionSizer, PPOPositionSizerConfig, - PPORecommendationMetrics, PPOTrainingConfig, RegimeAdaptationConfig, RewardComponents, - RewardFunctionConfig, -}; +// DO NOT RE-EXPORT - Use explicit imports at usage sites // Comprehensive tests #[cfg(test)] diff --git a/adaptive-strategy/src/risk/mod.rs.bak b/adaptive-strategy/src/risk/mod.rs.bak new file mode 100644 index 000000000..1279088ae --- /dev/null +++ b/adaptive-strategy/src/risk/mod.rs.bak @@ -0,0 +1,1378 @@ +//! Risk management and position sizing module +//! +//! This module provides comprehensive risk management capabilities including: +//! - Enhanced Kelly criterion with dynamic risk tolerance adjustment +//! - Portfolio concentration monitoring and correlation analysis +//! - Position sizing algorithms (Kelly criterion, risk parity, volatility targeting) +//! - Portfolio risk metrics (VaR, CVaR, maximum drawdown) +//! - Real-time risk monitoring and limits +//! - Dynamic risk adjustment based on market conditions +//! - Volatility-based position size optimization + +// Import core types +use common::types::Position; +use common::types::Symbol; +use common::types::Price; +use common::types::Quantity; +use common::types::Decimal; +use common::error::CommonError; +use common::error::CommonResult; +use common::types::Order; +use common::types::OrderId; +use common::types::HftTimestamp; +use common::types::TradeId; + +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use tracing::{debug, info, warn}; +use num_traits::ToPrimitive; +use uuid::Uuid; + +// Add missing core types +use crate::config::{PositionSizingMethod, RiskConfig}; +use crate::risk::kelly_position_sizer::{DrawdownTracker, VolatilityRegime}; +// Import the proper MarketRegime enum from common module (has Normal variant) +pub use common::types::MarketRegime; + +// Enhanced Kelly Criterion implementation +mod kelly_position_sizer; +pub use kelly_position_sizer::{ + ConcentrationMetrics, ConcentrationMonitor, DynamicRiskAdjuster, KellyConfig, + KellyPositionRecommendation, KellyPositionSizer, MarketData, RiskAdjustments, + VolatilityOptimizer, +}; + +// PPO-based position sizing implementation +mod ppo_position_sizer; +pub use ppo_position_sizer::{ + ContinuousPPOConfig, ContinuousPolicyConfig, ContinuousTrajectory, + KellyComparisonMetrics, KellyIntegrationConfig, MarketData as PPOMarketData, + PPOPositionSizeRecommendation, PPOPositionSizer, PPOPositionSizerConfig, + PPORecommendationMetrics, PPOTrainingConfig, RegimeAdaptationConfig, RewardComponents, + RewardFunctionConfig, +}; + +// Comprehensive tests +#[cfg(test)] +// mod tests; // Commented out - tests module defined below + +// PPO integration tests +#[cfg(test)] +mod ppo_integration_test; + +/// Risk management engine +/// +/// Coordinates all risk management activities including position sizing, +/// portfolio risk monitoring, and dynamic risk adjustment. +#[derive(Debug)] +pub struct RiskManager { + /// Risk configuration + config: RiskConfig, + /// Position sizing calculator + position_sizer: PositionSizer, + /// Enhanced Kelly Criterion position sizer + kelly_sizer: Option, + /// PPO-based position sizer + ppo_sizer: Option, + /// Portfolio risk monitor + portfolio_monitor: PortfolioRiskMonitor, + /// Risk metrics calculator + metrics_calculator: RiskMetricsCalculator, + /// Dynamic risk adjuster + risk_adjuster: DynamicRiskAdjuster, +} + +/// Position sizing engine +#[derive(Debug)] +pub struct PositionSizer { + /// Position sizing method + method: PositionSizingMethod, + /// Historical returns for Kelly calculation + historical_returns: Vec, + /// Volatility estimates + volatility_estimates: HashMap, + /// Correlation matrix for risk parity + correlation_matrix: Option, + /// Portfolio risk monitoring + portfolio_monitor: PortfolioRiskMonitor, +} + +/// Portfolio risk monitoring +#[derive(Debug)] +pub struct PortfolioRiskMonitor { + /// Current portfolio positions + positions: HashMap, + /// Risk limits and thresholds + risk_limits: RiskLimits, + /// Real-time P&L tracking + pnl_tracker: PnLTracker, + /// Drawdown calculator + drawdown_calculator: DrawdownCalculator, +} + +/// Risk limits and thresholds +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RiskLimits { + /// Maximum portfolio VaR + pub max_portfolio_var: f64, + /// Maximum position size as fraction of portfolio + pub max_position_size: f64, + /// Maximum portfolio leverage + pub max_leverage: f64, + /// Maximum drawdown threshold + pub max_drawdown: f64, + /// Maximum daily loss limit + pub max_daily_loss: f64, + /// Maximum concentration per asset + pub max_concentration: f64, +} + +/// P&L tracking +#[derive(Debug, Clone)] +pub struct PnLTracker { + /// Daily P&L history + daily_pnl: Vec, + /// Current session P&L + session_pnl: f64, + /// Total portfolio value + portfolio_value: f64, + /// High-water mark for drawdown calculation + high_water_mark: f64, +} + +/// Daily P&L record +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DailyPnL { + /// Date + pub date: chrono::NaiveDate, + /// Realized P&L + pub realized_pnl: f64, + /// Unrealized P&L + pub unrealized_pnl: f64, + /// Total P&L + pub total_pnl: f64, + /// Portfolio value at end of day + pub portfolio_value: f64, +} + +/// Drawdown calculation +#[derive(Debug, Clone)] +pub struct DrawdownCalculator { + /// Portfolio value history + value_history: Vec<(chrono::DateTime, f64)>, + /// Current drawdown + current_drawdown: f64, + /// Maximum drawdown + max_drawdown: f64, + /// High-water mark + high_water_mark: f64, + /// Drawdown start time + drawdown_start: Option>, +} + +/// Risk metrics calculator +#[derive(Debug)] +pub struct RiskMetricsCalculator { + /// Historical price data + price_history: HashMap>, + /// Portfolio returns history + portfolio_returns: Vec, + /// Confidence levels for VaR calculation + confidence_levels: Vec, +} + +/// Price point for historical data +#[derive(Debug, Clone)] +pub struct PricePoint { + /// Timestamp + pub timestamp: chrono::DateTime, + /// Price + pub price: f64, + /// Volume + pub volume: f64, +} + +/// Correlation matrix for risk calculations +#[derive(Debug, Clone)] +pub struct CorrelationMatrix { + /// Asset symbols + symbols: Vec, + /// Correlation coefficients + correlations: Vec>, + /// Last update timestamp + last_update: chrono::DateTime, +} + +/// Risk adjustment record +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RiskAdjustment { + /// Timestamp + pub timestamp: chrono::DateTime, + /// Previous regime + pub previous_regime: MarketRegime, + /// New regime + pub new_regime: MarketRegime, + /// Risk scaling factor + pub risk_scaling: f64, + /// Reason for adjustment + pub reason: String, +} + +/// Position sizing recommendation +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PositionSizeRecommendation { + /// Recommended position size + pub size: f64, + /// Confidence in recommendation (0-1) + pub confidence: f64, + /// Maximum allowed size based on risk limits + pub max_allowed_size: f64, + /// Sizing method used + pub method: String, + /// Risk metrics + pub risk_metrics: PositionRiskMetrics, + /// Timestamp + pub timestamp: chrono::DateTime, +} + +/// Risk metrics for a position +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PositionRiskMetrics { + /// Expected return + pub expected_return: f64, + /// Expected volatility + pub expected_volatility: f64, + /// Sharpe ratio + pub sharpe_ratio: f64, + /// Value at Risk (VaR) + pub var_95: f64, + /// Conditional Value at Risk (CVaR) + pub cvar_95: f64, + /// Maximum loss potential + pub max_loss: f64, +} + +/// Portfolio risk metrics +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PortfolioRiskMetrics { + /// Total portfolio VaR + pub portfolio_var: f64, + /// Portfolio CVaR + pub portfolio_cvar: f64, + /// Current leverage + pub leverage: f64, + /// Current drawdown + pub current_drawdown: f64, + /// Maximum drawdown + pub max_drawdown: f64, + /// Sharpe ratio + pub sharpe_ratio: f64, + /// Sortino ratio + pub sortino_ratio: f64, + /// Beta (if benchmark provided) + pub beta: Option, + /// Concentration risk + pub concentration_risk: f64, + /// Timestamp + pub timestamp: chrono::DateTime, +} + +impl RiskManager { + /// Create a new risk manager + /// + /// # Arguments + /// + /// * `config` - Risk management configuration + /// + /// # Returns + /// + /// A new `RiskManager` instance + pub fn new(config: RiskConfig) -> Result { + info!( + "Initializing risk manager with max VaR: {}", + config.max_portfolio_var + ); + + let position_sizer = PositionSizer::new(&config)?; + let portfolio_monitor = PortfolioRiskMonitor::new(&config)?; + let metrics_calculator = RiskMetricsCalculator::new()?; + let risk_adjuster = DynamicRiskAdjuster::new(&KellyConfig::default())?; + + // Initialize enhanced Kelly sizer if Kelly method is selected + let kelly_sizer = if matches!(config.position_sizing_method, PositionSizingMethod::Kelly) { + let kelly_config = KellyConfig { + max_fraction: config.kelly_fraction, + min_fraction: 0.01, + lookback_period: 252, + confidence_threshold: 0.6, + volatility_adjustment: true, + drawdown_protection: true, + dynamic_risk_scaling: true, + max_concentration: 0.20, + correlation_adjustment: 0.85, + base_kelly: config.kelly_fraction, + }; + Some(KellyPositionSizer::new(kelly_config)?) + } else { + None + }; + + // Initialize PPO sizer if PPO method is selected + let ppo_sizer = if matches!(config.position_sizing_method, PositionSizingMethod::PPO) { + let ppo_config = PPOPositionSizerConfig { + state_dim: 128, + ppo_config: ContinuousPPOConfig { + state_dim: 128, + action_dim: 1, + learning_rate: 3e-4, + policy_config: ContinuousPolicyConfig { + state_dim: 128, + hidden_dims: vec![256, 128, 64], + action_bounds: (0.0, 1.0), + min_log_std: -3.0, + max_log_std: 1.0, + init_log_std: -1.5, + learnable_std: true, + }, + value_hidden_dims: vec![256, 128, 64], + policy_learning_rate: 3e-4, + value_learning_rate: 3e-4, + clip_epsilon: 0.2, + value_loss_coeff: 0.5, + entropy_coeff: 0.01, + batch_size: 2048, + mini_batch_size: 64, + num_epochs: 10, + max_grad_norm: 0.5, + }, + reward_config: RewardFunctionConfig { + sharpe_weight: 2.0, + drawdown_penalty_weight: 5.0, + kelly_alignment_weight: 1.5, + concentration_penalty_weight: 3.0, + var_penalty_weight: 4.0, + return_scaling: 1.0, + risk_penalty_thresholds: ppo_position_sizer::RiskPenaltyThresholds { + max_sharpe_deviation: 0.5, + max_drawdown_threshold: config.max_drawdown_threshold, + max_concentration_threshold: 0.25, + var_limit_fraction: config.max_portfolio_var, + }, + }, + ..Default::default() + }; + Some( + PPOPositionSizer::new(ppo_config) + .map_err(|e| anyhow::anyhow!("Failed to create PPO position sizer: {}", e))?, + ) + } else { + None + }; + + Ok(Self { + config, + position_sizer, + kelly_sizer, + ppo_sizer, + portfolio_monitor, + metrics_calculator, + risk_adjuster, + }) + } + + /// Calculate position size for a trade + /// + /// # Arguments + /// + /// * `symbol` - Trading symbol + /// * `expected_return` - Expected return for the trade + /// * `confidence` - Confidence in the prediction (0-1) + /// * `current_price` - Current market price + /// + /// # Returns + /// + /// Position sizing recommendation + pub async fn calculate_position_size( + &mut self, + symbol: &str, + expected_return: f64, + confidence: f64, + current_price: f64, + ) -> Result { + debug!( + "Calculating position size for {} with expected return: {}", + symbol, expected_return + ); + + // Use enhanced Kelly sizer if available and method is Kelly + if let PositionSizingMethod::Kelly = &self.config.position_sizing_method { + if self.kelly_sizer.is_some() { + return self + .calculate_kelly_position_size( + symbol, + expected_return, + confidence, + current_price, + ) + .await; + } + } + + // Use PPO sizer if available and method is PPO + if let PositionSizingMethod::PPO = &self.config.position_sizing_method { + if self.ppo_sizer.is_some() { + return self + .calculate_ppo_position_size(symbol, expected_return, confidence, current_price) + .await; + } + } + + // Fallback to standard position sizing + let portfolio_metrics = self.get_portfolio_risk_metrics().await?; + + let base_size = self + .position_sizer + .calculate_size(symbol, expected_return, confidence, current_price) + .await?; + + let max_allowed = + self.calculate_max_allowed_size(symbol, current_price, &portfolio_metrics)?; + let recommended_size = base_size.min(max_allowed); + + let risk_metrics = self + .calculate_position_risk_metrics( + symbol, + recommended_size, + expected_return, + current_price, + ) + .await?; + + let adjusted_size = self + .risk_adjuster + .adjust_position_size(recommended_size, &risk_metrics, &portfolio_metrics) + .await?; + + Ok(PositionSizeRecommendation { + size: adjusted_size, + confidence, + max_allowed_size: max_allowed, + method: format!("{:?}", self.config.position_sizing_method), + risk_metrics, + timestamp: chrono::Utc::now(), + }) + } + + /// Update portfolio with new position + pub async fn update_position(&mut self, position: Position) -> Result<()> { + info!( + "Updating position for {}: quantity={}", + position.symbol, position.quantity + ); + + self.portfolio_monitor.update_position(position).await?; + + // Check risk limits after update + self.check_risk_limits().await?; + + Ok(()) + } + + /// Get current portfolio risk metrics + pub async fn get_portfolio_risk_metrics(&self) -> Result { + self.portfolio_monitor + .calculate_risk_metrics(&self.metrics_calculator) + .await + } + + /// Check if trade violates risk limits + pub async fn check_trade_risk(&self, symbol: &str, size: f64, price: f64) -> Result { + // Create temporary position to test + let test_position = Position { + id: uuid::Uuid::new_v4(), + symbol: symbol.into(), + quantity: Decimal::try_from(size).unwrap_or(Decimal::ZERO), + avg_price: Decimal::try_from(price).unwrap_or(Decimal::ZERO), + avg_cost: Decimal::try_from(price).unwrap_or(Decimal::ZERO), + basis: Decimal::try_from(size * price).unwrap_or(Decimal::ZERO), + average_price: Decimal::try_from(price).unwrap_or(Decimal::ZERO), + market_value: Decimal::try_from(size * price).unwrap_or(Decimal::ZERO), + unrealized_pnl: Decimal::ZERO, + realized_pnl: Decimal::ZERO, + created_at: chrono::Utc::now(), + updated_at: chrono::Utc::now(), + last_updated: chrono::Utc::now(), + current_price: Some(Decimal::try_from(price).unwrap_or(Decimal::ZERO)), + notional_value: Decimal::try_from(size * price).unwrap_or(Decimal::ZERO), + margin_requirement: Decimal::try_from(size * price * 0.1).unwrap_or(Decimal::ZERO), + }; + + // Check against risk limits + self.portfolio_monitor.check_position_limits(&test_position) + } + + // Note: update_market_regime method moved to comprehensive implementation below + + /// Get current risk limits status + pub async fn get_risk_limits_status(&self) -> Result> { + self.portfolio_monitor.get_limits_utilization().await + } + + /// Calculate position size using enhanced Kelly criterion + async fn calculate_kelly_position_size( + &mut self, + symbol: &str, + expected_return: f64, + confidence: f64, + current_price: f64, + ) -> Result { + info!( + "Using enhanced Kelly criterion for position sizing: {}", + symbol + ); + + // Gather historical returns (production - would come from market data service) + let historical_returns = self.get_historical_returns(symbol).await?; + + // Create market data for Kelly calculation + let market_data = self.build_market_data(symbol, current_price).await?; + + // Get Kelly recommendation + let kelly_recommendation = self + .kelly_sizer + .as_mut() + .unwrap() + .calculate_position_size( + symbol, + expected_return, + confidence, + &historical_returns, + &market_data, + ) + .await?; + + // Convert Kelly recommendation to standard PositionSizeRecommendation + let portfolio_value = self.portfolio_monitor.get_portfolio_value(); + let position_size = + kelly_recommendation.recommended_fraction * portfolio_value / current_price; + + // Build risk metrics from Kelly recommendation + let risk_metrics = PositionRiskMetrics { + expected_return: kelly_recommendation.expected_return, + expected_volatility: kelly_recommendation.volatility, + sharpe_ratio: kelly_recommendation.sharpe_ratio, + var_95: position_size * current_price * kelly_recommendation.volatility * 1.645, + cvar_95: position_size * current_price * kelly_recommendation.volatility * 1.645 * 1.28, + max_loss: position_size * current_price, + }; + + Ok(PositionSizeRecommendation { + size: position_size, + confidence: kelly_recommendation.confidence, + max_allowed_size: kelly_recommendation.max_allowed_fraction * portfolio_value + / current_price, + method: "Enhanced Kelly Criterion".to_string(), + risk_metrics, + timestamp: kelly_recommendation.timestamp, + }) + } + + /// Get historical returns for a symbol (production implementation) + async fn get_historical_returns(&self, _symbol: &str) -> Result> { + // In production, this would fetch from market data service + // For now, return sample data + Ok(vec![ + 0.05, -0.02, 0.08, -0.03, 0.06, -0.01, 0.04, -0.02, 0.07, -0.01, 0.03, -0.04, 0.09, + -0.02, 0.05, -0.03, 0.06, -0.01, 0.08, -0.02, + ]) + } + + /// Build market data for Kelly calculation + async fn build_market_data(&self, symbol: &str, current_price: f64) -> Result { + let mut prices = HashMap::new(); + prices.insert(symbol.to_string(), current_price); + + let mut volatilities = HashMap::new(); + volatilities.insert(symbol.to_string(), 0.20); // 20% default volatility + + Ok(MarketData { + prices, + volatilities, + correlations: HashMap::new(), + timestamp: chrono::Utc::now(), + volatility_index: Some(20.0), + sentiment_indicators: HashMap::new(), + }) + } + + /// Calculate position size using PPO with risk-aware continuous optimization + async fn calculate_ppo_position_size( + &mut self, + symbol: &str, + expected_return: f64, + confidence: f64, + current_price: f64, + ) -> Result { + info!("Using PPO for position sizing: {}", symbol); + + // Build market data for PPO + let market_data = self.build_ppo_market_data(symbol, current_price).await?; + + // Get current portfolio metrics + let portfolio_metrics = self.get_portfolio_risk_metrics().await?; + + // Get Kelly recommendation for comparison (if Kelly sizer available) + let kelly_recommendation = if self.kelly_sizer.is_some() { + let historical_returns = self.get_historical_returns(symbol).await?; + let kelly_market_data = self.build_market_data(symbol, current_price).await?; + + Some( + self.kelly_sizer + .as_mut() + .unwrap() + .calculate_position_size( + symbol, + expected_return, + confidence, + &historical_returns, + &kelly_market_data, + ) + .await?, + ) + } else { + None + }; + // Calculate PPO recommendation + let ppo_recommendation = self + .ppo_sizer + .as_mut() + .unwrap() + .calculate_position_size( + symbol, + &market_data, + &portfolio_metrics, + kelly_recommendation.as_ref(), + ) + .await + .map_err(|e| anyhow::anyhow!("PPO position sizing failed: {}", e))?; + + // Convert PPO recommendation to standard format + let mut recommendation = ppo_recommendation.base_recommendation; + + // Apply risk constraints + let max_allowed = + self.calculate_max_allowed_size(symbol, current_price, &portfolio_metrics)?; + recommendation.size = recommendation.size.min(max_allowed); + + // Add PPO-specific information to method field + recommendation.method = format!( + "PPO (confidence: {:.2}, Kelly blend: {:.2}, entropy: {:.3})", + ppo_recommendation.action_confidence, + ppo_recommendation.kelly_comparison.blended_recommendation, + ppo_recommendation.ppo_metrics.policy_entropy + ); + + info!( + "PPO position size for {}: {:.4} (vs Kelly: {:.4}, confidence: {:.2})", + symbol, + recommendation.size, + ppo_recommendation.kelly_comparison.kelly_optimal_size, + ppo_recommendation.action_confidence + ); + + Ok(recommendation) + } + + /// Build market data for PPO position sizing + async fn build_ppo_market_data( + &self, + symbol: &str, + current_price: f64, + ) -> Result { + let mut prices = HashMap::new(); + prices.insert(symbol.to_string(), current_price); + + let mut volatilities = HashMap::new(); + volatilities.insert(symbol.to_string(), 0.20); // 20% default volatility + + let mut sentiment_indicators = HashMap::new(); + sentiment_indicators.insert("market_sentiment".to_string(), 0.5); // Neutral sentiment + sentiment_indicators.insert("momentum".to_string(), 0.0); // No momentum bias + + Ok(PPOMarketData { + prices, + volatilities, + correlations: HashMap::new(), + timestamp: chrono::Utc::now(), + volatility_index: Some(20.0), // VIX-like indicator + sentiment_indicators, + }) + } + + // Convert local MarketRegime to ml::prelude::MarketRegime + fn convert_regime(regime: &crate::regime::MarketRegime) -> u32 { + match regime { + crate::regime::MarketRegime::Normal => 0, + crate::regime::MarketRegime::Trending => 1, + crate::regime::MarketRegime::Bull => 2, + crate::regime::MarketRegime::Bear => 3, + crate::regime::MarketRegime::Sideways => 4, + crate::regime::MarketRegime::HighVolatility => 5, + crate::regime::MarketRegime::LowVolatility => 6, + crate::regime::MarketRegime::Crisis => 7, + crate::regime::MarketRegime::Recovery => 8, + crate::regime::MarketRegime::Bubble => 9, + crate::regime::MarketRegime::Correction => 10, + crate::regime::MarketRegime::Unknown => 11, + } + } + + /// Update market regime for both Kelly and PPO sizing + pub async fn update_market_regime(&mut self, regime: MarketRegime) -> Result<()> { + if let Some(kelly_sizer) = &mut self.kelly_sizer { + // Convert common::types::MarketRegime to regime::MarketRegime + let local_regime = match regime { + common::types::MarketRegime::Normal => crate::regime::MarketRegime::Normal, + common::types::MarketRegime::Trending => crate::regime::MarketRegime::Trending, + common::types::MarketRegime::Sideways => crate::regime::MarketRegime::Sideways, + common::types::MarketRegime::Bull => crate::regime::MarketRegime::Bull, + common::types::MarketRegime::Bear => crate::regime::MarketRegime::Bear, + common::types::MarketRegime::Crisis => crate::regime::MarketRegime::Crisis, + common::types::MarketRegime::HighVolatility => crate::regime::MarketRegime::HighVolatility, + common::types::MarketRegime::LowVolatility => crate::regime::MarketRegime::LowVolatility, + common::types::MarketRegime::Volatile => crate::regime::MarketRegime::HighVolatility, // Alias + common::types::MarketRegime::Calm => crate::regime::MarketRegime::LowVolatility, // Alias + common::types::MarketRegime::Unknown => crate::regime::MarketRegime::Unknown, + common::types::MarketRegime::Recovery => crate::regime::MarketRegime::Recovery, + common::types::MarketRegime::Bubble => crate::regime::MarketRegime::Bubble, + common::types::MarketRegime::Correction => crate::regime::MarketRegime::Correction, + common::types::MarketRegime::Custom(_) => crate::regime::MarketRegime::Unknown, // Map custom to unknown + }; + kelly_sizer.update_market_regime(local_regime).await?; + } + + if let Some(ppo_sizer) = &mut self.ppo_sizer { + ppo_sizer + .update_market_regime(regime) + .await + .map_err(|e| anyhow::anyhow!("Failed to update PPO market regime: {}", e))?; + } + + Ok(()) + } + + /// Get Kelly performance metrics if available + pub async fn get_kelly_performance_metrics( + &self, + ) -> Result> { + if let Some(kelly_sizer) = &self.kelly_sizer { + Ok(Some(kelly_sizer.get_performance_metrics().await?)) + } else { + Ok(None) + } + } + + /// Get concentration metrics if Kelly sizer is available + pub async fn get_concentration_metrics(&self) -> Result> { + if let Some(kelly_sizer) = &self.kelly_sizer { + Ok(Some(kelly_sizer.get_concentration_metrics().await?)) + } else { + Ok(None) + } + } + + /// Update PPO policy with trading experience (if PPO sizer is available) + pub async fn update_ppo_policy( + &mut self, + trajectory: ContinuousTrajectory, + ) -> Result> { + if let Some(ppo_sizer) = &mut self.ppo_sizer { + let (policy_loss, value_loss) = ppo_sizer + .update_policy(trajectory) + .await + .map_err(|e| anyhow::anyhow!("Failed to update PPO policy: {}", e))?; + Ok(Some((policy_loss, value_loss))) + } else { + Ok(None) + } + } + + /// Get PPO performance metrics if available + pub fn get_ppo_performance_metrics( + &self, + ) -> Option<&ppo_position_sizer::PPOPerformanceTracker> { + self.ppo_sizer + .as_ref() + .map(|sizer| sizer.get_performance_metrics()) + } + + /// Get PPO configuration if available + pub fn get_ppo_config(&self) -> Option<&PPOPositionSizerConfig> { + self.ppo_sizer.as_ref().map(|sizer| sizer.get_config()) + } + + /// Check if PPO position sizing is enabled + pub fn is_ppo_enabled(&self) -> bool { + matches!( + self.config.position_sizing_method, + PositionSizingMethod::PPO + ) && self.ppo_sizer.is_some() + } + + /// Calculate maximum allowed position size + fn calculate_max_allowed_size( + &self, + symbol: &str, + price: f64, + portfolio_metrics: &PortfolioRiskMetrics, + ) -> Result { + let portfolio_value = self.portfolio_monitor.get_portfolio_value(); + + // Position size limit + let max_by_position_limit = + (portfolio_value * self.config.max_leverage * self.config.kelly_fraction) / price; + + // VaR limit + let remaining_var_capacity = + self.config.max_portfolio_var - portfolio_metrics.portfolio_var; + let max_by_var = if remaining_var_capacity > 0.0 { + remaining_var_capacity * portfolio_value / price + } else { + 0.0 + }; + + // Concentration limit + let max_by_concentration = (portfolio_value * self.config.kelly_fraction) / price; + + Ok([max_by_position_limit, max_by_var, max_by_concentration] + .iter() + .cloned() + .fold(f64::INFINITY, f64::min)) + } + + /// Calculate risk metrics for a specific position + async fn calculate_position_risk_metrics( + &self, + symbol: &str, + size: f64, + expected_return: f64, + price: f64, + ) -> Result { + let volatility = self + .position_sizer + .get_volatility_estimate(symbol) + .unwrap_or(0.02); + + // Calculate VaR and CVaR (simplified) + let var_95 = size * price * volatility * 1.645; // 95% VaR assuming normal distribution + let cvar_95 = var_95 * 1.28; // Approximate CVaR + + let sharpe_ratio = if volatility > 0.0 { + expected_return / volatility + } else { + 0.0 + }; + + Ok(PositionRiskMetrics { + expected_return, + expected_volatility: volatility, + sharpe_ratio, + var_95, + cvar_95, + max_loss: size * price, // Worst case: total loss + }) + } + + /// Check all risk limits + async fn check_risk_limits(&self) -> Result<()> { + let portfolio_metrics = self.get_portfolio_risk_metrics().await?; + + if portfolio_metrics.portfolio_var > self.config.max_portfolio_var { + warn!( + "Portfolio VaR exceeded: {} > {}", + portfolio_metrics.portfolio_var, self.config.max_portfolio_var + ); + } + + if portfolio_metrics.current_drawdown > self.config.max_drawdown_threshold { + warn!( + "Maximum drawdown exceeded: {} > {}", + portfolio_metrics.current_drawdown, self.config.max_drawdown_threshold + ); + } + + if portfolio_metrics.leverage > self.config.max_leverage { + warn!( + "Maximum leverage exceeded: {} > {}", + portfolio_metrics.leverage, self.config.max_leverage + ); + } + + Ok(()) + } +} + +impl PositionSizer { + /// Create a new position sizer + pub fn new(config: &RiskConfig) -> Result { + Ok(Self { + method: config.position_sizing_method.clone(), + historical_returns: Vec::new(), + volatility_estimates: HashMap::new(), + correlation_matrix: None, + portfolio_monitor: PortfolioRiskMonitor::new(config)?, + }) + } + + /// Calculate position size based on configured method + pub async fn calculate_size( + &self, + symbol: &str, + expected_return: f64, + confidence: f64, + price: f64, + ) -> Result { + match &self.method { + PositionSizingMethod::FixedFraction => { + self.calculate_fixed_fraction_size(symbol, price) + } + PositionSizingMethod::FixedFractional(fraction) => { + self.calculate_fixed_fractional_size(*fraction, symbol, price) + } + PositionSizingMethod::EqualWeight => { + self.calculate_equal_weight_size(symbol, price) + } + PositionSizingMethod::Kelly => self.calculate_kelly_size(expected_return, confidence), + PositionSizingMethod::PPO => { + // PPO position sizing is handled through the ppo_sizer + // This is a fallback for when PPO sizer is not available + self.calculate_fixed_fraction_size(symbol, price) + } + PositionSizingMethod::RiskParity => { + // Risk parity sizing - fallback to fixed fraction + self.calculate_fixed_fraction_size(symbol, price) + } + PositionSizingMethod::VolatilityTarget => { + // Volatility targeting - fallback to fixed fraction + self.calculate_fixed_fraction_size(symbol, price) + } + PositionSizingMethod::Custom(_) => { + // Custom sizing method - fallback to fixed fraction + self.calculate_fixed_fraction_size(symbol, price) + } + } + } + + /// Fixed fraction position sizing + fn calculate_fixed_fraction_size(&self, _symbol: &str, _price: f64) -> Result { + // Production implementation + Ok(1000.0) // Fixed size + } + + /// Fixed fractional position sizing + fn calculate_fixed_fractional_size(&self, fraction: f64, _symbol: &str, _price: f64) -> Result { + let portfolio_value = self.portfolio_monitor.get_portfolio_value(); + Ok(portfolio_value * fraction.clamp(0.0, 1.0)) + } + + /// Equal weight position sizing + fn calculate_equal_weight_size(&self, _symbol: &str, _price: f64) -> Result { + let portfolio_value = self.portfolio_monitor.get_portfolio_value(); + let position_count = self.portfolio_monitor.positions.len().max(1); + Ok(portfolio_value / position_count as f64) + } + + /// Kelly criterion position sizing (enhanced version available via KellyPositionSizer) + fn calculate_kelly_size(&self, expected_return: f64, confidence: f64) -> Result { + if self.historical_returns.is_empty() { + return Ok(0.0); + } + + // Basic Kelly calculation - for enhanced features use KellyPositionSizer + let win_rate = confidence; + let avg_win = expected_return; + let avg_loss = self + .historical_returns + .iter() + .filter(|&&r| r < 0.0) + .sum::() + / self.historical_returns.iter().filter(|&&r| r < 0.0).count() as f64; + + if avg_loss == 0.0 { + return Ok(0.0); + } + + let kelly_fraction = (win_rate * avg_win - (1.0 - win_rate) * avg_loss.abs()) / avg_win; + + // Apply safety margin + let conservative_kelly = kelly_fraction * 0.25; // Use quarter Kelly for safety + + Ok(conservative_kelly.max(0.0).min(1.0) * 10000.0) // Scale to position size + } + + /// Risk parity position sizing + fn calculate_risk_parity_size(&self, _symbol: &str) -> Result { + // Production implementation + Ok(1000.0) + } + + /// Volatility targeting position sizing + fn calculate_volatility_target_size(&self, symbol: &str, price: f64) -> Result { + let target_volatility = 0.15; // 15% annual volatility target + let estimated_volatility = self.get_volatility_estimate(symbol).unwrap_or(0.02); + + if estimated_volatility == 0.0 { + return Ok(0.0); + } + + let size = (target_volatility / estimated_volatility) * 1000.0 / price; + Ok(size) + } + + /// Custom position sizing method + fn calculate_custom_size( + &self, + _method: &str, + _symbol: &str, + _expected_return: f64, + _confidence: f64, + _price: f64, + ) -> Result { + // Production for custom sizing algorithms + Ok(1000.0) + } + + /// Get volatility estimate for symbol + pub fn get_volatility_estimate(&self, symbol: &str) -> Option { + self.volatility_estimates.get(symbol).copied() + } + + /// Update volatility estimate + pub fn update_volatility_estimate(&mut self, symbol: String, volatility: f64) { + self.volatility_estimates.insert(symbol, volatility); + } +} + +impl PortfolioRiskMonitor { + /// Create a new portfolio risk monitor + pub fn new(config: &RiskConfig) -> Result { + let risk_limits = RiskLimits { + max_portfolio_var: config.max_portfolio_var, + max_position_size: 0.1, // 10% of portfolio + max_leverage: config.max_leverage, + max_drawdown: config.max_drawdown_threshold, + max_daily_loss: 0.05, // 5% daily loss limit + max_concentration: 0.2, // 20% concentration limit + }; + + Ok(Self { + positions: HashMap::new(), + risk_limits, + pnl_tracker: PnLTracker::new(100000.0), // $100k initial portfolio + drawdown_calculator: DrawdownCalculator::new(), + }) + } + + /// Update position in portfolio + pub async fn update_position(&mut self, position: Position) -> Result<()> { + self.positions.insert(position.symbol.to_string(), position); + self.update_portfolio_value().await?; + Ok(()) + } + + /// Calculate current portfolio risk metrics + pub async fn calculate_risk_metrics( + &self, + metrics_calculator: &RiskMetricsCalculator, + ) -> Result { + let portfolio_value = self.get_portfolio_value(); + let leverage = self.calculate_leverage()?; + + // Calculate portfolio VaR (simplified) + let portfolio_var = self.calculate_portfolio_var(metrics_calculator)?; + + Ok(PortfolioRiskMetrics { + portfolio_var, + portfolio_cvar: portfolio_var * 1.28, // Approximate CVaR + leverage, + current_drawdown: self.drawdown_calculator.current_drawdown, + max_drawdown: self.drawdown_calculator.max_drawdown, + sharpe_ratio: self.calculate_sharpe_ratio()?, + sortino_ratio: self.calculate_sortino_ratio()?, + beta: None, // Would require benchmark data + concentration_risk: self.calculate_concentration_risk()?, + timestamp: chrono::Utc::now(), + }) + } + + /// Check if position violates limits + pub fn check_position_limits(&self, position: &Position) -> Result { + let portfolio_value = self.get_portfolio_value(); + let position_value = position.quantity.abs().to_f64().unwrap_or(0.0) * position.average_price.to_f64().unwrap_or(0.0); + let position_fraction = position_value / portfolio_value; + + Ok(position_fraction <= self.risk_limits.max_position_size) + } + + /// Get portfolio value + pub fn get_portfolio_value(&self) -> f64 { + self.pnl_tracker.portfolio_value + } + + /// Get risk limits utilization + pub async fn get_limits_utilization(&self) -> Result> { + let mut utilization = HashMap::new(); + + let portfolio_value = self.get_portfolio_value(); + let total_exposure: f64 = self + .positions + .values() + .map(|p| p.quantity.abs().to_f64().unwrap_or(0.0) * p.average_price.to_f64().unwrap_or(0.0)) + .sum(); + + let leverage = total_exposure / portfolio_value; + utilization.insert( + "leverage".to_string(), + leverage / self.risk_limits.max_leverage, + ); + + utilization.insert( + "drawdown".to_string(), + self.drawdown_calculator.current_drawdown / self.risk_limits.max_drawdown, + ); + + Ok(utilization) + } + + /// Update portfolio value + async fn update_portfolio_value(&mut self) -> Result<()> { + let total_value: f64 = self + .positions + .values() + .map(|p| (p.quantity.abs().to_f64().unwrap_or(0.0) * p.average_price.to_f64().unwrap_or(0.0))) + .sum(); + + self.pnl_tracker.portfolio_value = total_value; + self.drawdown_calculator.update(total_value); + + Ok(()) + } + + /// Calculate portfolio leverage + fn calculate_leverage(&self) -> Result { + let portfolio_value = self.get_portfolio_value(); + let total_exposure: f64 = self + .positions + .values() + .map(|p| p.quantity.abs().to_f64().unwrap_or(0.0) * p.average_price.to_f64().unwrap_or(0.0)) + .sum(); + + if portfolio_value == 0.0 { + Ok(0.0) + } else { + Ok(total_exposure / portfolio_value) + } + } + + /// Calculate portfolio VaR (simplified) + fn calculate_portfolio_var(&self, _metrics_calculator: &RiskMetricsCalculator) -> Result { + // Simplified VaR calculation - would be more sophisticated in production + let portfolio_value = self.get_portfolio_value(); + let estimated_volatility = 0.02; // 2% daily volatility assumption + + Ok(portfolio_value * estimated_volatility * 1.645) // 95% VaR + } + + /// Calculate Sharpe ratio + fn calculate_sharpe_ratio(&self) -> Result { + // Production implementation + Ok(1.5) + } + + /// Calculate Sortino ratio + fn calculate_sortino_ratio(&self) -> Result { + // Production implementation + Ok(1.8) + } + + /// Calculate concentration risk + fn calculate_concentration_risk(&self) -> Result { + if self.positions.is_empty() { + return Ok(0.0); + } + + let portfolio_value = self.get_portfolio_value(); + let max_position_value = self + .positions + .values() + .map(|p| (p.quantity.abs().to_f64().unwrap_or(0.0) * p.average_price.to_f64().unwrap_or(0.0)).abs()) + .fold(0.0f64, f64::max); + + Ok(max_position_value / portfolio_value) + } +} + +impl PnLTracker { + /// Create a new P&L tracker + pub fn new(initial_value: f64) -> Self { + Self { + daily_pnl: Vec::new(), + session_pnl: 0.0, + portfolio_value: initial_value, + high_water_mark: initial_value, + } + } +} + +impl DrawdownCalculator { + /// Create a new drawdown calculator + pub fn new() -> Self { + let initial_value = 100000.0; + Self { + value_history: Vec::new(), + current_drawdown: 0.0, + max_drawdown: 0.0, + high_water_mark: initial_value, + drawdown_start: None, + } + } + + /// Update with new portfolio value + pub fn update(&mut self, new_value: f64) { + let now = chrono::Utc::now(); + self.value_history.push((now, new_value)); + + // Update high water mark + if new_value > self.high_water_mark { + self.high_water_mark = new_value; + self.current_drawdown = 0.0; + self.drawdown_start = None; + } else { + // Calculate current drawdown + self.current_drawdown = (self.high_water_mark - new_value) / self.high_water_mark; + + if self.drawdown_start.is_none() { + self.drawdown_start = Some(now); + } + + // Update max drawdown + if self.current_drawdown > self.max_drawdown { + self.max_drawdown = self.current_drawdown; + } + } + + // Maintain history size + if self.value_history.len() > 10000 { + self.value_history.remove(0); + } + } +} + +impl RiskMetricsCalculator { + /// Create a new risk metrics calculator + pub fn new() -> Result { + Ok(Self { + price_history: HashMap::new(), + portfolio_returns: Vec::new(), + confidence_levels: vec![0.95, 0.99], // 95% and 99% confidence levels + }) + } + + /// Add price data for calculations + pub fn add_price_data(&mut self, symbol: String, price_point: PricePoint) { + let history = self.price_history.entry(symbol).or_insert_with(Vec::new); + history.push(price_point); + + // Maintain history size + if history.len() > 1000 { + history.remove(0); + } + } +} + +// DynamicRiskAdjuster implementation moved to kelly_position_sizer.rs to avoid duplication + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_risk_manager_creation() { + let config = RiskConfig { + max_portfolio_var: 0.02, + var_confidence_level: 0.95, + max_drawdown_threshold: 0.05, + position_sizing_method: PositionSizingMethod::Kelly, + kelly_fraction: 0.25, + max_leverage: 2.0, + stop_loss_pct: 0.02, + take_profit_pct: 0.04, + }; + + let risk_manager = RiskManager::new(config); + assert!(risk_manager.is_ok()); + } + + #[test] + fn test_position_sizer() { + let config = RiskConfig { + max_portfolio_var: 0.02, + var_confidence_level: 0.95, + max_drawdown_threshold: 0.05, + position_sizing_method: PositionSizingMethod::FixedFraction, + kelly_fraction: 0.25, + max_leverage: 2.0, + stop_loss_pct: 0.02, + take_profit_pct: 0.04, + }; + + let sizer = PositionSizer::new(&config); + assert!(sizer.is_ok()); + } + + #[test] + fn test_drawdown_calculator() { + let mut calc = DrawdownCalculator::new(); + + // Test increasing values (no drawdown) + calc.update(110000.0); + assert_eq!(calc.current_drawdown, 0.0); + + // Test drawdown + calc.update(95000.0); + assert!(calc.current_drawdown > 0.0); + assert!(calc.max_drawdown > 0.0); + } + + #[test] + fn test_dynamic_risk_adjuster() { + let adjuster = DynamicRiskAdjuster::new(&KellyConfig::default()).unwrap(); + + let position_metrics = PositionRiskMetrics { + expected_return: 0.05, + expected_volatility: 0.02, + sharpe_ratio: 2.5, + var_95: 1000.0, + cvar_95: 1280.0, + max_loss: 5000.0, + }; + + let portfolio_metrics = PortfolioRiskMetrics { + portfolio_var: 0.01, + portfolio_cvar: 0.013, + leverage: 1.5, + current_drawdown: 0.0, + max_drawdown: 0.02, + sharpe_ratio: 1.8, + sortino_ratio: 2.1, + beta: None, + concentration_risk: 0.15, + timestamp: chrono::Utc::now(), + }; + + let adjusted_size = + adjuster.adjust_position_size(1000.0, &position_metrics, &portfolio_metrics); + assert!(adjusted_size.is_ok()); + } +} diff --git a/backtesting/src/lib.rs b/backtesting/src/lib.rs index 10b129f98..cf839e940 100644 --- a/backtesting/src/lib.rs +++ b/backtesting/src/lib.rs @@ -83,8 +83,8 @@ pub mod strategy_tester; pub mod strategy_runner; -// Import OrderSide from common types -use trading_engine::types::events::MarketEvent; +// Import events from trading_engine directly +use trading_engine::events::MarketEvent; /// Main backtesting engine configuration #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/backtesting/src/replay_engine.rs b/backtesting/src/replay_engine.rs index b7d7b8101..32ebdab3c 100644 --- a/backtesting/src/replay_engine.rs +++ b/backtesting/src/replay_engine.rs @@ -13,7 +13,7 @@ use std::{ use anyhow::{Context, Result}; use chrono::{DateTime, Utc}; use common::{Timestamp, Symbol, Decimal, Quantity, Price}; -use trading_engine::types::events::MarketEvent; +use trading_engine::events::MarketEvent; use crossbeam_channel::{bounded, Receiver, Sender}; use dashmap::DashMap; use serde::{Deserialize, Serialize}; diff --git a/backtesting/src/strategy_runner.rs b/backtesting/src/strategy_runner.rs index d5cde8676..c6cc918b7 100644 --- a/backtesting/src/strategy_runner.rs +++ b/backtesting/src/strategy_runner.rs @@ -8,7 +8,7 @@ use async_trait::async_trait; use common::types::{OrderSide, OrderStatus, Position, Symbol, Quantity, Price, Order}; use rust_decimal::prelude::ToPrimitive; use rust_decimal::Decimal; -use trading_engine::types::events::MarketEvent; +use trading_engine::events::MarketEvent; // Use canonical types from ML module use ml::{Features, ModelPrediction}; diff --git a/backtesting/src/strategy_tester.rs b/backtesting/src/strategy_tester.rs index 1725899d2..79e74ba06 100644 --- a/backtesting/src/strategy_tester.rs +++ b/backtesting/src/strategy_tester.rs @@ -30,7 +30,7 @@ use common::types::Symbol; use common::types::TimeInForce; use common::types::OrderStatus; use common::types::OrderType; -use trading_engine::types::events::MarketEvent; +use trading_engine::events::MarketEvent; use uuid::Uuid; use rust_decimal::Decimal; // TECHNICAL DEBT ELIMINATED - Use String and DateTime directly diff --git a/common/src/constants.rs b/common/src/constants.rs index 6ecd0d217..aecdd8236 100644 --- a/common/src/constants.rs +++ b/common/src/constants.rs @@ -168,5 +168,4 @@ pub mod environments { } } -/// Re-export for backward compatibility -pub use ServiceDefaults as SERVICE_DEFAULTS; +// ELIMINATED: Re-exports removed to force explicit imports diff --git a/common/src/database.rs b/common/src/database.rs index 7c2896e68..14931dee9 100644 --- a/common/src/database.rs +++ b/common/src/database.rs @@ -8,8 +8,9 @@ use sqlx::{Pool, Postgres}; use std::time::Duration; use thiserror::Error; -// Re-export centralized database configuration -pub use config::DatabaseConfig; +// Import centralized database configuration +use config::database::DatabaseConfig; +use config::structures::BacktestingDatabaseConfig; /// Database-specific errors #[derive(Debug, Error)] @@ -115,8 +116,8 @@ impl Default for PerformanceConfig { } /// Convert from centralized config to common crate config with HFT optimizations -impl From for LocalDatabaseConfig { - fn from(config: config::DatabaseConfig) -> Self { +impl From for LocalDatabaseConfig { + fn from(config: DatabaseConfig) -> Self { Self { url: config.url, pool: PoolConfig { @@ -139,8 +140,8 @@ impl From for LocalDatabaseConfig { } /// Convert from backtesting config to common crate config with backtesting optimizations -impl From for LocalDatabaseConfig { - fn from(config: config::BacktestingDatabaseConfig) -> Self { +impl From for LocalDatabaseConfig { + fn from(config: BacktestingDatabaseConfig) -> Self { Self { url: config.postgres_url, pool: PoolConfig { diff --git a/common/src/trading.rs b/common/src/trading.rs index 0955d0f39..9aa3a575d 100644 --- a/common/src/trading.rs +++ b/common/src/trading.rs @@ -7,8 +7,7 @@ use serde::{Deserialize, Serialize}; use std::fmt; -// Re-export canonical types from common::types -pub use crate::types::{Currency, OrderSide, OrderStatus, OrderType, TimeInForce}; +// ELIMINATED: Re-exports removed to force explicit imports // REMOVED: TimeInForce duplicate - use canonical definition from common::types // Currency moved to canonical source: common::types::Currency diff --git a/common/src/types.rs b/common/src/types.rs index 3875b0d3b..d0563f6b3 100644 --- a/common/src/types.rs +++ b/common/src/types.rs @@ -6,8 +6,8 @@ use chrono::{DateTime, Utc}; use crate::error::ErrorCategory; -// Re-export Decimal for public use -pub use rust_decimal::Decimal; +// ELIMINATED: Re-exports removed to force explicit imports +use rust_decimal::Decimal; use serde::{Deserialize, Serialize}; use serde_json::Value; use std::collections::HashMap; diff --git a/common/src/types_backup.rs b/common/src/types_backup.rs index c5e2564b8..31454ae23 100644 --- a/common/src/types_backup.rs +++ b/common/src/types_backup.rs @@ -6,8 +6,7 @@ use chrono::{DateTime, Utc}; use crate::error::ErrorCategory; -// Re-export Decimal for public use -pub use rust_decimal::Decimal; +// ELIMINATED: Re-exports removed to force explicit imports use serde::{Deserialize, Serialize}; #[cfg(feature = "database")] diff --git a/crates/config/src/database.rs b/crates/config/src/database.rs index a4523d1f1..e5e55c228 100644 --- a/crates/config/src/database.rs +++ b/crates/config/src/database.rs @@ -8,7 +8,8 @@ //! - Configuration change notifications with atomic updates use crate::schemas::{ModelConfig, ModelLoadRequest, ModelLoadResponse, ModelVersion}; -use crate::{ConfigCategory, ConfigError, ConfigResult, ConfigSource, ConfigValue}; +use crate::error::{ConfigError, ConfigResult}; +use crate::{ConfigCategory, ConfigSource, ConfigValue}; use anyhow::Context; use chrono::Utc; use serde::{Deserialize, Serialize}; diff --git a/crates/config/src/manager.rs b/crates/config/src/manager.rs index 604eaa7af..2e9f99336 100644 --- a/crates/config/src/manager.rs +++ b/crates/config/src/manager.rs @@ -8,9 +8,10 @@ //! - Unified caching and change notifications use crate::schemas::{ConfigChangeNotification, ModelConfig, ModelVersion}; +use crate::database::{DatabaseConfig, PostgresConfigLoader}; +use crate::error::{ConfigError, ConfigResult}; use crate::{ - ConfigCategory, ConfigChange, ConfigError, ConfigHealth, ConfigResult, ConfigSource, - ConfigValue, DatabaseConfig, PostgresConfigLoader, + ConfigCategory, ConfigChange, ConfigHealth, ConfigSource, ConfigValue, }; // Vault types will be used when initializing VaultSecrets use chrono::Utc; diff --git a/crates/config/src/ml_config.rs b/crates/config/src/ml_config.rs index 3dda8787c..8404a2ba7 100644 --- a/crates/config/src/ml_config.rs +++ b/crates/config/src/ml_config.rs @@ -5,8 +5,8 @@ use serde::{Deserialize, Serialize}; -// Re-export commonly used types -pub use chrono::{DateTime, Utc}; +// REMOVED: All pub use statements eliminated per cleanup requirements +// Use direct import: chrono::{DateTime, Utc} // =============================== // CORE ML MODEL CONFIGURATIONS diff --git a/crates/config/src/vault.rs b/crates/config/src/vault.rs index 07657f349..0a2275701 100644 --- a/crates/config/src/vault.rs +++ b/crates/config/src/vault.rs @@ -9,7 +9,8 @@ //! - Environment-specific secret paths //! - Fallback to environment variables -use crate::{ConfigCategory, ConfigError, ConfigResult, ConfigSource, ConfigValue}; +use crate::error::{ConfigError, ConfigResult}; +use crate::{ConfigCategory, ConfigSource, ConfigValue}; use chrono::Utc; use serde::{Deserialize, Serialize}; use std::collections::HashMap; diff --git a/crates/model_loader/src/model_interfaces/mod.rs b/crates/model_loader/src/model_interfaces/mod.rs index 0ea7b80ae..216e7f3ea 100644 --- a/crates/model_loader/src/model_interfaces/mod.rs +++ b/crates/model_loader/src/model_interfaces/mod.rs @@ -6,40 +6,40 @@ // MAMBA-2 SSM interface pub mod mamba_interface; -pub use mamba_interface::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites MambaInferenceInput, MambaInferenceOutput, MambaModelConfig, MambaModelInterface, MambaModelWeights, MambaSSMMatrices, }; // TLOB Transformer interface pub mod tlob_interface; -pub use tlob_interface::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites OrderBookSnapshot, TlobModelConfig, TlobModelInterface, TlobPrediction, }; // DQN Deep Q-Learning interface pub mod dqn_interface; -pub use dqn_interface::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites DqnInferenceOutput, DqnModelConfig, DqnModelInterface, DqnPrediction, TradingAction, TradingState, }; // PPO Proximal Policy Optimization interface pub mod ppo_interface; -pub use ppo_interface::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites ContinuousTradingAction, MarketState, PpoInferenceOutput, PpoModelConfig, PpoModelInterface, PpoPrediction, }; // Liquid Networks interface pub mod liquid_interface; -pub use liquid_interface::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites LiquidModelConfig, LiquidModelInterface, LiquidPrediction, MarketFeatures, }; // Temporal Fusion Transformer interface pub mod tft_interface; -pub use tft_interface::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites TftModelConfig, TftModelInterface, TftPrediction, TimeSeriesInput, }; diff --git a/crates/model_loader/src/model_interfaces/mod.rs.bak b/crates/model_loader/src/model_interfaces/mod.rs.bak new file mode 100644 index 000000000..0ea7b80ae --- /dev/null +++ b/crates/model_loader/src/model_interfaces/mod.rs.bak @@ -0,0 +1,332 @@ +//! Model Interfaces Module +//! +//! This module provides standardized interfaces for all ML models used in the +//! Foxhunt HFT trading system. Each interface handles model loading, inference, +//! and performance monitoring with the production model loader. + +// MAMBA-2 SSM interface +pub mod mamba_interface; +pub use mamba_interface::{ + MambaInferenceInput, MambaInferenceOutput, MambaModelConfig, MambaModelInterface, + MambaModelWeights, MambaSSMMatrices, +}; + +// TLOB Transformer interface +pub mod tlob_interface; +pub use tlob_interface::{ + OrderBookSnapshot, TlobModelConfig, TlobModelInterface, TlobPrediction, +}; + +// DQN Deep Q-Learning interface +pub mod dqn_interface; +pub use dqn_interface::{ + DqnInferenceOutput, DqnModelConfig, DqnModelInterface, DqnPrediction, TradingAction, + TradingState, +}; + +// PPO Proximal Policy Optimization interface +pub mod ppo_interface; +pub use ppo_interface::{ + ContinuousTradingAction, MarketState, PpoInferenceOutput, PpoModelConfig, PpoModelInterface, + PpoPrediction, +}; + +// Liquid Networks interface +pub mod liquid_interface; +pub use liquid_interface::{ + LiquidModelConfig, LiquidModelInterface, LiquidPrediction, MarketFeatures, +}; + +// Temporal Fusion Transformer interface +pub mod tft_interface; +pub use tft_interface::{ + TftModelConfig, TftModelInterface, TftPrediction, TimeSeriesInput, +}; + +use crate::ModelType; +use anyhow::Result; +use std::collections::HashMap; + +/// Unified model interface trait for all ML models +#[async_trait::async_trait] +pub trait ModelInterface: Send + Sync { + /// Load the model from storage + async fn load_model(&mut self) -> Result<()>; + + /// Check if model is loaded and ready + fn is_loaded(&self) -> bool; + + /// Get model type + fn model_type(&self) -> ModelType; + + /// Get performance metrics + fn metrics(&self) -> &HashMap; + + /// Get model name + fn model_name(&self) -> &str; + + /// Get model version + fn model_version(&self) -> &str; +} + +// Implement trait for all model interfaces +#[async_trait::async_trait] +impl ModelInterface for MambaModelInterface { + async fn load_model(&mut self) -> Result<()> { + self.load_model().await + } + + fn is_loaded(&self) -> bool { + self.is_loaded() + } + + fn model_type(&self) -> ModelType { + self.model_type() + } + + fn metrics(&self) -> &HashMap { + self.metrics() + } + + fn model_name(&self) -> &str { + &self.config().model_name + } + + fn model_version(&self) -> &str { + &self.config().model_version + } +} + +#[async_trait::async_trait] +impl ModelInterface for TlobModelInterface { + async fn load_model(&mut self) -> Result<()> { + self.load_model().await + } + + fn is_loaded(&self) -> bool { + self.is_loaded() + } + + fn model_type(&self) -> ModelType { + self.model_type() + } + + fn metrics(&self) -> &HashMap { + self.metrics() + } + + fn model_name(&self) -> &str { + &self.config().model_name + } + + fn model_version(&self) -> &str { + &self.config().model_version + } +} + +#[async_trait::async_trait] +impl ModelInterface for DqnModelInterface { + async fn load_model(&mut self) -> Result<()> { + self.load_model().await + } + + fn is_loaded(&self) -> bool { + self.is_loaded() + } + + fn model_type(&self) -> ModelType { + self.model_type() + } + + fn metrics(&self) -> &HashMap { + self.metrics() + } + + fn model_name(&self) -> &str { + &self.config().model_name + } + + fn model_version(&self) -> &str { + &self.config().model_version + } +} + +#[async_trait::async_trait] +impl ModelInterface for PpoModelInterface { + async fn load_model(&mut self) -> Result<()> { + self.load_model().await + } + + fn is_loaded(&self) -> bool { + self.is_loaded() + } + + fn model_type(&self) -> ModelType { + self.model_type() + } + + fn metrics(&self) -> &HashMap { + self.metrics() + } + + fn model_name(&self) -> &str { + &self.config().model_name + } + + fn model_version(&self) -> &str { + &self.config().model_version + } +} + +#[async_trait::async_trait] +impl ModelInterface for LiquidModelInterface { + async fn load_model(&mut self) -> Result<()> { + self.load_model().await + } + + fn is_loaded(&self) -> bool { + self.is_loaded() + } + + fn model_type(&self) -> ModelType { + self.model_type() + } + + fn metrics(&self) -> &HashMap { + self.metrics() + } + + fn model_name(&self) -> &str { + &self.config().model_name + } + + fn model_version(&self) -> &str { + &self.config().model_version + } +} + +#[async_trait::async_trait] +impl ModelInterface for TftModelInterface { + async fn load_model(&mut self) -> Result<()> { + self.load_model().await + } + + fn is_loaded(&self) -> bool { + self.is_loaded() + } + + fn model_type(&self) -> ModelType { + self.model_type() + } + + fn metrics(&self) -> &HashMap { + self.metrics() + } + + fn model_name(&self) -> &str { + &self.config().model_name + } + + fn model_version(&self) -> &str { + &self.config().model_version + } +} + +/// Model factory for creating model interfaces +pub struct ModelFactory; + +impl ModelFactory { + /// Create a model interface based on model type + pub async fn create_interface( + model_type: ModelType, + loader: std::sync::Arc, + ) -> Result> { + match model_type { + ModelType::Mamba2 => { + let config = MambaModelConfig::default(); + let interface = MambaModelInterface::new(config, loader).await?; + Ok(Box::new(interface)) + } + ModelType::TlobTransformer => { + let config = TlobModelConfig::default(); + let interface = TlobModelInterface::new(config, loader).await?; + Ok(Box::new(interface)) + } + ModelType::Dqn => { + let config = DqnModelConfig::default(); + let interface = DqnModelInterface::new(config, loader).await?; + Ok(Box::new(interface)) + } + ModelType::Ppo => { + let config = PpoModelConfig::default(); + let interface = PpoModelInterface::new(config, loader).await?; + Ok(Box::new(interface)) + } + ModelType::Liquid => { + let config = LiquidModelConfig::default(); + let interface = LiquidModelInterface::new(config, loader).await?; + Ok(Box::new(interface)) + } + ModelType::Tft => { + let config = TftModelConfig::default(); + let interface = TftModelInterface::new(config, loader).await?; + Ok(Box::new(interface)) + } + _ => Err(anyhow::anyhow!("Unsupported model type: {:?}", model_type)), + } + } + + /// Create a model interface with custom configuration + pub async fn create_interface_with_config( + interface: T, + ) -> Result> + where + T: ModelInterface, + { + Ok(Box::new(interface)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_trading_action_enum() { + assert_eq!(TradingAction::Hold as usize, 0); + assert_eq!(TradingAction::Buy as usize, 1); + assert_eq!(TradingAction::Sell as usize, 2); + assert_eq!(TradingAction::Close as usize, 3); + assert_eq!(TradingAction::num_actions(), 4); + } + + #[test] + fn test_continuous_action_dim() { + assert_eq!(ContinuousTradingAction::dim(), 3); + } + + #[test] + fn test_model_configs() { + let mamba_config = MambaModelConfig::default(); + let tlob_config = TlobModelConfig::default(); + let dqn_config = DqnModelConfig::default(); + let ppo_config = PpoModelConfig::default(); + let liquid_config = LiquidModelConfig::default(); + let tft_config = TftModelConfig::default(); + + assert_eq!(mamba_config.model_name, "mamba2_hft"); + assert_eq!(tlob_config.model_name, "tlob_transformer"); + assert_eq!(dqn_config.model_name, "dqn_policy"); + assert_eq!(ppo_config.model_name, "ppo_policy"); + assert_eq!(liquid_config.model_name, "liquid_network"); + assert_eq!(tft_config.model_name, "tft_forecaster"); + + // All should have reasonable latency targets + assert!(mamba_config.target_latency_us <= 50); + assert!(tlob_config.target_latency_us <= 50); + assert!(dqn_config.target_latency_us <= 50); + assert!(ppo_config.target_latency_us <= 50); + assert!(liquid_config.target_latency_us <= 50); + assert!(tft_config.target_latency_us <= 50); + } +} \ No newline at end of file diff --git a/data/src/brokers/common.rs b/data/src/brokers/common.rs index 6d7c64d2e..60d98aefa 100644 --- a/data/src/brokers/common.rs +++ b/data/src/brokers/common.rs @@ -5,6 +5,7 @@ use std::collections::HashMap; // Import the unified broker interface (SINGLE SOURCE OF TRUTH) use trading_engine::trading::data_interface::BrokerError; +use trading_engine::events::{OrderEvent, OrderEventType}; /// Result type for broker operations pub type BrokerResult = std::result::Result; @@ -43,9 +44,8 @@ pub trait BrokerConfig: Send + Sync + Clone { fn connection_timeout(&self) -> std::time::Duration; } -// BrokerClient trait DELETED - Use BrokerInterface from core::trading::data_interface instead -// Import the unified BrokerInterface -pub use trading_engine::trading::data_interface::BrokerInterface as BrokerClient; +// BrokerClient trait DELETED - Use BrokerInterface from trading_engine::trading::data_interface directly +// Import removed - use trading_engine::trading::data_interface::BrokerInterface directly /// Connection status enumeration #[derive(Debug, Clone, PartialEq)] @@ -66,9 +66,9 @@ pub enum ConnectionStatus { #[derive(Debug)] pub struct OrderManager { /// Pending orders - pending_orders: HashMap, + pending_orders: HashMap, /// Order history - order_history: HashMap>, + order_history: HashMap>, } impl OrderManager { @@ -81,7 +81,7 @@ impl OrderManager { } /// Add a pending order - pub fn add_pending_order(&mut self, order: trading_engine::types::events::OrderEvent) { + pub fn add_pending_order(&mut self, order: OrderEvent) { self.pending_orders .insert(order.order_id.to_string(), order); } @@ -89,8 +89,8 @@ impl OrderManager { pub fn update_order_event( &mut self, order_id: &str, - event_type: trading_engine::types::events::OrderEventType, - ) -> Option { + event_type: OrderEventType, + ) -> Option { if let Some(mut order) = self.pending_orders.get(order_id).cloned() { // Update the order with new event type order.event_type = event_type.clone(); @@ -104,9 +104,9 @@ impl OrderManager { // Only keep in pending if not in a final state match event_type { - trading_engine::types::events::OrderEventType::Cancelled - | trading_engine::types::events::OrderEventType::Rejected - | trading_engine::types::events::OrderEventType::Expired => { + OrderEventType::Cancelled + | OrderEventType::Rejected + | OrderEventType::Expired => { self.pending_orders.remove(order_id); } _ => { @@ -125,12 +125,12 @@ impl OrderManager { pub fn get_pending_order( &self, order_id: &str, - ) -> Option<&trading_engine::types::events::OrderEvent> { + ) -> Option<&OrderEvent> { self.pending_orders.get(order_id) } /// Get all pending orders - pub fn get_all_pending_orders(&self) -> Vec<&trading_engine::types::events::OrderEvent> { + pub fn get_all_pending_orders(&self) -> Vec<&OrderEvent> { self.pending_orders.values().collect() } @@ -138,7 +138,7 @@ impl OrderManager { pub fn get_order_history( &self, order_id: &str, - ) -> Option<&Vec> { + ) -> Option<&Vec> { self.order_history.get(order_id) } } @@ -275,22 +275,21 @@ impl Drop for HeartbeatManager { mod tests { use super::*; use crate::types::*; - use trading_engine::types::events::OrderEventType; use common::dec; -use common::Decimal; -use common::OrderId; -use common::OrderSide; -use common::OrderStatus; -use common::OrderType; -use common::Quantity; -use common::Symbol; + use common::Decimal; + use common::OrderId; + use common::OrderSide; + use common::OrderStatus; + use common::OrderType; + use common::Quantity; + use common::Symbol; #[test] fn test_order_manager() { let mut manager = OrderManager::new(); - let order = trading_engine::types::events::OrderEvent { + let order = OrderEvent { order_id: OrderId::new(), symbol: Symbol::from("EURUSD"), order_type: OrderType::Market, @@ -301,7 +300,7 @@ use common::Symbol; price: None, timestamp: chrono::Utc::now(), strategy_id: "test_strategy".to_string(), - event_type: trading_engine::types::events::OrderEventType::Placed, + event_type: OrderEventType::Placed, previous_quantity: None, previous_price: None, reason: None, diff --git a/data/src/brokers/mod.rs b/data/src/brokers/mod.rs index 2253d4c63..3009bf31b 100644 --- a/data/src/brokers/mod.rs +++ b/data/src/brokers/mod.rs @@ -11,8 +11,6 @@ pub mod interactive_brokers; // Re-export commonly used types // Note: Using direct imports from common crate instead of broker-specific types -pub use interactive_brokers::{IBConfig, InteractiveBrokersAdapter}; -pub use trading_engine::trading::data_interface::BrokerError; // Create alias for BrokerAdapter (used in examples) // TODO: Re-enable when BrokerClient trait is implemented diff --git a/data/src/brokers/mod.rs.bak b/data/src/brokers/mod.rs.bak new file mode 100644 index 000000000..2253d4c63 --- /dev/null +++ b/data/src/brokers/mod.rs.bak @@ -0,0 +1,63 @@ +//! Broker integration modules +//! +//! This module provides integration with various brokers and trading platforms +//! using their native protocols (FIX, REST APIs, WebSockets, etc.). +//! +//! NOTE: Broker clients have been moved to core module for monolithic architecture. +//! This module now only provides data-specific broker adapters. + +pub mod common; +pub mod interactive_brokers; + +// Re-export commonly used types +// Note: Using direct imports from common crate instead of broker-specific types +pub use interactive_brokers::{IBConfig, InteractiveBrokersAdapter}; +pub use trading_engine::trading::data_interface::BrokerError; + +// Create alias for BrokerAdapter (used in examples) +// TODO: Re-enable when BrokerClient trait is implemented +// pub type BrokerAdapter = Box; + +/// Supported broker types +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub enum BrokerType { + /// ICMarkets FIX 4.4 + ICMarkets, + /// Interactive Brokers TWS API + InteractiveBrokers, + /// Alpaca REST API + Alpaca, + /// Mock broker for testing + Mock, +} + +/// Generic broker factory +pub struct BrokerFactory; + +impl BrokerFactory { + // TODO: Uncomment when BrokerClient trait is restored + /* + /// Create a broker client based on configuration + pub async fn create_client(broker_type: BrokerType, config: serde_json::Value) -> crate::Result> { + match broker_type { + BrokerType::ICMarkets => { + let icmarkets_config: ICMarketsConfig = serde_json::from_value(config)?; + let client = ICMarketsClient::new(icmarkets_config); + Ok(Box::new(client)) + } + BrokerType::InteractiveBrokers => { + // TODO: Implement IB client + Err(crate::DataError::configuration("Interactive Brokers not yet implemented")) + } + BrokerType::Alpaca => { + // TODO: Implement Alpaca client + Err(crate::DataError::configuration("Alpaca not yet implemented")) + } + BrokerType::Mock => { + // TODO: Implement mock client + Err(crate::DataError::configuration("Mock broker not yet implemented")) + } + } + } + */ +} diff --git a/data/src/error_consolidated.rs b/data/src/error_consolidated.rs index 43fb25849..fe412774a 100644 --- a/data/src/error_consolidated.rs +++ b/data/src/error_consolidated.rs @@ -3,11 +3,11 @@ //! This module demonstrates the consolidated error handling pattern //! using the common error system across all Foxhunt services. -// Re-export shared error types and utilities -pub use common::error::{CommonError, CommonResult, ErrorCategory, RetryStrategy, ErrorSeverity}; +// REMOVED: All pub use statements eliminated per cleanup requirements +// Use direct import: common::error::{CommonError, CommonResult, ErrorCategory, RetryStrategy, ErrorSeverity} /// Result type for data module operations using CommonError -pub type DataResult = CommonResult; +pub type DataResult = common::error::CommonResult; /// Data module specific error extensions /// For cases where we need domain-specific error information beyond CommonError @@ -15,7 +15,7 @@ pub type DataResult = CommonResult; pub enum DataServiceError { /// Common error with context #[error("Data service error: {0}")] - Common(#[from] CommonError), + Common(#[from] common::error::CommonError), /// FIX protocol specific error with detailed context #[error("FIX protocol error: {session_id} - {message}")] diff --git a/data/src/lib.rs b/data/src/lib.rs index d2cf3896f..afe33c1a3 100644 --- a/data/src/lib.rs +++ b/data/src/lib.rs @@ -152,7 +152,7 @@ use tracing::{error, info, warn}; use tokio::sync::broadcast; // Import configuration and event types that are actually used use config::DataModuleConfig; -use trading_engine::types::events::OrderEvent; +use trading_engine::events::OrderEvent; use crate::error::Result; use crate::brokers::{InteractiveBrokersAdapter, IBConfig}; use common::{MarketDataEvent, Subscription}; diff --git a/data/src/providers/benzinga/mod.rs b/data/src/providers/benzinga/mod.rs index fd63f0d35..9d013442d 100644 --- a/data/src/providers/benzinga/mod.rs +++ b/data/src/providers/benzinga/mod.rs @@ -270,22 +270,20 @@ pub mod ml_integration; pub mod integration; // Convenience re-exports for common types -pub use historical::{BenzingaConfig, BenzingaHistoricalProvider, NewsEvent, NewsEventType}; -pub use streaming::{BenzingaStreamingConfig, BenzingaStreamingProvider}; // Production provider re-exports -pub use production_historical::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites ProductionBenzingaHistoricalConfig, ProductionBenzingaHistoricalProvider, }; -pub use production_streaming::{ProductionBenzingaConfig, ProductionBenzingaProvider}; +// DO NOT RE-EXPORT - Use explicit imports at usage sites // ML integration re-exports -pub use ml_integration::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites BenzingaFeatureVector, BenzingaMLConfig, BenzingaMLExtractor, NormalizationMethod, }; // HFT integration re-exports -pub use integration::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites BenzingaHFTIntegration, MLModelIntegration, SignalConfig, TradingSignal, }; diff --git a/data/src/providers/benzinga/mod.rs.bak b/data/src/providers/benzinga/mod.rs.bak new file mode 100644 index 000000000..fd63f0d35 --- /dev/null +++ b/data/src/providers/benzinga/mod.rs.bak @@ -0,0 +1,441 @@ +//! # Benzinga Provider Module +//! +//! This module provides comprehensive integration with Benzinga Pro API for financial +//! news, sentiment analysis, analyst ratings, and unusual options activity. +//! +//! ## Components +//! +//! - **Streaming Provider**: Real-time WebSocket streaming for live data feeds +//! - **Historical Provider**: REST API access for historical news and events +//! - **Production Providers**: Enhanced versions with advanced features +//! - **ML Integration**: Feature extraction for machine learning models +//! - **HFT Integration**: Complete orchestration layer for high-frequency trading +//! +//! ## Architecture +//! +//! The Benzinga integration follows a multi-tier provider pattern: +//! - `BenzingaStreamingProvider`: Basic WebSocket streaming implementation +//! - `ProductionBenzingaProvider`: Production-grade with rate limiting, deduplication, circuit breakers +//! - `BenzingaHistoricalProvider`: Basic REST API access +//! - `ProductionBenzingaHistoricalProvider`: Production-grade with caching, retry logic, bulk operations +//! - `BenzingaMLExtractor`: ML feature extraction and time series preparation +//! - `BenzingaHFTIntegration`: Complete orchestration layer with trading signal generation +//! +//! ## Usage +//! +//! ### Production Real-time Streaming +//! +//! ```rust,no_run +//! use data::providers::benzinga::{ProductionBenzingaProvider, ProductionBenzingaConfig}; +//! use data::providers::traits::RealTimeProvider; +//! use common::Symbol; +//! +//! # async fn example() -> anyhow::Result<()> { +//! let config = ProductionBenzingaConfig { +//! api_key: "your-benzinga-api-key".to_string(), +//! enable_news: true, +//! enable_sentiment: true, +//! enable_ratings: true, +//! enable_options: true, +//! rate_limit_per_second: 100, +//! enable_ml_integration: true, +//! ..Default::default() +//! }; +//! +//! let mut provider = ProductionBenzingaProvider::new(config)?; +//! provider.connect().await?; +//! provider.subscribe(vec![Symbol::from("AAPL"), Symbol::from("SPY")]).await?; +//! +//! let mut stream = provider.stream().await?; +//! while let Some(event) = stream.next().await { +//! match event { +//! MarketDataEvent::NewsAlert(news) => { +//! println!("News: {} - Impact: {:?}", news.headline, news.impact_score); +//! } +//! MarketDataEvent::SentimentUpdate(sentiment) => { +//! println!("Sentiment for {}: {:.3}", sentiment.symbol, sentiment.sentiment_score); +//! } +//! MarketDataEvent::AnalystRating(rating) => { +//! println!("Rating: {} {} -> {}", rating.symbol, rating.action, rating.current_rating); +//! } +//! MarketDataEvent::UnusualOptions(options) => { +//! println!("Options: {} {:?} Vol: {}", options.symbol, options.activity_type, options.volume); +//! } +//! _ => {} +//! } +//! } +//! # Ok(()) +//! # } +//! ``` +//! +//! ### Production Historical Data +//! +//! ```rust,no_run +//! use data::providers::benzinga::{ProductionBenzingaHistoricalProvider, ProductionBenzingaHistoricalConfig}; +//! use chrono::{Utc, Duration}; +//! +//! # async fn example() -> anyhow::Result<()> { +//! let config = ProductionBenzingaHistoricalConfig { +//! api_key: "your-benzinga-api-key".to_string(), +//! enable_caching: true, +//! enable_bulk_download: true, +//! rate_limit_per_second: 10, +//! ..Default::default() +//! }; +//! +//! let provider = ProductionBenzingaHistoricalProvider::new(config)?; +//! let symbols = ["AAPL", "SPY"]; +//! let end = Utc::now(); +//! let start = end - Duration::days(7); +//! +//! // Get all events (news, ratings, earnings, options) in parallel +//! let events = provider.get_all_events(Some(&symbols), start, end).await?; +//! println!("Retrieved {} historical events", events.len()); +//! +//! // Get specific event types +//! let news = provider.get_news_events(Some(&symbols), start, end).await?; +//! let ratings = provider.get_rating_events(Some(&symbols), start, end).await?; +//! let options = provider.get_options_events(Some(&symbols), start, end).await?; +//! +//! # Ok(()) +//! # } +//! ``` +//! +//! ### ML Feature Extraction +//! +//! ```rust,no_run +//! use data::providers::benzinga::{BenzingaMLExtractor, BenzingaMLConfig}; +//! use data::providers::common::MarketDataEvent; +//! use chrono::Utc; +//! use common::Symbol; +//! +//! # async fn example() -> anyhow::Result<()> { +//! let config = BenzingaMLConfig { +//! feature_window_minutes: 60, +//! enable_nlp_features: true, +//! enable_sentiment_indicators: true, +//! normalization_method: data::providers::benzinga::NormalizationMethod::ZScore, +//! ..Default::default() +//! }; +//! +//! let mut extractor = BenzingaMLExtractor::new(config); +//! +//! // Process real-time events +//! let event = MarketDataEvent::NewsAlert(/* news event */); +//! extractor.process_event(&event).await?; +//! +//! // Extract features for ML models +//! let symbol = Symbol::from("AAPL"); +//! let features = extractor.extract_features(&symbol, Utc::now()).await?; +//! +//! println!("Feature vector dimension: {}", extractor.get_feature_dimension()); +//! println!("Feature names: {:?}", extractor.get_feature_names()); +//! +//! // Batch feature extraction +//! let symbols = vec![Symbol::from("AAPL"), Symbol::from("SPY")]; +//! let batch_features = extractor.extract_features_batch(&symbols, Utc::now()).await?; +//! +//! # Ok(()) +//! # } +//! ``` +//! +//! ### HFT Integration (Complete System) +//! +//! ```rust,no_run +//! use data::providers::benzinga::{BenzingaHFTIntegration, BenzingaIntegrationConfig, TradingSignal, TradingSignalType}; +//! use config::ConfigManager; +//! use common::Symbol; +//! use std::sync::Arc; +//! +//! # async fn example() -> anyhow::Result<()> { +//! let config = BenzingaIntegrationConfig { +//! enable_streaming: true, +//! enable_historical: true, +//! enable_ml_integration: true, +//! symbols: vec![Symbol::from("AAPL"), Symbol::from("SPY")], +//! signal_config: SignalConfig { +//! news_impact_threshold: 0.7, +//! sentiment_momentum_threshold: 0.5, +//! analyst_rating_enabled: true, +//! options_flow_threshold: 1000, +//! }, +//! ..Default::default() +//! }; +//! +//! // Create comprehensive HFT integration +//! let mut integration = BenzingaHFTIntegration::new(config).await?; +//! integration.start().await?; +//! +//! // Process trading signals in real-time +//! while let Some(signal) = integration.next_signal().await { +//! match signal.signal_type { +//! TradingSignalType::NewsImpact => { +//! println!("News Impact: {} - Strength: {:.3}", signal.symbol, signal.strength); +//! // Route to trading engine... +//! } +//! TradingSignalType::SentimentShift => { +//! println!("Sentiment Shift: {} - Direction: {}", signal.symbol, +//! if signal.strength > 0.0 { "Bullish" } else { "Bearish" }); +//! } +//! TradingSignalType::AnalystAction => { +//! println!("Analyst Action: {} - Confidence: {:.3}", signal.symbol, signal.confidence); +//! } +//! TradingSignalType::OptionsFlow => { +//! println!("Options Flow: {} - Activity: {:.0}", signal.symbol, signal.strength); +//! } +//! } +//! } +//! +//! integration.stop().await?; +//! # Ok(()) +//! # } +//! ``` +//! +//! ## Event Types +//! +//! The Benzinga providers emit the following `MarketDataEvent` types: +//! +//! - `NewsAlert`: Breaking financial news with impact scoring and smart categorization +//! - `SentimentUpdate`: AI-powered sentiment analysis scores with technical indicators +//! - `AnalystRating`: Analyst upgrades, downgrades, and price targets with consensus tracking +//! - `UnusualOptions`: Unusual options activity detection with sentiment analysis +//! - `ConnectionStatus`: Provider connection state changes +//! - `Error`: Provider error notifications with recovery information +//! +//! ## Production Features +//! +//! ### Streaming Provider +//! - Advanced rate limiting with token bucket algorithm +//! - Message deduplication using SHA-256 hashing +//! - Circuit breakers for fault tolerance +//! - Smart categorization with ML-enhanced classification +//! - Batch processing for efficiency +//! - Comprehensive metrics and monitoring +//! +//! ### Historical Provider +//! - Redis and in-memory caching with TTL +//! - Retry logic with exponential backoff +//! - Bulk data download capabilities +//! - Data quality validation and filtering +//! - Concurrent API requests with semaphore control +//! - Comprehensive event coverage (news, earnings, ratings, options, calendar) +//! +//! ### ML Integration +//! - 50+ engineered features for temporal ML models +//! - Real-time feature extraction for TFT and Liquid Networks +//! - Technical indicators applied to sentiment data +//! - NLP features with keyword and topic analysis +//! - Multiple normalization methods (Z-score, Min-Max, Robust) +//! - Batch processing and caching for performance +//! +//! ### HFT Integration +//! - Complete orchestration layer for high-frequency trading +//! - Real-time trading signal generation from news/sentiment events +//! - ML model integration with feature queues for TFT and Liquid Networks +//! - Event-driven architecture optimized for sub-millisecond latency +//! - Automated symbol monitoring and signal routing +//! - Performance metrics and latency monitoring +//! - Signal strength calibration and confidence scoring +//! +//! ## Configuration +//! +//! All providers require a Benzinga Pro API key. Set the `BENZINGA_API_KEY` +//! environment variable or provide it directly in the configuration. +//! +//! Optional Redis caching can be enabled by setting `REDIS_URL` environment variable. +//! +//! ## Rate Limits +//! +//! Benzinga Pro has rate limits that vary by subscription tier: +//! - Basic: 5 requests/second +//! - Professional: 20 requests/second +//! - Enterprise: 100+ requests/second +//! +//! The providers implement automatic rate limiting and respect API quotas. + +// Re-export the streaming provider +pub mod streaming; + +// Re-export the historical provider +pub mod historical; + +// Production-grade providers +pub mod production_historical; +pub mod production_streaming; + +// ML integration module +pub mod ml_integration; + +// HFT integration orchestration +pub mod integration; + +// Convenience re-exports for common types +pub use historical::{BenzingaConfig, BenzingaHistoricalProvider, NewsEvent, NewsEventType}; +pub use streaming::{BenzingaStreamingConfig, BenzingaStreamingProvider}; + +// Production provider re-exports +pub use production_historical::{ + ProductionBenzingaHistoricalConfig, ProductionBenzingaHistoricalProvider, +}; +pub use production_streaming::{ProductionBenzingaConfig, ProductionBenzingaProvider}; + +// ML integration re-exports +pub use ml_integration::{ + BenzingaFeatureVector, BenzingaMLConfig, BenzingaMLExtractor, NormalizationMethod, +}; + +// HFT integration re-exports +pub use integration::{ + BenzingaHFTIntegration, MLModelIntegration, SignalConfig, + TradingSignal, +}; + +/// Benzinga provider factory for creating provider instances +pub struct BenzingaProviderFactory; + +impl BenzingaProviderFactory { + /// Create a new production streaming provider with the given configuration + pub fn create_production_streaming_provider( + config: ProductionBenzingaConfig, + ) -> crate::error::Result { + ProductionBenzingaProvider::new(config) + } + + /// Create a new production historical provider with the given configuration + pub fn create_production_historical_provider( + config: ProductionBenzingaHistoricalConfig, + ) -> crate::error::Result { + ProductionBenzingaHistoricalProvider::new(config) + } + + /// Create ML feature extractor + pub fn create_ml_extractor(config: BenzingaMLConfig) -> BenzingaMLExtractor { + BenzingaMLExtractor::new(config) + } + + /// Create a basic streaming provider with the given configuration + pub fn create_streaming_provider( + config: BenzingaStreamingConfig, + ) -> crate::error::Result { + BenzingaStreamingProvider::new(config) + } + + /// Create a basic historical provider with the given configuration + pub fn create_historical_provider( + config: BenzingaConfig, + ) -> crate::error::Result { + BenzingaHistoricalProvider::new(config) + } + + /// Create a production streaming provider from environment variables + pub fn create_production_streaming_from_env() -> crate::error::Result + { + let config = ProductionBenzingaConfig::default(); + Self::create_production_streaming_provider(config) + } + + /// Create a production historical provider from environment variables + pub fn create_production_historical_from_env( + ) -> crate::error::Result { + let config = ProductionBenzingaHistoricalConfig::default(); + Self::create_production_historical_provider(config) + } + + /// Create ML extractor from environment + pub fn create_ml_extractor_from_env() -> BenzingaMLExtractor { + let config = BenzingaMLConfig::default(); + Self::create_ml_extractor(config) + } + + /// Create HFT integration instance + pub async fn create_hft_integration( + _config: BenzingaStreamingConfig, + ) -> crate::error::Result { + // Create a default config manager for now - this needs proper implementation + let config_manager = config::ConfigManager::new(None, None, None).await?; + BenzingaHFTIntegration::new(config_manager).await + } + + /// Create HFT integration from environment variables + pub async fn create_hft_integration_from_env() -> crate::error::Result { + let config = BenzingaStreamingConfig::default(); + Self::create_hft_integration(config).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_factory_creation_with_api_key() { + let streaming_config = ProductionBenzingaConfig { + api_key: "test-key".to_string(), + ..Default::default() + }; + + let result = + BenzingaProviderFactory::create_production_streaming_provider(streaming_config); + assert!(result.is_ok()); + + let historical_config = ProductionBenzingaHistoricalConfig { + api_key: "test-key".to_string(), + ..Default::default() + }; + + let result = + BenzingaProviderFactory::create_production_historical_provider(historical_config); + assert!(result.is_ok()); + } + + #[test] + fn test_factory_creation_without_api_key() { + let streaming_config = ProductionBenzingaConfig { + api_key: "".to_string(), + ..Default::default() + }; + + let result = + BenzingaProviderFactory::create_production_streaming_provider(streaming_config); + assert!(result.is_err()); + } + + #[test] + fn test_ml_extractor_creation() { + let config = BenzingaMLConfig::default(); + let extractor = BenzingaProviderFactory::create_ml_extractor(config); + + assert!(extractor.get_feature_dimension() > 0); + assert!(!extractor.get_feature_names().is_empty()); + } + + #[test] + fn test_factory_from_env() { + // These will use default values from environment variables + let streaming_result = BenzingaProviderFactory::create_production_streaming_from_env(); + let historical_result = BenzingaProviderFactory::create_production_historical_from_env(); + let ml_extractor = BenzingaProviderFactory::create_ml_extractor_from_env(); + + // May fail due to missing API key in test environment, but should not panic + // In production with proper API key, these would succeed + assert!(streaming_result.is_err() || streaming_result.is_ok()); + assert!(historical_result.is_err() || historical_result.is_ok()); + assert!(ml_extractor.get_feature_dimension() > 0); + } + + #[tokio::test] + async fn test_hft_integration_creation() { + use common::Symbol; + + let config = BenzingaStreamingConfig { + api_key: "test-key".to_string(), + enable_news: true, + enable_sentiment: true, + ..Default::default() + }; + + let result = BenzingaProviderFactory::create_hft_integration(config).await; + // May fail due to missing API key or other dependencies in test environment + assert!(result.is_err() || result.is_ok()); + } +} diff --git a/data/src/providers/common.rs b/data/src/providers/common.rs index 2764c5efb..913432592 100644 --- a/data/src/providers/common.rs +++ b/data/src/providers/common.rs @@ -15,12 +15,9 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use common::{Symbol, Decimal, Volume}; -// Re-export the canonical MarketDataEvent and event types -pub use crate::types::MarketDataEvent; -pub use common::{TradeEvent, QuoteEvent}; - -// Re-export ErrorCategory for provider modules -pub use common::error::ErrorCategory; +// Import canonical types - use direct imports instead of re-exports +// Use crate::types::MarketDataEvent, common::TradeEvent, common::QuoteEvent directly +// Use common::error::ErrorCategory directly // === PROVIDER-SPECIFIC STRUCTURES === // Only types that are NOT duplicated in types.rs should be defined here @@ -456,7 +453,7 @@ pub struct ErrorEvent { pub code: Option, /// Error category - pub category: ErrorCategory, + pub category: common::error::ErrorCategory, /// Whether the error is recoverable pub recoverable: bool, diff --git a/data/src/providers/databento/mod.rs b/data/src/providers/databento/mod.rs index 3c078a90a..e8fceacaa 100644 --- a/data/src/providers/databento/mod.rs +++ b/data/src/providers/databento/mod.rs @@ -76,7 +76,7 @@ pub mod types; pub mod websocket_client; // Import all major components -pub use self::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites client::{DatabentoClient, DatabentoClientBuilder, DatabentoClientConfig}, dbn_parser::{ DbnParser, ProcessedMessage, DbnParserMetrics, DbnParserMetricsSnapshot, @@ -111,7 +111,7 @@ pub use self::{ }; // Re-export from parent modules for convenience -pub use crate::providers::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites traits::{RealTimeProvider, HistoricalProvider, HistoricalSchema, ConnectionStatus, ConnectionState}, common::MarketDataEvent, }; diff --git a/data/src/providers/databento/mod.rs.bak b/data/src/providers/databento/mod.rs.bak new file mode 100644 index 000000000..3c078a90a --- /dev/null +++ b/data/src/providers/databento/mod.rs.bak @@ -0,0 +1,652 @@ +//! # Databento Market Data Provider - Production Integration +//! +//! High-performance, production-ready integration with Databento's market data services. +//! Provides both real-time streaming and historical data access with ultra-low latency +//! optimizations for HFT trading systems. +//! +//! ## Architecture Overview +//! +//! ```text +//! ┌─────────────────────────────────────────────────────────────────────────────┐ +//! │ Databento Integration Architecture │ +//! ├─────────────────────────────────────────────────────────────────────────────┤ +//! │ Real-Time Stream: WebSocket → DBN Parser → Lock-Free Queues → Events │ +//! │ Historical Data: REST API → JSON/DBN → Batch Processing → Storage │ +//! │ Connection Pool: Multiple Feeds → Load Balancing → Failover → Recovery │ +//! ├─────────────────────────────────────────────────────────────────────────────┤ +//! │ Performance: <1μs parsing, <5μs to trading engine, zero-copy operations │ +//! └─────────────────────────────────────────────────────────────────────────────┘ +//! ``` +//! +//! ## Key Features +//! +//! - **Ultra-Low Latency**: <1μs DBN parsing, <5μs end-to-end processing +//! - **Zero-Copy Operations**: Direct memory mapping, minimal allocations +//! - **Production Resilience**: Automatic reconnection, circuit breakers, health monitoring +//! - **Comprehensive Data Coverage**: L1/L2/L3 order books, trades, OHLCV bars, statistics +//! - **Enterprise Integration**: Core event system, lock-free queues, shared memory +//! +//! ## Usage Examples +//! +//! ### Real-Time Streaming +//! +//! ```rust +//! use data::providers::databento::{DatabentoStreamingProvider, DatabentoConfig}; +//! use trading_engine::events::EventProcessor; +//! +//! let config = DatabentoConfig::production(); +//! let mut provider = DatabentoStreamingProvider::new(config).await?; +//! +//! // Integrate with core event system +//! let event_processor = EventProcessor::new().await?; +//! provider.set_event_processor(event_processor).await; +//! +//! // Subscribe to symbols +//! provider.subscribe(vec!["SPY".into(), "QQQ".into()]).await?; +//! +//! // Stream processes automatically with <1μs latency +//! let stream = provider.stream().await?; +//! while let Some(event) = stream.next().await { +//! // Events automatically flow to trading engine via lock-free queues +//! } +//! ``` +//! +//! ### Historical Data +//! +//! ```rust +//! use data::providers::databento::{DatabentoHistoricalProvider, HistoricalSchema}; +//! use data::types::TimeRange; +//! +//! let provider = DatabentoHistoricalProvider::new(config).await?; +//! +//! let range = TimeRange::last_day(); +//! let trades = provider.fetch( +//! &"SPY".into(), +//! HistoricalSchema::Trade, +//! range +//! ).await?; +//! ``` + +// Module declarations +pub mod client; +pub mod dbn_parser; +pub mod parser; +pub mod stream; +pub mod types; +pub mod websocket_client; + +// Import all major components +pub use self::{ + client::{DatabentoClient, DatabentoClientBuilder, DatabentoClientConfig}, + dbn_parser::{ + DbnParser, ProcessedMessage, DbnParserMetrics, DbnParserMetricsSnapshot, + DbnMessageType, DbnMessageHeader, DbnTradeMessage, DbnQuoteMessage, + DbnOrderBookMessage, DbnOhlcvMessage, OrderBookAction + }, + parser::{BinaryParser, ParserConfig, ParserMetrics}, + stream::{ + DatabentoStreamHandler, StreamConfig, StreamState, StreamMetrics, + ReconnectionConfig, CircuitBreakerConfig, BackpressureConfig + }, + types::{ + // Market data types + DatabentoSymbol, DatabentoInstrument, DatabentoPublisher, + DatabentoDataset, DatabentoSchema, DatabentoSType, + + // Configuration types + DatabentoConfig, DatabentoWebSocketConfig, DatabentoHistoricalConfig, + ProductionConfig, TestingConfig, + + // Request/Response types + SubscriptionRequest, SubscriptionResponse, AuthenticationRequest, + HeartbeatMessage, StatusMessage, ErrorMessage, + + // Statistics and metrics + ConnectionStats, ProcessingStats, PerformanceMetrics + }, + websocket_client::{ + DatabentoWebSocketClient, WebSocketMetrics, WebSocketMetricsSnapshot, + SubscriptionState, ConnectionHealth, HealthMonitor + }, +}; + +// Re-export from parent modules for convenience +pub use crate::providers::{ + traits::{RealTimeProvider, HistoricalProvider, HistoricalSchema, ConnectionStatus, ConnectionState}, + common::MarketDataEvent, +}; + +// Import dependencies +use crate::error::{DataError, Result}; +use crate::types::TimeRange; +use common::{Symbol, TradeEvent, QuoteEvent, Decimal}; +use trading_engine::{ + events::EventProcessor, +}; +use async_trait::async_trait; +use tokio_stream::Stream; +use std::pin::Pin; +use std::sync::Arc; +use tracing::{info, warn, error, debug}; +use chrono::Utc; +/// Production-ready Databento streaming provider +/// +/// Implements the RealTimeProvider trait with enterprise-grade features: +/// - Ultra-low latency DBN parsing (<1μs) +/// - Lock-free message processing +/// - Automatic reconnection with circuit breakers +/// - Comprehensive health monitoring +/// - Integration with core event system +pub struct DatabentoStreamingProvider { + /// Core client for WebSocket connections + client: DatabentoWebSocketClient, + /// Configuration settings + config: DatabentoConfig, + /// Event processor integration + event_processor: Option>, + /// Connection status tracking + connection_status: Arc>, +} + +impl DatabentoStreamingProvider { + /// Create new streaming provider with production configuration + pub async fn new(config: DatabentoConfig) -> Result { + info!("Initializing Databento streaming provider"); + + // Convert to WebSocket config + let ws_config = config.to_websocket_config(); + + // Create WebSocket client + let client = DatabentoWebSocketClient::new(ws_config)?; + + let provider = Self { + client, + config, + event_processor: None, + connection_status: Arc::new(std::sync::RwLock::new(ConnectionStatus::disconnected())), + }; + + info!("Databento streaming provider initialized successfully"); + Ok(provider) + } + + /// Create with production-optimized settings + pub async fn production() -> Result { + Self::new(DatabentoConfig::production()).await + } + + /// Create with testing settings + pub async fn testing() -> Result { + Self::new(DatabentoConfig::testing()).await + } + + /// Set event processor for core system integration + pub async fn set_event_processor(&mut self, processor: Arc) { + self.event_processor = Some(processor.clone()); + self.client.set_event_processor(processor); + debug!("Event processor integration configured"); + } + + /// Get real-time performance metrics + pub fn get_performance_metrics(&self) -> PerformanceMetrics { + let ws_metrics = self.client.get_metrics(); + + PerformanceMetrics { + messages_per_second: ws_metrics.messages_per_second, + avg_latency_ns: ws_metrics.avg_processing_latency_ns, + error_rate: if ws_metrics.messages_received > 0 { + (ws_metrics.parse_errors + ws_metrics.event_errors) as f64 / ws_metrics.messages_received as f64 + } else { + 0.0 + }, + uptime_seconds: ws_metrics.uptime_s, + connection_stability: ws_metrics.connection_successes as f64 / + ws_metrics.connection_attempts.max(1) as f64, + } + } + + /// Check if performance targets are being met + pub fn validate_performance(&self) -> bool { + let metrics = self.get_performance_metrics(); + + // Production performance targets + let latency_target_ns = 1_000; // <1μs parsing + let error_rate_target = 0.001; // <0.1% error rate + let stability_target = 0.99; // >99% connection stability + + let meets_targets = metrics.avg_latency_ns <= latency_target_ns && + metrics.error_rate <= error_rate_target && + metrics.connection_stability >= stability_target; + + if !meets_targets { + warn!( + "Performance targets not met - Latency: {}ns (target: {}ns), \ + Error rate: {:.3}% (target: {:.3}%), \ + Stability: {:.2}% (target: {:.2}%)", + metrics.avg_latency_ns, latency_target_ns, + metrics.error_rate * 100.0, error_rate_target * 100.0, + metrics.connection_stability * 100.0, stability_target * 100.0 + ); + } + + meets_targets + } +} + +#[async_trait] +impl RealTimeProvider for DatabentoStreamingProvider { + async fn connect(&mut self) -> Result<()> { + info!("Connecting Databento streaming provider"); + + // Update connection status + { + let mut status = self.connection_status.write().unwrap(); + status.state = ConnectionState::Connecting; + status.last_connection_attempt = Some(Utc::now()); + } + + match self.client.connect().await { + Ok(()) => { + let mut status = self.connection_status.write().unwrap(); + *status = ConnectionStatus::connected(); + info!("Databento streaming provider connected successfully"); + Ok(()) + } + Err(e) => { + let mut status = self.connection_status.write().unwrap(); + status.state = ConnectionState::Failed; + error!("Failed to connect Databento streaming provider: {}", e); + Err(e) + } + } + } + + async fn disconnect(&mut self) -> Result<()> { + info!("Disconnecting Databento streaming provider"); + + match self.client.shutdown().await { + Ok(()) => { + let mut status = self.connection_status.write().unwrap(); + status.state = ConnectionState::Disconnected; + info!("Databento streaming provider disconnected successfully"); + Ok(()) + } + Err(e) => { + error!("Error during Databento disconnect: {}", e); + Err(e) + } + } + } + + async fn subscribe(&mut self, symbols: Vec) -> Result<()> { + info!("Subscribing to {} symbols", symbols.len()); + + let symbol_strings: Vec = symbols.iter().map(|s| s.to_string()).collect(); + + match self.client.subscribe(symbol_strings).await { + Ok(()) => { + // Update connection status with subscription count + { + let mut status = self.connection_status.write().unwrap(); + status.active_subscriptions = symbols.len(); + } + info!("Successfully subscribed to {} symbols", symbols.len()); + Ok(()) + } + Err(e) => { + error!("Failed to subscribe to symbols: {}", e); + Err(e) + } + } + } + + async fn unsubscribe(&mut self, symbols: Vec) -> Result<()> { + info!("Unsubscribing from {} symbols", symbols.len()); + + let symbol_strings: Vec = symbols.iter().map(|s| s.to_string()).collect(); + + match self.client.unsubscribe(symbol_strings).await { + Ok(()) => { + // Update connection status + { + let mut status = self.connection_status.write().unwrap(); + status.active_subscriptions = status.active_subscriptions.saturating_sub(symbols.len()); + } + info!("Successfully unsubscribed from {} symbols", symbols.len()); + Ok(()) + } + Err(e) => { + error!("Failed to unsubscribe from symbols: {}", e); + Err(e) + } + } + } + + async fn stream(&mut self) -> Result + Send>>> { + // Create a stream that bridges the WebSocket client to the MarketDataEvent stream + // This is a complex implementation that would integrate with the existing + // WebSocket client and DBN parser to produce the required stream format. + + // For now, return an error indicating this needs full implementation + Err(DataError::NotImplemented( + "Stream implementation requires integration with WebSocket message processing pipeline".to_string() + )) + } + + fn get_connection_status(&self) -> ConnectionStatus { + let base_status = self.connection_status.read().unwrap().clone(); + let metrics = self.client.get_metrics(); + + // Enhance with real-time metrics + ConnectionStatus { + state: base_status.state, + active_subscriptions: base_status.active_subscriptions, + events_per_second: metrics.messages_per_second as f64, + latency_micros: Some(metrics.avg_processing_latency_ns / 1000), + recent_error_count: metrics.parse_errors.saturating_add(metrics.event_errors) as u32, + last_message_time: Some(Utc::now()), // Would be actual last message time + last_connection_attempt: base_status.last_connection_attempt, + } + } + + fn get_provider_name(&self) -> &'static str { + "databento" + } +} + +/// Production-ready Databento historical provider +/// +/// Implements the HistoricalProvider trait with features for backtesting and analysis: +/// - Efficient batch data retrieval +/// - Multiple data schemas (trades, quotes, order books, OHLCV) +/// - Rate limiting and retry logic +/// - Comprehensive error handling +pub struct DatabentoHistoricalProvider { + /// Core client for REST API access + client: DatabentoClient, + /// Configuration settings + config: DatabentoConfig, +} + +impl DatabentoHistoricalProvider { + /// Create new historical provider + pub async fn new(config: DatabentoConfig) -> Result { + info!("Initializing Databento historical provider"); + + let client = DatabentoClient::new(config.clone()).await?; + + let provider = Self { + client, + config, + }; + + info!("Databento historical provider initialized successfully"); + Ok(provider) + } + + /// Create with production settings + pub async fn production() -> Result { + Self::new(DatabentoConfig::production()).await + } + + /// Convert types::MarketDataEvent to providers::common::MarketDataEvent + fn convert_to_common_event(&self, event: MarketDataEvent) -> MarketDataEvent { + match event { + MarketDataEvent::Trade(trade) => { + let common_trade = TradeEvent { + symbol: trade.symbol.into(), + price: trade.price, + size: trade.size, + timestamp: trade.timestamp, + trade_id: Some(trade.trade_id.unwrap_or_else(|| "UNKNOWN".to_string())), + exchange: Some(trade.exchange.unwrap_or_else(|| "UNKNOWN".to_string())), + conditions: vec![], + sequence: 0, + }; + MarketDataEvent::Trade(common_trade) + } + MarketDataEvent::Quote(quote) => { + let common_quote = QuoteEvent { + symbol: quote.symbol.into(), + bid: quote.bid, + ask: quote.ask, + bid_size: quote.bid_size, + ask_size: quote.ask_size, + timestamp: quote.timestamp, + exchange: quote.exchange.clone(), + bid_exchange: quote.exchange.clone(), + ask_exchange: quote.exchange, + conditions: vec![], + sequence: 0, + }; + MarketDataEvent::Quote(common_quote) + } + // Add other event types as needed + _ => { + // For unsupported event types, create a placeholder trade event + let placeholder_trade = TradeEvent { + symbol: "UNKNOWN".into(), + price: Decimal::ZERO, + size: Decimal::ZERO, + timestamp: Utc::now(), + trade_id: Some("placeholder".to_string()), + exchange: Some("UNKNOWN".to_string()), + conditions: vec![], + sequence: 0, + }; + MarketDataEvent::Trade(placeholder_trade) + } + } + } +} + +#[async_trait] +impl HistoricalProvider for DatabentoHistoricalProvider { + async fn fetch( + &self, + symbol: &Symbol, + schema: HistoricalSchema, + range: TimeRange, + ) -> Result> { + debug!("Fetching historical data for {} ({:?}) from {} to {}", + symbol, schema, range.start, range.end); + + // Convert schema to Databento format + let databento_schema = match schema { + HistoricalSchema::Trade => DatabentoSchema::Trades, + HistoricalSchema::Quote => DatabentoSchema::Tbbo, + HistoricalSchema::OrderBookL2 => DatabentoSchema::Mbp1, + HistoricalSchema::OrderBookL3 => DatabentoSchema::Mbo, + HistoricalSchema::OHLCV => DatabentoSchema::Ohlcv1M, + _ => { + return Err(DataError::Unsupported(format!( + "Schema {:?} not supported by Databento", schema + ))); + } + }; + + // Execute the fetch request + match self.client.fetch_historical(symbol, databento_schema, range).await { + Ok(events) => { + info!("Successfully fetched {} events for {}", events.len(), symbol); + // Events are already in providers::common::MarketDataEvent format + Ok(events) + } + Err(e) => { + error!("Failed to fetch historical data for {}: {}", symbol, e); + Err(e) + } + } + } + + async fn fetch_batch( + &self, + symbols: &[Symbol], + schema: HistoricalSchema, + range: TimeRange, + ) -> Result> { + info!("Fetching batch historical data for {} symbols", symbols.len()); + + // Use parallel fetching for efficiency + let mut all_events = Vec::new(); + + for symbol in symbols { + let mut events = self.fetch(symbol, schema, range).await?; + all_events.append(&mut events); + } + + // Sort by timestamp for proper chronological ordering + all_events.sort_by_key(|event| event.timestamp()); + + info!("Successfully fetched {} total events for {} symbols", + all_events.len(), symbols.len()); + + Ok(all_events) + } + + fn supports_schema(&self, schema: HistoricalSchema) -> bool { + matches!( + schema, + HistoricalSchema::Trade | + HistoricalSchema::Quote | + HistoricalSchema::OrderBookL2 | + HistoricalSchema::OrderBookL3 | + HistoricalSchema::OHLCV + ) + } + + fn max_range(&self) -> std::time::Duration { + // Databento allows large historical ranges, but we limit for practical reasons + std::time::Duration::from_secs(30 * 24 * 3600) // 30 days + } + + fn get_provider_name(&self) -> &'static str { + "databento" + } +} + +/// Factory for creating Databento providers +pub struct DatabentoProviderFactory; + +impl DatabentoProviderFactory { + /// Create streaming provider with production settings + pub async fn create_streaming_provider() -> Result { + DatabentoStreamingProvider::production().await + } + + /// Create historical provider with production settings + pub async fn create_historical_provider() -> Result { + DatabentoHistoricalProvider::production().await + } + + /// Create both providers with shared configuration + pub async fn create_providers() -> Result<(DatabentoStreamingProvider, DatabentoHistoricalProvider)> { + let config = DatabentoConfig::production(); + + let streaming = DatabentoStreamingProvider::new(config.clone()).await?; + let historical = DatabentoHistoricalProvider::new(config).await?; + + Ok((streaming, historical)) + } +} + +/// Integration utilities for core system +pub mod integration { + use super::*; + use trading_engine::events::EventProcessor; + + /// Set up complete Databento integration with the core trading system + pub async fn setup_production_integration( + event_processor: Arc + ) -> Result { + info!("Setting up production Databento integration"); + + let mut provider = DatabentoStreamingProvider::production().await?; + provider.set_event_processor(event_processor).await; + + // Connect and validate performance + provider.connect().await?; + + // Wait a moment for connection to stabilize + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + + if !provider.validate_performance() { + warn!("Performance targets not initially met - system will continue optimizing"); + } + + info!("Databento production integration setup complete"); + Ok(provider) + } + + /// Health check for Databento integration + pub async fn health_check(provider: &DatabentoStreamingProvider) -> Result<()> { + let metrics = provider.get_performance_metrics(); + let status = provider.get_connection_status(); + + if !status.is_healthy() { + return Err(DataError::Connection("Databento connection unhealthy".to_string())); + } + + if metrics.error_rate > 0.01 { // >1% error rate + return Err(DataError::internal(format!( + "High error rate: {:.2}%", metrics.error_rate * 100.0 + ))); + } + + info!("Databento health check passed - {:.2} msg/s, {}ns latency", + metrics.messages_per_second, metrics.avg_latency_ns); + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::test; + + #[test] + async fn test_streaming_provider_creation() { + let config = DatabentoConfig::testing(); + let provider = DatabentoStreamingProvider::new(config).await; + assert!(provider.is_ok()); + } + + #[test] + async fn test_historical_provider_creation() { + let config = DatabentoConfig::testing(); + let provider = DatabentoHistoricalProvider::new(config).await; + assert!(provider.is_ok()); + } + + #[test] + async fn test_factory_creation() { + // Note: These tests would require proper API keys in a real environment + let config = DatabentoConfig::testing(); + + let streaming = DatabentoStreamingProvider::new(config.clone()).await; + let historical = DatabentoHistoricalProvider::new(config).await; + + assert!(streaming.is_ok()); + assert!(historical.is_ok()); + } + + #[test] + fn test_schema_support() { + use crate::providers::traits::HistoricalSchema; + + let config = DatabentoConfig::testing(); + let provider = tokio::runtime::Runtime::new().unwrap().block_on(async { + DatabentoHistoricalProvider::new(config).await.unwrap() + }); + + assert!(provider.supports_schema(HistoricalSchema::Trade)); + assert!(provider.supports_schema(HistoricalSchema::Quote)); + assert!(provider.supports_schema(HistoricalSchema::OrderBookL2)); + assert!(provider.supports_schema(HistoricalSchema::OrderBookL3)); + assert!(provider.supports_schema(HistoricalSchema::OHLCV)); + + assert!(!provider.supports_schema(HistoricalSchema::News)); + assert!(!provider.supports_schema(HistoricalSchema::Sentiment)); + } +} \ No newline at end of file diff --git a/data/src/providers/mod.rs b/data/src/providers/mod.rs index b84796c5d..c3a0d7c3d 100644 --- a/data/src/providers/mod.rs +++ b/data/src/providers/mod.rs @@ -42,8 +42,6 @@ mod databento_old; pub mod databento_streaming; // Re-export the new traits and common types -pub use common::MarketDataEvent; -pub use traits::{ ConnectionState, ConnectionStatus, HistoricalProvider, HistoricalSchema, RealTimeProvider, }; diff --git a/data/src/providers/mod.rs.bak b/data/src/providers/mod.rs.bak new file mode 100644 index 000000000..b84796c5d --- /dev/null +++ b/data/src/providers/mod.rs.bak @@ -0,0 +1,392 @@ +//! # Market Data Providers Module +//! +//! This module contains implementations for various market data providers in the +//! Foxhunt HFT trading system with a focus on dual-provider architecture. +//! +//! ## Architecture +//! +//! The system uses a dual-provider approach: +//! - **Databento**: Market microstructure data (trades, quotes, L2/L3 order books) +//! - **Benzinga Pro**: News, sentiment, analyst ratings, unusual options activity +//! - **Polygon.io**: Legacy provider (being phased out) +//! +//! ## Provider Traits +//! +//! - `RealTimeProvider`: Streaming WebSocket data with sub-millisecond latency +//! - `HistoricalProvider`: Batch historical data retrieval with rate limiting +//! - `MarketDataProvider`: Legacy unified interface (backwards compatibility) +//! +//! ## Features +//! +//! - Zero-copy message parsing for maximum HFT performance +//! - Unified event types across all providers via `MarketDataEvent` +//! - Automatic reconnection with exponential backoff +//! - Provider-specific error handling and rate limiting +//! - Real-time connection health monitoring + +// Core trait definitions and common types +pub mod common; +pub mod traits; + +// Provider implementations +pub mod benzinga; + +// Databento provider - only available when feature is enabled +#[cfg(feature = "databento")] +pub mod databento; +// Legacy historical provider temporarily kept for reference +#[cfg(feature = "databento")] +#[allow(dead_code)] +mod databento_old; +#[cfg(feature = "databento")] +pub mod databento_streaming; + +// Re-export the new traits and common types +pub use common::MarketDataEvent; +pub use traits::{ + ConnectionState, ConnectionStatus, HistoricalProvider, HistoricalSchema, RealTimeProvider, +}; + +use crate::error::{DataError, Result}; +use crate::types::TimeRange; +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use tokio::sync::mpsc; +// use common::Symbol; + +/// Configuration for market data providers +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ProviderConfig { + /// Provider name (polygon, databento, benzinga) + pub name: String, + /// API endpoint URL + pub endpoint: String, + /// API key or credentials + pub api_key: String, + /// Enable real-time data streaming + pub enable_realtime: bool, + /// Maximum concurrent connections + pub max_connections: usize, + /// Rate limit (requests per second) + pub rate_limit: u32, + /// Connection timeout in milliseconds + pub timeout_ms: u64, + /// Enable Level 2 data + pub enable_level2: bool, + /// Subscription symbols + pub symbols: Vec, +} + +/// Legacy market data provider trait for backwards compatibility +/// +/// This trait provides a unified interface for providers that implement both +/// real-time and historical capabilities. New providers should implement +/// `RealTimeProvider` and/or `HistoricalProvider` directly for better +/// separation of concerns. +#[async_trait] +pub trait MarketDataProvider: Send + Sync { + /// Connect to the data provider + async fn connect(&mut self) -> Result<()>; + + /// Disconnect from the data provider + async fn disconnect(&mut self) -> Result<()>; + + /// Subscribe to real-time market data for symbols + async fn subscribe(&mut self, symbols: Vec) -> Result<()>; + + /// Unsubscribe from symbols + async fn unsubscribe(&mut self, symbols: Vec) -> Result<()>; + + /// Get historical market data + async fn get_historical_data( + &self, + symbol: &str, + timeframe: &str, + range: TimeRange, + ) -> Result>; + + /// Get current market status + async fn get_market_status(&self) -> Result; + + /// Get provider health status + fn get_health_status(&self) -> ProviderHealthStatus; + + /// Get provider name + fn get_name(&self) -> &str; +} + +/// Market status information +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MarketStatus { + /// Market is currently open + pub is_open: bool, + /// Next market open time + pub next_open: Option>, + /// Next market close time + pub next_close: Option>, + /// Market timezone + pub timezone: String, + /// Extended hours trading available + pub extended_hours: bool, +} + +/// Provider health status +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ProviderHealthStatus { + /// Provider is connected + pub connected: bool, + /// Last successful connection time + pub last_connected: Option>, + /// Number of active subscriptions + pub active_subscriptions: usize, + /// Messages received per second + pub messages_per_second: f64, + /// Connection latency in microseconds + pub latency_micros: Option, + /// Error count in last hour + pub error_count: u32, +} + +/// Provider factory for creating different provider instances +pub struct ProviderFactory; + +impl ProviderFactory { + /// Create a new provider instance based on configuration + pub fn create_provider( + config: ProviderConfig, + _event_tx: mpsc::UnboundedSender, + ) -> Result> { + match config.name.as_str() { + "databento" => { + // Databento streaming provider for real-time data + Err(DataError::Configuration { + field: "provider.name".to_string(), + message: "Use DatabentoStreamingProvider for real-time data or DatabentoHistoricalProvider for historical data.".to_string(), + }) + } + "benzinga" => { + // Benzinga news and sentiment provider + Err(DataError::Configuration { + field: "provider.name".to_string(), + message: "Use BenzingaProvider for news and sentiment data.".to_string(), + }) + } + _ => Err(DataError::Configuration { + field: "provider.name".to_string(), + message: format!( + "Unknown provider: {}. Available providers: databento, benzinga", + config.name + ), + }), + } + } +} + +/// Provider manager for coordinating multiple providers +pub struct ProviderManager { + providers: Vec>, + event_tx: mpsc::UnboundedSender, + health_monitor: HealthMonitor, +} + +impl ProviderManager { + /// Create a new provider manager + pub fn new(event_tx: mpsc::UnboundedSender) -> Self { + Self { + providers: Vec::new(), + event_tx, + health_monitor: HealthMonitor::new(), + } + } + + /// Add a provider to the manager + pub fn add_provider(&mut self, provider: Box) { + self.providers.push(provider); + } + + /// Connect all providers + pub async fn connect_all(&mut self) -> Result<()> { + for provider in &mut self.providers { + if let Err(e) = provider.connect().await { + tracing::error!("Failed to connect provider {}: {}", provider.get_name(), e); + continue; + } + tracing::info!("Connected to provider: {}", provider.get_name()); + } + Ok(()) + } + + /// Subscribe to symbols across all providers + pub async fn subscribe_all(&mut self, symbols: Vec) -> Result<()> { + for provider in &mut self.providers { + if let Err(e) = provider.subscribe(symbols.clone()).await { + tracing::error!( + "Failed to subscribe on provider {}: {}", + provider.get_name(), + e + ); + continue; + } + } + Ok(()) + } + + /// Get health status for all providers + pub fn get_all_health_status(&self) -> Vec<(String, ProviderHealthStatus)> { + self.providers + .iter() + .map(|p| (p.get_name().to_string(), p.get_health_status())) + .collect() + } + + /// Start health monitoring + pub async fn start_health_monitoring(&mut self) { + self.health_monitor.start(&self.providers).await; + } +} + +/// Health monitor for tracking provider status +struct HealthMonitor { + monitoring: bool, +} + +impl HealthMonitor { + fn new() -> Self { + Self { monitoring: false } + } + + async fn start(&mut self, _providers: &[Box]) { + if self.monitoring { + return; + } + + self.monitoring = true; + tracing::info!("Started provider health monitoring"); + + // Health monitoring implementation would go here + // This would periodically check provider status and emit alerts + } +} + +// Blanket implementation to provide backwards compatibility +// Any type that implements both RealTimeProvider and HistoricalProvider +// automatically implements the legacy MarketDataProvider trait +#[async_trait] +impl MarketDataProvider for T +where + T: RealTimeProvider + HistoricalProvider, +{ + async fn connect(&mut self) -> Result<()> { + RealTimeProvider::connect(self).await + } + + async fn disconnect(&mut self) -> Result<()> { + RealTimeProvider::disconnect(self).await + } + + async fn subscribe(&mut self, symbols: Vec) -> Result<()> { + let symbol_structs: Vec<::common::Symbol> = symbols.into_iter().map(|s| ::common::Symbol::from(s.as_str())).collect(); + RealTimeProvider::subscribe(self, symbol_structs).await + } + + async fn unsubscribe(&mut self, symbols: Vec) -> Result<()> { + let symbol_structs: Vec<::common::Symbol> = symbols.into_iter().map(|s| ::common::Symbol::from(s.as_str())).collect(); + RealTimeProvider::unsubscribe(self, symbol_structs).await + } + + async fn get_historical_data( + &self, + symbol: &str, + timeframe: &str, + range: TimeRange, + ) -> Result> { + // Convert timeframe string to HistoricalSchema + let schema = match timeframe.to_lowercase().as_str() { + "trades" | "trade" => HistoricalSchema::Trade, + "quotes" | "quote" => HistoricalSchema::Quote, + "orderbook" | "l2" => HistoricalSchema::OrderBookL2, + "mbo" | "l3" => HistoricalSchema::OrderBookL3, + "bars" | "ohlcv" | "candles" => HistoricalSchema::OHLCV, + "news" => HistoricalSchema::News, + "sentiment" => HistoricalSchema::Sentiment, + _ => HistoricalSchema::Trade, // Default fallback + }; + + // Convert string to Symbol + let symbol_struct = ::common::Symbol::from(symbol); + // Fetch data from the historical provider - already returns common::MarketDataEvent + let results = HistoricalProvider::fetch(self, &symbol_struct, schema, range).await?; + // No conversion needed - HistoricalProvider::fetch returns common::MarketDataEvent + Ok(results) + } + + async fn get_market_status(&self) -> Result { + // Default implementation - providers can override + Ok(MarketStatus { + is_open: true, + next_open: None, + next_close: None, + timezone: "US/Eastern".to_string(), + extended_hours: false, + }) + } + + fn get_health_status(&self) -> ProviderHealthStatus { + let connection_status = RealTimeProvider::get_connection_status(self); + ProviderHealthStatus { + connected: matches!(connection_status.state, ConnectionState::Connected), + last_connected: connection_status.last_connection_attempt, + active_subscriptions: connection_status.active_subscriptions, + messages_per_second: connection_status.events_per_second, + latency_micros: connection_status.latency_micros, + error_count: connection_status.recent_error_count, + } + } + + fn get_name(&self) -> &str { + RealTimeProvider::get_provider_name(self) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::sync::mpsc; + + #[tokio::test] + async fn test_provider_manager_creation() { + let (tx, _rx) = mpsc::unbounded_channel(); + let manager = ProviderManager::new(tx); + assert_eq!(manager.providers.len(), 0); + } + + #[test] + fn test_provider_config_serialization() { + let config = ProviderConfig { + name: "databento".to_string(), + endpoint: "wss://api.databento.com/ws".to_string(), + api_key: std::env::var("DATABENTO_API_KEY") + .unwrap_or_else(|_| "DATABENTO_API_KEY_REQUIRED".to_string()), + enable_realtime: true, + max_connections: 5, + rate_limit: 100, + timeout_ms: 5000, + enable_level2: true, + symbols: vec!["SPY".to_string(), "QQQ".to_string()], + }; + + let json = serde_json::to_string(&config).unwrap(); + let deserialized: ProviderConfig = serde_json::from_str(&json).unwrap(); + assert_eq!(config.name, deserialized.name); + } + + #[test] + fn test_historical_schema_conversion() { + use traits::HistoricalSchema; + + assert!(HistoricalSchema::Trade.is_market_data()); + assert!(!HistoricalSchema::News.is_market_data()); + assert!(HistoricalSchema::News.is_news_data()); + assert!(!HistoricalSchema::Trade.is_news_data()); + } +} diff --git a/data/src/types.rs b/data/src/types.rs index a9b5feff6..c4f57d9f3 100644 --- a/data/src/types.rs +++ b/data/src/types.rs @@ -27,14 +27,13 @@ pub enum MarketDataType { Status, } -// Use canonical MarketDataEvent from common crate -pub use common::MarketDataEvent; +// Import MarketDataEvent from common crate - use common::MarketDataEvent directly /// Extended market data event types with provider-specific events #[derive(Debug, Clone, Serialize, Deserialize)] pub enum ExtendedMarketDataEvent { /// Core market data event - Core(MarketDataEvent), + Core(common::MarketDataEvent), /// News alerts (Benzinga) NewsAlert(crate::providers::common::NewsEvent), /// Sentiment updates (Benzinga) @@ -45,8 +44,7 @@ pub enum ExtendedMarketDataEvent { UnusualOptions(crate::providers::common::UnusualOptionsEvent), } -// Use canonical event types from common crate -pub use common::QuoteEvent; +// Import canonical event types from common crate - use common::QuoteEvent directly use common::TradeEvent; use common::Aggregate; use common::BarEvent; @@ -178,7 +176,7 @@ impl ExtendedMarketDataEvent { /// For provider-specific events (NewsAlert, SentimentUpdate, etc.), /// returns None since they don't have equivalents in the core MarketDataEvent enum. /// For Core events, returns the wrapped MarketDataEvent. - pub fn into_core_event(self) -> Option { + pub fn into_core_event(self) -> Option { match self { ExtendedMarketDataEvent::Core(event) => Some(event), _ => None, // Provider-specific events don't have core equivalents @@ -188,7 +186,7 @@ impl ExtendedMarketDataEvent { /// Helper function to convert a Vec to Vec /// by extracting only the core events and filtering out provider-specific ones -pub fn extract_core_events(extended_events: Vec) -> Vec { +pub fn extract_core_events(extended_events: Vec) -> Vec { extended_events .into_iter() .filter_map(|event| event.into_core_event()) @@ -197,19 +195,19 @@ pub fn extract_core_events(extended_events: Vec) -> Vec /// Helper function to get timestamp from MarketDataEvent /// Since we can't implement methods on MarketDataEvent from common crate -pub fn get_event_timestamp(event: &MarketDataEvent) -> Option> { +pub fn get_event_timestamp(event: &common::MarketDataEvent) -> Option> { match event { - MarketDataEvent::Quote(q) => Some(q.timestamp), - MarketDataEvent::Trade(t) => Some(t.timestamp), - MarketDataEvent::Aggregate(a) => Some(a.end_timestamp), - MarketDataEvent::Bar(b) => Some(b.end_timestamp), - MarketDataEvent::Level2(l) => Some(l.timestamp), - MarketDataEvent::Status(s) => Some(s.timestamp), - MarketDataEvent::ConnectionStatus(c) => Some(c.timestamp), - MarketDataEvent::Error(e) => Some(e.timestamp), - MarketDataEvent::OrderBook(o) => Some(o.timestamp), - MarketDataEvent::OrderBookL2Snapshot(s) => Some(s.timestamp), - MarketDataEvent::OrderBookL2Update(u) => Some(u.timestamp), + common::MarketDataEvent::Quote(q) => Some(q.timestamp), + common::MarketDataEvent::Trade(t) => Some(t.timestamp), + common::MarketDataEvent::Aggregate(a) => Some(a.end_timestamp), + common::MarketDataEvent::Bar(b) => Some(b.end_timestamp), + common::MarketDataEvent::Level2(l) => Some(l.timestamp), + common::MarketDataEvent::Status(s) => Some(s.timestamp), + common::MarketDataEvent::ConnectionStatus(c) => Some(c.timestamp), + common::MarketDataEvent::Error(e) => Some(e.timestamp), + common::MarketDataEvent::OrderBook(o) => Some(o.timestamp), + common::MarketDataEvent::OrderBookL2Snapshot(s) => Some(s.timestamp), + common::MarketDataEvent::OrderBookL2Update(u) => Some(u.timestamp), } } @@ -222,15 +220,15 @@ mod tests { #[test] fn test_subscription_creation() { - let sub = Subscription::quotes(vec!["AAPL".to_string(), "GOOGL".to_string()]); + let sub = common::Subscription::quotes(vec!["AAPL".to_string(), "GOOGL".to_string()]); assert_eq!(sub.symbols.len(), 2); assert_eq!(sub.data_types.len(), 1); - assert!(matches!(sub.data_types[0], DataType::Quotes)); + assert!(matches!(sub.data_types[0], common::DataType::Quotes)); } #[test] fn test_market_data_event_symbol() { - let quote = MarketDataEvent::Quote(QuoteEvent { + let quote = common::MarketDataEvent::Quote(common::QuoteEvent { symbol: "AAPL".to_string(), bid: Some(Decimal::new(15000, 2)), // 150.00 ask: Some(Decimal::new(15001, 2)), // 150.01 diff --git a/data/tests/test_event_conversion_streaming.rs b/data/tests/test_event_conversion_streaming.rs index 5d14ee45e..828d5f342 100644 --- a/data/tests/test_event_conversion_streaming.rs +++ b/data/tests/test_event_conversion_streaming.rs @@ -9,12 +9,14 @@ use data::providers::benzinga::{ BenzingaEarnings, BenzingaNewsArticle, BenzingaRating, NewsEvent as BenzingaNewsEvent, }; use data::providers::common::{ - AggregateEvent, AnalystRatingEvent, ConnectionStatusEvent, ErrorCategory, ErrorEvent, + AggregateEvent, AnalystRatingEvent, ConnectionStatusEvent, ErrorEvent, MarketState, MarketStatusEvent, NewsEvent, OptionsContract, OptionsSentiment, OptionsType, OrderBookSide, OrderBookSnapshot, OrderBookUpdate, PriceLevel, - PriceLevelChange, PriceLevelChangeType, QuoteEvent, RatingAction, SentimentEvent, - SentimentPeriod, TradeEvent, UnusualOptionsEvent, UnusualOptionsType, + PriceLevelChange, PriceLevelChangeType, RatingAction, SentimentEvent, + SentimentPeriod, UnusualOptionsEvent, UnusualOptionsType, }; +use common::error::ErrorCategory; +use common::{QuoteEvent, TradeEvent}; use data::types::ExtendedMarketDataEvent; use common::types::MarketDataEvent; use data::providers::databento_streaming::{ diff --git a/data/tests/test_provider_traits.rs b/data/tests/test_provider_traits.rs index 2462da3fe..1601caeb5 100644 --- a/data/tests/test_provider_traits.rs +++ b/data/tests/test_provider_traits.rs @@ -7,12 +7,14 @@ use chrono::{DateTime, Duration as ChronoDuration, Utc}; use data::error::{DataError, Result}; use data::providers::common::{ - AggregateEvent, AnalystRatingEvent, ConnectionStatusEvent, ErrorCategory, ErrorEvent, + AggregateEvent, AnalystRatingEvent, ConnectionStatusEvent, ErrorEvent, MarketState, MarketStatusEvent, NewsEvent, OptionsContract, OptionsSentiment, OptionsType, OrderBookSide, OrderBookSnapshot, OrderBookUpdate, PriceLevel, - PriceLevelChange, PriceLevelChangeType, QuoteEvent, RatingAction, SentimentEvent, - SentimentPeriod, TradeEvent, UnusualOptionsEvent, UnusualOptionsType, + PriceLevelChange, PriceLevelChangeType, RatingAction, SentimentEvent, + SentimentPeriod, UnusualOptionsEvent, UnusualOptionsType, }; +use common::error::ErrorCategory; +use common::{QuoteEvent, TradeEvent}; use data::types::ExtendedMarketDataEvent; use common::types::MarketDataEvent; use data::providers::traits::{ diff --git a/ml/src/checkpoint/mod.rs b/ml/src/checkpoint/mod.rs index 70b2fcf8c..e174efc38 100644 --- a/ml/src/checkpoint/mod.rs +++ b/ml/src/checkpoint/mod.rs @@ -55,14 +55,10 @@ pub mod versioning; #[cfg(test)] pub mod integration_tests; -pub use compression::*; -pub use model_implementations::*; -pub use storage::*; -pub use validation::*; -pub use versioning::*; +// DO NOT RE-EXPORT - Use explicit imports at usage sites // Use canonical ModelType from crate root -pub use crate::ModelType; +// DO NOT RE-EXPORT - Use explicit imports at usage sites /// Checkpoint format options #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] diff --git a/ml/src/checkpoint/mod.rs.bak b/ml/src/checkpoint/mod.rs.bak new file mode 100644 index 000000000..70b2fcf8c --- /dev/null +++ b/ml/src/checkpoint/mod.rs.bak @@ -0,0 +1,1038 @@ +//! # Unified Model Weight Persistence System +//! +//! Comprehensive checkpoint system for all 5 AI models (DQN, MAMBA, TFT, TGGN, LNN) +//! with versioning, metadata, compression, and validation. +//! +//! ## Key Features +//! +//! - **Unified Interface**: Single API for all model checkpointing +//! - **Model Versioning**: Semantic versioning with compatibility checks +//! - **Metadata Management**: Training metrics, hyperparameters, performance stats +//! - **Compression**: Optional LZ4/Zstd compression for large models +//! - **Validation**: Checksum verification and corruption detection +//! - **Incremental Saves**: Delta checkpoints for memory efficiency +//! - **Async I/O**: Non-blocking checkpoint operations +//! - **Multi-format**: Binary, JSON, and custom formats +//! +//! ## Architecture +//! +//! ```text +//! ┌─────────────────────────────────────────────────────────────┐ +//! │ CheckpointManager │ +//! ├─────────────────┬─────────────────┬─────────────────────────┤ +//! │ Versioning │ Compression │ Storage Backend │ +//! │ │ │ │ +//! │ • Semantic Ver │ • LZ4/Zstd │ • FileSystem │ +//! │ • Compatibility │ • Delta Saves │ • Cloud Storage │ +//! │ • Migration │ • Streaming │ • Database │ +//! └─────────────────┴─────────────────┴─────────────────────────┘ +//! ``` + +use std::collections::HashMap; +use std::fs::{self}; +use std::io::{Read, Write}; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use dashmap::DashMap; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use tokio::sync::RwLock; +use tracing::{info, instrument, warn}; +use uuid::Uuid; + +use crate::MLError; + +pub mod compression; +pub mod model_implementations; +pub mod storage; +pub mod validation; +pub mod versioning; + +#[cfg(test)] +pub mod integration_tests; + +pub use compression::*; +pub use model_implementations::*; +pub use storage::*; +pub use validation::*; +pub use versioning::*; + +// Use canonical ModelType from crate root +pub use crate::ModelType; + +/// Checkpoint format options +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum CheckpointFormat { + /// Binary format (fastest) + Binary, + /// JSON format (human-readable) + JSON, + /// MessagePack format (compact) + MessagePack, + /// Custom optimized format + Custom, +} + +/// Compression algorithm options +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum CompressionType { + /// No compression + None, + /// LZ4 - fast compression + LZ4, + /// Zstandard - balanced compression + Zstd, + /// Gzip - high compression + Gzip, +} + +/// Checkpoint metadata containing model information and training state +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CheckpointMetadata { + /// Unique checkpoint identifier + pub checkpoint_id: String, + + /// Model type + pub model_type: ModelType, + + /// Model name/identifier + pub model_name: String, + + /// Model version (semantic versioning) + pub version: String, + + /// Creation timestamp + pub created_at: DateTime, + + /// Training epoch when checkpoint was created + pub epoch: Option, + + /// Training step when checkpoint was created + pub step: Option, + + /// Training loss at checkpoint time + pub loss: Option, + + /// Validation accuracy at checkpoint time + pub accuracy: Option, + + /// Model hyperparameters + pub hyperparameters: HashMap, + + /// Training metrics and statistics + pub metrics: HashMap, + + /// Model architecture information + pub architecture: HashMap, + + /// Checkpoint file format + pub format: CheckpointFormat, + + /// Compression algorithm used + pub compression: CompressionType, + + /// File size in bytes + pub file_size: u64, + + /// Compressed file size (if compressed) + pub compressed_size: Option, + + /// SHA-256 checksum for validation + pub checksum: String, + + /// Tags for organizing checkpoints + pub tags: Vec, + + /// Additional custom metadata + pub custom_metadata: HashMap, +} + +impl CheckpointMetadata { + /// Create new checkpoint metadata + pub fn new(model_type: ModelType, model_name: String, version: String) -> Self { + Self { + checkpoint_id: Uuid::new_v4().to_string(), + model_type, + model_name, + version, + created_at: Utc::now(), + epoch: None, + step: None, + loss: None, + accuracy: None, + hyperparameters: HashMap::new(), + metrics: HashMap::new(), + architecture: HashMap::new(), + format: CheckpointFormat::Binary, + compression: CompressionType::None, + file_size: 0, + compressed_size: None, + checksum: String::new(), + tags: Vec::new(), + custom_metadata: HashMap::new(), + } + } + + /// Add training state information + pub fn with_training_state( + mut self, + epoch: Option, + step: Option, + loss: Option, + accuracy: Option, + ) -> Self { + self.epoch = epoch; + self.step = step; + self.loss = loss; + self.accuracy = accuracy; + self + } + + /// Add hyperparameters + pub fn with_hyperparameters(mut self, hyperparams: HashMap) -> Self { + self.hyperparameters = hyperparams; + self + } + + /// Add metrics + pub fn with_metrics(mut self, metrics: HashMap) -> Self { + self.metrics = metrics; + self + } + + /// Add tags + pub fn with_tags(mut self, tags: Vec) -> Self { + self.tags = tags; + self + } + + /// Check if this is a newer version than another metadata + pub fn is_newer_than(&self, other: &CheckpointMetadata) -> bool { + if self.model_type != other.model_type || self.model_name != other.model_name { + return false; + } + + // Compare by epoch if available + if let (Some(self_epoch), Some(other_epoch)) = (self.epoch, other.epoch) { + return self_epoch > other_epoch; + } + + // Compare by step if available + if let (Some(self_step), Some(other_step)) = (self.step, other.step) { + return self_step > other_step; + } + + // Compare by timestamp + self.created_at > other.created_at + } + + /// Generate filename for this checkpoint + pub fn generate_filename(&self) -> String { + let timestamp = self.created_at.format("%Y%m%d_%H%M%S"); + let epoch_str = self.epoch.map(|e| format!("_e{}", e)).unwrap_or_default(); + let step_str = self.step.map(|s| format!("_s{}", s)).unwrap_or_default(); + let ext = self.model_type.file_extension(); + + format!( + "{}_{}_v{}{}{}_{}.{}", + self.model_type.file_extension(), + self.model_name, + self.version, + epoch_str, + step_str, + timestamp, + ext + ) + } +} + +/// Configuration for checkpoint operations +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CheckpointConfig { + /// Base directory for checkpoints + pub base_dir: PathBuf, + + /// Default compression type + pub compression: CompressionType, + + /// Default checkpoint format + pub format: CheckpointFormat, + + /// Maximum number of checkpoints to keep per model + pub max_checkpoints_per_model: usize, + + /// Automatic cleanup of old checkpoints + pub auto_cleanup: bool, + + /// Enable checksum validation + pub validate_checksums: bool, + + /// Enable incremental checkpoints (delta saves) + pub incremental_checkpoints: bool, + + /// Compression level (0-9, algorithm dependent) + pub compression_level: u32, + + /// Enable async I/O operations + pub async_io: bool, + + /// Buffer size for I/O operations + pub buffer_size: usize, +} + +impl Default for CheckpointConfig { + fn default() -> Self { + Self { + base_dir: PathBuf::from("./checkpoints"), + compression: CompressionType::LZ4, + format: CheckpointFormat::Binary, + max_checkpoints_per_model: 10, + auto_cleanup: true, + validate_checksums: true, + incremental_checkpoints: true, + compression_level: 3, + async_io: true, + buffer_size: 64 * 1024, // 64KB + } + } +} + +/// Statistics for checkpoint operations +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct CheckpointStats { + /// Total checkpoints saved + pub total_saved: AtomicU64, + + /// Total checkpoints loaded + pub total_loaded: AtomicU64, + + /// Total bytes saved + pub total_bytes_saved: AtomicU64, + + /// Total bytes loaded + pub total_bytes_loaded: AtomicU64, + + /// Total compression savings (bytes) + pub compression_savings: AtomicU64, + + /// Average save time (microseconds) + pub avg_save_time_us: AtomicU64, + + /// Average load time (microseconds) + pub avg_load_time_us: AtomicU64, + + /// Failed operations + pub failed_operations: AtomicU64, +} + +impl CheckpointStats { + /// Record a save operation + pub fn record_save(&self, bytes_saved: u64, save_time_us: u64, compression_savings: u64) { + self.total_saved.fetch_add(1, Ordering::Relaxed); + self.total_bytes_saved + .fetch_add(bytes_saved, Ordering::Relaxed); + self.compression_savings + .fetch_add(compression_savings, Ordering::Relaxed); + + // Update moving average + let count = self.total_saved.load(Ordering::Relaxed); + let current_avg = self.avg_save_time_us.load(Ordering::Relaxed); + let new_avg = ((current_avg * (count - 1)) + save_time_us) / count; + self.avg_save_time_us.store(new_avg, Ordering::Relaxed); + } + + /// Record a load operation + pub fn record_load(&self, bytes_loaded: u64, load_time_us: u64) { + self.total_loaded.fetch_add(1, Ordering::Relaxed); + self.total_bytes_loaded + .fetch_add(bytes_loaded, Ordering::Relaxed); + + // Update moving average + let count = self.total_loaded.load(Ordering::Relaxed); + let current_avg = self.avg_load_time_us.load(Ordering::Relaxed); + let new_avg = ((current_avg * (count - 1)) + load_time_us) / count; + self.avg_load_time_us.store(new_avg, Ordering::Relaxed); + } + + /// Record a failed operation + pub fn record_failure(&self) { + self.failed_operations.fetch_add(1, Ordering::Relaxed); + } + + /// Get statistics as a map + pub fn to_map(&self) -> HashMap { + let mut map = HashMap::new(); + map.insert( + "total_saved".to_string(), + self.total_saved.load(Ordering::Relaxed), + ); + map.insert( + "total_loaded".to_string(), + self.total_loaded.load(Ordering::Relaxed), + ); + map.insert( + "total_bytes_saved".to_string(), + self.total_bytes_saved.load(Ordering::Relaxed), + ); + map.insert( + "total_bytes_loaded".to_string(), + self.total_bytes_loaded.load(Ordering::Relaxed), + ); + map.insert( + "compression_savings".to_string(), + self.compression_savings.load(Ordering::Relaxed), + ); + map.insert( + "avg_save_time_us".to_string(), + self.avg_save_time_us.load(Ordering::Relaxed), + ); + map.insert( + "avg_load_time_us".to_string(), + self.avg_load_time_us.load(Ordering::Relaxed), + ); + map.insert( + "failed_operations".to_string(), + self.failed_operations.load(Ordering::Relaxed), + ); + map + } +} + +/// Trait that models must implement to support checkpointing +#[async_trait] +pub trait Checkpointable { + /// Get model type + fn model_type(&self) -> ModelType; + + /// Get model name/identifier + fn model_name(&self) -> &str; + + /// Get model version + fn model_version(&self) -> &str; + + /// Serialize model weights and state to bytes + async fn serialize_state(&self) -> Result, MLError>; + + /// Deserialize model weights and state from bytes + async fn deserialize_state(&mut self, data: &[u8]) -> Result<(), MLError>; + + /// Get current training state (epoch, step, loss, etc.) + fn get_training_state(&self) -> (Option, Option, Option, Option) { + (None, None, None, None) // Default implementation + } + + /// Get hyperparameters for metadata + fn get_hyperparameters(&self) -> HashMap { + HashMap::new() // Default implementation + } + + /// Get current metrics for metadata + fn get_metrics(&self) -> HashMap { + HashMap::new() // Default implementation + } + + /// Get architecture information + fn get_architecture_info(&self) -> HashMap { + HashMap::new() // Default implementation + } + + /// Validate loaded state (optional override for custom validation) + fn validate_loaded_state(&self) -> Result<(), MLError> { + Ok(()) // Default implementation + } +} + +/// Main checkpoint manager for all AI models +#[derive(Debug)] +pub struct CheckpointManager { + /// Configuration + config: CheckpointConfig, + + /// Storage backend + storage: Arc, + + /// Checkpoint metadata index + metadata_index: Arc>>, + + /// Statistics + stats: Arc, + + /// Version manager + version_manager: Arc, + + /// Compression manager + compression_manager: Arc, + + /// Validation manager + validation_manager: Arc, +} + +impl CheckpointManager { + /// Create a new checkpoint manager + pub fn new(config: CheckpointConfig) -> Result { + // Create base directory if it doesn't exist + if !config.base_dir.exists() { + fs::create_dir_all(&config.base_dir).map_err(|e| { + MLError::ModelError(format!("Failed to create checkpoint directory: {}", e)) + })?; + } + + let storage: Arc = + Arc::new(FileSystemStorage::new(config.base_dir.clone())); + let metadata_index = Arc::new(RwLock::new(DashMap::new())); + let stats = Arc::new(CheckpointStats::default()); + let version_manager = Arc::new(VersionManager::new()); + let compression_manager = Arc::new(CompressionManager::new()); + let validation_manager = Arc::new(ValidationManager::new()); + + Ok(Self { + config, + storage, + metadata_index, + stats, + version_manager, + compression_manager, + validation_manager, + }) + } + + /// Save a checkpoint for a model + #[instrument(skip(self, model))] + pub async fn save_checkpoint( + &self, + model: &M, + tags: Option>, + ) -> Result { + let start_time = std::time::Instant::now(); + + info!("Saving checkpoint for model: {}", model.model_name()); + + // Create metadata + let (epoch, step, loss, accuracy) = model.get_training_state(); + let mut metadata = CheckpointMetadata::new( + model.model_type(), + model.model_name().to_string(), + model.model_version().to_string(), + ) + .with_training_state(epoch, step, loss, accuracy) + .with_hyperparameters(model.get_hyperparameters()) + .with_metrics(model.get_metrics()); + + if let Some(tags) = tags { + metadata = metadata.with_tags(tags); + } + + metadata.format = self.config.format; + metadata.compression = self.config.compression; + + // Add architecture info + metadata.architecture = model.get_architecture_info(); + + // Serialize model state + let model_data = model.serialize_state().await?; + let original_size = model_data.len() as u64; + + // Compress if needed + let (final_data, compressed_size) = if self.config.compression != CompressionType::None { + let compressed = self.compression_manager.compress( + &model_data, + self.config.compression, + self.config.compression_level, + )?; + let comp_size = compressed.len() as u64; + (compressed, Some(comp_size)) + } else { + (model_data, None) + }; + + // Calculate checksum + let mut hasher = Sha256::new(); + hasher.update(&final_data); + let checksum = format!("{:x}", hasher.finalize()); + + // Update metadata + metadata.file_size = original_size; + metadata.compressed_size = compressed_size; + metadata.checksum = checksum; + + // Generate filename + let filename = metadata.generate_filename(); + + // Save to storage + self.storage + .save_checkpoint(&filename, &final_data, &metadata) + .await?; + + // Update index + { + let index = self.metadata_index.write().await; + index.insert(metadata.checkpoint_id.clone(), metadata.clone()); + } + + // Cleanup old checkpoints if needed + if self.config.auto_cleanup { + self.cleanup_old_checkpoints(model.model_type(), model.model_name()) + .await?; + } + + // Record statistics + let save_time_us = start_time.elapsed().as_micros() as u64; + let compression_savings = compressed_size.map(|cs| original_size - cs).unwrap_or(0); + self.stats + .record_save(original_size, save_time_us, compression_savings); + + info!( + "Checkpoint saved: {} ({} bytes, {}µs, {:.1}% compression)", + filename, + final_data.len(), + save_time_us, + if compressed_size.is_some() { + (compression_savings as f64 / original_size as f64) * 100.0 + } else { + 0.0 + } + ); + + Ok(metadata.checkpoint_id) + } + + /// Load a checkpoint into a model + #[instrument(skip(self, model))] + pub async fn load_checkpoint( + &self, + model: &mut M, + checkpoint_id: &str, + ) -> Result { + let start_time = std::time::Instant::now(); + + info!("Loading checkpoint: {}", checkpoint_id); + + // Get metadata + let metadata = { + let index = self.metadata_index.read().await; + index.get(checkpoint_id).map(|entry| entry.clone()) + }; + + let metadata = metadata.ok_or_else(|| { + MLError::ModelError(format!("Checkpoint not found: {}", checkpoint_id)) + })?; + + // Verify model compatibility + if metadata.model_type != model.model_type() { + return Err(MLError::ModelError(format!( + "Model type mismatch: expected {:?}, got {:?}", + model.model_type(), + metadata.model_type + ))); + } + + // Load data from storage + let filename = metadata.generate_filename(); + let data = self.storage.load_checkpoint(&filename).await?; + + // Validate checksum if enabled + if self.config.validate_checksums { + self.validation_manager + .validate_checksum(&data, &metadata.checksum)?; + } + + // Decompress if needed + let final_data = if metadata.compression != CompressionType::None { + self.compression_manager + .decompress(&data, metadata.compression)? + } else { + data + }; + + // Deserialize into model + model.deserialize_state(&final_data).await?; + + // Validate loaded state + model.validate_loaded_state()?; + + // Record statistics + let load_time_us = start_time.elapsed().as_micros() as u64; + self.stats + .record_load(final_data.len() as u64, load_time_us); + + info!( + "Checkpoint loaded: {} ({} bytes, {}µs)", + filename, + final_data.len(), + load_time_us + ); + + Ok(metadata) + } + + /// Load the latest checkpoint for a model + pub async fn load_latest_checkpoint( + &self, + model: &mut M, + ) -> Result, MLError> { + let model_type = model.model_type(); + let model_name = model.model_name(); + + // Find latest checkpoint + let latest_checkpoint = { + let index = self.metadata_index.read().await; + index + .iter() + .filter(|entry| { + let metadata = entry.value(); + metadata.model_type == model_type && metadata.model_name == model_name + }) + .max_by(|a, b| a.value().created_at.cmp(&b.value().created_at)) + .map(|entry| entry.value().clone()) + }; + + if let Some(metadata) = latest_checkpoint { + let loaded_metadata = self.load_checkpoint(model, &metadata.checkpoint_id).await?; + Ok(Some(loaded_metadata)) + } else { + Ok(None) + } + } + + /// List all checkpoints for a model + pub async fn list_checkpoints( + &self, + model_type: ModelType, + model_name: &str, + ) -> Vec { + let index = self.metadata_index.read().await; + let mut checkpoints: Vec<_> = index + .iter() + .filter(|entry| { + let metadata = entry.value(); + metadata.model_type == model_type && metadata.model_name == model_name + }) + .map(|entry| entry.value().clone()) + .collect(); + + // Sort by creation time (newest first) + checkpoints.sort_by(|a, b| b.created_at.cmp(&a.created_at)); + checkpoints + } + + /// Delete a checkpoint + pub async fn delete_checkpoint(&self, checkpoint_id: &str) -> Result<(), MLError> { + // Get metadata + let metadata = { + let index = self.metadata_index.read().await; + index.get(checkpoint_id).map(|entry| entry.clone()) + }; + + let metadata = metadata.ok_or_else(|| { + MLError::ModelError(format!("Checkpoint not found: {}", checkpoint_id)) + })?; + + // Delete from storage + let filename = metadata.generate_filename(); + self.storage.delete_checkpoint(&filename).await?; + + // Remove from index + { + let index = self.metadata_index.write().await; + index.remove(checkpoint_id); + } + + info!("Checkpoint deleted: {}", checkpoint_id); + Ok(()) + } + + /// Cleanup old checkpoints for a model + async fn cleanup_old_checkpoints( + &self, + model_type: ModelType, + model_name: &str, + ) -> Result<(), MLError> { + let mut checkpoints = self.list_checkpoints(model_type, model_name).await; + + if checkpoints.len() <= self.config.max_checkpoints_per_model { + return Ok(()); + } + + // Remove oldest checkpoints + checkpoints.sort_by(|a, b| a.created_at.cmp(&b.created_at)); + let to_remove = checkpoints.len() - self.config.max_checkpoints_per_model; + + for checkpoint in checkpoints.iter().take(to_remove) { + if let Err(e) = self.delete_checkpoint(&checkpoint.checkpoint_id).await { + warn!( + "Failed to delete old checkpoint {}: {}", + checkpoint.checkpoint_id, e + ); + } + } + + info!( + "Cleaned up {} old checkpoints for {}:{}", + to_remove, + model_type.file_extension(), + model_name + ); + Ok(()) + } + + /// Get checkpoint statistics + pub fn get_stats(&self) -> HashMap { + self.stats.to_map() + } + + /// Refresh metadata index from storage + pub async fn refresh_index(&self) -> Result<(), MLError> { + let all_metadata = self.storage.list_all_checkpoints().await?; + + let index = self.metadata_index.write().await; + index.clear(); + + for metadata in all_metadata { + index.insert(metadata.checkpoint_id.clone(), metadata); + } + + info!( + "Refreshed checkpoint index with {} checkpoints", + index.len() + ); + Ok(()) + } + + /// Get checkpoint by tags + pub async fn find_checkpoints_by_tags(&self, tags: &[String]) -> Vec { + let index = self.metadata_index.read().await; + index + .iter() + .filter(|entry| { + let metadata = entry.value(); + tags.iter().all(|tag| metadata.tags.contains(tag)) + }) + .map(|entry| entry.value().clone()) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::AtomicBool; + // use crate::safe_operations; // DISABLED - module not found + + // Mock model for testing + struct MockModel { + model_type: ModelType, + model_name: String, + version: String, + data: Vec, + trained: AtomicBool, + } + + impl MockModel { + fn new(model_type: ModelType, name: String, version: String) -> Self { + Self { + model_type, + model_name: name, + version, + data: vec![1, 2, 3, 4, 5], + trained: AtomicBool::new(false), + } + } + } + + #[async_trait] + impl Checkpointable for MockModel { + fn model_type(&self) -> ModelType { + self.model_type + } + + fn model_name(&self) -> &str { + &self.model_name + } + + fn model_version(&self) -> &str { + &self.version + } + + async fn serialize_state(&self) -> Result, MLError> { + Ok(self.data.clone()) + } + + async fn deserialize_state(&mut self, data: &[u8]) -> Result<(), MLError> { + self.data = data.to_vec(); + self.trained.store(true, Ordering::Relaxed); + Ok(()) + } + + fn get_training_state(&self) -> (Option, Option, Option, Option) { + (Some(10), Some(1000), Some(0.1), Some(0.95)) + } + } + + #[tokio::test] + async fn test_checkpoint_save_load() { + let temp_dir = tempfile::tempdir()?; + let config = CheckpointConfig { + base_dir: temp_dir.path().to_path_buf(), + compression: CompressionType::None, + ..Default::default() + }; + + let manager = CheckpointManager::new(config)?; + let mut model = MockModel::new( + ModelType::DQN, + "test_model".to_string(), + "1.0.0".to_string(), + ); + + // Save checkpoint + let checkpoint_id = manager + .save_checkpoint(&model, Some(vec!["test".to_string()])) + .await?; + + // Modify model data + model.data = vec![9, 8, 7, 6, 5]; + + // Load checkpoint + let metadata = manager.load_checkpoint(&mut model, &checkpoint_id).await?; + + // Verify data was restored + assert_eq!(model.data, vec![1, 2, 3, 4, 5]); + assert_eq!(metadata.model_type, ModelType::DQN); + assert_eq!(metadata.model_name, "test_model"); + assert_eq!(metadata.tags, vec!["test".to_string()]); + assert!(model.trained.load(Ordering::Relaxed)); + } + + #[tokio::test] + async fn test_checkpoint_compression() { + let temp_dir = tempfile::tempdir()?; + let config = CheckpointConfig { + base_dir: temp_dir.path().to_path_buf(), + compression: CompressionType::LZ4, + ..Default::default() + }; + + let manager = CheckpointManager::new(config)?; + let mut model = MockModel::new( + ModelType::MAMBA, + "test_model".to_string(), + "1.0.0".to_string(), + ); + + // Create larger data for compression test + model.data = vec![42; 1000]; + + let checkpoint_id = manager.save_checkpoint(&model, None).await?; + + // Clear data + model.data.clear(); + + // Load and verify + manager.load_checkpoint(&mut model, &checkpoint_id).await?; + assert_eq!(model.data, vec![42; 1000]); + } + + #[tokio::test] + async fn test_checkpoint_metadata() { + let metadata = CheckpointMetadata::new( + ModelType::TFT, + "transformer_model".to_string(), + "2.1.0".to_string(), + ) + .with_training_state(Some(50), Some(5000), Some(0.05), Some(0.98)) + .with_tags(vec!["production".to_string(), "validated".to_string()]); + + assert_eq!(metadata.model_type, ModelType::TFT); + assert_eq!(metadata.epoch, Some(50)); + assert_eq!(metadata.accuracy, Some(0.98)); + assert!(metadata.tags.contains(&"production".to_string())); + + let filename = metadata.generate_filename(); + assert!(filename.contains("tft")); + assert!(filename.contains("transformer_model")); + assert!(filename.contains("v2.1.0")); + assert!(filename.contains("e50")); + assert!(filename.contains("s5000")); + } + + #[tokio::test] + async fn test_list_and_cleanup_checkpoints() { + let temp_dir = tempfile::tempdir()?; + let config = CheckpointConfig { + base_dir: temp_dir.path().to_path_buf(), + max_checkpoints_per_model: 2, + auto_cleanup: false, + ..Default::default() + }; + + let manager = CheckpointManager::new(config)?; + let model = MockModel::new( + ModelType::TGGN, + "graph_model".to_string(), + "1.0.0".to_string(), + ); + + // Save multiple checkpoints + let id1 = manager.save_checkpoint(&model, None).await?; + tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; + let id2 = manager.save_checkpoint(&model, None).await?; + tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; + let id3 = manager.save_checkpoint(&model, None).await?; + + // List checkpoints + let checkpoints = manager + .list_checkpoints(ModelType::TGGN, "graph_model") + .await; + assert_eq!(checkpoints.len(), 3); + + // Manual cleanup + manager + .cleanup_old_checkpoints(ModelType::TGGN, "graph_model") + .await?; + + // Should have only 2 checkpoints now + let checkpoints = manager + .list_checkpoints(ModelType::TGGN, "graph_model") + .await; + assert_eq!(checkpoints.len(), 2); + + // The oldest checkpoint should be gone + assert!(manager + .load_checkpoint( + &mut MockModel::new( + ModelType::TGGN, + "graph_model".to_string(), + "1.0.0".to_string() + ), + &id1 + ) + .await + .is_err()); + assert!(manager + .load_checkpoint( + &mut MockModel::new( + ModelType::TGGN, + "graph_model".to_string(), + "1.0.0".to_string() + ), + &id2 + ) + .await + .is_ok()); + assert!(manager + .load_checkpoint( + &mut MockModel::new( + ModelType::TGGN, + "graph_model".to_string(), + "1.0.0".to_string() + ), + &id3 + ) + .await + .is_ok()); + } +} diff --git a/ml/src/common/mod.rs b/ml/src/common/mod.rs index 85e0d2c33..9e4484e65 100644 --- a/ml/src/common/mod.rs +++ b/ml/src/common/mod.rs @@ -13,8 +13,6 @@ pub mod config; pub mod metrics; pub mod performance; -pub use config::*; -pub use performance::*; // Production ML types #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/ml/src/common/mod.rs.bak b/ml/src/common/mod.rs.bak new file mode 100644 index 000000000..85e0d2c33 --- /dev/null +++ b/ml/src/common/mod.rs.bak @@ -0,0 +1,222 @@ +//! Common types and utilities for ML models + +// Price imported from crate root (lib.rs) +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::path::PathBuf; +use std::time::SystemTime; +use uuid::Uuid; +// Import from common types with explicit path +use common::types::{Price, Volume, Symbol, Decimal, Quantity}; + +pub mod config; +pub mod metrics; +pub mod performance; + +pub use config::*; +pub use performance::*; + +// Production ML types +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ModelVersion { + pub version: String, + pub model_id: Uuid, + pub created_at: SystemTime, + pub commit_hash: String, + pub model_type: String, + pub performance_metrics: PerformanceMetrics, + pub quantization_config: Option, + pub onnx_config: Option, + pub artifacts: ModelArtifacts, + pub validation_results: ValidationResults, + pub tags: Vec, + pub description: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PerformanceMetrics { + pub avg_latency_us: f64, + pub p99_latency_us: f64, + pub throughput_ips: f64, + pub memory_usage_mb: f64, + pub accuracy: f64, + pub energy_consumption_mj: Option, + pub hardware_metrics: HardwareMetrics, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HardwareMetrics { + pub cpu_utilization: f64, + pub gpu_utilization: Option, + pub memory_bandwidth: f64, + pub cache_metrics: HashMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ModelArtifacts { + pub pytorch_model: PathBuf, + pub onnx_model: Option, + pub quantized_model: Option, + pub tensorrt_engine: Option, + pub optimization_logs: Option, + pub calibration_data: Option, + pub config_file: PathBuf, + pub benchmark_results: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ValidationResults { + pub test_accuracy: f64, + pub business_metrics: HashMap, + pub latency_distribution: LatencyDistribution, + pub stress_test_passed: bool, + pub ab_test_results: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LatencyDistribution { + pub min_us: f64, + pub max_us: f64, + pub mean_us: f64, + pub median_us: f64, + pub p95_us: f64, + pub p99_us: f64, + pub p999_us: f64, + pub std_dev_us: f64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ABTestResults { + pub control_accuracy: f64, + pub treatment_accuracy: f64, + pub statistical_significance: f64, + pub confidence_interval: (f64, f64), + pub sample_size: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct QuantizationConfig { + pub precision: String, // "int8", "int4", "fp16" + pub calibration_samples: usize, + pub accuracy_threshold: f64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ONNXExportConfig { + pub opset_version: i64, + pub optimization_level: String, + pub enable_tensorrt: bool, + pub dynamic_axes: HashMap>, +} + +// Direct use of canonical types - no compatibility wrappers + +/// `Market` data structure compatible with ML models +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +/// MarketData component. +pub struct MarketData { + pub asset_id: Symbol, + pub price: Price, + pub volume: Volume, + pub bid: Price, + pub ask: Price, + pub bid_size: Volume, + pub ask_size: Volume, + pub timestamp: u64, // Unix timestamp in nanoseconds +} + +// Precision factor compatible with canonical Price type (8 decimal places) +/// `PRECISION_FACTOR`: component. +pub const PRECISION_FACTOR: i64 = 100_000_000; // 10^8 + +/// Systematic conversion utilities for interfacing with different precision systems +/// ELIMINATES IntegerPrice usage throughout ML crate +pub mod conversions { + use rust_decimal::prelude::{FromPrimitive, ToPrimitive}; + use super::*; + + /// Convert canonical Price to liquid submodule FixedPoint (8-decimal to 6-decimal precision) + pub fn price_to_liquid_fixed_point(price: Price) -> Result> { + let liquid_precision = 1_000_000_i64; // 6 decimal places + let canonical_precision = 100_000_000_i64; // 8 decimal places + + // Scale down from 8-decimal to 6-decimal precision with proper error handling + let price_f64 = price.to_f64().ok_or("Failed to convert Price to f64")?; + let scaled_value = (price_f64 * liquid_precision as f64) as i64; + Ok(crate::liquid::FixedPoint(scaled_value)) + } + + /// Convert liquid submodule FixedPoint to canonical Price (6-decimal to 8-decimal precision) + pub fn liquid_fixed_point_to_price(fixed_point: crate::liquid::FixedPoint) -> Result> { + let liquid_precision = 1_000_000_i64; // 6 decimal places + let canonical_precision = 100_000_000_i64; // 8 decimal places + + // Scale up from 6-decimal to 8-decimal precision with proper error handling + let value_f64 = fixed_point.0 as f64 / liquid_precision as f64; + Price::from_f64(value_f64).ok_or_else(|| "Failed to convert f64 to Price".into()) + } + + /// Convert `f64` to canonical Price with full 8-decimal precision + pub fn f64_to_price(value: f64) -> Result> { + // error_handling::TradingError replaced + Price::from_f64(value).ok_or_else(|| "Invalid f64 value for Price conversion".into()) + } + + /// Convert canonical Price to `f64` for ML model inputs + pub fn price_to_f64(price: Price) -> Result> { + price.to_f64().ok_or_else(|| "Failed to convert Price to f64 for ML model".into()) + } + + /// SYSTEMATIC CONVERSION TRAITS: Eliminate IntegerPrice usage throughout ML + /// These traits provide compile-time guaranteed conversions between types + + /// Convert Price to Decimal for database/API operations + pub fn price_to_decimal(price: Price) -> Result> { + Ok(price) // Price is already a Decimal + } + + /// Convert Decimal to Price for trading operations + pub fn decimal_to_price(decimal: Decimal) -> Price { + decimal // Price is already a Decimal + } + + /// Convert Volume to f64 for ML model inputs + pub fn volume_to_f64(volume: Volume) -> Result> { + volume.to_f64().ok_or_else(|| "Failed to convert Volume to f64 for ML model".into()) + } + + /// Convert f64 to Volume with validation + pub fn f64_to_volume(value: f64) -> Result> { + if value < 0.0 { + return Err("Volume cannot be negative".into()); + } + Volume::from_f64(value).ok_or_else(|| "Invalid volume value".into()) + } + + /// Convert Quantity to i64 for efficient processing + pub fn quantity_to_i64(quantity: Quantity) -> Result> { + let quantity_f64 = quantity.to_f64().ok_or("Failed to convert Quantity to f64")?; + Ok((quantity_f64 * 100_000_000.0) as i64) // Convert to integer with 8 decimal precision + } + + /// Convert i64 to Quantity with validation + pub fn i64_to_quantity(value: i64) -> Result> { + if value < 0 { + return Err("Quantity cannot be negative".into()); + } + Ok(Quantity::from_f64(value as f64 / 100_000_000.0).unwrap_or(Quantity::ZERO)) + } + + /// Batch convert prices to f64 vector for ML model inputs + pub fn prices_to_f64_vec(prices: &[Price]) -> Vec { + prices.iter().map(|p| p.to_f64().unwrap_or(0.0)).collect() + } + + /// Batch convert f64 vector to prices with validation + pub fn f64_vec_to_prices(values: &[f64]) -> Result, Box> { + values.iter() + .map(|&v| f64_to_price(v)) + .collect::, _>>() + } +} diff --git a/ml/src/deployment/mod.rs b/ml/src/deployment/mod.rs index 81011714d..d5557759e 100644 --- a/ml/src/deployment/mod.rs +++ b/ml/src/deployment/mod.rs @@ -35,13 +35,7 @@ pub mod validation; pub mod monitoring; pub mod endpoints; -pub use registry::*; -pub use versioning::*; -pub use hot_swap::*; -pub use ab_testing::*; -pub use validation::*; -pub use monitoring::*; -pub use endpoints::{ModelDeploymentService, ModelDeploymentServiceImpl, ModelFactory}; +// DO NOT RE-EXPORT - Use explicit imports at usage sites /// Model deployment status #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] diff --git a/ml/src/dqn/mod.rs b/ml/src/dqn/mod.rs index 06ceecba6..b6e702b5f 100644 --- a/ml/src/dqn/mod.rs +++ b/ml/src/dqn/mod.rs @@ -35,34 +35,26 @@ pub mod performance_tests; pub mod performance_validation; // Re-export original DQN components -pub use experience::{Experience, ExperienceBatch}; -pub use network::{QNetwork, QNetworkConfig}; -pub use replay_buffer::{ReplayBuffer, ReplayBufferConfig, ReplayBufferStats}; +// DO NOT RE-EXPORT - Use explicit imports at usage sites // Import agent types specifically to avoid conflicts -pub use agent::{AgentMetrics, DQNAgent, DQNConfig, TradingAction, TradingState}; +// DO NOT RE-EXPORT - Use explicit imports at usage sites // Re-export working DQN components -pub use dqn::{ExperienceReplayBuffer, Sequential, WorkingDQN, WorkingDQNConfig}; +// DO NOT RE-EXPORT - Use explicit imports at usage sites // Re-export reward types -pub use reward::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites calculate_batch_rewards, MarketData, RewardConfig, RewardFunction, RewardStats, RiskMetrics, }; // Re-export Rainbow DQN components -pub use distributional::{CategoricalDistribution, DistributionalConfig}; -pub use multi_step::{ compute_discounted_return, compute_effective_gamma, create_multi_step_transition, MultiStepBatch, MultiStepCalculator, MultiStepConfig, MultiStepReturn, MultiStepTransition, }; -pub use noisy_layers::{NoisyLinear, NoisyNetworkConfig, NoisyNetworkManager}; -pub use rainbow_agent_impl::RainbowAgent; -pub use rainbow_config::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites RainbowAgentConfig, RainbowAgentMetrics, RainbowDQNConfig, RainbowMetrics, TrainingResult, }; -pub use rainbow_integration::RainbowDQNAgent; -pub use rainbow_network::{ActivationType, RainbowNetwork, RainbowNetworkConfig}; // Re-export prioritized replay components // TEMPORARILY COMMENTED OUT - Fix imports later @@ -87,7 +79,7 @@ pub use rainbow_network::{ActivationType, RainbowNetwork, RainbowNetworkConfig}; // }; // Re-export DQN demo functionality -pub use demo_2025_dqn::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites cleanup_demo_environment, initialize_demo_environment, run_2025_dqn_demo, DemoConfig, DemoMode, DemoResults, }; diff --git a/ml/src/dqn/mod.rs.bak b/ml/src/dqn/mod.rs.bak new file mode 100644 index 000000000..06ceecba6 --- /dev/null +++ b/ml/src/dqn/mod.rs.bak @@ -0,0 +1,93 @@ +//! Deep Q-Learning Network implementation for trading +//! +//! This module includes both the original DQN implementation and the enhanced Rainbow DQN +//! with all 6 components: Double Q-learning, Dueling Networks, Prioritized Experience Replay, +//! Multi-step Learning, Distributional RL (C51), and Noisy Networks. + +// Original DQN components +pub mod agent; +pub mod dqn; +pub mod experience; +pub mod network; +pub mod replay_buffer; +pub mod reward; // Added working DQN implementation + +// Rainbow DQN components +pub mod distributional; +pub mod multi_step; +pub mod noisy_layers; +pub mod rainbow_agent; +pub mod rainbow_agent_impl; +pub mod rainbow_config; +pub mod rainbow_integration; +pub mod rainbow_network; + +// Missing modules that exist but weren't declared +pub mod prioritized_replay; + +pub mod demo_2025_dqn; +pub mod multi_step_new; +pub mod noisy_exploration; +pub mod self_supervised_pretraining; + +// Performance validation +pub mod performance_tests; +pub mod performance_validation; + +// Re-export original DQN components +pub use experience::{Experience, ExperienceBatch}; +pub use network::{QNetwork, QNetworkConfig}; +pub use replay_buffer::{ReplayBuffer, ReplayBufferConfig, ReplayBufferStats}; + +// Import agent types specifically to avoid conflicts +pub use agent::{AgentMetrics, DQNAgent, DQNConfig, TradingAction, TradingState}; + +// Re-export working DQN components +pub use dqn::{ExperienceReplayBuffer, Sequential, WorkingDQN, WorkingDQNConfig}; + +// Re-export reward types +pub use reward::{ + calculate_batch_rewards, MarketData, RewardConfig, RewardFunction, RewardStats, RiskMetrics, +}; + +// Re-export Rainbow DQN components +pub use distributional::{CategoricalDistribution, DistributionalConfig}; +pub use multi_step::{ + compute_discounted_return, compute_effective_gamma, create_multi_step_transition, + MultiStepBatch, MultiStepCalculator, MultiStepConfig, MultiStepReturn, MultiStepTransition, +}; +pub use noisy_layers::{NoisyLinear, NoisyNetworkConfig, NoisyNetworkManager}; +pub use rainbow_agent_impl::RainbowAgent; +pub use rainbow_config::{ + RainbowAgentConfig, RainbowAgentMetrics, RainbowDQNConfig, RainbowMetrics, TrainingResult, +}; +pub use rainbow_integration::RainbowDQNAgent; +pub use rainbow_network::{ActivationType, RainbowNetwork, RainbowNetworkConfig}; + +// Re-export prioritized replay components +// TEMPORARILY COMMENTED OUT - Fix imports later +// pub use prioritized_replay::{PrioritizedReplayBuffer, PrioritizedReplayConfig}; + +// Re-export noisy exploration components +// TEMPORARILY COMMENTED OUT - Missing implementations +// pub use noisy_exploration::{NoisyExplorationConfig, AdaptiveNoisyManager, AdaptiveNoisyLinear, NoiseExplorationMetrics}; + +// Re-export performance validation utilities +// TEMPORARILY COMMENTED OUT - Missing implementations +// pub use performance_tests::{ +// RainbowPerformanceValidator, PerformanceTestConfig, PerformanceResults, +// validate_rainbow_performance +// }; + +// Re-export comprehensive performance validation +// TEMPORARILY COMMENTED OUT - Missing implementations +// pub use performance_validation::{ +// DQNPerformanceValidator, PerformanceValidationConfig, PerformanceValidationResults, +// validate_dqn_performance +// }; + +// Re-export DQN demo functionality +pub use demo_2025_dqn::{ + cleanup_demo_environment, initialize_demo_environment, run_2025_dqn_demo, DemoConfig, DemoMode, + DemoResults, +}; diff --git a/ml/src/dqn/reward.rs b/ml/src/dqn/reward.rs index 87b88a5e1..9071cd0aa 100644 --- a/ml/src/dqn/reward.rs +++ b/ml/src/dqn/reward.rs @@ -8,8 +8,7 @@ use common::{Decimal, Price}; use super::TradingAction; use crate::MLError; -// Re-export TradingState for use in tests -pub use super::TradingState; +// NO RE-EXPORTS - Use explicit imports: super::TradingState /// Configuration for reward function #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/ml/src/ensemble/mod.rs b/ml/src/ensemble/mod.rs index e098ca8af..0bf895048 100644 --- a/ml/src/ensemble/mod.rs +++ b/ml/src/ensemble/mod.rs @@ -10,11 +10,7 @@ pub mod model; pub mod voting; pub mod weights; -pub use aggregator::*; -pub use confidence::*; -pub use model::*; -pub use voting::*; -pub use weights::*; +// DO NOT RE-EXPORT - Use explicit imports at usage sites /// Errors that can occur in ensemble operations #[derive(Error, Debug)] diff --git a/ml/src/ensemble/mod.rs.bak b/ml/src/ensemble/mod.rs.bak new file mode 100644 index 000000000..e098ca8af --- /dev/null +++ b/ml/src/ensemble/mod.rs.bak @@ -0,0 +1,40 @@ +//! Ensemble signal aggregation for trading models + +use std; + +use thiserror::Error; + +pub mod aggregator; +pub mod confidence; +pub mod model; +pub mod voting; +pub mod weights; + +pub use aggregator::*; +pub use confidence::*; +pub use model::*; +pub use voting::*; +pub use weights::*; + +/// Errors that can occur in ensemble operations +#[derive(Error, Debug)] +/// `EnsembleError` component. +pub enum EnsembleError { + #[error("Failed to acquire lock: {0}")] + LockAcquisitionFailed(String), + + #[error("Invalid ensemble configuration: {0}")] + InvalidConfiguration(String), + + #[error("Model not found: {0}")] + ModelNotFound(String), + + #[error("Insufficient models for ensemble: expected {expected}, got {actual}")] + InsufficientModels { expected: usize, actual: usize }, + + #[error("Weight calculation failed: {0}")] + WeightCalculationFailed(String), + + #[error("Aggregation failed: {0}")] + AggregationFailed(String), +} diff --git a/ml/src/error_consolidated.rs b/ml/src/error_consolidated.rs index 40abbe9b3..f3d7aadb5 100644 --- a/ml/src/error_consolidated.rs +++ b/ml/src/error_consolidated.rs @@ -3,11 +3,10 @@ //! This module demonstrates the consolidated error handling pattern //! using the common error system across all Foxhunt ML services. -// Re-export shared error types and utilities -pub use common::error::{CommonError, CommonResult, ErrorCategory, RetryStrategy, ErrorSeverity}; +use common::error::{CommonError, ErrorCategory, RetryStrategy, ErrorSeverity}; -/// Result type for ML operations using CommonError -pub type MLResult = CommonResult; +// NO TYPE ALIASES USING RE-EXPORTS - Define directly or import explicitly +// pub type MLResult = CommonResult; /// ML module specific error extensions /// For cases where we need domain-specific error information beyond CommonError diff --git a/ml/src/flash_attention/mod.rs.bak b/ml/src/flash_attention/mod.rs.bak new file mode 100644 index 000000000..7ef438692 --- /dev/null +++ b/ml/src/flash_attention/mod.rs.bak @@ -0,0 +1,460 @@ +//! # Flash Attention 3 for High-Frequency Trading +//! +//! State-of-the-art Flash Attention 3 implementation optimized for HFT applications. +//! Provides 8x faster attention computation for order book processing with +//! IO-aware algorithms, block-sparse patterns, and custom CUDA kernels. +//! +//! ## Key Features +//! +//! - **IO-Aware Attention**: Minimizes memory transfers between HBM and SRAM +//! - **Block-Sparse Patterns**: Optimized for order book sparsity patterns +//! - **8x Performance**: Dramatically faster than standard attention +//! - **Causal Masking**: Efficient causal attention for temporal sequences +//! - **Custom CUDA Kernels**: Hardware-optimized GPU acceleration +//! - **Mixed Precision**: FP16/BF16 support for maximum throughput +//! +//! ## Architecture Overview +//! +//! ```text +//! ┌─────────────────────────────────────────────────────────────────┐ +//! │ Flash Attention 3 Pipeline │ +//! ├─────────────────┬─────────────────┬─────────────────────────────┤ +//! │ IO-Aware │ Block-Sparse │ Causal Masking │ +//! │ Tiling │ Patterns │ & CUDA Kernels │ +//! │ │ │ │ +//! │ • Minimize HBM │ • Order Book │ • Efficient Causality │ +//! │ • SRAM Blocking │ Sparsity │ • Custom GPU Kernels │ +//! │ • Fused Ops │ • 90% Speedup │ • Mixed Precision │ +//! │ • Memory Coales │ • Adaptive │ • Memory Coalescing │ +//! │ -cing │ Patterns │ │ +//! └─────────────────┴─────────────────┴─────────────────────────────┘ +//! ``` +//! +//! ## Performance Targets +//! +//! - Attention Speed: 8x faster than standard implementations +//! - Memory Usage: 4x reduction through IO-aware tiling +//! - Latency: <10μs for order book attention (1024 tokens) +//! - Throughput: >50K attention operations/second +//! - GPU Utilization: >90% through optimized kernels + +use std::collections::HashMap; + +use candle_core::{Device, Tensor}; +use serde::{Deserialize, Serialize}; + +use crate::MLError; + +/// Block sparse pattern for attention optimization +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BlockSparsePattern { + pub block_size: usize, + pub sparsity_ratio: f32, + pub pattern_type: SparsePatternType, +} + +/// Types of sparse patterns +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum SparsePatternType { + OrderBook, + Causal, + Random, + Fixed, +} + +impl Default for BlockSparsePattern { + fn default() -> Self { + Self { + block_size: 64, + sparsity_ratio: 0.1, + pattern_type: SparsePatternType::OrderBook, + } + } +} + +/// Sparse attention mask +#[derive(Debug, Clone)] +pub struct SparseAttentionMask { + pub mask: Tensor, + pub block_pattern: BlockSparsePattern, +} + +impl SparseAttentionMask { + pub fn new( + pattern: BlockSparsePattern, + seq_len: usize, + device: &Device, + ) -> Result { + // Create a mock sparse mask + let mask_data = vec![1.0_f32; seq_len * seq_len]; + let mask = Tensor::from_slice(&mask_data, (seq_len, seq_len), device) + .map_err(|e| MLError::ModelError(format!("Failed to create mask: {}", e)))?; + + Ok(Self { + mask, + block_pattern: pattern, + }) + } +} + +/// Causal mask optimizer +#[derive(Debug, Clone)] +pub struct CausalMaskOptimizer { + pub cache_size: usize, + pub use_fast_path: bool, +} + +impl CausalMaskOptimizer { + pub fn new(cache_size: usize) -> Self { + Self { + cache_size, + use_fast_path: true, + } + } + + pub fn optimize_mask(&self, mask: &Tensor) -> Result { + // Return the mask as-is for now (production implementation) + Ok(mask.clone()) + } +} + +/// CUDA kernel manager (mock) +#[derive(Debug, Clone)] +pub struct CudaKernelManager { + pub kernels_loaded: bool, + pub optimization_level: u32, +} + +impl CudaKernelManager { + pub fn new() -> Self { + Self { + kernels_loaded: false, + optimization_level: 3, + } + } + + pub fn load_kernels(&mut self) -> Result<(), MLError> { + self.kernels_loaded = true; + Ok(()) + } +} + +/// IO-aware attention implementation +#[derive(Debug, Clone)] +pub struct IOAwareAttention { + pub tile_size: usize, + pub memory_budget_mb: usize, +} + +impl IOAwareAttention { + pub fn new(tile_size: usize, memory_budget_mb: usize) -> Self { + Self { + tile_size, + memory_budget_mb, + } + } + + pub fn compute_attention( + &self, + _q: &Tensor, + _k: &Tensor, + v: &Tensor, + ) -> Result { + // Production implementation - return V for now + Ok(v.clone()) + } +} + +/// Mixed precision configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MixedPrecisionConfig { + pub use_fp16: bool, + pub use_bf16: bool, + pub loss_scaling: f32, +} + +impl Default for MixedPrecisionConfig { + fn default() -> Self { + Self { + use_fp16: true, + use_bf16: false, + loss_scaling: 1.0, + } + } +} + +/// Flash Attention 3 configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FlashAttention3Config { + pub hidden_dim: usize, + pub num_heads: usize, + pub head_dim: usize, + pub max_seq_len: usize, + pub dropout_rate: f32, + pub use_sparse_patterns: bool, + pub sparse_pattern: BlockSparsePattern, + pub mixed_precision: MixedPrecisionConfig, + pub io_aware_tiling: bool, + pub cuda_optimization: bool, +} + +impl Default for FlashAttention3Config { + fn default() -> Self { + Self { + hidden_dim: 512, + num_heads: 8, + head_dim: 64, + max_seq_len: 1024, + dropout_rate: 0.1, + use_sparse_patterns: true, + sparse_pattern: BlockSparsePattern::default(), + mixed_precision: MixedPrecisionConfig::default(), + io_aware_tiling: true, + cuda_optimization: true, + } + } +} + +/// Flash Attention 3 implementation +pub struct FlashAttention3 { + pub config: FlashAttention3Config, + pub device: Device, + pub io_aware: IOAwareAttention, + pub causal_optimizer: CausalMaskOptimizer, + pub cuda_manager: CudaKernelManager, + pub attention_cache: HashMap, +} + +impl FlashAttention3 { + /// Create new Flash Attention 3 instance + pub fn new(config: FlashAttention3Config, device: Device) -> Result { + let io_aware = IOAwareAttention::new(64, 2048); // 64 tile size, 2GB memory budget + let causal_optimizer = CausalMaskOptimizer::new(1024); + let mut cuda_manager = CudaKernelManager::new(); + + if config.cuda_optimization { + cuda_manager.load_kernels()?; + } + + Ok(Self { + config, + device, + io_aware, + causal_optimizer, + cuda_manager, + attention_cache: HashMap::new(), + }) + } + + /// Compute attention using Flash Attention 3 + pub fn forward( + &mut self, + q: &Tensor, + k: &Tensor, + v: &Tensor, + mask: Option<&Tensor>, + ) -> Result { + let (_batch_size, _seq_len, _) = q + .dims3() + .map_err(|e| MLError::ModelError(format!("Invalid Q tensor dims: {}", e)))?; + + // Use IO-aware attention for computation + let output = if self.config.io_aware_tiling { + self.io_aware.compute_attention(q, k, v)? + } else { + // Fallback to standard attention computation + self.standard_attention(q, k, v, mask)? + }; + + Ok(output) + } + + fn standard_attention( + &self, + q: &Tensor, + k: &Tensor, + v: &Tensor, + mask: Option<&Tensor>, + ) -> Result { + // Compute Q @ K^T + let scores = q + .matmul(&k.transpose(1, 2)?) + .map_err(|e| MLError::ModelError(format!("QK computation failed: {}", e)))?; + + // Scale by sqrt(head_dim) + let scale = (self.config.head_dim as f64).sqrt(); + let scaled_scores = (&scores / scale) + .map_err(|e| MLError::ModelError(format!("Score scaling failed: {}", e)))?; + + // Apply mask if provided + let masked_scores = if let Some(mask) = mask { + (&scaled_scores + mask) + .map_err(|e| MLError::ModelError(format!("Mask application failed: {}", e)))? + } else { + scaled_scores + }; + + // Apply softmax + let attention_weights = candle_nn::ops::softmax(&masked_scores, 2) + .map_err(|e| MLError::ModelError(format!("Softmax failed: {}", e)))?; + + // Apply attention to values + let output = attention_weights + .matmul(v) + .map_err(|e| MLError::ModelError(format!("Attention application failed: {}", e)))?; + + Ok(output) + } + + /// Create sparse attention mask + pub fn create_sparse_mask(&self, seq_len: usize) -> Result { + SparseAttentionMask::new(self.config.sparse_pattern.clone(), seq_len, &self.device) + } + + /// Get attention statistics + pub fn get_stats(&self) -> AttentionStats { + AttentionStats { + cache_size: self.attention_cache.len(), + cuda_kernels_loaded: self.cuda_manager.kernels_loaded, + io_aware_enabled: self.config.io_aware_tiling, + mixed_precision_enabled: self.config.mixed_precision.use_fp16 + || self.config.mixed_precision.use_bf16, + } + } +} + +/// Attention performance statistics +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AttentionStats { + pub cache_size: usize, + pub cuda_kernels_loaded: bool, + pub io_aware_enabled: bool, + pub mixed_precision_enabled: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_flash_attention_creation() -> Result<(), MLError> { + let device = Device::cuda_if_available(0).map_err(|e| RealInferenceError::GpuRequired { + reason: format!("GPU required for flash attention: {}", e), + })?; + let config = FlashAttention3Config::default(); + let _attention = FlashAttention3::new(config, device)?; + Ok(()) + } + + #[test] + fn test_flash_attention_forward() -> Result<(), MLError> { + let device = Device::cuda_if_available(0).unwrap_or(Device::Cpu); + let config = FlashAttention3Config { + hidden_dim: 64, + num_heads: 2, + head_dim: 32, + max_seq_len: 16, + ..Default::default() + }; + + let mut attention = FlashAttention3::new(config, device.clone())?; + + // Create test tensors + let batch_size = 1; + let seq_len = 8; + let head_dim = 32; + + let q_data = vec![0.1f32; batch_size * seq_len * head_dim]; + let k_data = vec![0.2f32; batch_size * seq_len * head_dim]; + let v_data = vec![0.3f32; batch_size * seq_len * head_dim]; + + let q = Tensor::from_slice(&q_data, (batch_size, seq_len, head_dim), &device) + .map_err(|e| MLError::ModelError(e.to_string()))?; + let k = Tensor::from_slice(&k_data, (batch_size, seq_len, head_dim), &device) + .map_err(|e| MLError::ModelError(e.to_string()))?; + let v = Tensor::from_slice(&v_data, (batch_size, seq_len, head_dim), &device) + .map_err(|e| MLError::ModelError(e.to_string()))?; + + let output = attention.forward(&q, &k, &v, None)?; + + // Check output dimensions + assert_eq!(output.dims(), q.dims()); + + Ok(()) + } + + #[test] + fn test_sparse_mask_creation() -> Result<(), MLError> { + let device = Device::cuda_if_available(0).unwrap_or(Device::Cpu); + let config = FlashAttention3Config::default(); + let attention = FlashAttention3::new(config, device)?; + + let mask = attention.create_sparse_mask(128)?; + assert_eq!(mask.block_pattern.block_size, 64); + + Ok(()) + } + + #[test] + fn test_attention_stats() -> Result<(), MLError> { + let device = Device::Cpu; + let config = FlashAttention3Config::default(); + let attention = FlashAttention3::new(config, device)?; + + let stats = attention.get_stats(); + assert_eq!(stats.cache_size, 0); + assert!(stats.io_aware_enabled); + + Ok(()) + } + + #[test] + fn test_causal_optimizer() { + let optimizer = CausalMaskOptimizer::new(1024); + assert_eq!(optimizer.cache_size, 1024); + assert!(optimizer.use_fast_path); + } + + #[test] + fn test_cuda_kernel_manager() -> Result<(), MLError> { + let mut manager = CudaKernelManager::new(); + assert!(!manager.kernels_loaded); + + manager.load_kernels()?; + assert!(manager.kernels_loaded); + + Ok(()) + } + + #[test] + fn test_io_aware_attention() -> Result<(), MLError> { + let device = Device::Cpu; + let io_aware = IOAwareAttention::new(32, 1024); + + // Create dummy tensors + let data = vec![1.0f32; 64]; + let tensor = Tensor::from_slice(&data, (8, 8), &device) + .map_err(|e| MLError::ModelError(e.to_string()))?; + + let result = io_aware.compute_attention(&tensor, &tensor, &tensor)?; + assert_eq!(result.dims(), tensor.dims()); + + Ok(()) + } + + #[test] + fn test_mixed_precision_config() { + let config = MixedPrecisionConfig::default(); + assert!(config.use_fp16); + assert!(!config.use_bf16); + assert_eq!(config.loss_scaling, 1.0); + } + + #[test] + fn test_block_sparse_pattern() { + let pattern = BlockSparsePattern::default(); + assert_eq!(pattern.block_size, 64); + assert_eq!(pattern.sparsity_ratio, 0.1); + assert!(matches!(pattern.pattern_type, SparsePatternType::OrderBook)); + } +} diff --git a/ml/src/gpu_benchmarks/mod.rs b/ml/src/gpu_benchmarks/mod.rs index b89bcd494..ab06b527f 100644 --- a/ml/src/gpu_benchmarks/mod.rs +++ b/ml/src/gpu_benchmarks/mod.rs @@ -2,4 +2,4 @@ pub mod gpu_performance; -pub use gpu_performance::*; +// DO NOT RE-EXPORT - Use explicit imports at usage sites diff --git a/ml/src/gpu_benchmarks/mod.rs.bak b/ml/src/gpu_benchmarks/mod.rs.bak new file mode 100644 index 000000000..b89bcd494 --- /dev/null +++ b/ml/src/gpu_benchmarks/mod.rs.bak @@ -0,0 +1,5 @@ +//! Benchmarks module for ML models performance validation + +pub mod gpu_performance; + +pub use gpu_performance::*; diff --git a/ml/src/integration/mod.rs b/ml/src/integration/mod.rs index 6088543ed..404e1f676 100644 --- a/ml/src/integration/mod.rs +++ b/ml/src/integration/mod.rs @@ -118,7 +118,7 @@ pub enum ModelState { } // Use canonical ModelType from crate root -pub use crate::ModelType; +// DO NOT RE-EXPORT - Use explicit imports at usage sites /// Model search criteria #[derive(Debug, Clone)] diff --git a/ml/src/integration/mod.rs.bak b/ml/src/integration/mod.rs.bak new file mode 100644 index 000000000..6088543ed --- /dev/null +++ b/ml/src/integration/mod.rs.bak @@ -0,0 +1,196 @@ +//! # Enhanced ML Integration Hub +//! +//! Realistic and optimized ML integration architecture for Foxhunt HFT system. +//! Based on expert consensus analysis, this module implements a practical approach +//! that balances performance requirements with technical feasibility. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::SystemTime; + +use tokio::sync::RwLock; + +use super::*; +use crate::MLError; +// use crate::safe_operations; // DISABLED - module not found + +// Re-export integration submodules +pub mod coordinator; +pub mod distillation; +pub mod inference_engine; +pub mod model_registry; +pub mod performance_monitor; +pub mod strategy_dqn_bridge; + +/// Configuration for ML Integration Hub +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct IntegrationHubConfig { + /// Maximum number of concurrent models + pub max_concurrent_models: usize, + /// Default inference timeout in milliseconds + pub default_timeout_ms: u64, + /// Enable performance monitoring + pub enable_monitoring: bool, + /// Model cache size + pub cache_size: usize, +} + +impl Default for IntegrationHubConfig { + fn default() -> Self { + Self { + max_concurrent_models: 5, + default_timeout_ms: 1000, + enable_monitoring: true, + cache_size: 100, + } + } +} + +/// ML Integration Hub for coordinating model operations +#[derive(Debug)] +pub struct MLIntegrationHub { + config: IntegrationHubConfig, + active_models: Arc>>, +} + +impl MLIntegrationHub { + /// Create new ML Integration Hub + pub async fn new(config: IntegrationHubConfig) -> Result { + Ok(Self { + config, + active_models: Arc::new(RwLock::new(HashMap::new())), + }) + } + + /// Get configuration + pub fn config(&self) -> &IntegrationHubConfig { + &self.config + } +} + +/// Model deployment configuration +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct ModelDeployment { + /// Unique model identifier + pub model_id: String, + /// Model type + pub model_type: ModelType, + /// Model version + pub version: String, + /// Serving modes + pub serving_modes: Vec, + /// File path to model + pub file_path: String, + /// Target latency in microseconds + pub target_latency_us: u64, + /// Memory requirement in MB + pub memory_requirement_mb: usize, + /// Compute unit (CPU/GPU) + pub compute_unit: String, + /// Quantization settings + pub quantization: Option, + /// Warm up samples + pub warm_up_samples: usize, +} + +/// Model serving modes +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +pub enum ServingMode { + /// Ultra-low latency serving + UltraLowLatency, + /// Low latency serving + LowLatency, + /// High throughput serving + HighThroughput, +} + +/// Model state +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +pub enum ModelState { + /// Model is loading + Loading, + /// Model is active and ready + Active, + /// Model is inactive + Inactive, + /// Model has failed + Failed, +} + +// Use canonical ModelType from crate root +pub use crate::ModelType; + +/// Model search criteria +#[derive(Debug, Clone)] +pub struct ModelSearchCriteria { + /// Optional model type filter + pub model_type: Option, + /// Optional serving mode filter + pub serving_mode: Option, + /// Maximum latency in microseconds + pub max_latency_us: Option, + /// Minimum accuracy threshold + pub min_accuracy: Option, + /// Search tags + pub tags: Vec, + /// Status filter + pub status: Option, +} + +/// Model status information +#[derive(Debug, Clone)] +pub struct ModelStatus { + /// Model identifier + pub model_id: String, + /// Current state + pub status: ModelState, + /// Last health check time + pub last_health_check: SystemTime, + /// Deployment time + pub deployment_time: SystemTime, + /// Inference count + pub inference_count: u64, + /// Error count + pub error_count: u64, + /// Average latency in microseconds + pub avg_latency_us: f64, + /// Memory usage in MB + pub memory_usage_mb: f64, + /// CPU utilization percentage + pub cpu_utilization: f64, +} + +/// Inference priority levels +#[derive(Debug, Clone, Copy, PartialEq, PartialOrd, serde::Serialize, serde::Deserialize)] +pub enum InferencePriority { + /// Critical priority + Critical = 0, + /// High priority + High = 1, + /// Medium priority + Medium = 2, + /// Low priority + Low = 3, +} + +#[tokio::test] +async fn test_integration_hub_creation() { + let config = IntegrationHubConfig::default(); + let hub = MLIntegrationHub::new(config).await; + assert!(hub.is_ok()); +} + +#[test] +fn test_model_type_serialization() { + let model_type = crate::checkpoint::ModelType::DistilledMicroNet; + let serialized = serde_json::to_string(&model_type)?; + let deserialized: crate::checkpoint::ModelType = serde_json::from_str(&serialized)?; + assert_eq!(model_type, deserialized); +} + +#[test] +fn test_inference_priority_ordering() { + assert!(InferencePriority::Critical < InferencePriority::High); + assert!(InferencePriority::High < InferencePriority::Medium); + assert!(InferencePriority::Medium < InferencePriority::Low); +} diff --git a/ml/src/labeling/mod.rs b/ml/src/labeling/mod.rs index 4630219f1..e4d7c9ca5 100644 --- a/ml/src/labeling/mod.rs +++ b/ml/src/labeling/mod.rs @@ -38,32 +38,9 @@ pub mod triple_barrier; pub mod types; // validation_test moved to tests/ directory -// Re-export core types and functions -pub use self::types::{ - BarrierConfig, BarrierResult, EventLabel, FractionalDiffConfig, FractionalDiffResult, - MetaLabel, MetaLabelResult, WeightedSample, WeightingConfig, WeightingResult, -}; +// DO NOT RE-EXPORT - Use explicit imports at usage sites -pub use gpu_acceleration::LabelingError; - -pub use crate::labeling::types::BarrierTouchedFirst; -pub use triple_barrier::{BarrierTracker, TripleBarrierEngine}; - -pub use meta_labeling::{MetaLabelConfig, MetaLabelingEngine}; - -pub use fractional_diff::{FractionalDifferentiator, StreamingDifferentiator}; - -pub use sample_weights::SampleWeightCalculator; - -pub use gpu_acceleration::GPULabelingEngine; - -pub use concurrent_tracking::{BarrierTrackingState, ConcurrentBarrierTracker, TrackingMetrics}; - -pub use benchmarks::{ - ConcurrentTrackingBenchmark, FractionalDiffBenchmark, LabelingBenchmarkResults, - LabelingBenchmarkSuite, MetaLabelingBenchmark, SampleWeightsBenchmark, - SystemPerformanceBenchmark, TripleBarrierBenchmark, -}; +// DO NOT RE-EXPORT - Benchmarks should be imported explicitly /// Labeling module constants matching Python reference precision pub mod constants { diff --git a/ml/src/lib.rs b/ml/src/lib.rs index 1a183ac76..6b7e66bec 100644 --- a/ml/src/lib.rs +++ b/ml/src/lib.rs @@ -191,10 +191,6 @@ pub struct UpdateSummary { pub total_models: usize, } -/// Common imports for ML module consumers -pub mod prelude { - // All pub use statements removed per cleanup requirements -} use thiserror::Error; /// Machine Learning specific errors diff --git a/ml/src/liquid/cuda/mod.rs b/ml/src/liquid/cuda/mod.rs index fd3b8d51c..77aa0d758 100644 --- a/ml/src/liquid/cuda/mod.rs +++ b/ml/src/liquid/cuda/mod.rs @@ -19,9 +19,7 @@ pub mod bindings; pub mod memory; pub mod stream_manager; -pub use bindings::*; -pub use memory::*; -pub use stream_manager::*; +// DO NOT RE-EXPORT - Use explicit imports at usage sites /// CUDA-accelerated Liquid Network configuration #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/ml/src/liquid/cuda/mod.rs.bak b/ml/src/liquid/cuda/mod.rs.bak new file mode 100644 index 000000000..fd3b8d51c --- /dev/null +++ b/ml/src/liquid/cuda/mod.rs.bak @@ -0,0 +1,572 @@ +//! CUDA-accelerated Liquid Neural Networks +//! +//! GPU implementation of Liquid Time-constant (LTC) and Closed-form Continuous-time (CfC) +//! neural networks with optimized CUDA kernels for ultra-low latency inference. + +use std::collections::HashMap; +use std::ffi::c_void; +use std::ptr; +use std::sync::Arc; + +use cudarc::driver::{CudaDevice, CudaSlice, DevicePtr, LaunchAsync, LaunchConfig}; +use cudarc::nvrtc::Ptx; +use serde::{Deserialize, Serialize}; + +use super::{FixedPoint, LiquidError, MarketRegime, NetworkType, PerformanceMetrics, Result}; +use crate::{MLError, MLResult}; + +pub mod bindings; +pub mod memory; +pub mod stream_manager; + +pub use bindings::*; +pub use memory::*; +pub use stream_manager::*; + +/// CUDA-accelerated Liquid Network configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CudaLiquidConfig { + pub device_id: usize, + pub max_batch_size: usize, + pub use_shared_memory: bool, + pub stream_count: usize, + pub memory_pool_size_mb: usize, + pub enable_profiling: bool, +} + +impl Default for CudaLiquidConfig { + fn default() -> Self { + Self { + device_id: 0, + max_batch_size: 32, + use_shared_memory: true, + stream_count: 4, + memory_pool_size_mb: 256, + enable_profiling: false, + } + } +} + +/// GPU memory buffers for Liquid Networks +#[derive(Debug)] +pub struct CudaBuffers { + // Input/Output buffers + pub input: CudaSlice, + pub hidden_state: CudaSlice, + pub new_hidden_state: CudaSlice, + pub output: CudaSlice, + + // Weight buffers + pub input_weights: CudaSlice, + pub recurrent_weights: CudaSlice, + pub output_weights: CudaSlice, + pub bias: CudaSlice, + pub output_bias: CudaSlice, + + // Dynamic parameters + pub time_constants: CudaSlice, + pub base_time_constants: CudaSlice, + + // CfC-specific buffers + pub backbone_weights: Option>, + pub backbone_bias: Option>, + pub layer_sizes: Option>, +} + +/// CUDA-accelerated Liquid Neural Network +#[derive(Debug)] +pub struct CudaLiquidNetwork { + pub config: CudaLiquidConfig, + pub device: Arc, + pub buffers: CudaBuffers, + pub stream_manager: cudarc::driver::CudaStreamManager, + pub memory_manager: GpuMemoryManager, + + // Network parameters + pub network_type: NetworkType, + pub input_size: usize, + pub hidden_size: usize, + pub output_size: usize, + pub batch_size: usize, + + // Performance tracking + pub performance_metrics: PerformanceMetrics, + pub current_regime: MarketRegime, + + // CUDA function handles + ltc_forward_fn: cudarc::driver::CudaFunction, + cfc_forward_fn: Option, + output_fn: cudarc::driver::CudaFunction, + adapt_tau_fn: cudarc::driver::CudaFunction, +} + +impl CudaLiquidNetwork { + /// Create a new CUDA-accelerated Liquid Network + pub fn new( + network_type: NetworkType, + input_size: usize, + hidden_size: usize, + output_size: usize, + config: CudaLiquidConfig, + ) -> Result { + // Initialize CUDA device + let device = CudaDevice::new(config.device_id) + .map_err(|e| LiquidError::InferenceError(format!("Failed to initialize CUDA device: {}", e)))?; + let device = Arc::new(device); + + // Load CUDA kernels + let ptx = compile_liquid_kernels()?; + device.load_ptx(ptx, "liquid_kernels", &[ + "fused_ltc_forward", + "fused_cfc_forward", + "fused_liquid_output", + "adapt_time_constants" + ]).map_err(|e| LiquidError::InferenceError(format!("Failed to load CUDA kernels: {}", e)))?; + + // Get kernel functions + let ltc_forward_fn = device.get_func("liquid_kernels", "fused_ltc_forward") + .map_err(|e| LiquidError::InferenceError(format!("Failed to get LTC kernel: {}", e)))?; + let cfc_forward_fn = if matches!(network_type, NetworkType::CfC | NetworkType::Mixed) { + Some(device.get_func("liquid_kernels", "fused_cfc_forward") + .map_err(|e| LiquidError::InferenceError(format!("Failed to get CfC kernel: {}", e)))?) + } else { + None + }; + let output_fn = device.get_func("liquid_kernels", "fused_liquid_output") + .map_err(|e| LiquidError::InferenceError(format!("Failed to get output kernel: {}", e)))?; + let adapt_tau_fn = device.get_func("liquid_kernels", "adapt_time_constants") + .map_err(|e| LiquidError::InferenceError(format!("Failed to get adaptation kernel: {}", e)))?; + + // Initialize memory manager + let memory_manager = GpuMemoryManager::new( + device.clone(), + config.memory_pool_size_mb * 1024 * 1024, + )?; + + // Initialize stream manager + let stream_manager = cudarc::driver::CudaStreamManager::new(device.clone(), config.stream_count)?; + + // Allocate GPU buffers + let batch_size = config.max_batch_size; + let buffers = Self::allocate_buffers( + &device, + &memory_manager, + batch_size, + input_size, + hidden_size, + output_size, + &network_type, + )?; + + let performance_metrics = PerformanceMetrics { + total_inferences: 0, + average_inference_time_ns: 0, + average_inference_time_us: 0.0, + total_parameters: Self::calculate_parameter_count(input_size, hidden_size, output_size), + current_regime: MarketRegime::Normal, + regime_switches: 0, + last_adaptation_time: None, + }; + + Ok(Self { + config, + device, + buffers, + stream_manager, + memory_manager, + network_type, + input_size, + hidden_size, + output_size, + batch_size, + performance_metrics, + current_regime: MarketRegime::Normal, + ltc_forward_fn, + cfc_forward_fn, + output_fn, + adapt_tau_fn, + }) + } + + /// Allocate GPU memory buffers + fn allocate_buffers( + device: &CudaDevice, + memory_manager: &GpuMemoryManager, + batch_size: usize, + input_size: usize, + hidden_size: usize, + output_size: usize, + network_type: &NetworkType, + ) -> Result { + // Input/Output buffers + let input = device.alloc_zeros::(batch_size * input_size) + .map_err(|e| LiquidError::InferenceError(format!("Failed to allocate input buffer: {}", e)))?; + let hidden_state = device.alloc_zeros::(batch_size * hidden_size) + .map_err(|e| LiquidError::InferenceError(format!("Failed to allocate hidden state buffer: {}", e)))?; + let new_hidden_state = device.alloc_zeros::(batch_size * hidden_size) + .map_err(|e| LiquidError::InferenceError(format!("Failed to allocate new hidden state buffer: {}", e)))?; + let output = device.alloc_zeros::(batch_size * output_size) + .map_err(|e| LiquidError::InferenceError(format!("Failed to allocate output buffer: {}", e)))?; + + // Weight buffers + let input_weights = device.alloc_zeros::(hidden_size * input_size) + .map_err(|e| LiquidError::InferenceError(format!("Failed to allocate input weights: {}", e)))?; + let recurrent_weights = device.alloc_zeros::(hidden_size * hidden_size) + .map_err(|e| LiquidError::InferenceError(format!("Failed to allocate recurrent weights: {}", e)))?; + let output_weights = device.alloc_zeros::(output_size * hidden_size) + .map_err(|e| LiquidError::InferenceError(format!("Failed to allocate output weights: {}", e)))?; + let bias = device.alloc_zeros::(hidden_size) + .map_err(|e| LiquidError::InferenceError(format!("Failed to allocate bias: {}", e)))?; + let output_bias = device.alloc_zeros::(output_size) + .map_err(|e| LiquidError::InferenceError(format!("Failed to allocate output bias: {}", e)))?; + + // Dynamic parameters + let time_constants = device.alloc_zeros::(hidden_size) + .map_err(|e| LiquidError::InferenceError(format!("Failed to allocate time constants: {}", e)))?; + let base_time_constants = device.alloc_zeros::(hidden_size) + .map_err(|e| LiquidError::InferenceError(format!("Failed to allocate base time constants: {}", e)))?; + + // CfC-specific buffers + let (backbone_weights, backbone_bias, layer_sizes) = match network_type { + NetworkType::CfC | NetworkType::Mixed => { + // For now, allocate simple backbone (2 layers of size hidden_size each) + let backbone_layers = 2; + let max_layer_size = hidden_size; + let total_backbone_weights = backbone_layers * max_layer_size * (input_size + hidden_size); + + let backbone_weights = device.alloc_zeros::(total_backbone_weights) + .map_err(|e| LiquidError::InferenceError(format!("Failed to allocate backbone weights: {}", e)))?; + let backbone_bias = device.alloc_zeros::(backbone_layers * max_layer_size) + .map_err(|e| LiquidError::InferenceError(format!("Failed to allocate backbone bias: {}", e)))?; + let layer_sizes = device.alloc_zeros::(backbone_layers) + .map_err(|e| LiquidError::InferenceError(format!("Failed to allocate layer sizes: {}", e)))?; + + (Some(backbone_weights), Some(backbone_bias), Some(layer_sizes)) + } + _ => (None, None, None), + }; + + Ok(CudaBuffers { + input, + hidden_state, + new_hidden_state, + output, + input_weights, + recurrent_weights, + output_weights, + bias, + output_bias, + time_constants, + base_time_constants, + backbone_weights, + backbone_bias, + layer_sizes, + }) + } + + /// Forward pass through the CUDA-accelerated network + pub fn forward_gpu(&mut self, input: &[f32], batch_size: usize) -> Result> { + if batch_size > self.config.max_batch_size { + return Err(LiquidError::InvalidInput(format!( + "Batch size {} exceeds maximum {}", + batch_size, self.config.max_batch_size + ))); + } + + if input.len() != batch_size * self.input_size { + return Err(LiquidError::InvalidInput(format!( + "Input size mismatch: expected {}, got {}", + batch_size * self.input_size, + input.len() + ))); + } + + let start_time = std::time::Instant::now(); + + // Get a stream for this operation + let stream = self.stream_manager.get_stream()?; + + // Copy input to GPU + self.device.htod_sync_copy_into(input, &mut self.buffers.input) + .map_err(|e| LiquidError::InferenceError(format!("Failed to copy input to GPU: {}", e)))?; + + // Launch appropriate forward kernel based on network type + match self.network_type { + NetworkType::LTC => { + self.launch_ltc_forward(batch_size, stream)?; + } + NetworkType::CfC => { + self.launch_cfc_forward(batch_size, stream)?; + } + NetworkType::Mixed => { + // Run both LTC and CfC in parallel on different parts of hidden state + self.launch_ltc_forward(batch_size, stream)?; + // Wait for LTC to complete, then run CfC + self.device.synchronize() + .map_err(|e| LiquidError::InferenceError(format!("CUDA sync failed: {}", e)))?; + self.launch_cfc_forward(batch_size, stream)?; + } + } + + // Launch output layer kernel + self.launch_output_layer(batch_size, stream)?; + + // Copy result back to CPU + let mut output = vec![0.0f32; batch_size * self.output_size]; + self.device.dtoh_sync_copy_into(&self.buffers.output, &mut output) + .map_err(|e| LiquidError::InferenceError(format!("Failed to copy output from GPU: {}", e)))?; + + // Update performance metrics + let elapsed = start_time.elapsed(); + self.performance_metrics.total_inferences += 1; + let total_time_ns = self.performance_metrics.average_inference_time_ns + * (self.performance_metrics.total_inferences - 1) + + elapsed.as_nanos() as u64; + self.performance_metrics.average_inference_time_ns = + total_time_ns / self.performance_metrics.total_inferences; + self.performance_metrics.average_inference_time_us = + self.performance_metrics.average_inference_time_ns as f64 / 1000.0; + + Ok(output) + } + + /// Launch LTC forward kernel + fn launch_ltc_forward(&self, batch_size: usize, stream: &cudarc::driver::CudaStream) -> Result<()> { + let grid_x = (self.hidden_size + 15) / 16; + let grid_y = (batch_size + 15) / 16; + let grid_z = 1; + + let block_x = 16; + let block_y = 16; + let block_z = 1; + + let shared_mem_size = (self.input_size + self.hidden_size) * 4; // 4 bytes per f32 + + let config = LaunchConfig { + grid_dim: (grid_x as u32, grid_y as u32, grid_z as u32), + block_dim: (block_x as u32, block_y as u32, block_z as u32), + shared_mem_bytes: shared_mem_size as u32, + }; + + let params = ( + &self.buffers.input, + &self.buffers.hidden_state, + &self.buffers.input_weights, + &self.buffers.recurrent_weights, + &self.buffers.bias, + &self.buffers.time_constants, + &self.buffers.new_hidden_state, + 0.01f32, // dt + 0.5f32, // volatility + 3i32, // activation_type (Tanh) + batch_size as i32, + self.input_size as i32, + self.hidden_size as i32, + ); + + unsafe { + self.ltc_forward_fn.launch_async(config, params, stream) + .map_err(|e| LiquidError::InferenceError(format!("LTC kernel launch failed: {}", e)))?; + } + + Ok(()) + } + + /// Launch CfC forward kernel + fn launch_cfc_forward(&self, batch_size: usize, stream: &cudarc::driver::CudaStream) -> Result<()> { + let cfc_fn = self.cfc_forward_fn.as_ref() + .ok_or_else(|| LiquidError::InferenceError("CfC kernel not available".to_string()))?; + + let backbone_weights = self.buffers.backbone_weights.as_ref() + .ok_or_else(|| LiquidError::InferenceError("Backbone weights not allocated".to_string()))?; + let backbone_bias = self.buffers.backbone_bias.as_ref() + .ok_or_else(|| LiquidError::InferenceError("Backbone bias not allocated".to_string()))?; + let layer_sizes = self.buffers.layer_sizes.as_ref() + .ok_or_else(|| LiquidError::InferenceError("Layer sizes not allocated".to_string()))?; + + let grid_x = (self.hidden_size + 15) / 16; + let grid_y = (batch_size + 15) / 16; + let grid_z = 1; + + let block_x = 16; + let block_y = 16; + let block_z = 1; + + let shared_mem_size = 2 * self.hidden_size * 4; // Two layers in shared memory + + let config = LaunchConfig { + grid_dim: (grid_x as u32, grid_y as u32, grid_z as u32), + block_dim: (block_x as u32, block_y as u32, block_z as u32), + shared_mem_bytes: shared_mem_size as u32, + }; + + let params = ( + &self.buffers.input, + &self.buffers.hidden_state, + backbone_weights, + backbone_bias, + layer_sizes, + &self.buffers.output_weights, + &self.buffers.new_hidden_state, + 0.01f32, // dt + batch_size as i32, + self.input_size as i32, + self.hidden_size as i32, + 2i32, // num_layers + self.hidden_size as i32, // max_layer_size + ); + + unsafe { + cfc_fn.launch_async(config, params, stream) + .map_err(|e| LiquidError::InferenceError(format!("CfC kernel launch failed: {}", e)))?; + } + + Ok(()) + } + + /// Launch output layer kernel + fn launch_output_layer(&self, batch_size: usize, stream: &cudarc::driver::CudaStream) -> Result<()> { + let grid_x = (self.output_size + 15) / 16; + let grid_y = (batch_size + 15) / 16; + let grid_z = 1; + + let block_x = 16; + let block_y = 16; + let block_z = 1; + + let config = LaunchConfig { + grid_dim: (grid_x as u32, grid_y as u32, grid_z as u32), + block_dim: (block_x as u32, block_y as u32, block_z as u32), + shared_mem_bytes: 0, + }; + + let params = ( + &self.buffers.new_hidden_state, + &self.buffers.output_weights, + &self.buffers.output_bias, + &self.buffers.output, + 0i32, // activation_type (Linear) + batch_size as i32, + self.hidden_size as i32, + self.output_size as i32, + ); + + unsafe { + self.output_fn.launch_async(config, params, stream) + .map_err(|e| LiquidError::InferenceError(format!("Output kernel launch failed: {}", e)))?; + } + + Ok(()) + } + + /// Update market volatility and adapt network parameters + pub fn update_market_volatility_gpu(&mut self, volatility: f32) -> Result<()> { + let stream = self.stream_manager.get_stream()?; + + let grid_size = (self.hidden_size + 255) / 256; + let block_size = 256; + + let config = LaunchConfig { + grid_dim: (grid_size as u32, 1, 1), + block_dim: (block_size as u32, 1, 1), + shared_mem_bytes: 0, + }; + + let params = ( + &self.buffers.time_constants, + &self.buffers.base_time_constants, + volatility, + 0.01f32, // tau_min + 1.0f32, // tau_max + self.hidden_size as i32, + ); + + unsafe { + self.adapt_tau_fn.launch_async(config, params, stream) + .map_err(|e| LiquidError::InferenceError(format!("Adaptation kernel launch failed: {}", e)))?; + } + + // Update regime based on volatility + let new_regime = if volatility < 0.2 { + MarketRegime::Normal + } else if variance < FixedPoint(2 * PRECISION) { + MarketRegime::Sideways + } else if variance < FixedPoint(5 * PRECISION) { + MarketRegime::Trending + } else if variance < FixedPoint(10 * PRECISION) { + MarketRegime::Bull + } else { + MarketRegime::Crisis + }; + + if new_regime != self.current_regime { + self.current_regime = new_regime.clone(); + self.performance_metrics.current_regime = new_regime; + self.performance_metrics.regime_switches += 1; + self.performance_metrics.last_adaptation_time = Some( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|e| LiquidError::InferenceError(format!("System time error: {}", e)))? + .as_millis() as u64, + ); + } + + Ok(()) + } + + /// Initialize network weights from CPU network + pub fn load_weights_from_cpu( + &mut self, + input_weights: &[f32], + recurrent_weights: &[f32], + output_weights: &[f32], + bias: &[f32], + output_bias: &[f32], + time_constants: &[f32], + ) -> Result<()> { + // Copy weights to GPU + self.device.htod_sync_copy_into(input_weights, &mut self.buffers.input_weights) + .map_err(|e| LiquidError::InferenceError(format!("Failed to copy input weights: {}", e)))?; + self.device.htod_sync_copy_into(recurrent_weights, &mut self.buffers.recurrent_weights) + .map_err(|e| LiquidError::InferenceError(format!("Failed to copy recurrent weights: {}", e)))?; + self.device.htod_sync_copy_into(output_weights, &mut self.buffers.output_weights) + .map_err(|e| LiquidError::InferenceError(format!("Failed to copy output weights: {}", e)))?; + self.device.htod_sync_copy_into(bias, &mut self.buffers.bias) + .map_err(|e| LiquidError::InferenceError(format!("Failed to copy bias: {}", e)))?; + self.device.htod_sync_copy_into(output_bias, &mut self.buffers.output_bias) + .map_err(|e| LiquidError::InferenceError(format!("Failed to copy output bias: {}", e)))?; + self.device.htod_sync_copy_into(time_constants, &mut self.buffers.time_constants) + .map_err(|e| LiquidError::InferenceError(format!("Failed to copy time constants: {}", e)))?; + self.device.htod_sync_copy_into(time_constants, &mut self.buffers.base_time_constants) + .map_err(|e| LiquidError::InferenceError(format!("Failed to copy base time constants: {}", e)))?; + + Ok(()) + } + + /// Get performance metrics + pub fn get_performance_metrics(&self) -> &PerformanceMetrics { + &self.performance_metrics + } + + /// Calculate total parameter count + fn calculate_parameter_count(input_size: usize, hidden_size: usize, output_size: usize) -> usize { + let input_params = hidden_size * input_size; + let recurrent_params = hidden_size * hidden_size; + let output_params = output_size * hidden_size; + let bias_params = hidden_size + output_size; + let tau_params = hidden_size; + + input_params + recurrent_params + output_params + bias_params + tau_params + } +} + +/// Compile CUDA kernels from source +fn compile_liquid_kernels() -> Result { + // In a real implementation, this would compile the .cu file + // For now, we'll assume the kernels are pre-compiled + Err(LiquidError::InferenceError( + "CUDA kernel compilation not implemented in this demo".to_string(), + )) +} + +// TECHNICAL DEBT ELIMINATED - Use cudarc types directly \ No newline at end of file diff --git a/ml/src/liquid/mod.rs b/ml/src/liquid/mod.rs index a15b11ce8..b30135f7f 100644 --- a/ml/src/liquid/mod.rs +++ b/ml/src/liquid/mod.rs @@ -19,12 +19,7 @@ pub mod training; #[cfg(test)] mod tests; -// Re-export key types -pub use activation::*; -pub use cells::*; -pub use network::*; -pub use ode_solvers::*; -pub use training::*; +// DO NOT RE-EXPORT - Use explicit imports at usage sites /// Fixed-point arithmetic for ultra-low latency inference pub const PRECISION: i64 = 100_000_000; // 8 decimal places diff --git a/ml/src/liquid/training.rs b/ml/src/liquid/training.rs index 17b644757..74e89029c 100644 --- a/ml/src/liquid/training.rs +++ b/ml/src/liquid/training.rs @@ -7,7 +7,7 @@ use std::time::Instant; use serde::{Deserialize, Serialize}; -pub use super::activation::ActivationType; +// NO RE-EXPORTS - Use explicit imports: super::activation::ActivationType use super::network::LiquidNetwork; use super::{FixedPoint, LiquidError, MarketRegime, Result, PRECISION}; diff --git a/ml/src/mamba/mod.rs b/ml/src/mamba/mod.rs index e9eadf966..f923f80ff 100644 --- a/ml/src/mamba/mod.rs +++ b/ml/src/mamba/mod.rs @@ -38,10 +38,7 @@ mod scan_algorithms; mod selective_state; mod ssd_layer; -pub use hardware_aware::HardwareOptimizer; -pub use scan_algorithms::{ParallelScanEngine, ScanOperator}; -pub use selective_state::SelectiveStateSpace; -pub use ssd_layer::SSDLayer; +// DO NOT RE-EXPORT - Use explicit imports at usage sites use std::collections::HashMap; use std::sync::atomic::{AtomicU64, Ordering}; diff --git a/ml/src/mamba/mod.rs.bak b/ml/src/mamba/mod.rs.bak new file mode 100644 index 000000000..f923f80ff --- /dev/null +++ b/ml/src/mamba/mod.rs.bak @@ -0,0 +1,1559 @@ +//! # Mamba-2 State-Space Model for HFT +//! +//! Next-generation Mamba-2 implementation with Structured State Duality (SSD), +//! hardware-aware algorithms, and 5x performance improvements over Mamba-1. +//! +//! ## Key Features +//! +//! - **SSD Layers**: Structured State Duality for linear attention mechanisms +//! - **Hardware-aware**: Optimized memory access patterns and SIMD instructions +//! - **5x Faster**: Sub-linear memory usage and linear-time sequence modeling +//! - **Selective State Spaces**: Advanced state selection mechanisms +//! - **Sub-5μs**: Target inference latency for HFT applications +//! - **Integer Precision**: 10,000x scaling for financial precision +//! +//! ## Architecture Improvements +//! +//! ```text +//! ┌─────────────────────────────────────────────────────────────┐ +//! │ Mamba-2 Block │ +//! ├─────────────────┬─────────────────┬─────────────────────────┤ +//! │ SSD Layer │ Hardware-Aware │ Selective State │ +//! │ │ Optimization │ Mechanism │ +//! │ • Linear Attn │ • SIMD Vectors │ • Advanced Selection │ +//! │ • Structured │ • Cache-Friendly│ • Dynamic Parameters │ +//! │ Duality │ • Prefetching │ • State Compression │ +//! └─────────────────┴─────────────────┴─────────────────────────┘ +//! ``` +//! +//! ## Performance Targets +//! +//! - Inference: <5μs per sequence step (5x faster than Mamba-1) +//! - Memory: Sub-linear growth with sequence length +//! - Throughput: >1M sequences/sec +//! - Latency: 99.9% percentile <10μs + +mod hardware_aware; +mod scan_algorithms; +mod selective_state; +mod ssd_layer; + +// DO NOT RE-EXPORT - Use explicit imports at usage sites + +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant, SystemTime}; + +use candle_core::{DType, Device, Tensor}; +use candle_nn::{Dropout, Linear, VarBuilder}; +use crate::Module; +use candle_nn::LayerNorm; +use serde::{Deserialize, Serialize}; +use tracing::{debug, info, instrument, warn}; +use uuid::Uuid; + +use crate::MLError; +// use crate::safe_operations; // DISABLED - module not found + +/// Configuration for MAMBA-2 state-space model +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Mamba2Config { + /// Model dimension + pub d_model: usize, + /// State dimension + pub d_state: usize, + /// Head dimension for multi-head attention + pub d_head: usize, + /// Number of attention heads + pub num_heads: usize, + /// Expansion factor for inner dimension + pub expand: usize, + /// Number of layers + pub num_layers: usize, + /// Dropout rate + pub dropout: f64, + /// Use structured state duality + pub use_ssd: bool, + /// Use selective state mechanism + pub use_selective_state: bool, + /// Enable hardware optimizations + pub hardware_aware: bool, + /// Target latency in microseconds + pub target_latency_us: u64, + /// Maximum sequence length + pub max_seq_len: usize, + /// Learning rate + pub learning_rate: f64, + /// Weight decay + pub weight_decay: f64, + /// Gradient clipping threshold + pub grad_clip: f64, + /// Warmup steps + pub warmup_steps: usize, + /// Training batch size + pub batch_size: usize, + /// Sequence length for training + pub seq_len: usize, +} + +impl Default for Mamba2Config { + fn default() -> Self { + Self { + d_model: 512, + d_state: 64, + d_head: 64, + num_heads: 8, + expand: 2, + num_layers: 6, + dropout: 0.1, + use_ssd: true, + use_selective_state: true, + hardware_aware: true, + target_latency_us: 5, + max_seq_len: 2048, + learning_rate: 1e-4, + weight_decay: 1e-5, + grad_clip: 1.0, + warmup_steps: 1000, + batch_size: 32, + seq_len: 512, + } + } +} + +/// MAMBA-2 state container +#[derive(Debug, Clone)] +pub struct Mamba2State { + /// Hidden states for each layer + pub hidden_states: Vec, + /// Selective state components + pub selective_state: Vec, + /// State transition matrices A, B, C + pub ssm_states: Vec, + /// Compression indices for memory efficiency + pub compression_indices: Vec, + /// Performance metrics + pub metrics: HashMap, + /// Last update timestamp + pub last_update: Instant, +} + +/// State Space Model state matrices +#[derive(Debug, Clone)] +pub struct SSMState { + /// State transition matrix A (d_state × d_state) + pub A: Tensor, + /// Input matrix B (d_state × d_model) + pub B: Tensor, + /// Output matrix C (d_model × d_state) + pub C: Tensor, + /// Discretization parameter Δ (Delta) + pub delta: Tensor, + /// Current hidden state + pub hidden: Tensor, +} + +impl Mamba2State { + pub fn zeros(config: &Mamba2Config) -> Result { + let device = match Device::cuda_if_available(0) { + Ok(cuda_device) => { + debug!("Using CUDA device for Mamba2State"); + cuda_device + } + Err(_) => { + debug!("Using CPU device for Mamba2State"); + Device::Cpu + } + }; + let mut hidden_states = Vec::new(); + let mut ssm_states = Vec::new(); + + for layer_idx in 0..config.num_layers { + // Create hidden state with proper error handling + let hidden = Tensor::zeros((config.batch_size, config.d_model), DType::F32, &device) + .map_err(|e| MLError::TensorCreationError { + operation: format!("hidden state creation for layer {}", layer_idx), + reason: e.to_string(), + })?; + hidden_states.push(hidden); + + // Initialize SSM matrices with proper error handling + let A = Tensor::randn(0.0, 1.0, (config.d_state, config.d_state), &device).map_err( + |e| MLError::TensorCreationError { + operation: format!("SSM A matrix creation for layer {}", layer_idx), + reason: e.to_string(), + }, + )?; + + let B = Tensor::randn(0.0, 1.0, (config.d_state, config.d_model), &device).map_err( + |e| MLError::TensorCreationError { + operation: format!("SSM B matrix creation for layer {}", layer_idx), + reason: e.to_string(), + }, + )?; + + let C = Tensor::randn(0.0, 1.0, (config.d_model, config.d_state), &device).map_err( + |e| MLError::TensorCreationError { + operation: format!("SSM C matrix creation for layer {}", layer_idx), + reason: e.to_string(), + }, + )?; + + let delta = Tensor::ones((config.d_model,), DType::F32, &device).map_err(|e| { + MLError::TensorCreationError { + operation: format!("delta tensor creation for layer {}", layer_idx), + reason: e.to_string(), + } + })?; + + let ssm_hidden = + Tensor::zeros((config.batch_size, config.d_state), DType::F32, &device).map_err( + |e| MLError::TensorCreationError { + operation: format!("SSM hidden state creation for layer {}", layer_idx), + reason: e.to_string(), + }, + )?; + + ssm_states.push(SSMState { + A, + B, + C, + delta, + hidden: ssm_hidden, + }); + } + + Ok(Self { + hidden_states, + selective_state: vec![0.0; config.d_model * config.expand], + ssm_states, + compression_indices: Vec::new(), + metrics: HashMap::new(), + last_update: Instant::now(), + }) + } + + /// Compress state to reduce memory usage + pub fn compress(&mut self, compression_ratio: f64) { + let target_size = (self.selective_state.len() as f64 * compression_ratio) as usize; + + // Sort by magnitude and keep top components + let mut indexed_values: Vec<(usize, f64)> = self + .selective_state + .iter() + .enumerate() + .map(|(i, &v)| (i, v.abs())) + .collect(); + + indexed_values.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + + self.compression_indices.clear(); + for i in 0..target_size.min(indexed_values.len()) { + self.compression_indices.push(indexed_values[i].0); + } + + // Zero out non-selected components + for i in 0..self.selective_state.len() { + if !self.compression_indices.contains(&i) { + self.selective_state[i] = 0.0; + } + } + } +} + +/// Training metadata for MAMBA-2 model +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Mamba2Metadata { + pub model_id: String, + pub created_at: SystemTime, + pub version: String, + pub input_dim: usize, + pub output_dim: usize, + pub num_parameters: usize, + pub training_history: Vec, + pub performance_stats: HashMap, + pub last_checkpoint: Option, +} + +/// Training epoch information +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TrainingEpoch { + pub epoch: usize, + pub loss: f64, + pub accuracy: f64, + pub learning_rate: f64, + pub duration_seconds: f64, + pub timestamp: SystemTime, +} + +/// MAMBA-2 State-Space Model implementation +#[derive(Debug)] +pub struct Mamba2SSM { + pub config: Mamba2Config, + pub metadata: Mamba2Metadata, + pub state: Mamba2State, + pub ssd_layers: Vec, + pub selective_state: Option, + pub hardware_optimizer: Option, + pub scan_engine: Arc, + pub is_trained: bool, + + // Model parameters + pub input_projection: Linear, + pub output_projection: Linear, + pub layer_norms: Vec, + pub dropouts: Vec, + + // Training state + pub optimizer_state: HashMap, + pub gradients: HashMap, + pub grad_scaler: f64, + pub step_count: usize, + + // Performance counters + pub total_inferences: AtomicU64, + pub total_training_steps: AtomicU64, + pub latency_histogram: Vec, +} + +impl Mamba2SSM { + /// Create new MAMBA-2 model + pub fn new(config: Mamba2Config) -> Result { + let device = Device::Cpu; + let vs = candle_nn::VarMap::new(); + let vb = VarBuilder::from_varmap(&vs, DType::F32, &device); + + let input_projection = candle_nn::linear( + config.d_model, + config.d_model * config.expand, + vb.pp("input_proj"), + )?; + let output_projection = candle_nn::linear(config.d_model, 1, vb.pp("output_proj"))?; // Single output for regression + + let mut layer_norms = Vec::new(); + let mut dropouts = Vec::new(); + let mut ssd_layers = Vec::new(); + + for i in 0..config.num_layers { + let ln = candle_nn::layer_norm(config.d_model, 1e-5, vb.pp(&format!("ln_{}", i)))?; + layer_norms.push(ln); + + let dropout = Dropout::new(config.dropout as f32); + dropouts.push(dropout); + + let ssd_layer = SSDLayer::new(&config, i)?; + ssd_layers.push(ssd_layer); + } + + let selective_state = if config.use_selective_state { + Some(SelectiveStateSpace::new(&config)?) + } else { + None + }; + + let hardware_optimizer = if config.hardware_aware { + Some(HardwareOptimizer::new(&config)?) + } else { + None + }; + + let scan_engine = Arc::new(ParallelScanEngine::new(device, 1_000_000)); + + let metadata = Mamba2Metadata { + model_id: Uuid::new_v4().to_string(), + created_at: SystemTime::now(), + version: "2.0.0".to_string(), + input_dim: config.d_model, + output_dim: 1, + num_parameters: Self::count_parameters(&config), + training_history: Vec::new(), + performance_stats: HashMap::new(), + last_checkpoint: None, + }; + + let state = Mamba2State::zeros(&config)?; + + Ok(Self { + config, + metadata, + state, + ssd_layers, + selective_state, + hardware_optimizer, + scan_engine, + is_trained: false, + input_projection, + output_projection, + layer_norms, + dropouts, + optimizer_state: HashMap::new(), + gradients: HashMap::new(), + grad_scaler: 1.0, + step_count: 0, + total_inferences: AtomicU64::new(0), + total_training_steps: AtomicU64::new(0), + latency_histogram: Vec::new(), + }) + } + + /// Count total parameters in model + fn count_parameters(config: &Mamba2Config) -> usize { + let input_proj_params = config.d_model * (config.d_model * config.expand); + let output_proj_params = config.d_model * 1; + let layer_params = config.num_layers + * ( + config.d_model * 3 + // Layer norm + config.d_model * config.d_state * 3 + // A, B, C matrices + config.d_model + // Delta parameters + ); + + input_proj_params + output_proj_params + layer_params + } + + /// Create HFT-optimized configuration + pub fn default_hft() -> Result { + let config = Mamba2Config { + d_model: 256, + d_state: 32, + d_head: 32, + num_heads: 8, + expand: 2, + num_layers: 4, + target_latency_us: 3, + hardware_aware: true, + use_ssd: true, + use_selective_state: true, + max_seq_len: 1024, + batch_size: 16, + seq_len: 256, + ..Default::default() + }; + + Self::new(config) + } + + /// Forward pass through the model + #[instrument(skip(self, input))] + pub fn forward(&mut self, input: &Tensor) -> Result { + let start = Instant::now(); + + // Input projection + let mut hidden = self.input_projection.forward(input)?; + + // Process through each layer - collect indices first to avoid borrow conflicts + let num_layers = self.ssd_layers.len(); + for layer_idx in 0..num_layers { + // Layer normalization + let normalized = self.layer_norms[layer_idx].forward(&hidden)?; + + // SSD layer processing with selective scan + let layer_output = { + let ssd_layer = self.ssd_layers[layer_idx].clone(); + self.forward_ssd_layer(&ssd_layer, &normalized, layer_idx)? + }; + + // Residual connection + hidden = (&hidden + &layer_output)?; + + // Dropout + if self.config.dropout > 0.0 { + hidden = self.dropouts[layer_idx].forward(&hidden, true)?; + } + } + + // Output projection + let output = self.output_projection.forward(&hidden)?; + + // Update performance metrics + let inference_time = start.elapsed(); + self.total_inferences.fetch_add(1, Ordering::Relaxed); + self.latency_histogram.push(inference_time); + + if self.latency_histogram.len() > 10000 { + self.latency_histogram.remove(0); + } + + Ok(output) + } + + /// Forward pass through SSD layer with selective scan + #[instrument(skip(self, _ssd_layer, input))] + fn forward_ssd_layer( + &mut self, + _ssd_layer: &SSDLayer, + input: &Tensor, + layer_idx: usize, + ) -> Result { + // Extract needed data before borrowing to avoid conflicts + let dt = self.state.ssm_states[layer_idx].delta.clone(); + let A = self.state.ssm_states[layer_idx].A.clone(); + let B = self.state.ssm_states[layer_idx].B.clone(); + let C = self.state.ssm_states[layer_idx].C.clone(); + + // Discretize the continuous-time SSM + let A_discrete = self.discretize_ssm(&A, &dt)?; + let B_discrete = self.discretize_ssm_input(&B, &dt)?; + + // Selective scan algorithm + let scan_input = self.prepare_scan_input(input, &A_discrete, &B_discrete)?; + let scanned_states = self + .scan_engine + .parallel_prefix_scan(&scan_input, ScanOperator::SSMScan)?; + + // Apply output transformation + let output = scanned_states.matmul(&C.t()?)?; + + // Update hidden state + let _batch_size = input.dim(0)?; + let seq_len = input.dim(1)?; + if seq_len > 0 { + let last_state = scanned_states.narrow(1, seq_len - 1, 1)?.squeeze(1)?; + self.state.ssm_states[layer_idx].hidden = last_state; + } + + Ok(output) + } + + /// Discretize continuous-time SSM matrix A + fn discretize_ssm(&self, A_cont: &Tensor, dt: &Tensor) -> Result { + // A_discrete = exp(A_cont * dt) + // For simplicity, using first-order approximation: I + A_cont * dt + let dt_expanded = dt.unsqueeze(0)?.broadcast_as(A_cont.shape())?; + let A_scaled = (A_cont * &dt_expanded)?; + let identity = Tensor::eye(A_cont.dim(0)?, DType::F32, A_cont.device())?; + let A_discrete = (&identity + &A_scaled)?; + + Ok(A_discrete) + } + + /// Discretize continuous-time input matrix B + fn discretize_ssm_input(&self, B_cont: &Tensor, dt: &Tensor) -> Result { + // B_discrete = B_cont * dt + let dt_expanded = dt.unsqueeze(0)?.broadcast_as(B_cont.shape())?; + let B_discrete = (B_cont * &dt_expanded)?; + + Ok(B_discrete) + } + + /// Prepare input for selective scan algorithm + fn prepare_scan_input( + &self, + input: &Tensor, + _A: &Tensor, + B: &Tensor, + ) -> Result { + // Combine input with state transition matrices for scanning + let Bu = input.matmul(B)?; + Ok(Bu) + } + + /// Fast single prediction for HFT + pub fn predict_single_fast(&mut self, input: &[f64]) -> Result { + let start = Instant::now(); + + if input.len() != self.config.d_model { + return Err(MLError::InvalidInput(format!( + "Expected input dimension {}, got {}", + self.config.d_model, + input.len() + ))); + } + + let device = &Device::Cpu; + let input_tensor = Tensor::from_vec(input.to_vec(), (1, input.len()), device)?; + + let output = self.forward(&input_tensor)?; + let result: f32 = output.to_scalar()?; + + let elapsed = start.elapsed(); + if elapsed.as_micros() > self.config.target_latency_us as u128 { + warn!( + "Prediction exceeded target latency: {}μs", + elapsed.as_micros() + ); + } + + Ok(result as f64) + } + + /// Get performance metrics + pub fn get_performance_metrics(&self) -> HashMap { + let mut metrics = HashMap::new(); + + metrics.insert( + "total_inferences".to_string(), + self.total_inferences.load(Ordering::Relaxed) as f64, + ); + metrics.insert( + "total_training_steps".to_string(), + self.total_training_steps.load(Ordering::Relaxed) as f64, + ); + + if !self.latency_histogram.is_empty() { + let avg_latency = self + .latency_histogram + .iter() + .map(|d| d.as_micros() as f64) + .sum::() + / self.latency_histogram.len() as f64; + metrics.insert("avg_latency_us".to_string(), avg_latency); + + let throughput = 1_000_000.0 / avg_latency; // predictions per second + metrics.insert("throughput_pps".to_string(), throughput); + } + + // Hardware metrics + if let Some(hw_optimizer) = &self.hardware_optimizer { + let hw_metrics = hw_optimizer.get_performance_metrics(); + for (k, v) in hw_metrics { + metrics.insert(k, v); + } + } + + // Model-specific metrics + metrics.insert( + "model_parameters".to_string(), + self.metadata.num_parameters as f64, + ); + metrics.insert( + "compression_ratio".to_string(), + if self.state.selective_state.len() > 0 && !self.state.compression_indices.is_empty() { + self.state.compression_indices.len() as f64 + / self.state.selective_state.len() as f64 + } else { + 1.0 + }, + ); + + let latency_target_ratio = if !self.latency_histogram.is_empty() { + let avg_latency = self + .latency_histogram + .iter() + .map(|d| d.as_micros() as f64) + .sum::() + / self.latency_histogram.len() as f64; + avg_latency / self.config.target_latency_us as f64 + } else { + 0.0 + }; + metrics.insert("latency_target_ratio".to_string(), latency_target_ratio); + + // Additional production metrics for compatibility + metrics.insert("cache_hit_rate".to_string(), 0.95); + metrics.insert("simd_ops_per_inference".to_string(), 1000.0); + metrics.insert( + "state_compression_ratio".to_string(), + metrics.get("compression_ratio").copied().unwrap_or(1.0), + ); + + metrics + } + + /// Train the model with selective scan algorithm + #[instrument(skip(self, train_data, val_data))] + pub async fn train( + &mut self, + train_data: &[(Tensor, Tensor)], + val_data: &[(Tensor, Tensor)], + epochs: usize, + ) -> Result, MLError> { + info!("Starting MAMBA-2 training with {} epochs", epochs); + + let mut training_history = Vec::new(); + let mut best_val_loss = f64::INFINITY; + + // Initialize optimizer + self.initialize_optimizer()?; + + for epoch in 0..epochs { + let epoch_start = Instant::now(); + let mut epoch_loss = 0.0; + let mut epoch_accuracy = 0.0; + let mut batch_count = 0; + + // Training phase + for batch_idx in (0..train_data.len()).step_by(self.config.batch_size) { + let batch_end = (batch_idx + self.config.batch_size).min(train_data.len()); + let batch = &train_data[batch_idx..batch_end]; + + let batch_loss = self.train_batch(batch, epoch)?; + epoch_loss += batch_loss; + batch_count += 1; + + // Update learning rate + self.update_learning_rate(epoch, batch_idx)?; + + if batch_idx % 100 == 0 { + debug!( + "Epoch {}, Batch {}: Loss = {:.6}", + epoch, batch_idx, batch_loss + ); + } + } + + epoch_loss /= batch_count as f64; + + // Validation phase + let val_loss = self.validate(val_data)?; + epoch_accuracy = self.calculate_accuracy(val_data)?; + + // Update learning rate scheduler + let current_lr = self.get_current_learning_rate(); + + let epoch_duration = epoch_start.elapsed().as_secs_f64(); + let training_epoch = TrainingEpoch { + epoch, + loss: epoch_loss, + accuracy: epoch_accuracy, + learning_rate: current_lr, + duration_seconds: epoch_duration, + timestamp: SystemTime::now(), + }; + + training_history.push(training_epoch.clone()); + self.metadata.training_history.push(training_epoch); + + // Save checkpoint if best model + if val_loss < best_val_loss { + best_val_loss = val_loss; + self.save_checkpoint(&format!("best_epoch_{}.ckpt", epoch)) + .await?; + info!( + "New best validation loss: {:.6} at epoch {}", + val_loss, epoch + ); + } + + // Log epoch results + info!( + "Epoch {}/{}: Loss = {:.6}, Val Loss = {:.6}, Accuracy = {:.4}, LR = {:.2e}, Time = {:.2}s", + epoch + 1, epochs, epoch_loss, val_loss, epoch_accuracy, current_lr, epoch_duration + ); + + // Early stopping check + if self.should_early_stop(&training_history) { + info!("Early stopping triggered at epoch {}", epoch); + break; + } + } + + self.is_trained = true; + info!("Training completed with {} epochs", training_history.len()); + + Ok(training_history) + } + + /// Train a single batch with selective scan + #[instrument(skip(self, batch))] + fn train_batch(&mut self, batch: &[(Tensor, Tensor)], epoch: usize) -> Result { + let mut total_loss = 0.0; + + for (input, target) in batch { + // Zero gradients + self.zero_gradients()?; + + // Forward pass with selective scan + let output = self.forward_with_gradients(input)?; + + // Compute loss + let loss = self.compute_loss(&output, target)?; + total_loss += loss.to_scalar::()? as f64; + + // Backward pass - compute gradients for SSM parameters + self.backward_pass(&loss, input, target)?; + + // Update parameters + self.optimizer_step()?; + + // Update selective state based on gradients + if let Some(selective_state) = &mut self.selective_state { + selective_state.update_importance_scores(input, &mut self.state)?; + } + } + + self.total_training_steps.fetch_add(1, Ordering::Relaxed); + self.step_count += 1; + + Ok(total_loss / batch.len() as f64) + } + + /// Forward pass with gradient computation enabled + fn forward_with_gradients(&mut self, input: &Tensor) -> Result { + // Enable gradient tracking + let input = input.detach(); + + // Input projection with gradients + let mut hidden = self.input_projection.forward(&input)?; + + // Process through each layer with SSM gradients - collect indices first to avoid borrow conflicts + let num_layers = self.ssd_layers.len(); + for layer_idx in 0..num_layers { + // Layer normalization + let normalized = self.layer_norms[layer_idx].forward(&hidden)?; + + // SSD layer processing with selective scan and gradients + let layer_output = { + let ssd_layer = self.ssd_layers[layer_idx].clone(); + self.forward_ssd_layer_with_gradients(&ssd_layer, &normalized, layer_idx)? + }; + + // Residual connection + hidden = (&hidden + &layer_output)?; + + // Dropout (enabled during training) + if self.config.dropout > 0.0 { + hidden = self.dropouts[layer_idx].forward(&hidden, true)?; + } + } + + // Output projection + let output = self.output_projection.forward(&hidden)?; + + Ok(output) + } + + /// Forward pass through SSD layer with gradient tracking + fn forward_ssd_layer_with_gradients( + &mut self, + _ssd_layer: &SSDLayer, + input: &Tensor, + layer_idx: usize, + ) -> Result { + // Extract needed data before borrowing to avoid conflicts + let dt = self.state.ssm_states[layer_idx].delta.clone(); + let A = self.state.ssm_states[layer_idx].A.clone(); + let B = self.state.ssm_states[layer_idx].B.clone(); + let C = self.state.ssm_states[layer_idx].C.clone(); + + // Discretize with gradient tracking + let A_discrete = self.discretize_ssm_with_gradients(&A, &dt)?; + let B_discrete = self.discretize_ssm_input_with_gradients(&B, &dt)?; + + // Selective scan with gradient computation + let scan_input = self.prepare_scan_input_with_gradients(input, &A_discrete, &B_discrete)?; + let scanned_states = self.selective_scan_with_gradients(&scan_input, &A_discrete)?; + + // Output transformation with gradients + let output = scanned_states.matmul(&C.t()?)?; + + // Update hidden state + let _batch_size = input.dim(0)?; + let seq_len = input.dim(1)?; + if seq_len > 0 { + let last_state = scanned_states.narrow(1, seq_len - 1, 1)?.squeeze(1)?; + self.state.ssm_states[layer_idx].hidden = last_state; + } + + Ok(output) + } + + /// Selective scan algorithm with gradient computation + fn selective_scan_with_gradients(&self, input: &Tensor, A: &Tensor) -> Result { + let seq_len = input.dim(1)?; + let d_state = input.dim(2)?; + let device = input.device(); + + // Initialize state sequence + let mut states = Vec::new(); + let mut current_state = Tensor::zeros((input.dim(0)?, d_state), input.dtype(), device)?; + + // Sequential scan with state transitions (maintaining gradients) + for t in 0..seq_len { + let x_t = input.narrow(1, t, 1)?.squeeze(1)?; + + // State transition: h_t = A * h_{t-1} + B * x_t + // B is already incorporated in the input preparation + let state_dims = current_state.dims().len(); + current_state = (A + .matmul(¤t_state.unsqueeze(state_dims)?)? + .squeeze(state_dims)? + + &x_t)?; + states.push(current_state.unsqueeze(1)?); + } + + // Stack all states + let result = Tensor::cat(&states, 1)?; + Ok(result) + } + + /// Discretize SSM with gradient tracking + fn discretize_ssm_with_gradients( + &self, + A_cont: &Tensor, + dt: &Tensor, + ) -> Result { + // Use more accurate discretization: A_discrete = exp(A_cont * dt) + // For gradients, use matrix exponential approximation + let dt_expanded = dt.unsqueeze(0)?.broadcast_as(A_cont.shape())?; + let A_scaled = (A_cont * &dt_expanded)?; + + // Matrix exponential approximation: exp(A) ≈ I + A + A²/2 + A³/6 + let identity = Tensor::eye(A_cont.dim(0)?, DType::F32, A_cont.device())?; + let A2 = A_scaled.matmul(&A_scaled)?; + let A3 = A2.matmul(&A_scaled)?; + + let A_discrete = (&identity + &A_scaled + &(A2 * 0.5)? + &(A3 * (1.0 / 6.0))?)?; + + Ok(A_discrete) + } + + /// Discretize input matrix with gradients + fn discretize_ssm_input_with_gradients( + &self, + B_cont: &Tensor, + dt: &Tensor, + ) -> Result { + let dt_expanded = dt.unsqueeze(0)?.broadcast_as(B_cont.shape())?; + let B_discrete = (B_cont * &dt_expanded)?; + Ok(B_discrete) + } + + /// Prepare scan input with gradient tracking + fn prepare_scan_input_with_gradients( + &self, + input: &Tensor, + _A: &Tensor, + B: &Tensor, + ) -> Result { + // Multiply input by B matrix for state transition + let Bu = input.matmul(&B.t()?)?; + Ok(Bu) + } + + /// Compute training loss + fn compute_loss(&self, output: &Tensor, target: &Tensor) -> Result { + // Mean Squared Error for regression + let diff = (output - target)?; + let squared_diff = (&diff * &diff)?; + let loss = squared_diff.mean_all()?; + Ok(loss) + } + + /// Backward pass - compute gradients for SSM parameters + fn backward_pass( + &mut self, + loss: &Tensor, + _input: &Tensor, + _target: &Tensor, + ) -> Result<(), MLError> { + // Compute gradients using automatic differentiation + // The loss tensor should already have the computational graph attached + let _grad = loss.backward()?; + + // Apply gradient clipping for SSM stability + self.clip_gradients(self.config.grad_clip)?; + + // For MAMBA SSM, gradients flow through: + // 1. Output matrix C: δC += (∂L/∂y_t) · h_t^T + // 2. State transitions: backward recurrence with A_d^T + // 3. Input matrix B: δB += g_t · x_t^T + // 4. Discretization parameter Δ: chain rule from A_d, B_d + + // The actual gradient computation is handled by candle's automatic differentiation + // when we call backward() on the loss. The gradients are stored in each tensor's + // gradient field and will be used in optimizer_step(). + + // Additional SSM-specific gradient processing + let num_layers = self.state.ssm_states.len(); + for _layer_idx in 0..num_layers { + // Ensure gradients don't explode for SSM parameters + // A matrix needs special handling to maintain stability + if let Some(A_grad) = self.gradients.get("A") { + // Project A gradients to maintain spectral radius < 1 + let spectral_radius = self.compute_spectral_radius(&A_grad)?; + if spectral_radius > 1.0 { + let scale_factor = 0.99 / spectral_radius; + // Scale the gradient to maintain stability + let scale_tensor = Tensor::new(&[scale_factor as f32], A_grad.device())?; + let _scaled_grad = A_grad.mul(&scale_tensor)?; + // Note: In real candle implementation, we'd update the gradient directly + } + } + } + + Ok(()) + } + + /// Initialize optimizer state + fn initialize_optimizer(&mut self) -> Result<(), MLError> { + // Initialize Adam optimizer state + self.optimizer_state.clear(); + + // Add momentum and variance terms for each parameter + // In real implementation, this would be handled by candle's optimizers + + Ok(()) + } + + /// Zero gradients + fn zero_gradients(&mut self) -> Result<(), MLError> { + // Clear all gradients for SSM parameters + for _ssm_state in &mut self.state.ssm_states { + // Zero gradients for A, B, C matrices and delta parameter + if let Some(mut A_grad) = self.gradients.get("A").cloned() { + A_grad = A_grad.zeros_like()?; + } + if let Some(mut B_grad) = self.gradients.get("B").cloned() { + B_grad = B_grad.zeros_like()?; + } + if let Some(mut C_grad) = self.gradients.get("C").cloned() { + C_grad = C_grad.zeros_like()?; + } + if let Some(mut delta_grad) = self.gradients.get("delta").cloned() { + delta_grad = delta_grad.zeros_like()?; + } + } + + // Clear optimizer state gradients if they exist + for (param_name, tensor) in self.optimizer_state.iter_mut() { + if param_name.contains("grad") { + *tensor = tensor.zeros_like()?; + } + } + + Ok(()) + } + + /// Optimizer step + fn optimizer_step(&mut self) -> Result<(), MLError> { + // Adam hyperparameters + let beta1: f32 = 0.9; + let beta2: f32 = 0.999; + let eps = 1e-8; + let lr = self.config.learning_rate; + + // Increment step counter for bias correction + let step = self + .optimizer_state + .get("step") + .and_then(|t| t.to_scalar::().ok()) + .unwrap_or(0.0) + + 1.0; + + let step_tensor = Tensor::new(&[step as f32], &Device::Cpu)?; + self.optimizer_state.insert("step".to_string(), step_tensor); + + // Bias correction terms + let beta1_t = beta1.powf(step as f32); + let beta2_t = beta2.powf(step as f32); + let bias_correction1 = 1.0 - beta1_t; + let bias_correction2 = 1.0 - beta2_t; + + // Collect gradients first to avoid borrow checker issues + let a_grad = self.gradients.get("A").cloned(); + let b_grad = self.gradients.get("B").cloned(); + let c_grad = self.gradients.get("C").cloned(); + let delta_grad = self.gradients.get("delta").cloned(); + + // Apply Adam updates to all SSM parameters + let num_layers = self.state.ssm_states.len(); + for layer_idx in 0..num_layers { + // Update A matrix (state transition matrix) + if let Some(ref A_grad) = a_grad { + let mut A_param = self.state.ssm_states[layer_idx].A.clone(); + self.apply_adam_update( + &mut A_param, + A_grad, + layer_idx, + "A", + lr, + beta1 as f64, + beta2 as f64, + eps, + bias_correction1 as f64, + bias_correction2 as f64, + false, // No weight decay for A matrix (maintains stability) + )?; + self.state.ssm_states[layer_idx].A = A_param; + } + + // Update B matrix (input matrix) + if let Some(ref B_grad) = b_grad { + let mut B_param = self.state.ssm_states[layer_idx].B.clone(); + self.apply_adam_update( + &mut B_param, + B_grad, + layer_idx, + "B", + lr, + beta1 as f64, + beta2 as f64, + eps, + bias_correction1 as f64, + bias_correction2 as f64, + true, // Apply weight decay to B matrix + )?; + self.state.ssm_states[layer_idx].B = B_param; + } + + // Update C matrix (output matrix) + if let Some(ref C_grad) = c_grad { + let mut C_param = self.state.ssm_states[layer_idx].C.clone(); + self.apply_adam_update( + &mut C_param, + C_grad, + layer_idx, + "C", + lr, + beta1 as f64, + beta2 as f64, + eps, + bias_correction1 as f64, + bias_correction2 as f64, + true, // Apply weight decay to C matrix + )?; + self.state.ssm_states[layer_idx].C = C_param; + } + + // Update Delta parameter (discretization parameter) + if let Some(ref delta_grad) = delta_grad { + let mut delta_param = self.state.ssm_states[layer_idx].delta.clone(); + self.apply_adam_update( + &mut delta_param, + delta_grad, + layer_idx, + "delta", + lr, + beta1 as f64, + beta2 as f64, + eps, + bias_correction1 as f64, + bias_correction2 as f64, + false, // No weight decay for Delta (maintains discretization stability) + )?; + self.state.ssm_states[layer_idx].delta = delta_param; + } + } + + // After updating A matrices, project to maintain spectral radius < 1 + self.project_ssm_matrices()?; + + Ok(()) + } + + /// Update learning rate with warmup and decay + fn update_learning_rate(&mut self, epoch: usize, batch_idx: usize) -> Result<(), MLError> { + let total_steps = + epoch * (1000 / self.config.batch_size) + (batch_idx / self.config.batch_size); + + let _lr = if total_steps < self.config.warmup_steps { + // Linear warmup + self.config.learning_rate * (total_steps as f64 / self.config.warmup_steps as f64) + } else { + // Cosine decay + let progress = (total_steps - self.config.warmup_steps) as f64; + let total_decay_steps = 10000.0; // Total training steps + let decay_ratio = (progress / total_decay_steps).min(1.0); + self.config.learning_rate * 0.5 * (1.0 + (std::f64::consts::PI * decay_ratio).cos()) + }; + + // Update learning rate in optimizer + // In practice, this would update the candle optimizer + + Ok(()) + } + + /// Get current learning rate + fn get_current_learning_rate(&self) -> f64 { + // Return current learning rate from optimizer + self.config.learning_rate // Simplified + } + + /// Validate model on validation set + fn validate(&mut self, val_data: &[(Tensor, Tensor)]) -> Result { + let mut total_loss = 0.0; + let mut count = 0; + + // Disable dropout for validation + for (input, target) in val_data { + let output = self.forward(input)?; + let loss = self.compute_loss(&output, target)?; + total_loss += loss.to_scalar::()? as f64; + count += 1; + + if count >= 100 { + // Limit validation set size for speed + break; + } + } + + Ok(total_loss / count as f64) + } + + /// Calculate accuracy metric + fn calculate_accuracy(&mut self, val_data: &[(Tensor, Tensor)]) -> Result { + let mut correct = 0; + let mut total = 0; + + for (input, target) in val_data { + let output = self.forward(input)?; + + // For regression, use relative error as accuracy metric + let error = ((output.to_scalar::()? - target.to_scalar::()?) + / target.to_scalar::()?) + .abs(); + if error < 0.1 { + // Within 10% is considered "correct" + correct += 1; + } + total += 1; + + if total >= 100 { + break; + } + } + + Ok(correct as f64 / total as f64) + } + + /// Check for early stopping + fn should_early_stop(&self, history: &[TrainingEpoch]) -> bool { + if history.len() < 5 { + return false; + } + + // Check if validation loss has stopped improving + let recent_losses: Vec = history + .iter() + .rev() + .take(5) + .map(|epoch| epoch.loss) + .collect(); + + let min_recent = recent_losses.iter().fold(f64::INFINITY, |a, &b| a.min(b)); + let max_recent = recent_losses + .iter() + .fold(f64::NEG_INFINITY, |a, &b| a.max(b)); + + // Stop if loss variation is very small + (max_recent - min_recent) < 1e-6 + } + + /// Save model checkpoint + pub async fn save_checkpoint(&mut self, path: &str) -> Result<(), MLError> { + info!("Saving checkpoint to {}", path); + + // Update metadata + self.metadata.last_checkpoint = Some(path.to_string()); + self.metadata.performance_stats = self.get_performance_metrics(); + + // In real implementation, would serialize all model parameters + // For now, just log the checkpoint + debug!( + "Checkpoint saved with {} parameters", + self.metadata.num_parameters + ); + + Ok(()) + } + + /// Load model checkpoint + pub async fn load_checkpoint(&mut self, path: &str) -> Result<(), MLError> { + info!("Loading checkpoint from {}", path); + + // In real implementation, would deserialize and load all parameters + self.is_trained = true; + self.metadata.last_checkpoint = Some(path.to_string()); + + Ok(()) + } + + /// Apply gradient clipping to prevent exploding gradients + fn clip_gradients(&mut self, max_norm: f64) -> Result<(), MLError> { + if max_norm <= 0.0 { + return Ok(()); + } + + let mut total_norm_squared = 0.0_f64; + + // Calculate total gradient norm across all SSM parameters + for _ssm_state in &self.state.ssm_states { + if let Some(A_grad) = self.gradients.get("A") { + let grad_norm_sq = A_grad.powf(2.0)?.sum_all()?.to_scalar::()? as f64; + total_norm_squared += grad_norm_sq; + } + if let Some(B_grad) = self.gradients.get("B") { + let grad_norm_sq = B_grad.powf(2.0)?.sum_all()?.to_scalar::()? as f64; + total_norm_squared += grad_norm_sq; + } + if let Some(C_grad) = self.gradients.get("C") { + let grad_norm_sq = C_grad.powf(2.0)?.sum_all()?.to_scalar::()? as f64; + total_norm_squared += grad_norm_sq; + } + if let Some(delta_grad) = self.gradients.get("delta") { + let grad_norm_sq = delta_grad.powf(2.0)?.sum_all()?.to_scalar::()? as f64; + total_norm_squared += grad_norm_sq; + } + } + + let total_norm = total_norm_squared.sqrt(); + + // Clip gradients if necessary + if total_norm > max_norm { + let clip_factor = (max_norm / total_norm) as f32; + let clip_scalar = Tensor::new(&[clip_factor], &Device::Cpu)?; + + // Apply clipping to all gradients + for _ssm_state in &mut self.state.ssm_states { + if let Some(A_grad) = self.gradients.get("A") { + let _clipped_grad = A_grad.mul(&clip_scalar)?; + // Note: In real candle implementation, we'd set the gradient directly + } + if let Some(B_grad) = self.gradients.get("B") { + let _clipped_grad = B_grad.mul(&clip_scalar)?; + // Note: In real candle implementation, we'd set the gradient directly + } + if let Some(C_grad) = self.gradients.get("C") { + let _clipped_grad = C_grad.mul(&clip_scalar)?; + // Note: In real candle implementation, we'd set the gradient directly + } + if let Some(delta_grad) = self.gradients.get("delta") { + let _clipped_grad = delta_grad.mul(&clip_scalar)?; + // Note: In real candle implementation, we'd set the gradient directly + } + } + } + + Ok(()) + } + + /// Apply Adam optimizer update to a single parameter + fn apply_adam_update( + &mut self, + param: &mut Tensor, + grad: &Tensor, + layer_idx: usize, + param_name: &str, + lr: f64, + beta1: f64, + beta2: f64, + eps: f64, + bias_correction1: f64, + bias_correction2: f64, + apply_weight_decay: bool, + ) -> Result<(), MLError> { + // Create unique keys for momentum and variance + let m_key = format!( + "layer_{}_{}_{}_m", + layer_idx, + param_name, + param.dims().len() + ); + let v_key = format!( + "layer_{}_{}_{}_v", + layer_idx, + param_name, + param.dims().len() + ); + + // Initialize momentum and variance if not present + if !self.optimizer_state.contains_key(&m_key) { + let m_init = grad.zeros_like()?; + let v_init = grad.zeros_like()?; + self.optimizer_state.insert(m_key.clone(), m_init); + self.optimizer_state.insert(v_key.clone(), v_init); + } + + // Get momentum and variance tensors separately to avoid double borrow + let m_tensor = self + .optimizer_state + .get(&m_key) + .ok_or_else(|| { + MLError::ModelError(format!("Missing momentum tensor for key: {}", m_key)) + })? + .clone(); + let v_tensor = self + .optimizer_state + .get(&v_key) + .ok_or_else(|| { + MLError::ModelError(format!("Missing variance tensor for key: {}", v_key)) + })? + .clone(); + + // Apply weight decay if specified + let effective_grad = if apply_weight_decay && self.config.weight_decay > 0.0 { + let weight_decay_term = param.mul(&Tensor::new( + &[self.config.weight_decay as f32], + &Device::Cpu, + )?)?; + grad.add(&weight_decay_term)? + } else { + grad.clone() + }; + + // Update biased first moment estimate: m_t = β1 * m_{t-1} + (1 - β1) * g_t + let beta1_tensor = Tensor::new(&[beta1 as f32], &Device::Cpu)?; + let one_minus_beta1 = Tensor::new(&[(1.0 - beta1) as f32], &Device::Cpu)?; + let new_m = m_tensor + .mul(&beta1_tensor)? + .add(&effective_grad.mul(&one_minus_beta1)?)?; + + // Update biased second moment estimate: v_t = β2 * v_{t-1} + (1 - β2) * g_t^2 + let beta2_tensor = Tensor::new(&[beta2 as f32], &Device::Cpu)?; + let one_minus_beta2 = Tensor::new(&[(1.0 - beta2) as f32], &Device::Cpu)?; + let grad_squared = effective_grad.mul(&effective_grad)?; + let new_v = v_tensor + .mul(&beta2_tensor)? + .add(&grad_squared.mul(&one_minus_beta2)?)?; + + // Compute bias-corrected estimates + let bias_correction1_tensor = Tensor::new(&[bias_correction1 as f32], &Device::Cpu)?; + let bias_correction2_tensor = Tensor::new(&[bias_correction2 as f32], &Device::Cpu)?; + let m_hat = new_m.div(&bias_correction1_tensor)?; + let v_hat = new_v.div(&bias_correction2_tensor)?; + + // Compute parameter update: θ = θ - lr * m_hat / (√(v_hat) + ε) + let eps_tensor = Tensor::new(&[eps as f32], &Device::Cpu)?; + let lr_tensor = Tensor::new(&[lr as f32], &Device::Cpu)?; + let sqrt_v_hat = v_hat.sqrt()?; + let denominator = sqrt_v_hat.add(&eps_tensor)?; + let update = m_hat.div(&denominator)?.mul(&lr_tensor)?; + + // Update parameter: θ_{t+1} = θ_t - update + *param = param.sub(&update)?; + + // Store updated momentum and variance back + self.optimizer_state.insert(m_key, new_m); + self.optimizer_state.insert(v_key, new_v); + + Ok(()) + } + + /// Project SSM matrices to maintain stability + fn project_ssm_matrices(&mut self) -> Result<(), MLError> { + // Avoid borrow checker issues by processing each state separately + for i in 0..self.state.ssm_states.len() { + // Ensure A matrix has spectral radius < 1 for stability + let spectral_radius = { + let ssm_state = &self.state.ssm_states[i]; + self.compute_spectral_radius(&ssm_state.A)? + }; + if spectral_radius >= 1.0 { + let scale_factor = 0.99 / spectral_radius; + self.state.ssm_states[i].A = self.state.ssm_states[i] + .A + .mul(&Tensor::new(&[scale_factor as f32], &Device::Cpu)?)?; + } + + // Ensure Delta parameter stays positive and reasonable + // Apply softplus-like projection: delta = log(1 + exp(delta_raw)) + let delta_min = Tensor::new(&[1e-6_f32], &Device::Cpu)?; + let delta_max = Tensor::new(&[1.0_f32], &Device::Cpu)?; + self.state.ssm_states[i].delta = self.state.ssm_states[i] + .delta + .clamp(&delta_min, &delta_max)?; + } + + Ok(()) + } + + /// Compute spectral radius (largest eigenvalue magnitude) of a matrix + fn compute_spectral_radius(&self, matrix: &Tensor) -> Result { + // For simplicity, use Frobenius norm as approximation + // In production, we'd compute actual eigenvalues + let frobenius_norm = matrix.powf(2.0)?.sum_all()?.to_scalar::()?.sqrt(); + + // Frobenius norm upper bounds spectral radius + // For better approximation, we scale by sqrt of matrix size + let dims = matrix.dims(); + if dims.len() >= 2 { + let size = (dims[0].min(dims[1]) as f32).sqrt(); + Ok((frobenius_norm / size) as f64) + } else { + Ok(frobenius_norm as f64) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use anyhow::Result; + + #[tokio::test] + async fn test_mamba_creation() -> Result<()> { + let config = Mamba2Config { + d_model: 8, + d_state: 4, + d_head: 4, + num_heads: 2, + ..Default::default() + }; + + let model = + Mamba2SSM::new(config).map_err(|_| anyhow::anyhow!("Failed to create MAMBA model"))?; + assert_eq!(model.metadata.input_dim, 8); + assert_eq!(model.metadata.output_dim, 1); + Ok(()) + } + + #[test] + fn test_mamba_config_default() -> Result<()> { + let config = Mamba2Config::default(); + assert!(config.d_model > 0); + assert!(config.d_state > 0); + assert!(config.num_heads > 0); + Ok(()) + } + + #[test] + fn test_mamba_state_creation() -> Result<()> { + let config = Mamba2Config { + d_model: 4, + d_state: 2, + d_head: 2, + num_heads: 2, + ..Default::default() + }; + + let state = Mamba2State::zeros(&config) + .map_err(|_| anyhow::anyhow!("Failed to create MAMBA state"))?; + assert_eq!(state.ssm_states.len(), config.num_layers); + assert!(!state.selective_state.is_empty()); + Ok(()) + } + + #[test] + fn test_mamba_performance_metrics() -> Result<()> { + let config = Mamba2Config { + d_model: 4, + target_latency_us: 5, + ..Default::default() + }; + + let model = + Mamba2SSM::new(config).map_err(|_| anyhow::anyhow!("Failed to create MAMBA model"))?; + let metrics = model.get_performance_metrics(); + + assert!(metrics.contains_key("total_inferences")); + assert!(metrics.contains_key("model_parameters")); + assert!(metrics.contains_key("compression_ratio")); + Ok(()) + } + + #[test] + fn test_mamba_hft_config() -> Result<()> { + let model = Mamba2SSM::default_hft() + .map_err(|_| anyhow::anyhow!("Failed to create HFT MAMBA model"))?; + assert_eq!(model.config.target_latency_us, 3); + assert!(model.config.hardware_aware); + assert!(model.config.use_ssd); + assert!(model.config.use_selective_state); + Ok(()) + } +} + +#[test] +fn test_mamba_parameter_count() -> anyhow::Result<()> { + let config = Mamba2Config { + d_model: 8, + num_layers: 2, + ..Default::default() + }; + + let param_count = Mamba2SSM::count_parameters(&config); + assert!(param_count > 0); + Ok(()) +} diff --git a/ml/src/microstructure/mod.rs b/ml/src/microstructure/mod.rs index 72ed936ec..a0e508b81 100644 --- a/ml/src/microstructure/mod.rs +++ b/ml/src/microstructure/mod.rs @@ -43,7 +43,7 @@ pub mod vpin_implementation; // Re-export VPIN types for public API -pub use vpin_implementation::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites MarketDataUpdate, TradeDirection, VPINCalculator, VPINConfig, VPINMetrics, VPINPerformanceMetrics, }; diff --git a/ml/src/microstructure/mod.rs.bak b/ml/src/microstructure/mod.rs.bak new file mode 100644 index 000000000..72ed936ec --- /dev/null +++ b/ml/src/microstructure/mod.rs.bak @@ -0,0 +1,112 @@ +//! # Advanced Microstructure ML Module +//! +//! Comprehensive machine learning enhanced market microstructure analysis for +//! high-frequency trading alpha generation. All components target <25μs latency +//! with advanced ML models for superior prediction accuracy. +//! +//! ## Core ML Models (7 Advanced Models) +//! +//! - **Order Flow Imbalance Predictor**: LSTM-Transformer hybrid for OFI prediction +//! - **Liquidity Provision Optimizer**: Multi-output neural network for optimal liquidity provision +//! - **Spread Predictor**: Time series transformer for bid-ask spread forecasting +//! - **Market Impact Estimator**: Temporal CNN for price impact estimation +//! - **Adverse Selection Detector**: Deep learning model for toxicity detection +//! - **Price Discovery Model**: Information flow analysis and efficiency measurement +//! - **Hidden Liquidity Detector**: Pattern recognition for dark pools and icebergs +//! +//! ## Integration Components +//! +//! - **ML Ensemble**: Unified ensemble combining all 7 models for robust predictions +//! - **Training Pipeline**: Comprehensive training system with unified data providers +//! - **Portfolio Integration**: Seamless integration with Portfolio Transformer +//! - **Performance Optimization**: Sub-25μs inference with real-time deployment +//! +//! ## Classical Microstructure Analytics +//! +//! - **VPIN Calculator**: Volume-synchronized probability of informed trading +//! - **Kyle's Lambda**: Price impact measurement and information asymmetry detection +//! - **Amihud Illiquidity**: Liquidity measurement via price impact per volume +//! - **Roll Spread Estimator**: Bid-ask spread estimation from price autocovariance +//! - **Hasbrouck Information Share**: Price discovery attribution analysis +//! +//! ## Performance Targets +//! +//! - ML Inference latency: <25μs for ensemble predictions +//! - Classical calculation latency: <25μs for all metrics +//! - Throughput: 100K+ calculations/second +//! - Memory efficiency: Zero-allocation hot paths +//! - Integer arithmetic: 10,000x scaling for financial precision + +// use error_handling::{AppResult, ErrorSeverity, FoxhuntError}; // Commented out - crate doesn't exist + +// VPIN Implementation Module +pub mod vpin_implementation; + +// Re-export VPIN types for public API +pub use vpin_implementation::{ + MarketDataUpdate, TradeDirection, VPINCalculator, VPINConfig, VPINMetrics, + VPINPerformanceMetrics, +}; + +#[test] +fn test_trade_direction_classification() { + // Test Lee-Ready algorithm + let direction = TradeDirection::classify_lee_ready( + 105000, // trade price (10.50) + 104000, // bid (10.40) + 106000, // ask (10.60) + 104500, // prev price (10.45) + ); + assert_eq!(direction, TradeDirection::Buy); + + // Test tick rule + let direction = TradeDirection::classify_tick_rule(105000, 104000); + assert_eq!(direction, TradeDirection::Buy); +} + +// TODO: Fix test_volume_bucket - requires MarketDataUpdate struct definition +/* +#[test] +fn test_volume_bucket() { + let mut bucket = VolumeBucket::new(0, 1000, 1000000); + // Test implementation needed after MarketDataUpdate is defined +} +*/ + +#[test] +fn test_ring_buffer() { + let mut buffer = RingBuffer::new(3); + + buffer.push(1); + buffer.push(2); + buffer.push(3); + + assert_eq!(buffer.len(), 3); + assert_eq!(buffer.get(0), Some(&1)); + assert_eq!(buffer.get(1), Some(&2)); + assert_eq!(buffer.get(2), Some(&3)); + + buffer.push(4); + assert_eq!(buffer.len(), 3); + assert_eq!(buffer.get(0), Some(&2)); + assert_eq!(buffer.get(1), Some(&3)); + assert_eq!(buffer.get(2), Some(&4)); +} + +#[test] +fn test_utils_functions() { + let prices = vec![100000, 101000, 99000, 102000]; + let returns = utils::calculate_returns(&prices); + assert_eq!(returns.len(), 3); + + let values = vec![1000, 2000, 3000, 4000, 5000]; + let ma = utils::moving_average(&values, 3); + assert_eq!(ma.len(), 3); + assert_eq!(ma[0], 2000); // (1000 + 2000 + 3000) / 3 + + let cov = utils::autocovariance(&values, 1); + assert!(cov > 0); // Should be positive for trending series + + let sqrt_val = utils::fast_sqrt(10000); + assert_eq!(sqrt_val, 100); +} diff --git a/ml/src/models_demo.rs b/ml/src/models_demo.rs index 0d01a6584..2fd36053b 100644 --- a/ml/src/models_demo.rs +++ b/ml/src/models_demo.rs @@ -38,8 +38,7 @@ impl Default for ModelDemoConfig { } } -// Use canonical ModelType from crate root -pub use crate::ModelType; +// NO RE-EXPORTS - Use explicit imports: crate::ModelType /// Performance metrics for model demonstrations #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/ml/src/observability/mod.rs b/ml/src/observability/mod.rs index df8322951..f1b8bd2b6 100644 --- a/ml/src/observability/mod.rs +++ b/ml/src/observability/mod.rs @@ -7,11 +7,11 @@ pub mod alerts; pub mod dashboards; pub mod metrics; -pub use metrics::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites get_metrics_collector, initialize_metrics, record_inference_timing, AlertThresholds, MLMetricsCollector, MLMetricsReport, MLPerformanceMonitor, PerformanceHealthCheck, }; -pub use alerts::{Alert, AlertChannel, AlertManager, AlertRule, AlertSeverity}; +// DO NOT RE-EXPORT - Use explicit imports at usage sites -pub use dashboards::{DashboardConfig, DashboardWidget, MetricsDashboard, WidgetType}; +// DO NOT RE-EXPORT - Use explicit imports at usage sites diff --git a/ml/src/observability/mod.rs.bak b/ml/src/observability/mod.rs.bak new file mode 100644 index 000000000..df8322951 --- /dev/null +++ b/ml/src/observability/mod.rs.bak @@ -0,0 +1,17 @@ +//! Production observability and monitoring for ML systems +//! +//! This module provides comprehensive monitoring, metrics collection, and alerting +//! for ML models in production HFT environments. + +pub mod alerts; +pub mod dashboards; +pub mod metrics; + +pub use metrics::{ + get_metrics_collector, initialize_metrics, record_inference_timing, AlertThresholds, + MLMetricsCollector, MLMetricsReport, MLPerformanceMonitor, PerformanceHealthCheck, +}; + +pub use alerts::{Alert, AlertChannel, AlertManager, AlertRule, AlertSeverity}; + +pub use dashboards::{DashboardConfig, DashboardWidget, MetricsDashboard, WidgetType}; diff --git a/ml/src/ppo/mod.rs b/ml/src/ppo/mod.rs index f31720419..a46d591a5 100644 --- a/ml/src/ppo/mod.rs +++ b/ml/src/ppo/mod.rs @@ -16,11 +16,7 @@ pub mod trajectories; pub mod continuous_demo; // Re-export main components -pub use continuous_policy::{ContinuousAction, ContinuousPolicyConfig, ContinuousPolicyNetwork}; -pub use continuous_ppo::{ collect_continuous_trajectories, ContinuousPPO, ContinuousPPOConfig, ContinuousTrajectory, ContinuousTrajectoryBatch, ContinuousTrajectoryStep, }; -pub use gae::{compute_gae, GAEConfig}; -pub use ppo::{PPOConfig, PolicyNetwork, ValueNetwork, WorkingPPO}; -pub use trajectories::{collect_trajectories, Trajectory, TrajectoryBatch, TrajectoryStep}; +// DO NOT RE-EXPORT - Use explicit imports at usage sites diff --git a/ml/src/ppo/mod.rs.bak b/ml/src/ppo/mod.rs.bak new file mode 100644 index 000000000..f31720419 --- /dev/null +++ b/ml/src/ppo/mod.rs.bak @@ -0,0 +1,26 @@ +//! Proximal Policy Optimization (PPO) Implementation +//! +//! This module provides a complete PPO implementation with: +//! - Actor-Critic architecture with separate policy and value networks +//! - Generalized Advantage Estimation (GAE) +//! - Clipped surrogate objective +//! - Trajectory collection and processing +//! - Real mathematical operations using candle-core v0.9.1 + +pub mod continuous_policy; +pub mod continuous_ppo; +pub mod gae; +pub mod ppo; +pub mod trajectories; +// pub mod continuous_example; // Module file not found +pub mod continuous_demo; + +// Re-export main components +pub use continuous_policy::{ContinuousAction, ContinuousPolicyConfig, ContinuousPolicyNetwork}; +pub use continuous_ppo::{ + collect_continuous_trajectories, ContinuousPPO, ContinuousPPOConfig, ContinuousTrajectory, + ContinuousTrajectoryBatch, ContinuousTrajectoryStep, +}; +pub use gae::{compute_gae, GAEConfig}; +pub use ppo::{PPOConfig, PolicyNetwork, ValueNetwork, WorkingPPO}; +pub use trajectories::{collect_trajectories, Trajectory, TrajectoryBatch, TrajectoryStep}; diff --git a/ml/src/risk/mod.rs b/ml/src/risk/mod.rs index 1f0c2b564..02743c009 100644 --- a/ml/src/risk/mod.rs +++ b/ml/src/risk/mod.rs @@ -11,26 +11,24 @@ pub mod position_sizing; pub mod var_models; // Export types from modules that actually exist -pub use circuit_breakers::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites CircuitBreakerConfig, CircuitBreakerState, CircuitBreakerType, MLCircuitBreaker, MarketDataPoint, }; -pub use graph_risk_model::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites CreditRating, EdgeType, FinancialRiskGraph, GraphRiskModel, NodeType, RiskEdge, RiskNode, SystemicRiskIndicators, TGATConfig, TemporalGraphAttentionNetwork, }; -pub use kelly_optimizer::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites KellyCriterionOptimizer, KellyOptimizerConfig, KellyPositionRecommendation, }; -pub use kelly_position_sizing_service::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites EnhancedPositionSizingRecommendation, KellyPositionSizingService, KellyServiceConfig, KellyServiceMetrics, PositionSizingRequest, RiskTolerance, }; -pub use position_sizing::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites PositionSizingConfig, PositionSizingNetwork, PositionSizingRecommendation, }; -pub use common::trading::MarketRegime; -pub use var_models::{ FeatureScaler, LinearLayer, NeuralVarConfig, NeuralVarModel, StressTestResults, VarFeatures, VarPrediction, }; diff --git a/ml/src/risk/mod.rs.bak b/ml/src/risk/mod.rs.bak new file mode 100644 index 000000000..1f0c2b564 --- /dev/null +++ b/ml/src/risk/mod.rs.bak @@ -0,0 +1,218 @@ +//! # Neural Risk Management System +//! +//! Advanced ML-driven risk management with neural VaR models, Kelly criterion optimization, +//! and real-time regime detection for production HFT systems using canonical types. + +pub mod circuit_breakers; +pub mod graph_risk_model; +pub mod kelly_optimizer; +pub mod kelly_position_sizing_service; +pub mod position_sizing; +pub mod var_models; + +// Export types from modules that actually exist +pub use circuit_breakers::{ + CircuitBreakerConfig, CircuitBreakerState, CircuitBreakerType, MLCircuitBreaker, + MarketDataPoint, +}; +pub use graph_risk_model::{ + CreditRating, EdgeType, FinancialRiskGraph, GraphRiskModel, NodeType, RiskEdge, RiskNode, + SystemicRiskIndicators, TGATConfig, TemporalGraphAttentionNetwork, +}; +pub use kelly_optimizer::{ + KellyCriterionOptimizer, KellyOptimizerConfig, KellyPositionRecommendation, +}; +pub use kelly_position_sizing_service::{ + EnhancedPositionSizingRecommendation, KellyPositionSizingService, KellyServiceConfig, + KellyServiceMetrics, PositionSizingRequest, RiskTolerance, +}; +pub use position_sizing::{ + PositionSizingConfig, PositionSizingNetwork, PositionSizingRecommendation, +}; +pub use common::trading::MarketRegime; +pub use var_models::{ + FeatureScaler, LinearLayer, NeuralVarConfig, NeuralVarModel, StressTestResults, VarFeatures, + VarPrediction, +}; + +use std::collections::HashMap; + +use chrono::{DateTime, Utc}; +use ndarray::Array2; +use serde::{Deserialize, Serialize}; + +use crate::{MLResult, Price, Volume}; + +// Create temporary AssetId type - will be replaced with proper import later +#[derive(Debug, Clone, Serialize, Deserialize, Hash, PartialEq, Eq)] +pub struct AssetId(String); + +/// Risk assessment levels +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, PartialOrd)] +/// RiskLevel component. +pub enum RiskLevel { + VeryLow, + Low, + Medium, + High, + VeryHigh, + Critical, +} + +/// Portfolio risk profile +#[derive(Debug, Clone, Serialize, Deserialize)] +/// RiskProfile component. +pub struct RiskProfile { + pub total_var_95: f64, // 1-day 95% VaR + pub total_var_99: f64, // 1-day 99% VaR + pub expected_shortfall: f64, // Expected tail loss + pub maximum_drawdown: f64, // Historical maximum drawdown + pub current_drawdown: f64, // Current drawdown from peak + pub sharpe_ratio: f64, // Risk-adjusted return + pub sortino_ratio: f64, // Downside risk-adjusted return + pub calmar_ratio: f64, // Return over maximum drawdown + pub beta: f64, // Market beta + pub tracking_error: f64, // Volatility vs benchmark + pub concentration_risk: f64, // Single position concentration + pub correlation_risk: f64, // Average correlation risk + pub liquidity_risk: f64, // Liquidity-adjusted risk + pub regime_risk: f64, // Market regime risk factor + pub overall_risk_score: f64, // Combined ML risk score (0-1) + pub risk_level: RiskLevel, // Categorical risk assessment + pub timestamp: DateTime, +} + +/// Position-level risk metrics +#[derive(Debug, Clone, Serialize, Deserialize)] +/// PositionRisk component. +pub struct PositionRisk { + pub asset_id: AssetId, + pub position_size: f64, // Current position size + pub market_value: f64, // Current market value + pub var_contribution: f64, // Contribution to portfolio VaR + pub marginal_var: f64, // Marginal VaR (change in portfolio VaR) + pub component_var: f64, // Component VaR (allocation of portfolio VaR) + pub standalone_var: f64, // Position VaR in isolation + pub beta_to_portfolio: f64, // Beta relative to portfolio + pub correlation_risk: f64, // Correlation with other positions + pub liquidity_horizon: f64, // Days to liquidate position + pub concentration_weight: f64, // Weight in portfolio + pub stress_loss: f64, // Loss under stress scenarios + pub kelly_optimal_size: f64, // Kelly optimal position size + pub recommended_size: f64, // ML recommended position size + pub risk_score: f64, // Individual risk score (0-1) + pub timestamp: DateTime, +} + +/// `Market` data for risk calculations +#[derive(Debug, Clone, Serialize, Deserialize)] +/// MarketData component. +pub struct MarketData { + pub timestamp: DateTime, + pub prices: HashMap, + pub volumes: HashMap, + pub volatilities: HashMap, + pub correlations: Array2, + pub market_cap_weights: HashMap, + pub sector_exposures: HashMap, +} + +/// Risk limits and constraints +#[derive(Debug, Clone, Serialize, Deserialize)] +/// RiskLimits component. +pub struct RiskLimits { + pub max_portfolio_var: f64, // Maximum portfolio VaR + pub max_position_size: f64, // Maximum single position size + pub max_sector_exposure: f64, // Maximum sector exposure + pub max_correlation: f64, // Maximum position correlation + pub max_drawdown: f64, // Maximum allowed drawdown + pub min_liquidity_days: f64, // Minimum liquidity (days to exit) + pub max_leverage: f64, // Maximum portfolio leverage + pub var_limit_buffer: f64, // VaR limit buffer (e.g., 0.8 of limit) +} + +impl Default for RiskLimits { + fn default() -> Self { + Self { + max_portfolio_var: 0.02, // 2% daily VaR limit + max_position_size: 0.05, // 5% maximum position size + max_sector_exposure: 0.20, // 20% maximum sector exposure + max_correlation: 0.70, // 70% maximum correlation + max_drawdown: 0.10, // 10% maximum drawdown + min_liquidity_days: 2.0, // 2 days maximum liquidation time + max_leverage: 3.0, // 3x maximum leverage + var_limit_buffer: 0.80, // Use 80% of VaR limit + } + } +} + +/// Comprehensive risk configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +/// RiskConfig component. +pub struct RiskConfig { + pub var_confidence_levels: Vec, + pub lookback_days: usize, + pub monte_carlo_simulations: usize, + pub stress_scenarios: usize, + pub regime_detection_window: usize, + pub update_frequency_seconds: u32, + pub enable_neural_var: bool, + pub enable_regime_detection: bool, + pub enable_ml_position_sizing: bool, + pub enable_dynamic_hedging: bool, + pub risk_limits: RiskLimits, +} + +impl Default for RiskConfig { + fn default() -> Self { + Self { + var_confidence_levels: vec![0.95, 0.99], + lookback_days: 252, + monte_carlo_simulations: 10_000, + stress_scenarios: 1_000, + regime_detection_window: 60, + update_frequency_seconds: 60, + enable_neural_var: true, + enable_regime_detection: true, + enable_ml_position_sizing: true, + enable_dynamic_hedging: true, + risk_limits: RiskLimits::default(), + } + } +} + +/// Main neural risk management system +pub struct NeuralRiskManager { + config: RiskConfig, + var_model: NeuralVarModel, + kelly_optimizer: KellyCriterionOptimizer, + position_sizer: PositionSizingNetwork, + circuit_breaker: MLCircuitBreaker, +} + +impl NeuralRiskManager { + /// Create new neural risk management system + pub fn new(config: RiskConfig) -> MLResult { + let var_config = NeuralVarConfig { + confidence_levels: config.var_confidence_levels.clone(), + lookback_days: config.lookback_days, + lookback_period: config.lookback_days, + monte_carlo_simulations: config.monte_carlo_simulations, + enable_stress_testing: true, + hidden_layers: vec![128, 64, 32], + }; + + let var_model = NeuralVarModel::new(var_config)?; + let kelly_optimizer = KellyCriterionOptimizer::new(Default::default())?; + let position_sizer = PositionSizingNetwork::new(Default::default())?; + let circuit_breaker = MLCircuitBreaker::new(Default::default())?; + + Ok(Self { + config, + var_model, + kelly_optimizer, + position_sizer, + circuit_breaker, + }) + } +} diff --git a/ml/src/risk/monitor.rs b/ml/src/risk/monitor.rs index 18bba2398..a01b599ca 100644 --- a/ml/src/risk/monitor.rs +++ b/ml/src/risk/monitor.rs @@ -36,8 +36,7 @@ impl Default for MonitorConfig { } } -// NO DUPLICATES - SINGLE TYPE SYSTEM -pub use common::Position; +// NO RE-EXPORTS - Use explicit imports: common::Position /// Exposure metrics #[derive(Debug, Clone)] diff --git a/ml/src/safety/mod.rs b/ml/src/safety/mod.rs index d3417b0f5..b9e13b1e5 100644 --- a/ml/src/safety/mod.rs +++ b/ml/src/safety/mod.rs @@ -32,7 +32,7 @@ pub mod timeout_manager; use bounds_checker::BoundsChecker; use drift_detector::ModelDriftDetector; use financial_validator::FinancialValidator; -pub use gradient_safety::{GradientSafetyConfig, GradientSafetyManager, GradientStatistics}; +// DO NOT RE-EXPORT - Use explicit imports at usage sites use math_ops::SafeMathOps; use memory_manager::SafeMemoryManager; use tensor_ops::SafeTensorOps; diff --git a/ml/src/safety/mod.rs.bak b/ml/src/safety/mod.rs.bak new file mode 100644 index 000000000..d3417b0f5 --- /dev/null +++ b/ml/src/safety/mod.rs.bak @@ -0,0 +1,638 @@ +//! Comprehensive ML Safety Framework +//! +//! This module provides enterprise-grade safety controls for all ML operations +//! in the Foxhunt trading system. All ML components MUST use these safety +//! controls to prevent trading system failures. + +use common::Price; +use std; + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use candle_core::{Device, Result as CandleResult, Tensor}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use tokio::sync::RwLock; +use tracing::{debug, error, info, warn}; + +// IntegerPrice replaced with common::Price + +// Re-export safety modules +pub mod bounds_checker; +pub mod drift_detector; +pub mod financial_validator; +pub mod gradient_safety; +pub mod math_ops; +pub mod memory_manager; +pub mod tensor_ops; +pub mod timeout_manager; + +use bounds_checker::BoundsChecker; +use drift_detector::ModelDriftDetector; +use financial_validator::FinancialValidator; +pub use gradient_safety::{GradientSafetyConfig, GradientSafetyManager, GradientStatistics}; +use math_ops::SafeMathOps; +use memory_manager::SafeMemoryManager; +use tensor_ops::SafeTensorOps; +use timeout_manager::TimeoutManager; + +/// Global safety configuration for all ML operations +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MLSafetyConfig { + /// Enable comprehensive safety checks (never disable in production) + pub safety_enabled: bool, + /// Maximum tensor size in elements (prevent OOM) + pub max_tensor_elements: usize, + /// Maximum model inference timeout in milliseconds + pub max_inference_timeout_ms: u64, + /// Maximum GPU memory per operation in bytes + pub max_gpu_memory_bytes: usize, + /// Drift detection sensitivity (0.0 = disabled, 1.0 = highest) + pub drift_sensitivity: f64, + /// Financial validation precision (decimal places) + pub financial_precision: u8, + /// Enable NaN/Infinity checks for all operations + pub nan_infinity_checks: bool, + /// Maximum allowed model prediction value + pub max_prediction_value: f64, + /// Minimum allowed model prediction value + pub min_prediction_value: f64, + /// Enable bounds checking for all array/tensor operations + pub bounds_checking: bool, + /// Enable automatic fallback mechanisms + pub auto_fallback: bool, + /// Maximum number of retries for failed operations + pub max_retries: u32, +} + +impl Default for MLSafetyConfig { + fn default() -> Self { + Self { + safety_enabled: true, + max_tensor_elements: 100_000_000, // 100M elements + max_inference_timeout_ms: 5000, // 5 seconds + max_gpu_memory_bytes: 8 * 1024 * 1024 * 1024, // 8GB + drift_sensitivity: 0.7, + financial_precision: 6, + nan_infinity_checks: true, + max_prediction_value: 1e6, + min_prediction_value: -1e6, + bounds_checking: true, + auto_fallback: true, + max_retries: 3, + } + } +} + +/// Safety errors for ML operations +#[derive(Error, Debug)] +pub enum MLSafetyError { + #[error("Mathematical safety violation: {reason}")] + MathSafety { reason: String }, + + #[error("Tensor safety violation: {reason}")] + TensorSafety { reason: String }, + + #[error("Financial validation failed: {reason}")] + FinancialValidation { reason: String }, + + #[error("Bounds check failed: index {index} >= length {length}")] + BoundsCheck { index: usize, length: usize }, + + #[error("Memory safety violation: {reason}")] + MemorySafety { reason: String }, + + #[error("Timeout exceeded: {timeout_ms}ms")] + Timeout { timeout_ms: u64 }, + + #[error("Model drift detected: {drift_score:.3} > threshold {threshold:.3}")] + ModelDrift { drift_score: f64, threshold: f64 }, + + #[error("GPU operation failed: {reason}")] + GPUFailure { reason: String }, + + #[error("NaN or Infinity detected in operation: {operation}")] + InvalidFloat { operation: String }, + + #[error("Prediction value out of bounds: {value} not in [{min}, {max}]")] + PredictionOutOfBounds { value: f64, min: f64, max: f64 }, + + #[error("Resource unavailable: {resource}")] + ResourceUnavailable { resource: String }, + + #[error("Candle framework error: {0}")] + CandleError(#[from] candle_core::Error), + + #[error("System resource exhausted: {resource}")] + ResourceExhausted { resource: String }, + + #[error("Validation error: {message}")] + ValidationError { message: String }, +} + +// Implement From for MLSafetyError +impl From for MLSafetyError { + fn from(err: crate::training_pipeline::ProductionTrainingError) -> Self { + match err { + crate::training_pipeline::ProductionTrainingError::ConfigError { reason } => { + MLSafetyError::ValidationError { + message: format!("Config error: {}", reason), + } + } + crate::training_pipeline::ProductionTrainingError::ArchitectureError { reason } => { + MLSafetyError::ValidationError { + message: format!("Architecture error: {}", reason), + } + } + crate::training_pipeline::ProductionTrainingError::DataError { reason } => { + MLSafetyError::ValidationError { + message: format!("Data error: {}", reason), + } + } + crate::training_pipeline::ProductionTrainingError::OptimizationError { reason } => { + MLSafetyError::ValidationError { + message: format!("Optimization error: {}", reason), + } + } + crate::training_pipeline::ProductionTrainingError::FinancialError { reason } => { + MLSafetyError::FinancialValidation { reason } + } + crate::training_pipeline::ProductionTrainingError::SafetyViolation { reason } => { + MLSafetyError::ValidationError { + message: format!("Safety violation: {}", reason), + } + } + crate::training_pipeline::ProductionTrainingError::ConvergenceError { reason } => { + MLSafetyError::ValidationError { + message: format!("Convergence error: {}", reason), + } + } + crate::training_pipeline::ProductionTrainingError::ResourceError { reason } => { + MLSafetyError::ResourceUnavailable { resource: reason } + } + crate::training_pipeline::ProductionTrainingError::GpuRequired { reason } => { + MLSafetyError::ResourceUnavailable { + resource: format!("GPU: {}", reason), + } + } + } + } +} + +pub type SafetyResult = Result; + +/// Safety status for operations +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum SafetyStatus { + Safe, + Warning { reason: String }, + Danger { reason: String }, + Critical { reason: String }, +} + +/// Comprehensive ML Safety Manager +/// +/// This is the central safety coordinator for all ML operations. +/// ALL ML operations MUST go through this manager. +pub struct MLSafetyManager { + config: MLSafetyConfig, + math_ops: SafeMathOps, + tensor_ops: SafeTensorOps, + bounds_checker: BoundsChecker, + memory_manager: Arc>, + drift_detector: Arc>, + financial_validator: FinancialValidator, + timeout_manager: TimeoutManager, + operation_history: Arc>>>, +} + +impl MLSafetyManager { + /// Create a new ML safety manager with configuration + pub fn new(config: MLSafetyConfig) -> Self { + Self { + math_ops: SafeMathOps::new(&config), + tensor_ops: SafeTensorOps::new(&config), + bounds_checker: BoundsChecker::new(&config), + memory_manager: Arc::new(RwLock::new(SafeMemoryManager::new(&config))), + drift_detector: Arc::new(RwLock::new(ModelDriftDetector::new(&config))), + financial_validator: FinancialValidator::new(&config), + timeout_manager: TimeoutManager::new(&config), + operation_history: Arc::new(RwLock::new(HashMap::new())), + config, + } + } + + /// Validate and execute a mathematical operation safely + pub async fn safe_math_operation( + &self, + operation_name: &str, + operation: F, + ) -> SafetyResult + where + F: FnOnce() -> SafetyResult + Send + 'static, + T: Send + 'static, + { + if !self.config.safety_enabled { + return operation(); + } + + // Record operation start time + self.record_operation_start(operation_name).await; + + // Execute with timeout + let result = self + .timeout_manager + .execute_with_timeout( + operation_name, + operation, + Duration::from_millis(self.config.max_inference_timeout_ms), + ) + .await?; + + // Validate result + self.validate_operation_result(operation_name, &result) + .await?; + + Ok(result) + } + + /// Safely create and validate a tensor + pub async fn safe_tensor_create( + &self, + data: Vec, + shape: &[usize], + device: &Device, + operation_context: &str, + ) -> SafetyResult { + if !self.config.safety_enabled { + return Ok(Tensor::from_vec(data, shape, device)?); + } + + // Validate tensor size + let total_elements: usize = shape.iter().product(); + if total_elements > self.config.max_tensor_elements { + return Err(MLSafetyError::TensorSafety { + reason: format!( + "Tensor too large: {} elements > {} limit in {}", + total_elements, self.config.max_tensor_elements, operation_context + ), + }); + } + + // Validate data length matches shape + if data.len() != total_elements { + return Err(MLSafetyError::TensorSafety { + reason: format!( + "Data length {} doesn't match shape size {} in {}", + data.len(), + total_elements, + operation_context + ), + }); + } + + // Check for NaN/Infinity in data + if self.config.nan_infinity_checks { + for (i, &value) in data.iter().enumerate() { + if !value.is_finite() { + return Err(MLSafetyError::InvalidFloat { + operation: format!( + "{}: Non-finite value {} at index {}", + operation_context, value, i + ), + }); + } + } + } + + // Check memory availability + let mut memory_manager = self.memory_manager.write().await; + memory_manager.check_memory_availability(total_elements * 8, device)?; // 8 bytes per f64 + drop(memory_manager); + + // Create tensor safely + self.tensor_ops.safe_from_vec(data, shape, device).await + } + + /// Safely perform tensor operations with bounds checking + pub async fn safe_tensor_operation( + &self, + operation_name: &str, + tensor: &Tensor, + operation: F, + ) -> SafetyResult + where + F: FnOnce(&Tensor) -> CandleResult + Send, + T: Send, + { + if !self.config.safety_enabled { + return operation(tensor).map_err(MLSafetyError::CandleError); + } + + // Validate input tensor + self.tensor_ops + .validate_tensor(tensor, operation_name) + .await?; + + // Execute operation directly to avoid lifetime issues with closures + let result = operation(tensor).map_err(MLSafetyError::CandleError)?; + + Ok(result) + } + + /// Validate financial values and convert to safe types + pub async fn validate_financial_prediction( + &self, + prediction: f64, + context: &str, + ) -> SafetyResult { + if !self.config.safety_enabled { + return Ok(Price::from_f64(prediction).unwrap()); + } + + // Check for NaN/Infinity + if !prediction.is_finite() { + return Err(MLSafetyError::InvalidFloat { + operation: format!("Financial prediction in {}: {}", context, prediction), + }); + } + + // Check prediction bounds + if prediction < self.config.min_prediction_value + || prediction > self.config.max_prediction_value + { + return Err(MLSafetyError::PredictionOutOfBounds { + value: prediction, + min: self.config.min_prediction_value, + max: self.config.max_prediction_value, + }); + } + + // Validate through financial validator + self.financial_validator + .validate_price(prediction, context) + .await?; + + // Convert to safe financial type + Ok(Price::from_f64(prediction).unwrap()) + } + + /// Validate and convert price with currency support + pub async fn validate_and_convert_price( + &self, + prediction: f64, + currency: &str, + ) -> SafetyResult { + if !self.config.safety_enabled { + return Ok(Price::from_f64(prediction).unwrap()); + } + + // Check for NaN/Infinity + if !prediction.is_finite() { + return Err(MLSafetyError::InvalidFloat { + operation: format!("Price prediction in {}: {}", currency, prediction), + }); + } + + // Check prediction bounds + if prediction < self.config.min_prediction_value + || prediction > self.config.max_prediction_value + { + return Err(MLSafetyError::PredictionOutOfBounds { + value: prediction, + min: self.config.min_prediction_value, + max: self.config.max_prediction_value, + }); + } + + // Validate through financial validator with currency context + let context = format!("price_prediction_{}_{}", currency, prediction); + self.financial_validator + .validate_price(prediction, &context) + .await?; + + // Convert to safe financial type + Ok(Price::from_f64(prediction).unwrap()) + } + + /// Validate financial value for safety + pub fn validate_financial_value(&self, value: f64) -> SafetyResult<()> { + if !self.config.safety_enabled { + return Ok(()); + } + + // Check for NaN/Infinity + if !value.is_finite() { + return Err(MLSafetyError::InvalidFloat { + operation: format!("Financial value validation: {}", value), + }); + } + + // Check prediction bounds + if value < self.config.min_prediction_value || value > self.config.max_prediction_value { + return Err(MLSafetyError::PredictionOutOfBounds { + value, + min: self.config.min_prediction_value, + max: self.config.max_prediction_value, + }); + } + + Ok(()) + } + + /// Check for model drift and update detection + pub async fn check_model_drift( + &self, + model_id: &str, + predictions: &[f64], + actual_values: Option<&[f64]>, + ) -> SafetyResult { + if !self.config.safety_enabled { + return Ok(SafetyStatus::Safe); + } + + let mut drift_detector = self.drift_detector.write().await; + let drift_score = drift_detector + .update_and_check(model_id, predictions, actual_values) + .await?; + + if drift_score > self.config.drift_sensitivity { + warn!( + "Model drift detected for {}: score {:.3} > threshold {:.3}", + model_id, drift_score, self.config.drift_sensitivity + ); + + return Ok(SafetyStatus::Danger { + reason: format!( + "Model drift: score {:.3} > threshold {:.3}", + drift_score, self.config.drift_sensitivity + ), + }); + } + + if drift_score > self.config.drift_sensitivity * 0.7 { + return Ok(SafetyStatus::Warning { + reason: format!("Model drift warning: score {:.3}", drift_score), + }); + } + + Ok(SafetyStatus::Safe) + } + + /// Get comprehensive safety status + pub async fn get_safety_status(&self) -> HashMap { + let mut status = HashMap::new(); + + // Memory status + let memory_manager = self.memory_manager.read().await; + status.insert("memory".to_string(), memory_manager.get_status().await); + drop(memory_manager); + + // Drift detection status + let drift_detector = self.drift_detector.read().await; + status.insert( + "drift_detection".to_string(), + drift_detector.get_status().await, + ); + drop(drift_detector); + + // Operation history status + let history = self.operation_history.read().await; + let recent_operations = history.values().map(|ops| ops.len()).sum::(); + + let ops_status = if recent_operations > 10000 { + SafetyStatus::Warning { + reason: format!("High operation count: {}", recent_operations), + } + } else { + SafetyStatus::Safe + }; + status.insert("operations".to_string(), ops_status); + + status + } + + /// Emergency shutdown all ML operations + pub async fn emergency_shutdown(&self, reason: &str) -> SafetyResult<()> { + error!("ML Safety Manager emergency shutdown: {}", reason); + + // Stop all ongoing operations + self.timeout_manager.shutdown_all().await; + + // Clear all caches and memory + let mut memory_manager = self.memory_manager.write().await; + memory_manager.emergency_cleanup().await?; + + // Reset drift detection + let mut drift_detector = self.drift_detector.write().await; + drift_detector.reset_all().await; + + // Clear operation history + let mut history = self.operation_history.write().await; + history.clear(); + + info!("ML Safety Manager emergency shutdown completed"); + Ok(()) + } + + /// Record operation start for monitoring + async fn record_operation_start(&self, operation_name: &str) { + let mut history = self.operation_history.write().await; + let operations = history + .entry(operation_name.to_string()) + .or_insert_with(Vec::new); + operations.push(Instant::now()); + + // Keep only recent operations + let cutoff = Instant::now() - Duration::from_secs(300); // 5 minutes + operations.retain(|&time| time > cutoff); + } + + /// Validate operation result + async fn validate_operation_result( + &self, + operation_name: &str, + _result: &T, + ) -> SafetyResult<()> { + debug!("Operation {} completed successfully", operation_name); + Ok(()) + } +} + +/// Global ML safety manager instance +static GLOBAL_SAFETY_MANAGER: once_cell::sync::Lazy = + once_cell::sync::Lazy::new(|| MLSafetyManager::new(MLSafetyConfig::default())); + +/// Get the global ML safety manager +pub fn get_global_safety_manager() -> &'static MLSafetyManager { + &GLOBAL_SAFETY_MANAGER +} + +/// Initialize ML safety with custom configuration +pub fn initialize_ml_safety(_config: MLSafetyConfig) -> &'static MLSafetyManager { + // Note: This is a simplified version. In production, we would use + // proper initialization that allows configuration override + &GLOBAL_SAFETY_MANAGER +} + +#[cfg(test)] +mod tests { + use super::*; + use candle_core::Device; + + #[tokio::test] + async fn test_safe_tensor_creation() { + let manager = MLSafetyManager::new(MLSafetyConfig::default()); + let device = Device::Cpu; + + // Valid tensor creation + let data = vec![1.0, 2.0, 3.0, 4.0]; + let shape = &[2, 2]; + let tensor = manager + .safe_tensor_create(data, shape, &device, "test_creation") + .await; + assert!(tensor.is_ok()); + + // Invalid tensor - NaN data + let bad_data = vec![1.0, f64::NAN, 3.0, 4.0]; + let bad_tensor = manager + .safe_tensor_create(bad_data, shape, &device, "test_nan") + .await; + assert!(bad_tensor.is_err()); + } + + #[tokio::test] + async fn test_financial_validation() { + let manager = MLSafetyManager::new(MLSafetyConfig::default()); + + // Valid prediction + let valid_pred = manager + .validate_financial_prediction(123.45, "test_valid") + .await; + assert!(valid_pred.is_ok()); + + // Invalid prediction - NaN + let invalid_pred = manager + .validate_financial_prediction(f64::NAN, "test_nan") + .await; + assert!(invalid_pred.is_err()); + + // Invalid prediction - out of bounds + let oob_pred = manager + .validate_financial_prediction(1e10, "test_oob") + .await; + assert!(oob_pred.is_err()); + } + + #[tokio::test] + async fn test_safety_status() { + let manager = MLSafetyManager::new(MLSafetyConfig::default()); + let status = manager.get_safety_status().await; + + assert!(status.contains_key("memory")); + assert!(status.contains_key("drift_detection")); + assert!(status.contains_key("operations")); + } +} diff --git a/ml/src/stress_testing/mod.rs b/ml/src/stress_testing/mod.rs index 46f444e0c..9aafa93f8 100644 --- a/ml/src/stress_testing/mod.rs +++ b/ml/src/stress_testing/mod.rs @@ -22,9 +22,7 @@ use rust_decimal::prelude::ToPrimitive; use crate::{Features, MLModel, ModelPrediction, ModelType}; -pub use load_generator::{LoadGenerator, LoadProfile, TrafficPattern}; -pub use market_simulator::{MarketCondition, MarketDataSimulator, SimulatorConfig}; -pub use performance_analyzer::{LatencyStats, PerformanceAnalyzer, StressTestReport}; +// DO NOT RE-EXPORT - Use explicit imports at usage sites /// Comprehensive stress test configuration #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/ml/src/stress_testing/mod.rs.bak b/ml/src/stress_testing/mod.rs.bak new file mode 100644 index 000000000..46f444e0c --- /dev/null +++ b/ml/src/stress_testing/mod.rs.bak @@ -0,0 +1,568 @@ +//! Stress testing framework for ML models under high-volume market data conditions +//! +//! This module provides comprehensive stress testing capabilities to validate ML model +//! performance under realistic HFT market conditions with high-frequency data feeds. + +// Import types from crate root (lib.rs) +use common::types::Price; + +pub mod load_generator; +pub mod market_simulator; +pub mod performance_analyzer; + +use anyhow::Result; +use futures::stream::StreamExt; +use serde::{Deserialize, Serialize}; +use std::time::{Duration, Instant}; +use tokio::sync::mpsc; + +// Price and Decimal imported above +use common::Decimal; +use rust_decimal::prelude::ToPrimitive; + +use crate::{Features, MLModel, ModelPrediction, ModelType}; + +pub use load_generator::{LoadGenerator, LoadProfile, TrafficPattern}; +pub use market_simulator::{MarketCondition, MarketDataSimulator, SimulatorConfig}; +pub use performance_analyzer::{LatencyStats, PerformanceAnalyzer, StressTestReport}; + +/// Comprehensive stress test configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StressTestConfig { + /// Test duration in seconds + pub duration_seconds: u64, + /// Target requests per second + pub target_rps: u32, + /// Number of concurrent connections + pub concurrent_connections: u32, + /// Market data feed rate (updates per second) + pub market_data_rate: u32, + /// Test phases with different load patterns + pub test_phases: Vec, + /// Models to test + pub models_to_test: Vec, + /// Market conditions to simulate + pub market_conditions: Vec, + /// Performance requirements + pub requirements: PerformanceRequirements, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TestPhase { + pub name: String, + pub duration_seconds: u64, + pub load_multiplier: f64, + pub market_volatility: f64, + pub error_injection_rate: f64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PerformanceRequirements { + /// Maximum acceptable latency in microseconds + pub max_latency_us: u64, + /// 95th percentile latency requirement + pub p95_latency_us: u64, + /// 99th percentile latency requirement + pub p99_latency_us: u64, + /// Maximum acceptable error rate + pub max_error_rate: f64, + /// Minimum throughput (predictions per second) + pub min_throughput: u32, + /// Maximum memory usage in MB + pub max_memory_mb: u64, + /// Maximum CPU utilization percentage + pub max_cpu_percent: f64, +} + +impl Default for PerformanceRequirements { + fn default() -> Self { + Self { + max_latency_us: 100, // 100μs max latency + p95_latency_us: 50, // 50μs 95th percentile + p99_latency_us: 80, // 80μs 99th percentile + max_error_rate: 0.01, // 1% max error rate + min_throughput: 10000, // 10k predictions/sec + max_memory_mb: 1024, // 1GB max memory + max_cpu_percent: 80.0, // 80% max CPU + } + } +} + +/// Main stress testing orchestrator +pub struct StressTestOrchestrator { + config: StressTestConfig, + market_simulator: MarketDataSimulator, + load_generator: LoadGenerator, + performance_analyzer: PerformanceAnalyzer, +} + +impl StressTestOrchestrator { + /// Create new stress test orchestrator + pub fn new(config: StressTestConfig) -> Result { + let simulator_config = SimulatorConfig { + symbols: vec!["AAPL".to_string(), "MSFT".to_string(), "GOOGL".to_string()], + update_rate_hz: config.market_data_rate, + volatility: 0.02, + trend: 0.0, + }; + + let market_simulator = MarketDataSimulator::new(simulator_config)?; + let load_generator = LoadGenerator::new(config.target_rps, config.concurrent_connections)?; + let performance_analyzer = PerformanceAnalyzer::new(); + + Ok(Self { + config, + market_simulator, + load_generator, + performance_analyzer, + }) + } + + /// Run comprehensive stress test + pub async fn run_stress_test( + &mut self, + models: Vec>, + ) -> Result { + tracing::info!("Starting stress test with {} models", models.len()); + + // Create channels for communication + let (market_tx, mut market_rx) = mpsc::channel(10000); + let (prediction_tx, _prediction_rx) = mpsc::channel(10000); + + // Start market data simulation + let simulator_handle = { + let mut simulator = self.market_simulator.clone(); + tokio::spawn(async move { simulator.start_simulation(market_tx).await }) + }; + + // Start performance monitoring + let analyzer = self.performance_analyzer.clone(); + let monitor_handle = tokio::spawn(async move { analyzer.start_monitoring().await }); + + // Execute test phases + let mut phase_results = Vec::new(); + let test_start = Instant::now(); + + // Clone the test phases to avoid borrowing conflicts + let test_phases = self.config.test_phases.clone(); + for (phase_idx, phase) in test_phases.iter().enumerate() { + tracing::info!("Starting test phase {}: {}", phase_idx + 1, phase.name); + + let phase_result = self + .run_test_phase(phase, &models, &mut market_rx, &prediction_tx) + .await?; + + phase_results.push(phase_result); + } + + // Stop simulation and monitoring + simulator_handle.abort(); + monitor_handle.abort(); + + // Generate comprehensive report + let total_duration = test_start.elapsed(); + let report = self + .generate_stress_test_report(phase_results, total_duration) + .await?; + + tracing::info!( + "Stress test completed in {:.2}s", + total_duration.as_secs_f64() + ); + Ok(report) + } + + /// Run individual test phase + async fn run_test_phase( + &mut self, + phase: &TestPhase, + models: &[std::sync::Arc], + market_rx: &mut mpsc::Receiver, + prediction_tx: &mpsc::Sender, + ) -> Result { + let phase_start = Instant::now(); + let phase_duration = Duration::from_secs(phase.duration_seconds); + + let mut phase_stats = PhaseStats::new(); + + // Adjust load generator for this phase + self.load_generator + .set_load_multiplier(phase.load_multiplier); + + while phase_start.elapsed() < phase_duration { + // Process market data updates + while let Ok(market_update) = market_rx.try_recv() { + // Convert market data to features + let features = self.convert_market_data_to_features(&market_update)?; + + // Run predictions on all models + for model in models { + let model_start = Instant::now(); + + match model.predict(&features).await { + Ok(prediction) => { + let latency_us = model_start.elapsed().as_micros() as u64; + + phase_stats.record_successful_prediction(latency_us); + + let result = PredictionResult { + model_name: model.name().to_string(), + model_type: model.model_type(), + prediction, + latency_us, + timestamp: std::time::SystemTime::now(), + success: true, + error_message: None, + }; + + let _ = prediction_tx.send(result).await; + } + Err(e) => { + let latency_us = model_start.elapsed().as_micros() as u64; + phase_stats.record_failed_prediction(latency_us); + + let result = PredictionResult { + model_name: model.name().to_string(), + model_type: model.model_type(), + prediction: ModelPrediction::new( + model.name().to_string(), + 0.0, + 0.0, + ), + latency_us, + timestamp: std::time::SystemTime::now(), + success: false, + error_message: Some(e.to_string()), + }; + + let _ = prediction_tx.send(result).await; + } + } + } + } + + // Small delay to prevent busy waiting + tokio::time::sleep(Duration::from_micros(100)).await; + } + + Ok(PhaseResult { + phase_name: phase.name.clone(), + duration: phase_start.elapsed(), + stats: phase_stats, + }) + } + + /// Convert market data to ML features + fn convert_market_data_to_features(&self, market_data: &MarketDataUpdate) -> Result { + let values = vec![ + market_data.price.to_f64(), + market_data.volume.to_f64().unwrap_or(0.0), + market_data.bid.to_f64(), + market_data.ask.to_f64(), + market_data.spread().to_f64(), + market_data.mid_price().to_f64(), + ]; + + let names = vec![ + "price".to_string(), + "volume".to_string(), + "bid".to_string(), + "ask".to_string(), + "spread".to_string(), + "mid_price".to_string(), + ]; + + Ok(Features::new(values, names).with_symbol(market_data.symbol.clone())) + } + + /// Generate comprehensive stress test report + async fn generate_stress_test_report( + &self, + phase_results: Vec, + total_duration: Duration, + ) -> Result { + let mut total_predictions = 0; + let mut total_errors = 0; + let mut all_latencies = Vec::new(); + + for phase in &phase_results { + total_predictions += + phase.stats.successful_predictions + phase.stats.failed_predictions; + total_errors += phase.stats.failed_predictions; + all_latencies.extend(&phase.stats.latencies); + } + + let error_rate = if total_predictions > 0 { + total_errors as f64 / total_predictions as f64 + } else { + 0.0 + }; + + let latency_stats = self.calculate_latency_statistics(&all_latencies); + + // Check if requirements are met + let requirements_met = self.check_requirements(&latency_stats, error_rate); + let recommendations = self.generate_recommendations(&latency_stats, error_rate); + + Ok(StressTestReport { + config: self.config.clone(), + total_duration, + phase_results, + total_predictions: total_predictions as u64, + total_errors: total_errors as u64, + error_rate, + latency_stats: latency_stats.clone(), + requirements_met, + throughput_achieved: total_predictions as f64 / total_duration.as_secs_f64(), + recommendations, + }) + } + + fn calculate_latency_statistics(&self, latencies: &[u64]) -> LatencyStats { + if latencies.is_empty() { + return LatencyStats::default(); + } + + let mut sorted_latencies = latencies.to_vec(); + sorted_latencies.sort_unstable(); + + let len = sorted_latencies.len(); + let mean = sorted_latencies.iter().sum::() as f64 / len as f64; + let min = sorted_latencies[0]; + let max = sorted_latencies[len - 1]; + let p50 = sorted_latencies[len * 50 / 100]; + let p95 = sorted_latencies[len * 95 / 100]; + let p99 = sorted_latencies[len * 99 / 100]; + + LatencyStats { + mean, + min, + max, + p50, + p95, + p99, + count: len as u64, + } + } + + fn check_requirements( + &self, + latency_stats: &LatencyStats, + error_rate: f64, + ) -> RequirementsCheck { + RequirementsCheck { + latency_ok: latency_stats.max <= self.config.requirements.max_latency_us, + p95_latency_ok: latency_stats.p95 <= self.config.requirements.p95_latency_us, + p99_latency_ok: latency_stats.p99 <= self.config.requirements.p99_latency_us, + error_rate_ok: error_rate <= self.config.requirements.max_error_rate, + overall_pass: latency_stats.max <= self.config.requirements.max_latency_us + && latency_stats.p95 <= self.config.requirements.p95_latency_us + && latency_stats.p99 <= self.config.requirements.p99_latency_us + && error_rate <= self.config.requirements.max_error_rate, + } + } + + fn generate_recommendations( + &self, + latency_stats: &LatencyStats, + error_rate: f64, + ) -> Vec { + let mut recommendations = Vec::new(); + + if latency_stats.p99 > self.config.requirements.p99_latency_us { + recommendations.push(format!( + "P99 latency ({}μs) exceeds requirement ({}μs). Consider model optimization or hardware upgrades.", + latency_stats.p99, self.config.requirements.p99_latency_us + )); + } + + if error_rate > self.config.requirements.max_error_rate { + recommendations.push(format!( + "Error rate ({:.2}%) exceeds requirement ({:.2}%). Investigate model reliability.", + error_rate * 100.0, + self.config.requirements.max_error_rate * 100.0 + )); + } + + if latency_stats.mean > 50.0 { + recommendations + .push("Consider enabling GPU acceleration for better performance.".to_string()); + } + + if recommendations.is_empty() { + recommendations + .push("All performance requirements met. System ready for production.".to_string()); + } + + recommendations + } +} + +/// Market data update structure +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MarketDataUpdate { + pub symbol: String, + pub price: Price, + pub volume: Decimal, + pub bid: Price, + pub ask: Price, + pub timestamp: std::time::SystemTime, +} + +impl MarketDataUpdate { + pub fn spread(&self) -> Price { + Price::from_f64(self.ask.to_f64() - self.bid.to_f64()).unwrap() + } + + pub fn mid_price(&self) -> Price { + Price::from_f64((self.bid.to_f64() + self.ask.to_f64()) / 2.0).unwrap() + } +} + +/// Prediction result with timing information +#[derive(Debug, Clone)] +pub struct PredictionResult { + pub model_name: String, + pub model_type: ModelType, + pub prediction: ModelPrediction, + pub latency_us: u64, + pub timestamp: std::time::SystemTime, + pub success: bool, + pub error_message: Option, +} + +/// Phase execution statistics +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PhaseStats { + pub successful_predictions: u64, + pub failed_predictions: u64, + pub latencies: Vec, +} + +impl PhaseStats { + pub fn new() -> Self { + Self { + successful_predictions: 0, + failed_predictions: 0, + latencies: Vec::new(), + } + } + + pub fn record_successful_prediction(&mut self, latency_us: u64) { + self.successful_predictions += 1; + self.latencies.push(latency_us); + } + + pub fn record_failed_prediction(&mut self, latency_us: u64) { + self.failed_predictions += 1; + self.latencies.push(latency_us); + } +} + +/// Phase execution result +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct PhaseResult { + pub phase_name: String, + pub duration: Duration, + pub stats: PhaseStats, +} + +/// Requirements check result +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RequirementsCheck { + pub latency_ok: bool, + pub p95_latency_ok: bool, + pub p99_latency_ok: bool, + pub error_rate_ok: bool, + pub overall_pass: bool, +} + +/// Create default HFT stress test configuration +pub fn create_hft_stress_test_config() -> StressTestConfig { + StressTestConfig { + duration_seconds: 300, // 5 minutes + target_rps: 50000, // 50k requests per second + concurrent_connections: 100, + market_data_rate: 10000, // 10k market updates per second + test_phases: vec![ + TestPhase { + name: "warmup".to_string(), + duration_seconds: 60, + load_multiplier: 0.5, + market_volatility: 0.01, + error_injection_rate: 0.0, + }, + TestPhase { + name: "normal_load".to_string(), + duration_seconds: 120, + load_multiplier: 1.0, + market_volatility: 0.02, + error_injection_rate: 0.001, + }, + TestPhase { + name: "peak_load".to_string(), + duration_seconds: 60, + load_multiplier: 2.0, + market_volatility: 0.05, + error_injection_rate: 0.005, + }, + TestPhase { + name: "stress_load".to_string(), + duration_seconds: 60, + load_multiplier: 5.0, + market_volatility: 0.1, + error_injection_rate: 0.01, + }, + ], + models_to_test: vec![ + "TLOB_Transformer".to_string(), + "MAMBA_SSM".to_string(), + "DQN_Agent".to_string(), + ], + market_conditions: vec![ + MarketCondition::Normal, + MarketCondition::HighVolatility, + MarketCondition::Flash, + ], + requirements: PerformanceRequirements::default(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_stress_test_config_creation() { + let config = create_hft_stress_test_config(); + assert_eq!(config.test_phases.len(), 4); + assert_eq!(config.target_rps, 50000); + } + + #[test] + fn test_market_data_calculations() { + let update = MarketDataUpdate { + symbol: "AAPL".to_string(), + price: 150.0, + volume: 1000.0, + bid: 149.95, + ask: 150.05, + timestamp: std::time::SystemTime::now(), + }; + + assert_eq!(update.spread(), 0.10); + assert_eq!(update.mid_price(), 150.0); + } + + #[test] + fn test_phase_stats() { + let mut stats = PhaseStats::new(); + stats.record_successful_prediction(25); + stats.record_successful_prediction(30); + stats.record_failed_prediction(100); + + assert_eq!(stats.successful_predictions, 2); + assert_eq!(stats.failed_predictions, 1); + assert_eq!(stats.latencies.len(), 3); + } +} diff --git a/ml/src/tft/mod.rs b/ml/src/tft/mod.rs index 461dd73f7..98c4d4ae3 100644 --- a/ml/src/tft/mod.rs +++ b/ml/src/tft/mod.rs @@ -40,14 +40,8 @@ pub mod temporal_attention; pub mod training; pub mod variable_selection; -pub use gated_residual::{GRNStack, GatedLinearUnit, GatedResidualNetwork}; -pub use hft_optimizations::{ HFTOptimizationConfig, HFTOptimizedTFT, HFTPerformanceMetrics, QuantizedTFT, }; -pub use quantile_outputs::QuantileLayer; -pub use temporal_attention::{AttentionConfig, PositionalEncoding, TemporalSelfAttention}; -pub use training::{TFTDataLoader, TFTTrainer, TFTTrainingConfig, TrainingMetrics}; -pub use variable_selection::VariableSelectionNetwork; /// TFT Configuration #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/ml/src/tft/mod.rs.bak b/ml/src/tft/mod.rs.bak new file mode 100644 index 000000000..461dd73f7 --- /dev/null +++ b/ml/src/tft/mod.rs.bak @@ -0,0 +1,734 @@ +//! # Temporal Fusion Transformer (TFT) for HFT +//! +//! State-of-the-art multi-horizon forecasting with variable selection networks, +//! temporal self-attention, gated residual networks, and uncertainty quantification. +//! +//! ## Key Features +//! +//! - Multi-horizon forecasting (1-tick to 100-tick ahead) +//! - Variable selection networks for feature importance +//! - Gated residual networks for improved gradient flow +//! - Quantile outputs for uncertainty estimation +//! - Temporal self-attention for sequential modeling +//! - Sub-50μs inference latency optimized for HFT +//! +//! ## Performance Targets +//! +//! - Inference: <50μs per prediction +//! - Accuracy improvement: +15% over baseline +//! - Memory usage: <1GB +//! - Throughput: >100K predictions/sec + +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Instant, SystemTime}; + +use candle_core::{DType, Device, Module, Tensor}; +use candle_nn::{linear, Linear, VarBuilder}; +use ndarray::{Array1, Array2}; +use serde::{Deserialize, Serialize}; +use tracing::{debug, info, instrument, warn}; +use uuid::Uuid; + +use crate::MLError; + +// Import TFT components +pub mod gated_residual; +pub mod hft_optimizations; +pub mod quantile_outputs; +pub mod temporal_attention; +pub mod training; +pub mod variable_selection; + +pub use gated_residual::{GRNStack, GatedLinearUnit, GatedResidualNetwork}; +pub use hft_optimizations::{ + HFTOptimizationConfig, HFTOptimizedTFT, HFTPerformanceMetrics, QuantizedTFT, +}; +pub use quantile_outputs::QuantileLayer; +pub use temporal_attention::{AttentionConfig, PositionalEncoding, TemporalSelfAttention}; +pub use training::{TFTDataLoader, TFTTrainer, TFTTrainingConfig, TrainingMetrics}; +pub use variable_selection::VariableSelectionNetwork; + +/// TFT Configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TFTConfig { + // Model architecture + pub input_dim: usize, + pub hidden_dim: usize, + pub num_heads: usize, + pub num_layers: usize, + + // Forecasting parameters + pub prediction_horizon: usize, + pub sequence_length: usize, + pub num_quantiles: usize, + + // Feature types + pub num_static_features: usize, + pub num_known_features: usize, + pub num_unknown_features: usize, + + // Training parameters + pub learning_rate: f64, + pub batch_size: usize, + pub dropout_rate: f64, + pub l2_regularization: f64, + + // HFT optimization + pub use_flash_attention: bool, + pub mixed_precision: bool, + pub memory_efficient: bool, + + // Performance constraints + pub max_inference_latency_us: u64, + pub target_throughput_pps: u64, +} + +impl Default for TFTConfig { + fn default() -> Self { + Self { + input_dim: 64, + hidden_dim: 128, + num_heads: 8, + num_layers: 3, + prediction_horizon: 10, + sequence_length: 50, + num_quantiles: 9, + num_static_features: 5, + num_known_features: 10, + num_unknown_features: 20, + learning_rate: 1e-3, + batch_size: 64, + dropout_rate: 0.1, + l2_regularization: 1e-4, + use_flash_attention: true, + mixed_precision: true, + memory_efficient: true, + max_inference_latency_us: 50, + target_throughput_pps: 100_000, + } + } +} + +/// TFT Model State for incremental processing +#[derive(Debug, Clone)] +pub struct TFTState { + pub hidden_state: Option, + pub attention_cache: HashMap, + pub last_update: u64, +} + +impl TFTState { + pub fn zeros(_config: &TFTConfig) -> Result { + Ok(Self { + hidden_state: None, + attention_cache: HashMap::new(), + last_update: 0, + }) + } +} + +/// TFT Model Metadata +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TFTMetadata { + pub model_id: String, + pub version: String, + pub input_dim: usize, + pub output_dim: usize, + pub created_at: SystemTime, + pub last_trained: Option, + pub training_samples: u64, + pub performance_metrics: HashMap, +} + +/// Multi-horizon prediction result +#[derive(Debug, Clone)] +pub struct MultiHorizonPrediction { + pub predictions: Vec, // Point predictions for each horizon + pub quantiles: Vec>, // Quantile predictions [horizon][quantile] + pub uncertainty: Vec, // Uncertainty estimates + pub confidence_intervals: Vec<(f64, f64)>, // 90% confidence intervals + pub attention_weights: HashMap>, // Attention interpretability + pub feature_importance: Vec, // Variable importance scores + pub latency_us: u64, // Inference latency +} + +/// Complete Temporal Fusion Transformer +#[derive(Debug)] +pub struct TemporalFusionTransformer { + pub config: TFTConfig, + pub metadata: TFTMetadata, + pub is_trained: bool, + + // Core TFT components + static_variable_selection: VariableSelectionNetwork, + historical_variable_selection: VariableSelectionNetwork, + future_variable_selection: VariableSelectionNetwork, + + // Encoding layers + static_encoder: GRNStack, + historical_encoder: GRNStack, + future_encoder: GRNStack, + + // Temporal processing + lstm_encoder: Linear, // Simplified LSTM representation + lstm_decoder: Linear, + + // Attention mechanism + temporal_attention: TemporalSelfAttention, + + // Output layers + quantile_outputs: QuantileLayer, + + // Performance tracking + inference_count: AtomicU64, + total_latency_us: AtomicU64, + max_latency_us: AtomicU64, + + device: Device, +} + +impl TemporalFusionTransformer { + pub fn new(config: TFTConfig) -> Result { + let device = Device::cuda_if_available(0).unwrap_or(Device::Cpu); + let vs = VarBuilder::zeros(DType::F32, &device); + + // Create variable selection networks + let static_variable_selection = VariableSelectionNetwork::new( + config.num_static_features, + config.hidden_dim, + vs.pp("static_vsn"), + )?; + + let historical_variable_selection = VariableSelectionNetwork::new( + config.num_unknown_features, + config.hidden_dim, + vs.pp("historical_vsn"), + )?; + + let future_variable_selection = VariableSelectionNetwork::new( + config.num_known_features, + config.hidden_dim, + vs.pp("future_vsn"), + )?; + + // Create encoding stacks + let static_encoder = GRNStack::new( + config.hidden_dim, + config.hidden_dim, + config.hidden_dim, + config.num_layers, + vs.pp("static_encoder"), + )?; + + let historical_encoder = GRNStack::new( + config.hidden_dim, + config.hidden_dim, + config.hidden_dim, + config.num_layers, + vs.pp("historical_encoder"), + )?; + + let future_encoder = GRNStack::new( + config.hidden_dim, + config.hidden_dim, + config.hidden_dim, + config.num_layers, + vs.pp("future_encoder"), + )?; + + // Simplified LSTM layers (in practice, would use proper LSTM) + let lstm_encoder = linear(config.hidden_dim, config.hidden_dim, vs.pp("lstm_encoder"))?; + let lstm_decoder = linear(config.hidden_dim, config.hidden_dim, vs.pp("lstm_decoder"))?; + + // Temporal attention + let temporal_attention = TemporalSelfAttention::new( + config.hidden_dim, + config.num_heads, + config.dropout_rate, + config.use_flash_attention, + vs.pp("temporal_attention"), + )?; + + // Quantile output layer + let quantile_outputs = QuantileLayer::new( + config.hidden_dim, + config.prediction_horizon, + config.num_quantiles, + vs.pp("quantile_outputs"), + )?; + + // Metadata + let metadata = TFTMetadata { + model_id: Uuid::new_v4().to_string(), + version: "1.0.0".to_string(), + input_dim: config.input_dim, + output_dim: config.prediction_horizon, + created_at: SystemTime::now(), + last_trained: None, + training_samples: 0, + performance_metrics: HashMap::new(), + }; + + Ok(Self { + config, + metadata, + is_trained: false, + static_variable_selection, + historical_variable_selection, + future_variable_selection, + static_encoder, + historical_encoder, + future_encoder, + lstm_encoder, + lstm_decoder, + temporal_attention, + quantile_outputs, + inference_count: AtomicU64::new(0), + total_latency_us: AtomicU64::new(0), + max_latency_us: AtomicU64::new(0), + device, + }) + } + + /// Forward pass through the complete TFT architecture + #[instrument(skip(self, static_features, historical_features, future_features))] + pub fn forward( + &mut self, + static_features: &Tensor, + historical_features: &Tensor, + future_features: &Tensor, + ) -> Result { + let start_time = Instant::now(); + + // 1. Variable Selection Networks + let static_selected = self + .static_variable_selection + .forward(static_features, None)?; + let historical_selected = self + .historical_variable_selection + .forward(historical_features, None)?; + let future_selected = self + .future_variable_selection + .forward(future_features, None)?; + + // 2. Feature Encoding + let static_encoded = self.static_encoder.forward(&static_selected, None)?; + let historical_encoded = self + .historical_encoder + .forward(&historical_selected, None)?; + let future_encoded = self.future_encoder.forward(&future_selected, None)?; + + // 3. Temporal Processing (Simplified LSTM) + let historical_temporal = self.lstm_encoder.forward(&historical_encoded)?; + let future_temporal = self.lstm_decoder.forward(&future_encoded)?; + + // 4. Combine temporal representations + let combined_temporal = + self.combine_temporal_features(&historical_temporal, &future_temporal)?; + + // 5. Self-Attention + let attended = self.temporal_attention.forward(&combined_temporal, true)?; + + // 6. Final processing with static context + let contextualized = self.apply_static_context(&attended, &static_encoded)?; + + // 7. Quantile Outputs + let quantile_preds = self.quantile_outputs.forward(&contextualized)?; + + // Update performance metrics + let latency = start_time.elapsed().as_micros() as u64; + self.update_performance_metrics(latency); + + Ok(quantile_preds) + } + + fn combine_temporal_features( + &self, + historical: &Tensor, + future: &Tensor, + ) -> Result { + // Concatenate historical and future features along the time dimension + let combined = Tensor::cat(&[historical, future], 1)?; + Ok(combined) + } + + fn apply_static_context( + &self, + temporal: &Tensor, + static_context: &Tensor, + ) -> Result { + let (batch_size, seq_len, hidden_dim) = temporal.dims3()?; + + // Broadcast static context to match temporal dimensions + let static_expanded = static_context.unsqueeze(1)?; // [batch, 1, hidden] + let static_broadcast = static_expanded.broadcast_as((batch_size, seq_len, hidden_dim))?; + + // Add static context to temporal features + let contextualized = (temporal + &static_broadcast)?; + + Ok(contextualized) + } + + /// Multi-horizon prediction interface + pub fn predict_horizons( + &mut self, + static_features: &Array1, + historical_features: &Array2, + future_features: &Array2, + ) -> Result { + if !self.is_trained { + return Err(MLError::ModelError("Model not trained".to_string())); + } + + let start_time = Instant::now(); + + // Convert ndarray to tensors + let static_tensor = self.array_to_tensor_1d(static_features)?; + let historical_tensor = self.array_to_tensor_2d(historical_features)?; + let future_tensor = self.array_to_tensor_2d(future_features)?; + + // Add batch dimension + let static_batched = static_tensor.unsqueeze(0)?; + let historical_batched = historical_tensor.unsqueeze(0)?; + let future_batched = future_tensor.unsqueeze(0)?; + + // Forward pass + let quantile_preds = self.forward(&static_batched, &historical_batched, &future_batched)?; + + // Extract predictions and process outputs + let pred_data = quantile_preds.squeeze(0)?.to_vec2::()?; // [horizon, quantiles] + + let mut predictions = Vec::new(); + let mut quantiles = Vec::new(); + let mut uncertainty = Vec::new(); + let mut confidence_intervals = Vec::new(); + + for horizon in 0..self.config.prediction_horizon { + let horizon_quantiles = &pred_data[horizon]; + + // Point prediction (median) + let median_idx = self.config.num_quantiles / 2; + predictions.push(horizon_quantiles[median_idx] as f64); + + // All quantiles for this horizon + quantiles.push(horizon_quantiles.iter().map(|&x| x as f64).collect()); + + // Uncertainty (IQR) + let q75_idx = (self.config.num_quantiles * 3) / 4; + let q25_idx = self.config.num_quantiles / 4; + let iqr = horizon_quantiles[q75_idx] - horizon_quantiles[q25_idx]; + uncertainty.push(iqr as f64); + + // 90% confidence interval + let lower_idx = self.config.num_quantiles / 10; // ~10th percentile + let upper_idx = (self.config.num_quantiles * 9) / 10; // ~90th percentile + let ci = ( + horizon_quantiles[lower_idx] as f64, + horizon_quantiles[upper_idx] as f64, + ); + confidence_intervals.push(ci); + } + + // Get feature importance and attention weights + let feature_importance = self.static_variable_selection.get_importance_scores()?; + let mut attention_weights = HashMap::new(); + let weights = self.temporal_attention.get_attention_weights(); + for (key, weight) in weights { + attention_weights.insert(key, vec![weight]); + } + + let latency = start_time.elapsed().as_micros() as u64; + + Ok(MultiHorizonPrediction { + predictions, + quantiles, + uncertainty, + confidence_intervals, + attention_weights, + feature_importance, + latency_us: latency, + }) + } + + fn array_to_tensor_1d(&self, arr: &Array1) -> Result { + let data: Vec = arr.iter().map(|&x| x as f32).collect(); + let tensor = Tensor::from_slice(&data, arr.len(), &self.device)?; + Ok(tensor) + } + + fn array_to_tensor_2d(&self, arr: &Array2) -> Result { + let data: Vec = arr.iter().map(|&x| x as f32).collect(); + let shape = arr.shape(); + let tensor = Tensor::from_slice(&data, (shape[0], shape[1]), &self.device)?; + Ok(tensor) + } + + fn update_performance_metrics(&self, latency_us: u64) { + self.inference_count.fetch_add(1, Ordering::Relaxed); + self.total_latency_us + .fetch_add(latency_us, Ordering::Relaxed); + + // Update max latency atomically + let mut current_max = self.max_latency_us.load(Ordering::Relaxed); + while latency_us > current_max { + match self.max_latency_us.compare_exchange_weak( + current_max, + latency_us, + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(new_max) => current_max = new_max, + } + } + } + + /// Get performance metrics + pub fn get_metrics(&self) -> HashMap { + let inference_count = self.inference_count.load(Ordering::Relaxed); + let total_latency = self.total_latency_us.load(Ordering::Relaxed); + let max_latency = self.max_latency_us.load(Ordering::Relaxed); + + let avg_latency = if inference_count > 0 { + total_latency as f64 / inference_count as f64 + } else { + 0.0 + }; + + let throughput = if avg_latency > 0.0 { + 1_000_000.0 / avg_latency // predictions per second + } else { + 0.0 + }; + + let mut metrics = HashMap::new(); + metrics.insert("total_inferences".to_string(), inference_count as f64); + metrics.insert("avg_latency_us".to_string(), avg_latency); + metrics.insert("max_latency_us".to_string(), max_latency as f64); + metrics.insert("throughput_pps".to_string(), throughput); + + metrics + } + + /// Training interface (simplified) + pub async fn train( + &mut self, + training_data: &[(Array1, Array2, Array2, Array1)], // (static, historical, future, targets) + validation_data: &[(Array1, Array2, Array2, Array1)], + epochs: usize, + ) -> Result<(), MLError> { + info!("Starting TFT training for {} epochs", epochs); + + for epoch in 0..epochs { + let mut epoch_loss = 0.0; + + for (_i, (static_feat, hist_feat, fut_feat, targets)) in + training_data.iter().enumerate() + { + // Convert to tensors + let static_tensor = self.array_to_tensor_1d(static_feat)?.unsqueeze(0)?; + let hist_tensor = self.array_to_tensor_2d(hist_feat)?.unsqueeze(0)?; + let fut_tensor = self.array_to_tensor_2d(fut_feat)?.unsqueeze(0)?; + let target_tensor = self.array_to_tensor_1d(targets)?.unsqueeze(0)?; + + // Forward pass + let predictions = self.forward(&static_tensor, &hist_tensor, &fut_tensor)?; + + // Compute quantile loss + let loss = self + .quantile_outputs + .quantile_loss(&predictions, &target_tensor)?; + epoch_loss += loss.to_vec0::()? as f64; + + // Backward pass would go here (simplified) + // In practice, would use proper optimizer and backpropagation + } + + let avg_epoch_loss = epoch_loss / training_data.len() as f64; + debug!("Epoch {}: Average Loss = {:.6}", epoch, avg_epoch_loss); + + // Validation + if epoch % 10 == 0 { + let val_loss = self.validate(validation_data).await?; + info!("Epoch {}: Validation Loss = {:.6}", epoch, val_loss); + } + } + + self.is_trained = true; + self.metadata.last_trained = Some(SystemTime::now()); + self.metadata.training_samples = training_data.len() as u64; + + info!("TFT training completed successfully"); + Ok(()) + } + + async fn validate( + &mut self, + validation_data: &[(Array1, Array2, Array2, Array1)], + ) -> Result { + let mut total_loss = 0.0; + + for (static_feat, hist_feat, fut_feat, targets) in validation_data { + let static_tensor = self.array_to_tensor_1d(static_feat)?.unsqueeze(0)?; + let hist_tensor = self.array_to_tensor_2d(hist_feat)?.unsqueeze(0)?; + let fut_tensor = self.array_to_tensor_2d(fut_feat)?.unsqueeze(0)?; + let target_tensor = self.array_to_tensor_1d(targets)?.unsqueeze(0)?; + + let predictions = self.forward(&static_tensor, &hist_tensor, &fut_tensor)?; + let loss = self + .quantile_outputs + .quantile_loss(&predictions, &target_tensor)?; + total_loss += loss.to_vec0::()? as f64; + } + + Ok(total_loss / validation_data.len() as f64) + } + + /// HFT-optimized inference + pub fn predict_fast( + &mut self, + static_features: &[f32], + historical_features: &[f32], + future_features: &[f32], + ) -> Result, MLError> { + let start = Instant::now(); + + // Convert to tensors (optimized path) + let static_tensor = + Tensor::from_slice(static_features, static_features.len(), &self.device)? + .unsqueeze(0)?; + + let hist_len = self.config.sequence_length; + let hist_dim = self.config.num_unknown_features; + let historical_tensor = + Tensor::from_slice(historical_features, (hist_len, hist_dim), &self.device)? + .unsqueeze(0)?; + + let fut_len = self.config.prediction_horizon; + let fut_dim = self.config.num_known_features; + let future_tensor = + Tensor::from_slice(future_features, (fut_len, fut_dim), &self.device)?.unsqueeze(0)?; + + // Forward pass + let quantile_preds = self.forward(&static_tensor, &historical_tensor, &future_tensor)?; + + // Extract median predictions + let pred_data = quantile_preds.squeeze(0)?.to_vec2::()?; + let median_idx = self.config.num_quantiles / 2; + let predictions: Vec = pred_data + .iter() + .map(|horizon_quantiles| horizon_quantiles[median_idx]) + .collect(); + + let latency = start.elapsed().as_micros() as u64; + self.update_performance_metrics(latency); + + if latency > self.config.max_inference_latency_us { + warn!( + "Inference latency {}μs exceeds target {}μs", + latency, self.config.max_inference_latency_us + ); + } + + Ok(predictions) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use anyhow::Result; + + #[tokio::test] + async fn test_tft_creation() -> Result<()> { + let config = TFTConfig { + input_dim: 10, + hidden_dim: 32, + num_heads: 4, + num_quantiles: 5, + prediction_horizon: 5, + sequence_length: 20, + num_static_features: 2, + num_known_features: 3, + num_unknown_features: 5, + ..Default::default() + }; + + let tft = TemporalFusionTransformer::new(config) + .map_err(|_| anyhow::anyhow!("Failed to create TFT"))?; + assert_eq!(tft.metadata.input_dim, 10); + assert_eq!(tft.metadata.output_dim, 5); + Ok(()) + } + + #[test] + fn test_tft_state_creation() -> Result<()> { + let config = TFTConfig { + hidden_dim: 32, + sequence_length: 20, + num_heads: 4, + ..Default::default() + }; + + let state = + TFTState::zeros(&config).map_err(|_| anyhow::anyhow!("Failed to create state"))?; + assert!(state.last_update == 0); + Ok(()) + } + + #[test] + fn test_tft_config_default() -> Result<()> { + let config = TFTConfig::default(); + assert!(config.input_dim > 0); + assert!(config.hidden_dim > 0); + assert!(config.num_heads > 0); + Ok(()) + } + + #[test] + fn test_tft_performance_metrics() -> Result<()> { + let config = TFTConfig { + input_dim: 10, + hidden_dim: 32, + ..Default::default() + }; + + let tft = TemporalFusionTransformer::new(config) + .map_err(|_| anyhow::anyhow!("Failed to create TFT"))?; + let metrics = tft.get_metrics(); + + assert!(metrics.contains_key("total_inferences")); + assert!(metrics.contains_key("avg_latency_us")); + assert!(metrics.contains_key("max_latency_us")); + assert!(metrics.contains_key("throughput_pps")); + Ok(()) + } + + #[test] + fn test_tft_training_state() -> Result<()> { + let config = TFTConfig::default(); + let mut tft = TemporalFusionTransformer::new(config) + .map_err(|_| anyhow::anyhow!("Failed to create TFT"))?; + + assert!(!tft.is_trained); + tft.is_trained = true; + assert!(tft.is_trained); + Ok(()) + } + + #[test] + fn test_tft_metadata() -> Result<()> { + let config = TFTConfig { + input_dim: 15, + prediction_horizon: 12, + ..Default::default() + }; + + let tft = TemporalFusionTransformer::new(config) + .map_err(|_| anyhow::anyhow!("Failed to create TFT"))?; + assert_eq!(tft.metadata.input_dim, 15); + assert_eq!(tft.metadata.output_dim, 12); + Ok(()) + } +} diff --git a/ml/src/tgnn/mod.rs b/ml/src/tgnn/mod.rs index 5003590f8..d99d8dd33 100644 --- a/ml/src/tgnn/mod.rs +++ b/ml/src/tgnn/mod.rs @@ -24,11 +24,7 @@ pub mod message_passing; pub mod traits; pub mod types; -// Re-exports -pub use gating::*; -pub use graph::*; -pub use message_passing::*; -pub use traits::*; +// DO NOT RE-EXPORT - Use explicit imports at usage sites // Import types from main crate - this fixes the circular dependency use crate::{InferenceResult, MLError, ModelMetadata, ModelType, PRECISION_FACTOR}; diff --git a/ml/src/tgnn/mod.rs.bak b/ml/src/tgnn/mod.rs.bak new file mode 100644 index 000000000..d99d8dd33 --- /dev/null +++ b/ml/src/tgnn/mod.rs.bak @@ -0,0 +1,1111 @@ +//! # Temporal Graph Gated Networks (TGNN) for HFT +//! +//! Ultra-low latency implementation of TGNN for market microstructure analysis. +//! +//! ## Key Features +//! +//! - Sub-1μs graph neural network inference +//! - Real-time order book graph construction +//! - Market maker and liquidity flow modeling +//! - Cache-friendly graph operations +//! - Integer arithmetic for precision +//! +//! ## Performance Targets +//! +//! - Graph construction: <500ns from order book +//! - GNN inference: <1μs per prediction +//! - Node updates: <100ns per update +//! - Memory: Minimal allocations + +// Module imports +pub mod gating; +pub mod graph; +pub mod message_passing; +pub mod traits; +pub mod types; + +// DO NOT RE-EXPORT - Use explicit imports at usage sites + +// Import types from main crate - this fixes the circular dependency +use crate::{InferenceResult, MLError, ModelMetadata, ModelType, PRECISION_FACTOR}; +// Import types from this module +use types::{TrainingMetrics, ValidationMetrics}; + +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Instant, SystemTime}; + +// Import RNG utilities from types crate +use rand::prelude::*; // Replace common::rng with standard rand + +use async_trait::async_trait; +use dashmap::DashMap; +use ndarray::{s, Array1, Array2}; +use serde::{Deserialize, Serialize}; +use tracing::{debug, info, warn}; + +/// Node types in market microstructure graph +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum NodeType { + /// Price level in order book + PriceLevel, + /// Market maker entity + MarketMaker, + /// Liquidity pool + LiquidityPool, + /// Order cluster + OrderCluster, +} + +/// Edge types representing market relationships +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum EdgeType { + /// Price proximity relationship + PriceProximity, + /// Liquidity flow + LiquidityFlow, + /// Market maker connection + MarketMaking, + /// Order correlation + OrderCorrelation, +} + +/// Market node identifier +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct NodeId { + pub node_type: NodeType, + pub id: String, +} + +impl NodeId { + pub fn price_level(price: i64) -> Self { + Self { + node_type: NodeType::PriceLevel, + id: format!("price_{}", price), + } + } + + pub fn market_maker(name: impl Into) -> Self { + Self { + node_type: NodeType::MarketMaker, + id: name.into(), + } + } +} + +/// Market edge with temporal properties +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MarketEdge { + pub edge_type: EdgeType, + pub weight: i64, + pub strength: f64, + pub timestamp: u64, + pub decay_factor: f64, +} + +impl MarketEdge { + pub fn new(edge_type: EdgeType, weight: i64, strength: f64) -> Self { + Self { + edge_type, + weight, + strength, + timestamp: SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() as u64, + decay_factor: 0.99, + } + } + + pub fn apply_temporal_decay(&mut self, current_time: u64) { + let age = current_time.saturating_sub(self.timestamp); + let decay = self.decay_factor.powf(age as f64 / 1_000_000_000.0); // per second + self.strength *= decay; + self.weight = (self.weight as f64 * decay) as i64; + } +} + +/// TGGN model configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TGGNConfig { + /// Maximum number of nodes + pub max_nodes: usize, + + /// Maximum number of edges + pub max_edges: usize, + + /// Node feature dimension + pub node_dim: usize, + + /// Edge feature dimension + pub edge_dim: usize, + + /// Hidden dimension for GNN layers + pub hidden_dim: usize, + + /// Number of message passing layers + pub num_layers: usize, + + /// Temporal decay factor + pub temporal_decay: f64, + + /// Graph update frequency (nanoseconds) + pub update_frequency_ns: u64, + + /// Enable SIMD optimizations + pub use_simd: bool, +} + +impl Default for TGGNConfig { + fn default() -> Self { + Self { + max_nodes: 1000, + max_edges: 10000, + node_dim: 32, + edge_dim: 16, + hidden_dim: 64, + num_layers: 3, + temporal_decay: 0.99, + update_frequency_ns: 1_000_000, // 1ms + use_simd: true, + } + } +} + +/// Temporal Graph Gated Networks for market microstructure +pub struct TGGN { + /// Model configuration + config: TGGNConfig, + + /// Model metadata + pub metadata: ModelMetadata, + + /// Market graph structure + graph: MarketGraph, + + /// Gating mechanism + gating: GatingMechanism, + + /// Message passing layers + message_passing: Vec, + + /// Node embeddings cache + node_embeddings: DashMap>, + + /// Edge embeddings cache + edge_embeddings: DashMap<(NodeId, NodeId), Array1>, + + /// Whether model is trained + is_trained: bool, + + /// Performance counters + inference_count: AtomicU64, + total_latency_ns: AtomicU64, + max_latency_ns: AtomicU64, + graph_updates: AtomicU64, + + /// Last update timestamp + last_update: AtomicU64, +} + +impl TGGN { + /// Create new TGGN model + pub fn new(config: TGGNConfig) -> Result { + let mut metadata = ModelMetadata::new( + ModelType::TGNN, + "1.0.0".to_string(), + config.node_dim, + 1.0, // Single output for prediction + ); + metadata.add_metadata("max_nodes", config.max_nodes.to_string()); + metadata.add_metadata("max_edges", config.max_edges.to_string()); + metadata.add_metadata("hidden_dim", config.hidden_dim.to_string()); + metadata.add_metadata("num_layers", config.num_layers.to_string()); + + let graph = MarketGraph::new(config.max_nodes, config.max_edges)?; + let gating = GatingMechanism::new(config.hidden_dim)?; + + // Initialize message passing layers + let mut message_passing = Vec::with_capacity(config.num_layers); + for layer in 0..config.num_layers { + let input_dim = if layer == 0 { + config.node_dim + } else { + config.hidden_dim + }; + message_passing.push(MessagePassing::new(input_dim, config.hidden_dim)?); + } + + info!( + "Initialized TGGN with {} nodes, {} layers", + config.max_nodes, config.num_layers + ); + + Ok(Self { + config, + metadata, + graph, + gating, + message_passing, + node_embeddings: DashMap::new(), + edge_embeddings: DashMap::new(), + is_trained: false, + inference_count: AtomicU64::new(0), + total_latency_ns: AtomicU64::new(0), + max_latency_ns: AtomicU64::new(0), + graph_updates: AtomicU64::new(0), + last_update: AtomicU64::new(0), + }) + } + + /// Create with default configuration + pub fn default() -> Result { + Self::new(TGGNConfig::default()) + } + + /// Update graph from order book data + pub fn update_from_order_book( + &mut self, + bids: &[(i64, i64)], // (price, volume) pairs + asks: &[(i64, i64)], + timestamp: u64, + ) -> Result<(), MLError> { + let start = Instant::now(); + + // Clear old nodes and edges + self.graph + .clear_temporal_data(timestamp, self.config.temporal_decay)?; + + // Add price level nodes for bids + for (i, &(price, volume)) in bids.iter().enumerate() { + let node_id = NodeId::price_level(price); + let features = self.create_price_level_features(price, volume, true, i)?; + self.graph.add_node(node_id.clone(), features.to_vec())?; + self.node_embeddings.insert(node_id, features); + } + + // Add price level nodes for asks + for (i, &(price, volume)) in asks.iter().enumerate() { + let node_id = NodeId::price_level(price); + let features = self.create_price_level_features(price, volume, false, i)?; + self.graph.add_node(node_id.clone(), features.to_vec())?; + self.node_embeddings.insert(node_id, features); + } + + // Create edges between nearby price levels + self.create_proximity_edges(bids, asks)?; + + // Create liquidity flow edges + self.create_liquidity_edges(bids, asks)?; + + let elapsed = start.elapsed(); + self.graph_updates.fetch_add(1, Ordering::Relaxed); + self.last_update.store(timestamp, Ordering::Relaxed); + + debug!( + "Updated graph in {}ns: {} nodes, {} edges", + elapsed.as_nanos(), + self.graph.node_count(), + self.graph.edge_count() + ); + + // Check latency target + let latency_ns = elapsed.as_nanos() as u64; + if latency_ns > 500 { + // 500ns target + warn!("Graph update {}ns exceeds target 500ns", latency_ns); + } + + Ok(()) + } + + /// Perform graph neural network inference + pub fn gnn_inference( + &mut self, + target_nodes: &[NodeId], + ) -> Result, MLError> { + let start = Instant::now(); + + let mut predictions = HashMap::new(); + + // Get current node embeddings + let mut node_features = HashMap::new(); + for node_id in target_nodes { + if let Some(embedding) = self.node_embeddings.get(node_id) { + node_features.insert(node_id.clone(), embedding.clone()); + } else { + // Create default features if node not found + let default_features = Array1::zeros(self.config.node_dim); + node_features.insert(node_id.clone(), default_features); + } + } + + // Apply message passing layers + for (layer_idx, layer) in self.message_passing.iter().enumerate() { + let layer_start = Instant::now(); + + // Collect messages from neighbors + for node_id in target_nodes { + if let Some(neighbors) = self.graph.get_neighbors(node_id) { + let messages = self.collect_messages(node_id, &neighbors, &node_features)?; + + // Apply gating mechanism + let gated_messages = self.gating.apply(&messages)?; + + // Update node features with gated messages + if let Some(current_features) = node_features.get_mut(node_id) { + let updated = layer.forward(current_features, &gated_messages)?; + *current_features = updated; + } + } + } + + debug!( + "Layer {} completed in {}ns", + layer_idx, + layer_start.elapsed().as_nanos() + ); + } + + // Generate predictions from final node features + for node_id in target_nodes { + if let Some(features) = node_features.get(node_id) { + // Simple prediction: weighted sum of features + let prediction = features.sum() / features.len() as f64; + predictions.insert(node_id.clone(), prediction); + } + } + + let elapsed = start.elapsed(); + self.inference_count.fetch_add(1, Ordering::Relaxed); + self.total_latency_ns + .fetch_add(elapsed.as_nanos() as u64, Ordering::Relaxed); + + let latency_ns = elapsed.as_nanos() as u64; + let current_max = self.max_latency_ns.load(Ordering::Relaxed); + if latency_ns > current_max { + self.max_latency_ns + .compare_exchange_weak( + current_max, + latency_ns, + Ordering::Relaxed, + Ordering::Relaxed, + ) + .ok(); + } + + // Check sub-1μs target + if latency_ns > 1000 { + // 1μs = 1000ns + warn!("GNN inference {}ns exceeds target 1000ns", latency_ns); + } else { + debug!("GNN inference completed in {}ns", latency_ns); + } + + Ok(predictions) + } + + /// Create features for price level nodes + fn create_price_level_features( + &self, + price: i64, + volume: i64, + is_bid: bool, + depth_level: usize, + ) -> Result, MLError> { + let mut features = Array1::zeros(self.config.node_dim); + + // Normalize price and volume + let price_norm = (price as f64) / PRECISION_FACTOR as f64; + let volume_norm = (volume as f64) / PRECISION_FACTOR as f64; + + // Feature 0-3: Basic price/volume info + if features.len() > 0 { + features[0] = price_norm; + } + if features.len() > 1 { + features[1] = volume_norm; + } + if features.len() > 2 { + features[2] = if is_bid { 1.0 } else { -1.0 }; + } + if features.len() > 3 { + features[3] = depth_level as f64 / 10.0; + } + + // Feature 4-7: Statistical features + if features.len() > 4 { + features[4] = price_norm.ln(); + } // Log price + if features.len() > 5 { + features[5] = volume_norm.sqrt(); + } // Sqrt volume + if features.len() > 6 { + features[6] = price_norm * volume_norm; + } // Price * volume + if features.len() > 7 { + features[7] = volume_norm / (price_norm + 1e-8); + } // Volume/price ratio + + // Feature 8-15: Technical indicators (simplified) + for i in 8..features.len().min(16) { + let phase = (i as f64 * std::f64::consts::PI) / 8.0; + features[i] = (price_norm * phase.cos() + volume_norm * phase.sin()) / 10.0; + } + + // Feature 16+: Reserved for market microstructure + for i in 16..features.len() { + features[i] = thread_rng().gen::() * 0.01; // Small random noise + } + + Ok(features) + } + + /// Create edges between nearby price levels + fn create_proximity_edges( + &mut self, + bids: &[(i64, i64)], + asks: &[(i64, i64)], + ) -> Result<(), MLError> { + // Connect adjacent price levels within same side + for window in bids.windows(2) { + let node1 = NodeId::price_level(window[0].0); + let node2 = NodeId::price_level(window[1].0); + let weight = ((window[0].1 + window[1].1) / 2) as i64; // Avg volume + let edge = MarketEdge::new(EdgeType::PriceProximity, weight, 0.8); + self.graph.add_edge(&node1, &node2, edge)?; + } + + for window in asks.windows(2) { + let node1 = NodeId::price_level(window[0].0); + let node2 = NodeId::price_level(window[1].0); + let weight = ((window[0].1 + window[1].1) / 2) as i64; + let edge = MarketEdge::new(EdgeType::PriceProximity, weight, 0.8); + self.graph.add_edge(&node1, &node2, edge)?; + } + + // Connect best bid and ask + if !bids.is_empty() && !asks.is_empty() { + let best_bid = NodeId::price_level(bids[0].0); + let best_ask = NodeId::price_level(asks[0].0); + let spread_weight = (asks[0].0 - bids[0].0).abs(); + let edge = MarketEdge::new(EdgeType::PriceProximity, spread_weight, 0.9); + self.graph.add_edge(&best_bid, &best_ask, edge)?; + } + + Ok(()) + } + + /// Create liquidity flow edges + fn create_liquidity_edges( + &mut self, + bids: &[(i64, i64)], + asks: &[(i64, i64)], + ) -> Result<(), MLError> { + // Create flow edges based on volume imbalance + let total_bid_volume: i64 = bids.iter().map(|(_, v)| v).sum(); + let total_ask_volume: i64 = asks.iter().map(|(_, v)| v).sum(); + + let imbalance = total_bid_volume - total_ask_volume; + let flow_strength = + (imbalance.abs() as f64) / (total_bid_volume + total_ask_volume + 1) as f64; + + // Connect high-volume levels with flow edges + for &(price, volume) in bids.iter().take(3) { + for &(ask_price, ask_volume) in asks.iter().take(3) { + if volume > total_bid_volume / 10 && ask_volume > total_ask_volume / 10 { + let node1 = NodeId::price_level(price); + let node2 = NodeId::price_level(ask_price); + let weight = (volume.min(ask_volume)) as i64; + let edge = MarketEdge::new(EdgeType::LiquidityFlow, weight, flow_strength); + self.graph.add_edge(&node1, &node2, edge)?; + } + } + } + + Ok(()) + } + + /// Collect messages from neighboring nodes + fn collect_messages( + &self, + node_id: &NodeId, + neighbors: &[NodeId], + node_features: &HashMap>, + ) -> Result>, MLError> { + let mut messages = Vec::new(); + + for neighbor in neighbors { + if let Some(neighbor_features) = node_features.get(neighbor) { + // Get edge weight if available + let edge_weight = self.graph.get_edge_weight(node_id, neighbor).unwrap_or(1.0); + + // Weight neighbor features by edge strength + let weighted_message = neighbor_features.mapv(|x| x * edge_weight); + messages.push(weighted_message); + } + } + + Ok(messages) + } + + /// Get performance statistics + pub fn get_performance_stats(&self) -> HashMap { + let mut stats = HashMap::new(); + + let inference_count = self.inference_count.load(Ordering::Relaxed); + let total_latency = self.total_latency_ns.load(Ordering::Relaxed); + let max_latency = self.max_latency_ns.load(Ordering::Relaxed); + let graph_updates = self.graph_updates.load(Ordering::Relaxed); + + stats.insert("inference_count".to_string(), inference_count as f64); + stats.insert("graph_updates".to_string(), graph_updates as f64); + stats.insert("max_latency_ns".to_string(), max_latency as f64); + + if inference_count > 0 { + stats.insert( + "avg_latency_ns".to_string(), + total_latency as f64 / inference_count as f64, + ); + } + + stats.insert("node_count".to_string(), self.graph.node_count() as f64); + stats.insert("edge_count".to_string(), self.graph.edge_count() as f64); + + stats + } + + /// Public getters for checkpoint operations + pub fn node_embeddings(&self) -> &DashMap> { + &self.node_embeddings + } + + pub fn edge_embeddings(&self) -> &DashMap<(NodeId, NodeId), Array1> { + &self.edge_embeddings + } + + pub fn config(&self) -> &TGGNConfig { + &self.config + } + + pub fn inference_count(&self) -> &AtomicU64 { + &self.inference_count + } + + pub fn graph_updates(&self) -> &AtomicU64 { + &self.graph_updates + } + + pub fn is_trained(&self) -> bool { + self.is_trained + } + + /// Get graph statistics for monitoring and checkpointing + pub fn get_graph_stats(&self) -> (usize, usize) { + (self.graph.node_count(), self.graph.edge_count()) + } + + /// Restore node embeddings from checkpoint state + pub fn restore_node_embeddings( + &mut self, + embeddings: &HashMap>, + ) -> Result<(), MLError> { + for (node_id_str, embedding) in embeddings { + // Convert f32 to f64 + let embedding_f64: Vec = embedding.iter().map(|&x| x as f64).collect(); + let array = Array1::from_vec(embedding_f64); + // Parse the node ID string to create proper NodeId + let node_id = if node_id_str.starts_with("price_") { + let price = node_id_str + .strip_prefix("price_") + .and_then(|s| s.parse::().ok()) + .unwrap_or(0); + NodeId::price_level(price) + } else if node_id_str.starts_with("mm_") { + NodeId::market_maker(&node_id_str[3..]) + } else { + // Default case - use the string as-is with generic type + NodeId { + node_type: NodeType::PriceLevel, + id: node_id_str.clone(), + } + }; + self.node_embeddings.insert(node_id, array); + } + Ok(()) + } + + /// Restore edge embeddings from checkpoint state + pub fn restore_edge_embeddings( + &mut self, + embeddings: &HashMap>, + ) -> Result<(), MLError> { + for (edge_key, embedding) in embeddings { + // Convert f32 to f64 + let embedding_f64: Vec = embedding.iter().map(|&x| x as f64).collect(); + let array = Array1::from_vec(embedding_f64); + + // Parse edge key (assuming format like "from_id->to_id") + if let Some((from_str, to_str)) = edge_key.split_once("->") { + // Parse from node + let from_node = if from_str.starts_with("price_") { + let price = from_str + .strip_prefix("price_") + .and_then(|s| s.parse::().ok()) + .unwrap_or(0); + NodeId::price_level(price) + } else if from_str.starts_with("mm_") { + NodeId::market_maker(&from_str[3..]) + } else { + NodeId { + node_type: NodeType::PriceLevel, + id: from_str.to_string(), + } + }; + + // Parse to node + let to_node = if to_str.starts_with("price_") { + let price = to_str + .strip_prefix("price_") + .and_then(|s| s.parse::().ok()) + .unwrap_or(0); + NodeId::price_level(price) + } else if to_str.starts_with("mm_") { + NodeId::market_maker(&to_str[3..]) + } else { + NodeId { + node_type: NodeType::PriceLevel, + id: to_str.to_string(), + } + }; + + self.edge_embeddings.insert((from_node, to_node), array); + } + } + Ok(()) + } + + /// Restore graph statistics from checkpoint state + pub fn restore_graph_statistics( + &mut self, + _stats: &HashMap, + ) -> Result<(), MLError> { + // Production implementation for now - graph statistics would be restored here + Ok(()) + } + + /// Restore message passing weights from checkpoint state + pub fn restore_message_passing_weights( + &mut self, + _weights: &Vec>, + ) -> Result<(), MLError> { + // Production implementation for now - message passing weights would be restored here + Ok(()) + } +} + +#[async_trait] +impl MLModel for TGGN { + type Config = serde_json::Value; + + fn metadata(&self) -> &ModelMetadata { + &self.metadata + } + + fn is_ready(&self) -> bool { + self.is_trained + } + + async fn train( + &mut self, + features: &Array2, + targets: &Array2, + ) -> Result { + info!("Starting TGGN training with {} samples", features.nrows()); + + let start = Instant::now(); + let _n_samples = features.nrows(); + + // For TGGN, training involves learning message passing weights with real gradients + let learning_rate = 0.001; + + // Prepare batch data for message passing training (moved outside loop) + let mut node_features_batch = Vec::new(); + let mut neighbor_messages_batch = Vec::new(); + let mut targets_batch = Vec::new(); + + // Convert features to node features and targets for each layer + for (layer_idx, layer) in self.message_passing.iter_mut().enumerate() { + info!("Training layer {} with real backpropagation", layer_idx); + + // Clear batch data for this layer + node_features_batch.clear(); + neighbor_messages_batch.clear(); + targets_batch.clear(); + + for sample_idx in 0..features.nrows().min(targets.nrows()) { + let node_features = features.row(sample_idx).to_owned(); + let target = targets + .row(sample_idx) + .slice(s![..self.config.hidden_dim.min(targets.ncols())]) + .to_owned(); + + // For training, create synthetic neighbor messages from nearby samples + let mut neighbor_messages = Vec::new(); + for neighbor_idx in 0..3.min(features.nrows()) { + // Use up to 3 neighbors + if neighbor_idx != sample_idx { + let neighbor_features = features.row(neighbor_idx).to_owned(); + neighbor_messages.push(neighbor_features); + } + } + + node_features_batch.push(node_features); + neighbor_messages_batch.push(neighbor_messages); + targets_batch.push(target); + } + + // Train layer with proper backpropagation + layer + .train_weights( + &node_features_batch, + &neighbor_messages_batch, + &targets_batch, + learning_rate, + ) + .map_err(|e| MLError::TrainingError(format!("Layer training failed: {}", e)))?; + } + + // Update gating mechanism with real gradients + if !node_features_batch.is_empty() { + // Prepare inputs and targets for gating mechanism + let gating_inputs = node_features_batch.clone(); + let gating_targets = targets_batch.clone(); + + self.gating + .update_weights(&gating_inputs, &gating_targets, learning_rate) + .map_err(|e| MLError::TrainingError(format!("Gating training failed: {}", e)))?; + } + + self.is_trained = true; + self.metadata.mark_trained(); + + let training_time = start.elapsed().as_secs_f64(); + + info!("TGGN training completed in {:.2}s", training_time); + + Ok(TrainingMetrics { + loss: 0.1, + accuracy: 0.9, + precision: 0.88, + recall: 0.85, + f1_score: 0.865, + training_time_seconds: training_time, + epochs_trained: 1, + convergence_achieved: true, + additional_metrics: HashMap::new(), + }) + } + + async fn predict(&self, features: &[f64]) -> Result { + if !self.is_trained { + return Err(MLError::NotTrained("TGGN not trained".to_string())); + } + + let start = Instant::now(); + + // Simple prediction based on features + let prediction = features.iter().sum::() / features.len() as f64; + let confidence = 0.9; // High confidence for graph-based predictions + + let result = InferenceResult::new( + "tgnn_1.0".to_string(), + prediction, + confidence, + start.elapsed().as_micros() as u64, + start.elapsed().as_nanos() as u64, + self.metadata.clone(), + ); + + Ok(result) + } + + async fn validate( + &self, + features: &Array2, + targets: &Array2, + ) -> Result { + let mut total_error = 0.0; + let mut correct_predictions = 0; + + for i in 0..features.nrows() { + let row_features: Vec = features.row(i).to_vec(); + let prediction_result = self.predict(&row_features).await?; + let prediction = prediction_result.prediction_as_float(); + + let target = targets[[i, 0]]; + let error = (prediction - target).abs(); + total_error += error; + + if error < 0.1 { + // Threshold for "correct" + correct_predictions += 1; + } + } + + let mse = total_error / features.nrows() as f64; + let accuracy = correct_predictions as f64 / features.nrows() as f64; + + Ok(ValidationMetrics { + validation_loss: mse, + validation_accuracy: accuracy, + validation_precision: accuracy * 0.95, + validation_recall: accuracy * 0.93, + validation_f1_score: accuracy * 0.94, + samples_validated: features.nrows(), + additional_metrics: HashMap::new(), + }) + } + + async fn update( + &mut self, + features: &Array2, + targets: &Array2, + ) -> Result<(), MLError> { + // Online learning for TGGN + self.train(features, targets).await?; + Ok(()) + } + + async fn save(&self, path: &str) -> Result<(), MLError> { + let data = serde_json::json!({ + "config": self.config, + "metadata": self.metadata, + "is_trained": self.is_trained, + "performance_stats": self.get_performance_stats(), + }); + + let serialized = + serde_json::to_string_pretty(&data).map_err(|e| MLError::SerializationError { + reason: e.to_string(), + })?; + tokio::fs::write(path, serialized) + .await + .map_err(|e| MLError::SerializationError { + reason: e.to_string(), + })?; + + info!("Saved TGGN model to {}", path); + Ok(()) + } + + async fn load(&mut self, path: &str) -> Result<(), MLError> { + let content = + tokio::fs::read_to_string(path) + .await + .map_err(|e| MLError::SerializationError { + reason: e.to_string(), + })?; + + let data: serde_json::Value = + serde_json::from_str(&content).map_err(|e| MLError::SerializationError { + reason: e.to_string(), + })?; + + self.config = serde_json::from_value(data["config"].clone()).map_err(|e| { + MLError::SerializationError { + reason: e.to_string(), + } + })?; + + self.metadata = serde_json::from_value(data["metadata"].clone()).map_err(|e| { + MLError::SerializationError { + reason: e.to_string(), + } + })?; + + self.is_trained = data["is_trained"].as_bool().unwrap_or(false); + + info!("Loaded TGGN model from {}", path); + Ok(()) + } + + fn config(&self) -> Self::Config { + serde_json::to_value(&self.config).unwrap_or_default() + } + + fn set_config(&mut self, config: Self::Config) -> Result<(), MLError> { + self.config = serde_json::from_value(config).map_err(|e| MLError::ConfigError { + reason: e.to_string(), + })?; + Ok(()) + } +} + +/// Training pipeline for TGGN with order book data +pub struct TGGNTrainingPipeline { + pub model: TGGN, + pub training_data: Vec<(Vec<(i64, i64)>, Vec<(i64, i64)>, f64)>, // (bids, asks, target) +} + +impl TGGNTrainingPipeline { + pub fn new(config: TGGNConfig) -> Result { + let model = TGGN::new(config)?; + Ok(Self { + model, + training_data: Vec::new(), + }) + } + + pub fn add_training_sample( + &mut self, + bids: Vec<(i64, i64)>, + asks: Vec<(i64, i64)>, + target: f64, + ) { + self.training_data.push((bids, asks, target)); + } + + pub async fn train_from_order_book_data(&mut self) -> Result { + info!( + "Training TGGN from {} order book samples", + self.training_data.len() + ); + + // Convert order book data to feature matrices + let mut features_vec = Vec::new(); + let mut targets_vec = Vec::new(); + + for (bids, asks, target) in &self.training_data { + // Update graph with order book data + let timestamp = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() as u64; + + self.model.update_from_order_book(bids, asks, timestamp)?; + + // Extract features from updated graph + let graph_features = self.extract_graph_features()?; + features_vec.push(graph_features); + targets_vec.push(vec![*target]); + } + + // Convert to ndarray format + let features = Array2::from_shape_vec( + (features_vec.len(), features_vec[0].len()), + features_vec.into_iter().flatten().collect(), + ) + .map_err(|e| MLError::DimensionMismatch { + expected: self.model.config.node_dim, + actual: e.to_string().len(), + })?; + + let targets = Array2::from_shape_vec( + (targets_vec.len(), 1), + targets_vec.into_iter().flatten().collect(), + ) + .map_err(|e| MLError::DimensionMismatch { + expected: 1, + actual: e.to_string().len(), + })?; + + // Train the model (convert types::MLError to MLError) + self.model + .train(&features, &targets) + .await + .map_err(|e| MLError::TrainingError(format!("TGNN training failed: {}", e))) + } + + fn extract_graph_features(&self) -> Result, MLError> { + let stats = self.model.graph.get_stats(); + let mut features = Vec::new(); + + // Graph topology features + features.push(stats.node_count as f64); + features.push(stats.edge_count as f64); + features.push(stats.density); + features.push(stats.average_degree); + + // Fill remaining features with zeros if needed + while features.len() < self.model.config.node_dim { + features.push(0.0); + } + + Ok(features) + } +} + +#[cfg(test)] +mod tests { + use super::*; + // use crate::safe_operations; // DISABLED - module not found + + #[tokio::test] + async fn test_tggn_creation() { + let config = TGGNConfig::default(); + let model = TGGN::new(config)?; + + assert_eq!(model.config.max_nodes, 1000); + assert_eq!(model.config.num_layers, 3); + assert!(!model.is_trained); + } + + #[tokio::test] + async fn test_order_book_update() { + let config = TGGNConfig::default(); + let mut model = TGGN::new(config)?; + + let bids = vec![(100_00000000, 1000_00000000), (99_00000000, 500_00000000)]; + let asks = vec![(101_00000000, 800_00000000), (102_00000000, 600_00000000)]; + let timestamp = 1234567890; + + let result = model.update_from_order_book(&bids, &asks, timestamp); + assert!(result.is_ok()); + + assert_eq!(model.graph.node_count(), 4); // 2 bids + 2 asks + assert!(model.graph.edge_count() > 0); + } + + #[tokio::test] + async fn test_gnn_inference() { + let config = TGGNConfig::default(); + let mut model = TGGN::new(config)?; + + // Setup graph with some nodes + let bids = vec![(100_00000000, 1000_00000000)]; + let asks = vec![(101_00000000, 800_00000000)]; + let timestamp = 1234567890; + + model.update_from_order_book(&bids, &asks, timestamp)?; + + let target_nodes = vec![NodeId::price_level(100_00000000)]; + let predictions = model.gnn_inference(&target_nodes)?; + + assert_eq!(predictions.len(), 1); + assert!(predictions.contains_key(&NodeId::price_level(100_00000000))); + } + + #[tokio::test] + async fn test_training_pipeline() { + let config = TGGNConfig::default(); + let mut pipeline = TGGNTrainingPipeline::new(config)?; + + // Add some training samples + pipeline.add_training_sample( + vec![(100_00000000, 1000_00000000)], + vec![(101_00000000, 800_00000000)], + 0.5, + ); + + pipeline.add_training_sample( + vec![(99_00000000, 1200_00000000)], + vec![(100_00000000, 900_00000000)], + -0.3, + ); + + let metrics = pipeline.train_from_order_book_data().await?; + assert!(metrics.training_time_seconds > 0.0); + assert!(pipeline.model.is_trained); + } +} diff --git a/ml/src/tgnn/types.rs b/ml/src/tgnn/types.rs index 9fbbf7828..7e136084c 100644 --- a/ml/src/tgnn/types.rs +++ b/ml/src/tgnn/types.rs @@ -3,11 +3,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; -// Re-export canonical types from the main crate -pub use crate::{InferenceResult, ModelMetadata, ModelType}; - -// Also re-export for compatibility -pub use crate::ModelType as MLModelType; +// NO RE-EXPORTS - Use explicit imports: crate::{InferenceResult, ModelMetadata, ModelType} /// Training metrics for model performance tracking #[derive(Debug, Clone, Serialize, Deserialize)] @@ -51,5 +47,4 @@ pub struct ValidationMetrics { pub additional_metrics: HashMap, } -// Use the main crate's MLError type to avoid conflicts -pub use crate::MLError; +// NO RE-EXPORTS - Use explicit imports: crate::MLError diff --git a/ml/src/tlob/mod.rs b/ml/src/tlob/mod.rs index 72f12be44..2fc1f3e0e 100644 --- a/ml/src/tlob/mod.rs +++ b/ml/src/tlob/mod.rs @@ -8,7 +8,3 @@ pub mod features; pub mod performance; pub mod transformer; -pub use analytics::{OrderFlowAnalytics, VolumeImbalanceCalculator}; -pub use features::{FeatureVector, TLOBFeatureExtractor, TLOBFeatures}; -pub use performance::{LatencyMetrics, TLOBBenchmark}; -pub use transformer::{TLOBConfig, TLOBMetrics, TLOBTransformer}; diff --git a/ml/src/tlob/mod.rs.bak b/ml/src/tlob/mod.rs.bak new file mode 100644 index 000000000..72f12be44 --- /dev/null +++ b/ml/src/tlob/mod.rs.bak @@ -0,0 +1,14 @@ +//! Time Limit Order Book (TLOB) Transformer +//! +//! High-performance TLOB analysis for HFT systems with sub-50μs latency requirements. +//! Based on advanced order flow analytics from institutional trading systems. + +pub mod analytics; +pub mod features; +pub mod performance; +pub mod transformer; + +pub use analytics::{OrderFlowAnalytics, VolumeImbalanceCalculator}; +pub use features::{FeatureVector, TLOBFeatureExtractor, TLOBFeatures}; +pub use performance::{LatencyMetrics, TLOBBenchmark}; +pub use transformer::{TLOBConfig, TLOBMetrics, TLOBTransformer}; diff --git a/ml/src/training.rs b/ml/src/training.rs index 4d7de3bf8..5b35b593c 100644 --- a/ml/src/training.rs +++ b/ml/src/training.rs @@ -13,12 +13,7 @@ // Sub-modules for specialized training components pub mod unified_data_loader; -// Re-export key types from unified data loader -pub use unified_data_loader::{ - BenzingaConfig, BenzingaHistoricalProvider, DatabentoConfig, DatabentoHistoricalProvider, - MarketDataContainer, NewsSentimentData, PriceData, TrainingDataset, TrainingSample, - UnifiedDataLoader, UnifiedDataLoaderConfig, VolumeData, -}; +// NO RE-EXPORTS - Use explicit imports: unified_data_loader::{...} use std::collections::HashMap; use std::time::Instant; diff --git a/ml/src/traits.rs b/ml/src/traits.rs index 2e80e9318..4916a95aa 100644 --- a/ml/src/traits.rs +++ b/ml/src/traits.rs @@ -7,9 +7,7 @@ use async_trait::async_trait; use ndarray::Array2; use serde::{Deserialize, Serialize}; -// Re-export types from correct modules -pub use crate::tgnn::types::{TrainingMetrics, ValidationMetrics}; -pub use crate::{InferenceResult, ModelMetadata}; +// DO NOT RE-EXPORT - Use explicit imports at usage sites // Note: MLModel trait is defined separately in tgnn::traits /// Core ML model trait for all models in the system diff --git a/ml/src/transformers/mod.rs.bak b/ml/src/transformers/mod.rs.bak new file mode 100644 index 000000000..30256dc77 --- /dev/null +++ b/ml/src/transformers/mod.rs.bak @@ -0,0 +1,269 @@ +//! # State-of-the-Art Transformer Models for HFT +//! +//! This module implements cutting-edge transformer architectures optimized for +//! ultra-low latency financial market prediction targeting sub-100μs inference. +//! +//! ## Key Innovations for 2025 HFT Applications +//! +//! - **Minimal Architecture**: 1-2 layers, 1-2 heads, optimized for speed +//! - **FlashAttention 2.0**: Memory-efficient attention via Candle +//! - **Financial Features**: Market microstructure, order book, trade flow +//! - **GPU Acceleration**: Pre-allocated tensors, zero-copy operations +//! - **Quantization Ready**: Support for INT8/INT4 optimization +//! - **LoRA Fine-tuning**: Efficient market adaptation +//! +//! ## Architecture Philosophy +//! +//! Based on 2025 HFT requirements, these transformers prioritize: +//! 1. **Latency over Accuracy**: Sub-100μs inference is paramount +//! 2. **Hardware Optimization**: Custom kernels, CUDA graphs +//! 3. **Feature Engineering**: Alpha captured in features, not model complexity +//! 4. **Memory Efficiency**: Pre-allocated GPU memory pools +//! +//! ## Performance Targets +//! +//! - **Inference Latency**: <100μs end-to-end +//! - **Feature Processing**: <20μs for market data normalization +//! - **Model Forward Pass**: <50μs for transformer computation +//! - **Memory Usage**: <256MB GPU memory footprint + +// Core modules that compile successfully +pub mod attention; + +// Re-export core types that work (commented out until implemented) +// pub use attention::{ +// AttentionConfig, AttentionMask, CrossModalAttention, MultiHeadAttention, +// }; + +/// Transformer model types optimized for different `HFT` use cases +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// TransformerType component. +pub enum TransformerType { + /// Ultra-minimal transformer for <50μs inference + Minimal, + /// Temporal Fusion Transformer for multi-horizon forecasting + TemporalFusion, + /// Sparse transformer for efficiency with longer sequences + Sparse, + /// Cross-modal transformer for `price`/`volume`/news fusion + CrossModal, +} + +/// Model size presets optimized for different latency requirements +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// ModelSize component. +pub enum ModelSize { + /// Ultra-fast: 1 layer, 1 head, 32 dims - target <25μs + Nano, + /// Fast: 1 layer, 2 heads, 64 dims - target <50μs + Micro, + /// Balanced: 2 layers, 2 heads, 128 dims - target <100μs + Small, + /// Custom size configuration + Custom, +} + +impl ModelSize { + /// Get the configuration parameters for each model size + pub const fn config(self) -> (usize, usize, usize) { + match self { + Self::Nano => (1, 1, 32), // (layers, heads, dim) + Self::Micro => (1, 2, 64), // (layers, heads, dim) + Self::Small => (2, 2, 128), // (layers, heads, dim) + Self::Custom => (1, 1, 32), // Default to Nano + } + } + + /// Get expected inference latency in microseconds + pub const fn expected_latency_us(self) -> u64 { + match self { + Self::Nano => 25, + Self::Micro => 50, + Self::Small => 100, + Self::Custom => 50, + } + } +} + +/// Device types for computation +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// DeviceType component. +pub enum DeviceType { + /// `CPU` computation + CPU, + /// `CUDA` `GPU` computation + Cuda, + /// Metal `GPU` computation (Apple) + Metal, +} + +/// Configuration for `HFT`-optimized transformers +#[derive(Debug, Clone, Copy)] +/// HFTTransformerConfig component. +pub struct HFTTransformerConfig { + /// Model type and architecture + pub model_type: TransformerType, + + /// Model size preset + pub model_size: ModelSize, + + /// Custom dimensions (if ModelSize::Custom) + pub num_layers: usize, + pub num_heads: usize, + pub hidden_dim: usize, + pub ff_dim: usize, + + /// Sequence length for market data + pub seq_len: usize, + + /// Feature configuration + pub feature_dim: usize, + pub use_market_microstructure: bool, + pub use_order_book_features: bool, + pub use_trade_flow_features: bool, + + /// Optimization settings + pub use_flash_attention: bool, + pub use_sparse_attention: bool, + pub attention_sparsity: f32, + + /// Memory optimization + pub pre_allocate_tensors: bool, + pub memory_pool_size: usize, + + /// Quantization + pub use_quantization: bool, + pub quantization_bits: u8, // 8, 4, or 2 bits + + /// Hardware settings + pub device_type: DeviceType, + pub use_cuda_graphs: bool, + pub enable_profiling: bool, +} + +impl Default for HFTTransformerConfig { + fn default() -> Self { + Self { + model_type: TransformerType::Minimal, + model_size: ModelSize::Micro, + num_layers: 1, + num_heads: 2, + hidden_dim: 64, + ff_dim: 256, + seq_len: 64, + feature_dim: 32, + use_market_microstructure: true, + use_order_book_features: true, + use_trade_flow_features: true, + use_flash_attention: true, + use_sparse_attention: false, + attention_sparsity: 0.1, + pre_allocate_tensors: true, + memory_pool_size: 1024 * 1024 * 64, // 64MB + use_quantization: false, + quantization_bits: 8, + device_type: DeviceType::Cuda, + use_cuda_graphs: false, // Enable after validation + enable_profiling: false, + } + } +} + +impl HFTTransformerConfig { + /// Create configuration for ultra-low latency (Nano model) + pub fn nano() -> Self { + let (layers, heads, dim) = ModelSize::Nano.config(); + Self { + model_size: ModelSize::Nano, + num_layers: layers, + num_heads: heads, + hidden_dim: dim, + ff_dim: dim * 2, + seq_len: 32, + feature_dim: 16, + ..Default::default() + } + } + + /// Create configuration for balanced latency/accuracy (Micro model) + pub fn micro() -> Self { + let (layers, heads, dim) = ModelSize::Micro.config(); + Self { + model_size: ModelSize::Micro, + num_layers: layers, + num_heads: heads, + hidden_dim: dim, + ff_dim: dim * 4, + ..Default::default() + } + } + + /// Create configuration for maximum accuracy within 100μs (Small model) + pub fn small() -> Self { + let (layers, heads, dim) = ModelSize::Small.config(); + Self { + model_size: ModelSize::Small, + num_layers: layers, + num_heads: heads, + hidden_dim: dim, + ff_dim: dim * 4, + seq_len: 128, + feature_dim: 64, + ..Default::default() + } + } + + /// Enable all optimizations for production deployment + pub fn production() -> Self { + Self { + use_flash_attention: true, + pre_allocate_tensors: true, + use_quantization: true, + quantization_bits: 8, + use_cuda_graphs: true, + ..Self::micro() + } + } + + /// Configuration for benchmarking and validation + pub fn benchmark() -> Self { + Self { + enable_profiling: true, + ..Self::micro() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_model_size_config() { + assert_eq!(ModelSize::Nano.config(), (1, 1, 32)); + assert_eq!(ModelSize::Micro.config(), (1, 2, 64)); + assert_eq!(ModelSize::Small.config(), (2, 2, 128)); + } + + #[test] + fn test_latency_expectations() { + assert_eq!(ModelSize::Nano.expected_latency_us(), 25); + assert_eq!(ModelSize::Micro.expected_latency_us(), 50); + assert_eq!(ModelSize::Small.expected_latency_us(), 100); + } + + #[test] + fn test_config_presets() { + let nano = HFTTransformerConfig::nano(); + assert_eq!(nano.model_size, ModelSize::Nano); + assert_eq!(nano.num_layers, 1); + assert_eq!(nano.num_heads, 1); + assert_eq!(nano.hidden_dim, 32); + + let production = HFTTransformerConfig::production(); + assert!(production.use_flash_attention); + assert!(production.pre_allocate_tensors); + assert!(production.use_quantization); + assert!(production.use_cuda_graphs); + } +} diff --git a/monitoring/mod.rs b/monitoring/mod.rs index 022302aec..4a2cee15a 100644 --- a/monitoring/mod.rs +++ b/monitoring/mod.rs @@ -6,7 +6,7 @@ pub mod metrics; pub mod server; -pub use metrics::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites FoxhuntMetrics, record_order, record_order_fill, @@ -23,7 +23,7 @@ pub use metrics::{ update_throughput, }; -pub use server::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites MetricsServer, MetricsServerConfig, MetricsError, diff --git a/monitoring/mod.rs.bak b/monitoring/mod.rs.bak new file mode 100644 index 000000000..022302aec --- /dev/null +++ b/monitoring/mod.rs.bak @@ -0,0 +1,32 @@ +//! Foxhunt Monitoring Module +//! +//! Provides comprehensive Prometheus metrics collection for the HFT trading system. +//! Optimized for minimal latency impact in critical trading paths. + +pub mod metrics; +pub mod server; + +pub use metrics::{ + FoxhuntMetrics, + record_order, + record_order_fill, + record_order_rejection, + record_latency, + record_order_processing_latency, + record_market_data_latency, + record_risk_breach, + update_position_value, + update_var, + update_active_orders, + record_market_data_message, + record_ml_prediction, + update_throughput, +}; + +pub use server::{ + MetricsServer, + MetricsServerConfig, + MetricsError, + start_metrics_server, + start_metrics_server_with_config, +}; \ No newline at end of file diff --git a/risk/src/error.rs b/risk/src/error.rs index 121ed9a34..e9a0ba146 100644 --- a/risk/src/error.rs +++ b/risk/src/error.rs @@ -223,8 +223,7 @@ use common::Price; } } -/// Re-export safe conversion functions -pub use safe_conversions::*; +// ELIMINATED: Re-exports removed to force explicit imports // Also support FoxhuntResult for consistency with error-handling framework // Removed core dependency - use core instead diff --git a/risk/src/error_consolidated.rs b/risk/src/error_consolidated.rs index 343774347..0bd7bcfc2 100644 --- a/risk/src/error_consolidated.rs +++ b/risk/src/error_consolidated.rs @@ -3,8 +3,7 @@ //! This module demonstrates the consolidated error handling pattern //! using the common error system across all Foxhunt Risk services. -// Re-export shared error types and utilities -pub use common::{CommonError, CommonResult, ErrorCategory}; +// ELIMINATED: Re-exports removed to force explicit imports /// Result type for risk operations using CommonError pub type RiskResult = CommonResult; diff --git a/risk/src/lib.rs b/risk/src/lib.rs index 6a4eeceab..9b8f3c03c 100644 --- a/risk/src/lib.rs +++ b/risk/src/lib.rs @@ -116,19 +116,7 @@ pub mod drawdown_monitor; pub mod safety; -/// Prelude module for convenient imports -pub mod prelude { - //! Prelude module that re-exports the most commonly used types and functions - //! - //! This module provides a convenient way to import all the essential risk management - //! components with a single use statement: - //! - //! ```rust - //! // Import specific types directly from modules - //! use risk::kelly_sizing::KellySizer; - //! use risk::risk_engine::RiskEngine; - //! ``` -} +// ELIMINATED: Prelude module removed to force explicit imports /// Library version pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/risk/src/risk_engine.rs b/risk/src/risk_engine.rs index eda55fe24..ec8375653 100644 --- a/risk/src/risk_engine.rs +++ b/risk/src/risk_engine.rs @@ -18,7 +18,7 @@ use num::{FromPrimitive, ToPrimitive}; use uuid::Uuid; use std::marker::Send; use std::sync::Arc; -use crate::prelude::*; +// ELIMINATED: Prelude import removed to force explicit imports use common::{Position, Symbol, Price, Decimal, OrderSide}; use std::time::Instant; use tokio::sync::broadcast; diff --git a/risk/src/risk_types.rs b/risk/src/risk_types.rs index f3eddaea4..b0df19bb5 100644 --- a/risk/src/risk_types.rs +++ b/risk/src/risk_types.rs @@ -9,8 +9,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -// Re-export commonly used types for convenience -pub use common::Price; +// ELIMINATED: Re-exports removed to force explicit imports use common::Quantity; use common::Symbol; use common::Volume; diff --git a/risk/src/safety/mod.rs b/risk/src/safety/mod.rs index 9b727b286..fd580cebe 100644 --- a/risk/src/safety/mod.rs +++ b/risk/src/safety/mod.rs @@ -19,16 +19,7 @@ pub mod safety_coordinator; pub mod trading_gate; pub mod unix_socket_kill_switch; -pub use kill_switch::*; -pub use emergency_response::*; -pub use performance_tests::*; -pub use position_limiter::*; -pub use safety_coordinator::*; -pub use trading_gate::*; -pub use unix_socket_kill_switch::*; - -// Re-export types from the types module for convenience -pub use crate::risk_types::KillSwitchScope; +// DO NOT RE-EXPORT - Use explicit imports at usage sites // REMOVED: Direct Decimal usage - use canonical types diff --git a/risk/src/var_calculator/mod.rs b/risk/src/var_calculator/mod.rs index 75643a436..cda491311 100644 --- a/risk/src/var_calculator/mod.rs +++ b/risk/src/var_calculator/mod.rs @@ -7,10 +7,4 @@ pub mod monte_carlo; pub mod parametric; pub mod var_engine; -pub use expected_shortfall::ExpectedShortfall; -pub use historical_simulation::HistoricalSimulationVaR; -pub use monte_carlo::MonteCarloVaR; -pub use parametric::ParametricVaR; -pub use var_engine::{ - CircuitBreakerCondition, ComprehensiveVaRResult, RealVaREngine, VaRMethodology, -}; +// ELIMINATED: All re-exports removed to force explicit imports diff --git a/risk/src/var_calculator/mod.rs.bak b/risk/src/var_calculator/mod.rs.bak new file mode 100644 index 000000000..75643a436 --- /dev/null +++ b/risk/src/var_calculator/mod.rs.bak @@ -0,0 +1,16 @@ +//! Value at Risk (`VaR`) calculation engine +//! Eliminates zero `VaR` mocks with enterprise-grade risk calculations + +pub mod expected_shortfall; +pub mod historical_simulation; +pub mod monte_carlo; +pub mod parametric; +pub mod var_engine; + +pub use expected_shortfall::ExpectedShortfall; +pub use historical_simulation::HistoricalSimulationVaR; +pub use monte_carlo::MonteCarloVaR; +pub use parametric::ParametricVaR; +pub use var_engine::{ + CircuitBreakerCondition, ComprehensiveVaRResult, RealVaREngine, VaRMethodology, +}; diff --git a/services/trading_service/src/error.rs b/services/trading_service/src/error.rs index c122ab423..a7559a588 100644 --- a/services/trading_service/src/error.rs +++ b/services/trading_service/src/error.rs @@ -1,7 +1,7 @@ //! Error types for the Trading Service - Using Shared Library Types -// Re-export shared error types and utilities -pub use common::error::CommonError; +// REMOVED: All pub use statements eliminated per cleanup requirements +// Use direct import: common::error::CommonError use common::error::CommonResult; use common::error::ErrorCategory; use common::error::ErrorSeverity; @@ -13,7 +13,7 @@ use common::error::RetryStrategy; pub enum TradingServiceError { /// Shared library error with context #[error("Trading service error: {0}")] - Common(#[from] CommonError), + Common(#[from] common::error::CommonError), /// Order validation failed with specific trading context #[error("Order validation failed: {reason}")] @@ -68,33 +68,33 @@ impl From for tonic::Status { TradingServiceError::Common(common_err) => { // Leverage shared error to gRPC status conversion match common_err { - CommonError::Validation(ref msg) => { + common::error::CommonError::Validation(ref msg) => { tonic::Status::invalid_argument(format!("Validation error: {}", msg)) } - CommonError::Configuration(ref msg) => { + common::error::CommonError::Configuration(ref msg) => { tonic::Status::invalid_argument(format!("Configuration error: {}", msg)) } - CommonError::Network(ref msg) => { + common::error::CommonError::Network(ref msg) => { tonic::Status::unavailable(format!("Network error: {}", msg)) } - CommonError::Database(ref db_err) => { + common::error::CommonError::Database(ref db_err) => { tonic::Status::internal(format!("Database error: {}", db_err)) } - CommonError::Service { category, message } => { + common::error::CommonError::Service { category, message } => { match category { - crate::error::ErrorCategory::Authentication => { + common::error::ErrorCategory::Authentication => { tonic::Status::unauthenticated(message.clone()) } - crate::error::ErrorCategory::Resource => { + common::error::ErrorCategory::Resource => { tonic::Status::not_found(message.clone()) } - crate::error::ErrorCategory::Validation => { + common::error::ErrorCategory::Validation => { tonic::Status::invalid_argument(message.clone()) } _ => tonic::Status::internal(format!("{}: {}", category, message)), } } - CommonError::Timeout { actual_ms, max_ms } => { + common::error::CommonError::Timeout { actual_ms, max_ms } => { tonic::Status::deadline_exceeded(format!( "Operation timed out: {}ms (max: {}ms)", actual_ms, max_ms diff --git a/services/trading_service/src/event_streaming/mod.rs b/services/trading_service/src/event_streaming/mod.rs index 1e681a2b0..4831a9913 100644 --- a/services/trading_service/src/event_streaming/mod.rs +++ b/services/trading_service/src/event_streaming/mod.rs @@ -24,10 +24,6 @@ pub mod filters; pub mod publisher; pub mod subscriber; -pub use events::*; -pub use filters::*; -pub use publisher::*; -pub use subscriber::*; /// Trading event streaming system #[derive(Debug, Clone)] diff --git a/services/trading_service/src/event_streaming/mod.rs.bak b/services/trading_service/src/event_streaming/mod.rs.bak new file mode 100644 index 000000000..1e681a2b0 --- /dev/null +++ b/services/trading_service/src/event_streaming/mod.rs.bak @@ -0,0 +1,418 @@ +//! # Trading Service Event Streaming Module +//! +//! This module provides event streaming capabilities for the trading service, +//! allowing real-time broadcasting of trading events to subscribers. +//! +//! ## Features +//! +//! - Publishing trading events (orders, fills, cancellations) +//! - Risk management event notifications +//! - Market data event streaming +//! - Performance metrics and system events +//! - Event filtering and subscription management + +use crate::error::{Result, TradingServiceError}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::{broadcast, RwLock}; +use tracing::{debug, error, info, warn}; + +pub mod events; +pub mod filters; +pub mod publisher; +pub mod subscriber; + +pub use events::*; +pub use filters::*; +pub use publisher::*; +pub use subscriber::*; + +/// Trading event streaming system +#[derive(Debug, Clone)] +pub struct TradingEventStreamer { + /// Event publisher for broadcasting events + pub publisher: EventPublisher, + /// Subscription manager for handling client subscriptions + pub subscription_manager: Arc>, + /// Event buffer for reliable delivery + pub event_buffer: Arc>, + /// Configuration for the streaming system + pub config: StreamingConfig, +} + +impl TradingEventStreamer { + /// Create a new trading event streamer + pub fn new(config: StreamingConfig) -> Self { + let (sender, _receiver) = broadcast::channel(config.max_subscribers); + + Self { + publisher: EventPublisher::new(sender), + subscription_manager: Arc::new(RwLock::new(SubscriptionManager::new())), + event_buffer: Arc::new(RwLock::new(EventBuffer::new(config.buffer_size))), + config, + } + } + + /// Start the event streaming system + pub async fn start(&self) -> Result<()> { + info!("Starting trading event streaming system"); + + // Initialize event buffer cleanup task + let buffer = self.event_buffer.clone(); + let cleanup_interval = self.config.cleanup_interval_secs; + + tokio::spawn(async move { + let mut interval = + tokio::time::interval(std::time::Duration::from_secs(cleanup_interval)); + + loop { + interval.tick().await; + + let mut buffer = buffer.write().await; + buffer.cleanup_expired_events(); + } + }); + + info!("Trading event streaming system started successfully"); + Ok(()) + } + + /// Publish a trading event + pub async fn publish_event(&self, event: TradingEvent) -> Result<()> { + // Store event in buffer for replay + { + let mut buffer = self.event_buffer.write().await; + buffer.add_event(event.clone()).await?; + } + + // Publish event to subscribers + self.publisher.publish(event.clone()).await?; + + debug!("Published trading event: {:?}", event.event_type); + Ok(()) + } + + /// Subscribe to trading events with a filter + pub async fn subscribe(&self, filter: EventFilter) -> Result { + let receiver = self.publisher.subscribe()?; + let subscription_id = uuid::Uuid::new_v4().to_string(); + + { + let mut manager = self.subscription_manager.write().await; + manager.add_subscription(subscription_id.clone(), filter.clone()); + } + + Ok(TradingEventReceiver::new(subscription_id, receiver, filter)) + } + + /// Unsubscribe from trading events + pub async fn unsubscribe(&self, subscription_id: &str) -> Result<()> { + let mut manager = self.subscription_manager.write().await; + manager.remove_subscription(subscription_id); + + debug!("Unsubscribed client: {}", subscription_id); + Ok(()) + } + + /// Get streaming system metrics + pub async fn get_metrics(&self) -> StreamingMetrics { + let manager = self.subscription_manager.read().await; + let buffer = self.event_buffer.read().await; + + StreamingMetrics { + active_subscriptions: manager.subscription_count(), + events_published: self.publisher.get_published_count(), + events_buffered: buffer.len(), + buffer_memory_usage: buffer.memory_usage(), + uptime_seconds: self.publisher.get_uptime().as_secs(), + } + } + + /// Get historical events from buffer + pub async fn get_historical_events( + &self, + filter: EventFilter, + limit: Option, + ) -> Result> { + let buffer = self.event_buffer.read().await; + Ok(buffer.get_filtered_events(filter, limit)) + } + + /// Shutdown the streaming system + pub async fn shutdown(&self) -> Result<()> { + info!("Shutting down trading event streaming system"); + + // Clear all subscriptions + { + let mut manager = self.subscription_manager.write().await; + manager.clear_all_subscriptions(); + } + + // Clear event buffer + { + let mut buffer = self.event_buffer.write().await; + buffer.clear(); + } + + info!("Trading event streaming system shutdown complete"); + Ok(()) + } +} + +/// Configuration for the trading event streaming system +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StreamingConfig { + /// Maximum number of concurrent subscribers + pub max_subscribers: usize, + /// Event buffer size for replay capability + pub buffer_size: usize, + /// Event cleanup interval in seconds + pub cleanup_interval_secs: u64, + /// Maximum event age in seconds before cleanup + pub max_event_age_secs: u64, + /// Enable event compression for storage + pub enable_compression: bool, + /// Maximum memory usage for event buffer (bytes) + pub max_buffer_memory: usize, +} + +impl Default for StreamingConfig { + fn default() -> Self { + Self { + max_subscribers: 1000, + buffer_size: 10000, + cleanup_interval_secs: 60, + max_event_age_secs: 3600, // 1 hour + enable_compression: true, + max_buffer_memory: 100 * 1024 * 1024, // 100MB + } + } +} + +/// Streaming system metrics +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StreamingMetrics { + pub active_subscriptions: usize, + pub events_published: u64, + pub events_buffered: usize, + pub buffer_memory_usage: usize, + pub uptime_seconds: u64, +} + +/// Event buffer for storing recent events +#[derive(Debug)] +pub struct EventBuffer { + events: Vec, + max_size: usize, + memory_usage: usize, +} + +impl EventBuffer { + fn new(max_size: usize) -> Self { + Self { + events: Vec::with_capacity(max_size), + max_size, + memory_usage: 0, + } + } + + async fn add_event(&mut self, event: TradingEvent) -> Result<()> { + let timestamped = TimestampedEvent { + event, + stored_at: Utc::now(), + }; + + // Estimate memory usage (rough approximation) + let event_size = std::mem::size_of::() + timestamped.event.payload.len(); + + // Remove old events if buffer is full + while self.events.len() >= self.max_size { + let removed = self.events.remove(0); + self.memory_usage = self.memory_usage.saturating_sub( + std::mem::size_of::() + removed.event.payload.len(), + ); + } + + self.events.push(timestamped); + self.memory_usage += event_size; + + Ok(()) + } + + fn cleanup_expired_events(&mut self) { + let now = Utc::now(); + let max_age = chrono::Duration::seconds(3600); // 1 hour + + self.events.retain(|event| { + let age = now - event.stored_at; + if age <= max_age { + true + } else { + self.memory_usage = self.memory_usage.saturating_sub( + std::mem::size_of::() + event.event.payload.len(), + ); + false + } + }); + } + + fn get_filtered_events(&self, filter: EventFilter, limit: Option) -> Vec { + let mut filtered: Vec = self + .events + .iter() + .filter(|timestamped| filter.matches(×tamped.event)) + .map(|timestamped| timestamped.event.clone()) + .collect(); + + // Sort by timestamp (newest first) + filtered.sort_by(|a, b| b.timestamp.cmp(&a.timestamp)); + + if let Some(limit) = limit { + filtered.truncate(limit); + } + + filtered + } + + fn len(&self) -> usize { + self.events.len() + } + + fn memory_usage(&self) -> usize { + self.memory_usage + } + + fn clear(&mut self) { + self.events.clear(); + self.memory_usage = 0; + } +} + +/// Event with storage timestamp +#[derive(Debug, Clone)] +struct TimestampedEvent { + event: TradingEvent, + stored_at: DateTime, +} + +/// Subscription manager for handling client subscriptions +#[derive(Debug)] +pub struct SubscriptionManager { + subscriptions: HashMap, +} + +impl SubscriptionManager { + fn new() -> Self { + Self { + subscriptions: HashMap::new(), + } + } + + fn add_subscription(&mut self, id: String, filter: EventFilter) { + self.subscriptions.insert(id, filter); + } + + fn remove_subscription(&mut self, id: &str) { + self.subscriptions.remove(id); + } + + fn subscription_count(&self) -> usize { + self.subscriptions.len() + } + + fn clear_all_subscriptions(&mut self) { + self.subscriptions.clear(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_event_streamer_creation() { + let config = StreamingConfig::default(); + let streamer = TradingEventStreamer::new(config); + + assert!(streamer.start().await.is_ok()); + assert!(streamer.shutdown().await.is_ok()); + } + + #[tokio::test] + async fn test_event_publishing() { + let config = StreamingConfig::default(); + let streamer = TradingEventStreamer::new(config); + + streamer.start().await.unwrap(); + + let event = TradingEvent::new( + TradingEventType::OrderSubmitted, + "test_order".to_string(), + "Test order submission".to_string(), + ); + + assert!(streamer.publish_event(event).await.is_ok()); + + let metrics = streamer.get_metrics().await; + assert_eq!(metrics.events_published, 1); + assert_eq!(metrics.events_buffered, 1); + + streamer.shutdown().await.unwrap(); + } + + #[tokio::test] + async fn test_subscription_management() { + let config = StreamingConfig::default(); + let streamer = TradingEventStreamer::new(config); + + streamer.start().await.unwrap(); + + let filter = EventFilter::for_event_types(vec![TradingEventType::OrderSubmitted]); + let _subscription = streamer.subscribe(filter).await.unwrap(); + + let metrics = streamer.get_metrics().await; + assert_eq!(metrics.active_subscriptions, 1); + + streamer.shutdown().await.unwrap(); + } + + #[test] + fn test_event_buffer() { + let mut buffer = EventBuffer::new(3); + + let event1 = TradingEvent::new( + TradingEventType::OrderSubmitted, + "order1".to_string(), + "First order".to_string(), + ); + + let event2 = TradingEvent::new( + TradingEventType::OrderFilled, + "order1".to_string(), + "Order filled".to_string(), + ); + + tokio::runtime::Runtime::new().unwrap().block_on(async { + buffer.add_event(event1).await.unwrap(); + buffer.add_event(event2).await.unwrap(); + + assert_eq!(buffer.len(), 2); + assert!(buffer.memory_usage() > 0); + }); + } + + #[test] + fn test_subscription_manager() { + let mut manager = SubscriptionManager::new(); + + let filter = EventFilter::for_event_types(vec![TradingEventType::OrderSubmitted]); + manager.add_subscription("sub1".to_string(), filter); + + assert_eq!(manager.subscription_count(), 1); + + manager.remove_subscription("sub1"); + assert_eq!(manager.subscription_count(), 0); + } +} diff --git a/services/trading_service/src/lib.rs b/services/trading_service/src/lib.rs index 6232595a2..1113adc62 100644 --- a/services/trading_service/src/lib.rs +++ b/services/trading_service/src/lib.rs @@ -87,6 +87,3 @@ pub mod tls_config; /// Utility functions and helpers pub mod utils; -/// Re-exports for convenient access -pub mod prelude { -} diff --git a/services/trading_service/src/services/mod.rs b/services/trading_service/src/services/mod.rs index 3e593e959..ace54fa45 100644 --- a/services/trading_service/src/services/mod.rs +++ b/services/trading_service/src/services/mod.rs @@ -10,9 +10,3 @@ pub mod ml_fallback_manager; pub mod ml_performance_monitor; // ConfigServiceImpl doesn't exist - using ConfigManager directly -pub use enhanced_ml::EnhancedMLServiceImpl; -pub use ml_fallback_manager::MLFallbackManager; -pub use ml_performance_monitor::MLPerformanceMonitor; -pub use monitoring::MonitoringServiceImpl; -pub use risk::RiskServiceImpl; -pub use trading::TradingServiceImpl; diff --git a/services/trading_service/src/services/mod.rs.bak b/services/trading_service/src/services/mod.rs.bak new file mode 100644 index 000000000..3e593e959 --- /dev/null +++ b/services/trading_service/src/services/mod.rs.bak @@ -0,0 +1,18 @@ +//! gRPC service implementations for the Trading Service + +pub mod enhanced_ml; +pub mod monitoring; +pub mod risk; +pub mod trading; + +// ML performance monitoring and fallback components +pub mod ml_fallback_manager; +pub mod ml_performance_monitor; + +// ConfigServiceImpl doesn't exist - using ConfigManager directly +pub use enhanced_ml::EnhancedMLServiceImpl; +pub use ml_fallback_manager::MLFallbackManager; +pub use ml_performance_monitor::MLPerformanceMonitor; +pub use monitoring::MonitoringServiceImpl; +pub use risk::RiskServiceImpl; +pub use trading::TradingServiceImpl; diff --git a/tests/chaos/examples/mod.rs b/tests/chaos/examples/mod.rs index 946a1f65c..268c554ce 100644 --- a/tests/chaos/examples/mod.rs +++ b/tests/chaos/examples/mod.rs @@ -2,4 +2,4 @@ pub mod usage_examples; -pub use usage_examples::*; +// DO NOT RE-EXPORT - Use explicit imports at usage sites diff --git a/tests/chaos/examples/mod.rs.bak b/tests/chaos/examples/mod.rs.bak new file mode 100644 index 000000000..946a1f65c --- /dev/null +++ b/tests/chaos/examples/mod.rs.bak @@ -0,0 +1,5 @@ +//! Chaos Engineering Examples Module + +pub mod usage_examples; + +pub use usage_examples::*; diff --git a/tests/chaos/mod.rs b/tests/chaos/mod.rs index 037366755..7360b99bb 100644 --- a/tests/chaos/mod.rs +++ b/tests/chaos/mod.rs @@ -9,9 +9,7 @@ pub mod examples; pub mod ml_training_chaos; pub mod nightly_chaos_runner; -pub use chaos_framework::*; -pub use ml_training_chaos::*; -pub use nightly_chaos_runner::*; +// DO NOT RE-EXPORT - Use explicit imports at usage sites use anyhow::Result; use chrono; diff --git a/tests/chaos/mod.rs.bak b/tests/chaos/mod.rs.bak new file mode 100644 index 000000000..037366755 --- /dev/null +++ b/tests/chaos/mod.rs.bak @@ -0,0 +1,108 @@ +//! Chaos Engineering Module for Foxhunt HFT Trading System +//! +//! This module provides comprehensive chaos engineering capabilities specifically +//! designed for high-frequency trading systems with sub-100ms recovery requirements. + +pub mod chaos_cli; +pub mod chaos_framework; +pub mod examples; +pub mod ml_training_chaos; +pub mod nightly_chaos_runner; + +pub use chaos_framework::*; +pub use ml_training_chaos::*; +pub use nightly_chaos_runner::*; + +use anyhow::Result; +use chrono; +use std::path::PathBuf; +use tracing::info; + +/// Initialize chaos engineering for the Foxhunt system +pub async fn initialize_foxhunt_chaos() -> Result { + info!("Initializing Foxhunt chaos engineering framework"); + + // Configure chaos testing for HFT requirements + let chaos_config = NightlyChaosConfig { + enabled: true, + schedule_time: chrono::NaiveTime::from_hms_opt(2, 0, 0).unwrap(), // 2 AM UTC + timezone: "UTC".to_string(), + max_duration_hours: 3, // Complete chaos testing within 3 hours + notification_webhook: std::env::var("CHAOS_WEBHOOK_URL").ok(), + report_storage_path: PathBuf::from("./chaos_reports"), + ml_chaos_config: MLChaosConfig { + ml_service_endpoint: std::env::var("ML_SERVICE_ENDPOINT") + .unwrap_or_else(|_| "http://localhost:8080".to_string()), + checkpoint_base_path: PathBuf::from("./ml_checkpoints"), + model_types: vec![ + ModelType::TLOB, // Ultra-low latency transformer + ModelType::MAMBA2, // State space model + ModelType::DQN, // Deep Q-learning + ModelType::PPO, // Policy optimization + ModelType::Liquid, // Liquid neural networks + ModelType::TFT, // Temporal fusion transformer + ], + training_timeout_secs: 300, // 5 minutes max training time + max_recovery_time_ms: 100, // HFT requirement: sub-100ms recovery + gpu_memory_threshold_mb: 8192, // 8GB GPU memory threshold + }, + exclude_weekends: true, // Skip weekends for production safety + retry_on_failure: true, + max_retries: 2, + }; + + let runner = NightlyChaosRunner::new(chaos_config); + + info!("Foxhunt chaos engineering framework initialized"); + Ok(runner) +} + +/// Quick chaos test for development/CI +pub async fn run_quick_chaos_test() -> Result> { + info!("Running quick chaos test for CI/development"); + + let ml_config = MLChaosConfig { + ml_service_endpoint: "http://localhost:8080".to_string(), + checkpoint_base_path: PathBuf::from("/tmp/test_checkpoints"), + model_types: vec![ModelType::TLOB], // Just test TLOB for speed + training_timeout_secs: 60, // 1 minute for quick test + max_recovery_time_ms: 100, + gpu_memory_threshold_mb: 2048, // Lower threshold for CI + }; + + let ml_chaos = MLTrainingChaosTests::new(ml_config); + + // Run a subset of chaos tests + let experiment_ids = ml_chaos.initialize_experiments().await?; + let first_experiment = experiment_ids + .into_iter() + .next() + .ok_or_else(|| anyhow::anyhow!("No experiments available"))?; + + // Execute just one experiment for quick testing + let orchestrator = ChaosOrchestrator::new(1); + let _result = orchestrator.execute_experiment(first_experiment).await?; + + // Return empty results for now (would implement actual quick test) + Ok(vec![]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_chaos_initialization() { + let runner = initialize_foxhunt_chaos().await; + assert!(runner.is_ok()); + } + + #[tokio::test] + async fn test_quick_chaos_test() { + // This would require actual ML service running + // For now just test that the function exists + let result = run_quick_chaos_test().await; + // In CI without services running, this might fail, so we don't assert success + println!("Quick chaos test result: {:?}", result); + } +} diff --git a/tests/e2e/src/mocks/mod.rs b/tests/e2e/src/mocks/mod.rs index da1f27830..0da163e53 100644 --- a/tests/e2e/src/mocks/mod.rs +++ b/tests/e2e/src/mocks/mod.rs @@ -9,7 +9,7 @@ pub mod dual_provider_mocks; // Re-export commonly used types -pub use dual_provider_mocks::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites DualProviderMockOrchestrator, MockDataProvider, MockDatabentoProvider, diff --git a/tests/e2e/src/mocks/mod.rs.bak b/tests/e2e/src/mocks/mod.rs.bak new file mode 100644 index 000000000..da1f27830 --- /dev/null +++ b/tests/e2e/src/mocks/mod.rs.bak @@ -0,0 +1,19 @@ +//! Mock Infrastructure Module +//! +//! Provides comprehensive mocking infrastructure for E2E testing including: +//! - Dual-provider mocks (Databento/Benzinga) +//! - Market data generators +//! - News data generators +//! - Provider failover simulation + +pub mod dual_provider_mocks; + +// Re-export commonly used types +pub use dual_provider_mocks::{ + DualProviderMockOrchestrator, + MockDataProvider, + MockDatabentoProvider, + MockBenzingaProvider, + MarketDataGenerator, + NewsGenerator, +}; \ No newline at end of file diff --git a/tests/e2e/tests/mod.rs b/tests/e2e/tests/mod.rs index a07fbb729..e0515c421 100644 --- a/tests/e2e/tests/mod.rs +++ b/tests/e2e/tests/mod.rs @@ -9,13 +9,7 @@ pub mod ml_model_integration_tests; pub mod order_lifecycle_risk_tests; pub mod performance_validation_tests; -pub use compliance_regulatory_tests::ComplianceRegulatoryTests; -pub use comprehensive_trading_workflows::ComprehensiveTradingWorkflows; -pub use data_flow_performance_tests::DataFlowPerformanceTests; -pub use emergency_shutdown_failover_tests::EmergencyShutdownFailoverTests; -pub use ml_model_integration_tests::MLModelIntegrationTests; -pub use order_lifecycle_risk_tests::OrderLifecycleRiskTests; -pub use performance_validation_tests::PerformanceValidationTests; +// DO NOT RE-EXPORT - Use explicit imports at usage sites /// Comprehensive E2E Test Suite Summary /// diff --git a/tests/e2e/tests/mod.rs.bak b/tests/e2e/tests/mod.rs.bak new file mode 100644 index 000000000..a07fbb729 --- /dev/null +++ b/tests/e2e/tests/mod.rs.bak @@ -0,0 +1,345 @@ +// End-to-End Test Suite Integration +// Complete HFT trading system validation with 20+ comprehensive scenarios + +pub mod compliance_regulatory_tests; +pub mod comprehensive_trading_workflows; +pub mod data_flow_performance_tests; +pub mod emergency_shutdown_failover_tests; +pub mod ml_model_integration_tests; +pub mod order_lifecycle_risk_tests; +pub mod performance_validation_tests; + +pub use compliance_regulatory_tests::ComplianceRegulatoryTests; +pub use comprehensive_trading_workflows::ComprehensiveTradingWorkflows; +pub use data_flow_performance_tests::DataFlowPerformanceTests; +pub use emergency_shutdown_failover_tests::EmergencyShutdownFailoverTests; +pub use ml_model_integration_tests::MLModelIntegrationTests; +pub use order_lifecycle_risk_tests::OrderLifecycleRiskTests; +pub use performance_validation_tests::PerformanceValidationTests; + +/// Comprehensive E2E Test Suite Summary +/// +/// **TOTAL TEST SCENARIOS: 25+ comprehensive end-to-end workflows** +/// +/// ## 1. Comprehensive Trading Workflows (5 scenarios, 37 steps) +/// - Complete HFT workflow with sub-50μs latency validation (12 steps) +/// - Multi-asset order lifecycle with portfolio management (15 steps) +/// - Advanced emergency scenarios with disaster recovery (10 steps) +/// +/// ## 2. ML Model Integration Tests (3 scenarios, 30 steps) +/// - MAMBA-2 state space model integration (10 steps) +/// - TLOB transformer order book analysis (9 steps) +/// - DQN reinforcement learning pipeline (11 steps) +/// +/// ## 3. Data Flow Performance Tests (2 scenarios, 22 steps) +/// - Real-time data ingestion pipeline (12 steps) +/// - Sub-50μs end-to-end latency validation (10 steps) +/// +/// ## 4. Order Lifecycle Risk Tests (3 scenarios, 37 steps) +/// - Complete order lifecycle from creation to settlement (15 steps) +/// - Multi-order risk aggregation and limits (12 steps) +/// - Emergency kill switch activation scenarios (10 steps) +/// +/// ## 5. Emergency Shutdown Failover Tests (3 scenarios, 36 steps) +/// - Graceful shutdown sequence with order preservation (12 steps) +/// - Hard kill switch activation with immediate termination (10 steps) +/// - Failover to backup service with state transfer (14 steps) +/// +/// ## 6. Compliance Regulatory Tests (3 scenarios, 34 steps) +/// - MiFID II transaction reporting workflow (13 steps) +/// - SOX compliance and financial controls (11 steps) +/// - Cross-jurisdiction regulatory compliance (10 steps) +/// +/// ## 7. Performance Validation Tests (2 scenarios, 22 steps) +/// - Critical path sub-50μs latency validation (12 steps) +/// - Throughput and scalability benchmarks (10 steps) +/// +/// **TOTAL: 21 major test scenarios with 218+ individual validation steps** +/// +/// ## Key Performance Requirements Validated: +/// - **Sub-50μs end-to-end latency** (critical HFT requirement) +/// - **100,000+ operations/second throughput** +/// - **RDTSC hardware timing precision (≤50ns resolution)** +/// - **Lock-free data structure performance (≤500ns operations)** +/// - **SIMD optimization validation (≤2μs vector operations)** +/// - **Multi-threaded scaling efficiency (>70% up to 8 threads)** +/// - **Memory bandwidth utilization (>10 GB/s)** +/// - **Database write performance (>5,000 writes/sec)** +/// - **Network message processing (>50,000 messages/sec)** +/// +/// ## ML Model Coverage: +/// - **MAMBA-2 State Space Models** for sequence prediction +/// - **TLOB Transformer** for order book microstructure analysis +/// - **Deep Q-Network (DQN)** with Rainbow enhancements +/// - **PPO (Proximal Policy Optimization)** for reinforcement learning +/// - **Liquid Neural Networks** for adaptive learning +/// - **Temporal Fusion Transformer (TFT)** for time series forecasting +/// +/// ## Risk Management Validation: +/// - **VaR calculations** with correlation adjustments +/// - **Kelly criterion position sizing** +/// - **Kill switch activation** (<100μs response time) +/// - **Emergency position flattening** +/// - **Multi-asset risk aggregation** +/// - **Stress testing scenarios** +/// +/// ## Compliance Framework Coverage: +/// - **MiFID II transaction reporting** (T+1 regulatory deadlines) +/// - **SOX financial controls** and segregation of duties +/// - **Best execution monitoring** and venue analysis +/// - **Cross-jurisdiction coordination** (EU, US, UK, APAC) +/// - **Audit trail completeness** and regulatory inquiry preparation +/// - **Data privacy compliance** (GDPR, cross-border transfers) +/// +/// ## Infrastructure Resilience: +/// - **Graceful shutdown** with order preservation +/// - **Hard kill switch** with immediate termination (<2s total time) +/// - **Automatic failover** with state synchronization +/// - **Service discovery** and client redirection +/// - **Data consistency** across service boundaries +/// - **Performance continuity** during failover operations + +#[cfg(test)] +mod integration_tests { + use super::*; + + /// Run all E2E test scenarios in sequence + #[tokio::test] + async fn test_complete_e2e_suite() { + println!("🚀 Starting Comprehensive E2E Test Suite"); + println!("📊 Target: 20+ scenarios with sub-50μs latency validation"); + + // Initialize all test suites + let trading_tests = ComprehensiveTradingWorkflows::new() + .await + .expect("Failed to initialize trading tests"); + let ml_tests = MLModelIntegrationTests::new() + .await + .expect("Failed to initialize ML tests"); + let data_flow_tests = DataFlowPerformanceTests::new() + .await + .expect("Failed to initialize data flow tests"); + let lifecycle_tests = OrderLifecycleRiskTests::new() + .await + .expect("Failed to initialize lifecycle tests"); + let emergency_tests = EmergencyShutdownFailoverTests::new() + .await + .expect("Failed to initialize emergency tests"); + let compliance_tests = ComplianceRegulatoryTests::new() + .await + .expect("Failed to initialize compliance tests"); + let performance_tests = PerformanceValidationTests::new() + .await + .expect("Failed to initialize performance tests"); + + let mut all_results = Vec::new(); + + // 1. Trading Workflow Tests (5 scenarios) + println!("\n🔄 Testing Trading Workflows..."); + all_results.push( + trading_tests + .test_complete_hft_workflow() + .await + .expect("HFT workflow failed"), + ); + all_results.push( + trading_tests + .test_multi_asset_order_lifecycle() + .await + .expect("Multi-asset lifecycle failed"), + ); + all_results.push( + trading_tests + .test_advanced_emergency_scenarios() + .await + .expect("Emergency scenarios failed"), + ); + + // 2. ML Model Integration Tests (3 scenarios) + println!("\n🧠 Testing ML Model Integration..."); + all_results.push( + ml_tests + .test_mamba_integration() + .await + .expect("MAMBA integration failed"), + ); + all_results.push( + ml_tests + .test_tlob_transformer_integration() + .await + .expect("TLOB integration failed"), + ); + all_results.push( + ml_tests + .test_dqn_reinforcement_learning() + .await + .expect("DQN integration failed"), + ); + + // 3. Data Flow Performance Tests (2 scenarios) + println!("\n📊 Testing Data Flow Performance..."); + all_results.push( + data_flow_tests + .test_real_time_data_ingestion() + .await + .expect("Data ingestion failed"), + ); + all_results.push( + data_flow_tests + .test_sub_50us_latency_validation() + .await + .expect("Latency validation failed"), + ); + + // 4. Order Lifecycle Risk Tests (3 scenarios) + println!("\n⚖️ Testing Order Lifecycle & Risk..."); + all_results.push( + lifecycle_tests + .test_complete_order_lifecycle() + .await + .expect("Order lifecycle failed"), + ); + all_results.push( + lifecycle_tests + .test_multi_order_risk_aggregation() + .await + .expect("Risk aggregation failed"), + ); + all_results.push( + lifecycle_tests + .test_emergency_kill_switch() + .await + .expect("Kill switch failed"), + ); + + // 5. Emergency Shutdown Failover Tests (3 scenarios) + println!("\n🚨 Testing Emergency & Failover..."); + all_results.push( + emergency_tests + .test_graceful_shutdown_sequence() + .await + .expect("Graceful shutdown failed"), + ); + all_results.push( + emergency_tests + .test_hard_kill_switch_activation() + .await + .expect("Hard kill failed"), + ); + all_results.push( + emergency_tests + .test_failover_with_state_transfer() + .await + .expect("Failover failed"), + ); + + // 6. Compliance Regulatory Tests (3 scenarios) + println!("\n📋 Testing Compliance & Regulatory..."); + all_results.push( + compliance_tests + .test_mifid_ii_transaction_reporting() + .await + .expect("MiFID II failed"), + ); + all_results.push( + compliance_tests + .test_sox_compliance_controls() + .await + .expect("SOX compliance failed"), + ); + all_results.push( + compliance_tests + .test_cross_jurisdiction_compliance() + .await + .expect("Cross-jurisdiction failed"), + ); + + // 7. Performance Validation Tests (2 scenarios) + println!("\n⚡ Testing Performance Validation..."); + all_results.push( + performance_tests + .test_critical_path_latency_validation() + .await + .expect("Critical path latency failed"), + ); + all_results.push( + performance_tests + .test_throughput_scalability_benchmarks() + .await + .expect("Throughput benchmarks failed"), + ); + + // Validate all tests passed + let total_scenarios = all_results.len(); + let successful_scenarios = all_results.iter().filter(|r| r.success).count(); + let total_steps = all_results.iter().map(|r| r.steps.len()).sum::(); + + println!("\n✅ E2E Test Suite Complete!"); + println!("📈 Results Summary:"); + println!(" • Total Scenarios: {}", total_scenarios); + println!(" • Successful: {}", successful_scenarios); + println!(" • Total Steps: {}", total_steps); + println!( + " • Success Rate: {:.1}%", + (successful_scenarios as f64 / total_scenarios as f64) * 100.0 + ); + + // Collect critical performance metrics + let mut critical_latencies = Vec::new(); + let mut throughput_metrics = Vec::new(); + + for result in &all_results { + if let Some(e2e_latency) = result.metrics.get("e2e_critical_p95_ns") { + critical_latencies.push(*e2e_latency); + } + if let Some(throughput) = result.metrics.get("single_thread_ops_per_sec") { + throughput_metrics.push(*throughput); + } + } + + if !critical_latencies.is_empty() { + let max_latency = critical_latencies.iter().fold(0.0_f64, |a, &b| a.max(b)); + println!( + " • Max Critical Path Latency: {:.0}ns ({:.1}μs)", + max_latency, + max_latency / 1000.0 + ); + assert!( + max_latency < 50_000.0, + "Critical path latency requirement failed: {}ns > 50μs", + max_latency + ); + } + + if !throughput_metrics.is_empty() { + let max_throughput = throughput_metrics.iter().fold(0.0_f64, |a, &b| a.max(b)); + println!(" • Max Throughput: {:.0} ops/sec", max_throughput); + assert!( + max_throughput > 100_000.0, + "Throughput requirement failed: {} ops/sec < 100k", + max_throughput + ); + } + + // All scenarios must pass + assert_eq!( + successful_scenarios, total_scenarios, + "Some E2E scenarios failed" + ); + assert!( + total_scenarios >= 20, + "Should have at least 20 test scenarios, got {}", + total_scenarios + ); + + println!("🎉 ALL E2E REQUIREMENTS VALIDATED SUCCESSFULLY!"); + println!(" ✅ 20+ comprehensive test scenarios"); + println!(" ✅ Sub-50μs critical path latency"); + println!(" ✅ 100k+ operations/second throughput"); + println!(" ✅ ML model integration (MAMBA, DQN, PPO, etc.)"); + println!(" ✅ Risk management and kill switches"); + println!(" ✅ Emergency shutdown and failover"); + println!(" ✅ Regulatory compliance (MiFID II, SOX)"); + println!(" ✅ Hardware-level performance optimization"); + } +} diff --git a/tests/fixtures/lib.rs b/tests/fixtures/lib.rs index f5a61143b..4ec3f51cb 100644 --- a/tests/fixtures/lib.rs +++ b/tests/fixtures/lib.rs @@ -244,5 +244,5 @@ macro_rules! fixture { }; } -// Re-export everything for easy access -pub use test_data::*; \ No newline at end of file +// REMOVED: All pub use statements eliminated per cleanup requirements +// Tests must import from canonical sources: tests::fixtures::test_data \ No newline at end of file diff --git a/tests/fixtures/mod.rs b/tests/fixtures/mod.rs index 43cc5ca0e..990e2e5ef 100644 --- a/tests/fixtures/mod.rs +++ b/tests/fixtures/mod.rs @@ -19,10 +19,6 @@ pub mod test_database; pub mod mock_services; pub mod test_data; -pub use test_config::*; -pub use test_database::*; -pub use mock_services::*; -pub use test_data::*; /// Integration test configuration #[derive(Debug, Clone)] diff --git a/tests/fixtures/mod.rs.bak b/tests/fixtures/mod.rs.bak new file mode 100644 index 000000000..43cc5ca0e --- /dev/null +++ b/tests/fixtures/mod.rs.bak @@ -0,0 +1,469 @@ +//! Test Fixtures and Mock Services +//! +//! This module provides comprehensive test fixtures, mock services, and test data +//! for integration testing of the Foxhunt HFT trading system. + +use std::collections::HashMap; +use std::sync::{Arc, atomic::{AtomicU16, AtomicU64, Ordering}}; +use std::time::{Duration, Instant}; + +use tokio::sync::{mpsc, RwLock, Mutex}; +use uuid::Uuid; +use serde_json::json; +use chrono::{DateTime, Utc, Duration as ChronoDuration}; + +use tli::prelude::*; + +pub mod test_config; +pub mod test_database; +pub mod mock_services; +pub mod test_data; + +pub use test_config::*; +pub use test_database::*; +pub use mock_services::*; +pub use test_data::*; + +/// Integration test configuration +#[derive(Debug, Clone)] +pub struct IntegrationTestConfig { + // Performance requirements + pub max_latency_ns: u64, + pub max_db_latency_ns: u64, + pub max_risk_latency_ns: u64, + pub max_ml_inference_latency_ns: u64, + pub max_init_latency_ns: u64, + pub min_throughput_ops_per_sec: f64, + pub min_db_throughput_ops_per_sec: f64, + pub min_backtest_throughput: f64, + + // Test parameters + pub request_timeout_ms: u64, + pub max_retry_attempts: u32, + pub circuit_breaker_threshold: u32, + pub concurrent_order_count: usize, + pub parallel_backtest_count: usize, + pub concurrent_operation_count: usize, + pub batch_size: usize, + pub stream_buffer_size: usize, + pub stress_test_duration_secs: u64, + + // Database configuration + pub test_db_url: String, + pub test_db_max_connections: u32, + pub enable_database_cleanup: bool, + + // Mock service configuration + pub mock_service_latency_ms: u64, + pub mock_failure_rate: f64, + pub enable_chaos_testing: bool, +} + +impl Default for IntegrationTestConfig { + fn default() -> Self { + Self { + // HFT Performance requirements + max_latency_ns: 50_000, // 50µs max latency + max_db_latency_ns: 100_000, // 100µs max DB latency + max_risk_latency_ns: 25_000, // 25µs max risk validation + max_ml_inference_latency_ns: 50_000, // 50µs max ML inference + max_init_latency_ns: 1_000_000, // 1ms max initialization + min_throughput_ops_per_sec: 10_000.0, // 10K ops/sec minimum + min_db_throughput_ops_per_sec: 5_000.0, // 5K DB ops/sec minimum + min_backtest_throughput: 10.0, // 10 backtests/sec minimum + + // Test parameters + request_timeout_ms: 5_000, // 5 second timeout + max_retry_attempts: 3, + circuit_breaker_threshold: 5, + concurrent_order_count: 100, + parallel_backtest_count: 20, + concurrent_operation_count: 50, + batch_size: 1000, + stream_buffer_size: 10_000, + stress_test_duration_secs: 30, + + // Database configuration + test_db_url: std::env::var("TEST_DATABASE_URL") + .unwrap_or_else(|_| "postgresql://foxhunt_test:test_password@localhost:5432/foxhunt_test".to_string()), + test_db_max_connections: 20, + enable_database_cleanup: true, + + // Mock service configuration + mock_service_latency_ms: 10, + mock_failure_rate: 0.01, // 1% failure rate + enable_chaos_testing: false, + } + } +} + +/// Base test result structure +#[derive(Debug, Clone)] +pub struct TestResult { + pub name: String, + pub passed: bool, + pub execution_time: Duration, + pub assertions: Vec, + pub errors: Vec, + pub metadata: HashMap, +} + +impl TestResult { + pub fn new(name: &str) -> Self { + Self { + name: name.to_string(), + passed: false, + execution_time: Duration::default(), + assertions: Vec::new(), + errors: Vec::new(), + metadata: HashMap::new(), + } + } + + pub fn add_assertion(&mut self, description: &str, passed: bool) { + self.assertions.push(Assertion { + description: description.to_string(), + passed, + }); + } + + pub fn add_error(&mut self, error: String) { + self.errors.push(error); + } + + pub fn set_passed(&mut self, passed: bool) { + self.passed = passed; + } +} + +/// Individual test assertion +#[derive(Debug, Clone)] +pub struct Assertion { + pub description: String, + pub passed: bool, +} + +/// Test suite containing multiple test results +#[derive(Debug, Clone)] +pub struct TestSuite { + pub name: String, + pub tests: Vec, + pub passed_tests: usize, + pub total_tests: usize, + pub passed: bool, + pub execution_time: Duration, + pub metadata: HashMap, +} + +impl TestSuite { + pub fn new(name: &str) -> Self { + Self { + name: name.to_string(), + tests: Vec::new(), + passed_tests: 0, + total_tests: 0, + passed: false, + execution_time: Duration::default(), + metadata: HashMap::new(), + } + } + + pub fn add_test_result(&mut self, test: TestResult) { + if test.passed { + self.passed_tests += 1; + } + self.total_tests += 1; + self.tests.push(test); + } + + pub fn set_passed(&mut self, passed: bool) { + self.passed = passed; + } +} + +/// Port manager for test services +#[derive(Debug)] +pub struct TestPortManager { + next_port: AtomicU16, + allocated_ports: RwLock>, +} + +impl TestPortManager { + pub fn new() -> Self { + Self { + next_port: AtomicU16::new(50000), // Start from port 50000 + allocated_ports: RwLock::new(Vec::new()), + } + } + + pub async fn allocate_port(&self) -> u16 { + loop { + let port = self.next_port.fetch_add(1, Ordering::Relaxed); + if port > 65000 { + // Reset if we've used too many ports + self.next_port.store(50000, Ordering::Relaxed); + continue; + } + + // Check if port is available + if let Ok(listener) = tokio::net::TcpListener::bind(format!("127.0.0.1:{}", port)).await { + drop(listener); // Release the port + self.allocated_ports.write().await.push(port); + return port; + } + } + } + + pub async fn release_port(&self, port: u16) { + let mut allocated = self.allocated_ports.write().await; + if let Some(pos) = allocated.iter().position(|&p| p == port) { + allocated.remove(pos); + } + } +} + +impl Default for TestPortManager { + fn default() -> Self { + Self::new() + } +} + +/// Global test port manager instance +lazy_static::lazy_static! { + pub static ref TEST_PORT_MANAGER: TestPortManager = TestPortManager::new(); +} + +/// Test event publisher for streaming tests +pub struct TestEventPublisher { + _event_sender: mpsc::UnboundedSender, + event_receiver: Arc>>, + published_events: AtomicU64, +} + +impl TestEventPublisher { + pub async fn new() -> TliResult { + let (sender, receiver) = mpsc::unbounded_channel(); + + Ok(Self { + _event_sender: sender, + event_receiver: Arc::new(Mutex::new(receiver)), + published_events: AtomicU64::new(0), + }) + } + + pub async fn publish_event(&self, event: TliEvent) -> TliResult<()> { + self._event_sender.send(event) + .map_err(|e| TliError::InternalError(format!("Failed to publish event: {}", e)))?; + + self.published_events.fetch_add(1, Ordering::Relaxed); + Ok(()) + } + + pub async fn publish_market_data_burst(&self, symbol: &str, count: usize) -> TliResult<()> { + for i in 0..count { + let event = TliEvent { + id: Uuid::new_v4(), + event_type: EventType::MarketData, + timestamp: Utc::now(), + data: json!({ + "symbol": symbol, + "price": 150.0 + (i as f64 * 0.01), + "volume": 100 + i, + "sequence": i + }), + source: "test_publisher".to_string(), + }; + + self.publish_event(event).await?; + } + + Ok(()) + } + + pub async fn publish_order_lifecycle(&self, order_id: &str) -> TliResult<()> { + let states = vec!["pending", "partially_filled", "filled"]; + + for (i, state) in states.iter().enumerate() { + let event = TliEvent { + id: Uuid::new_v4(), + event_type: EventType::OrderUpdate, + timestamp: Utc::now(), + data: json!({ + "order_id": order_id, + "status": state, + "filled_quantity": (i + 1) * 50, + "remaining_quantity": 100 - ((i + 1) * 50) + }), + source: "test_lifecycle".to_string(), + }; + + self.publish_event(event).await?; + + // Small delay between state changes + tokio::time::sleep(Duration::from_millis(100)).await; + } + + Ok(()) + } + + pub fn get_published_count(&self) -> u64 { + self.published_events.load(Ordering::Relaxed) + } + + pub async fn receive_event(&self) -> Option { + let mut receiver = self.event_receiver.lock().await; + receiver.recv().await + } +} + +/// Performance metrics collector for tests +#[derive(Debug, Default)] +pub struct TestMetricsCollector { + latency_measurements: RwLock>>, + throughput_measurements: RwLock>>, + error_counts: RwLock>, + custom_metrics: RwLock>, +} + +impl TestMetricsCollector { + pub fn new() -> Self { + Self::default() + } + + pub async fn record_latency(&self, operation: &str, latency_ns: u64) { + let mut latencies = self.latency_measurements.write().await; + latencies.entry(operation.to_string()).or_insert_with(Vec::new).push(latency_ns); + } + + pub async fn record_throughput(&self, operation: &str, ops_per_sec: f64) { + let mut throughputs = self.throughput_measurements.write().await; + throughputs.entry(operation.to_string()).or_insert_with(Vec::new).push(ops_per_sec); + } + + pub async fn record_error(&self, operation: &str) { + let mut errors = self.error_counts.write().await; + *errors.entry(operation.to_string()).or_insert(0) += 1; + } + + pub async fn record_custom_metric(&self, name: &str, value: serde_json::Value) { + let mut metrics = self.custom_metrics.write().await; + metrics.insert(name.to_string(), value); + } + + pub async fn get_latency_stats(&self, operation: &str) -> Option { + let latencies = self.latency_measurements.read().await; + if let Some(measurements) = latencies.get(operation) { + if measurements.is_empty() { + return None; + } + + let mut sorted = measurements.clone(); + sorted.sort_unstable(); + + let len = sorted.len(); + let avg = sorted.iter().sum::() / len as u64; + let p50 = sorted[len * 50 / 100]; + let p95 = sorted[len * 95 / 100]; + let p99 = sorted[len * 99 / 100]; + let max = sorted[len - 1]; + + Some(LatencyStats { avg, p50, p95, p99, max }) + } else { + None + } + } + + pub async fn get_summary(&self) -> serde_json::Value { + let latencies = self.latency_measurements.read().await; + let throughputs = self.throughput_measurements.read().await; + let errors = self.error_counts.read().await; + let custom = self.custom_metrics.read().await; + + let mut latency_summary = serde_json::Map::new(); + for (operation, measurements) in latencies.iter() { + if let Some(stats) = self.get_latency_stats(operation).await { + latency_summary.insert(operation.clone(), json!({ + "count": measurements.len(), + "avg_ns": stats.avg, + "p50_ns": stats.p50, + "p95_ns": stats.p95, + "p99_ns": stats.p99, + "max_ns": stats.max + })); + } + } + + let mut throughput_summary = serde_json::Map::new(); + for (operation, measurements) in throughputs.iter() { + if !measurements.is_empty() { + let avg = measurements.iter().sum::() / measurements.len() as f64; + let max = measurements.iter().fold(0.0f64, |a, &b| a.max(b)); + let min = measurements.iter().fold(f64::INFINITY, |a, &b| a.min(b)); + + throughput_summary.insert(operation.clone(), json!({ + "count": measurements.len(), + "avg_ops_per_sec": avg, + "max_ops_per_sec": max, + "min_ops_per_sec": min + })); + } + } + + json!({ + "latencies": latency_summary, + "throughput": throughput_summary, + "errors": errors.clone(), + "custom_metrics": custom.clone() + }) + } +} + +/// Latency statistics structure +#[derive(Debug, Clone)] +pub struct LatencyStats { + pub avg: u64, + pub p50: u64, + pub p95: u64, + pub p99: u64, + pub max: u64, +} + +/// Test environment setup and cleanup +pub struct TestEnvironment { + pub config: IntegrationTestConfig, + pub metrics: Arc, + pub port_manager: Arc, + pub event_publisher: Arc, + cleanup_tasks: Vec std::pin::Pin + Send>> + Send + Sync>>, +} + +impl TestEnvironment { + pub async fn new(config: IntegrationTestConfig) -> TliResult { + let metrics = Arc::new(TestMetricsCollector::new()); + let port_manager = Arc::new(TestPortManager::new()); + let event_publisher = Arc::new(TestEventPublisher::new().await?); + + Ok(Self { + config, + metrics, + port_manager, + event_publisher, + cleanup_tasks: Vec::new(), + }) + } + + pub fn add_cleanup_task(&mut self, task: F) + where + F: Fn() -> Fut + Send + Sync + 'static, + Fut: std::future::Future + Send + 'static, + { + let boxed_task = Box::new(move || Box::pin(task()) as std::pin::Pin + Send>>); + self.cleanup_tasks.push(boxed_task); + } + + pub async fn cleanup(self) { + for task in self.cleanup_tasks { + task().await; + } + } +} diff --git a/tests/framework/mod.rs b/tests/framework/mod.rs index 9183ddfdd..a15b0a3df 100644 --- a/tests/framework/mod.rs +++ b/tests/framework/mod.rs @@ -25,10 +25,6 @@ pub mod mocks; pub mod metrics; pub mod services; -pub use orchestrator::*; -pub use mocks::*; -pub use metrics::*; -pub use services::*; use std::collections::HashMap; use std::sync::Arc; diff --git a/tests/framework/mod.rs.bak b/tests/framework/mod.rs.bak new file mode 100644 index 000000000..9183ddfdd --- /dev/null +++ b/tests/framework/mod.rs.bak @@ -0,0 +1,173 @@ +//! Enhanced Integration Testing Framework for Foxhunt HFT System +//! +//! This module provides a unified testing framework that orchestrates all three services +//! (Trading, Backtesting, ML Training) along with TLI client testing, database hot-reload +//! validation, and kill switch system verification. +//! +//! ## Key Features: +//! - Unified service lifecycle management +//! - Centralized mock implementations +//! - Performance metrics collection +//! - Cross-service integration validation +//! - Kill switch emergency testing +//! - Database hot-reload verification +//! +//! ## Usage: +//! ```rust +//! use tests::framework::TestOrchestrator; +//! +//! let orchestrator = TestOrchestrator::new().await?; +//! orchestrator.run_integration_tests().await?; +//! ``` + +pub mod orchestrator; +pub mod mocks; +pub mod metrics; +pub mod services; + +pub use orchestrator::*; +pub use mocks::*; +pub use metrics::*; +pub use services::*; + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::{RwLock, broadcast, mpsc}; +use tokio::time::timeout; +use tracing::{info, warn, error, debug}; +use uuid::Uuid; + +use trading_engine::prelude::*; +use risk::prelude::*; + +/// Test framework configuration +#[derive(Debug, Clone)] +pub struct TestFrameworkConfig { + /// Maximum test execution timeout + pub max_test_timeout: Duration, + /// Service startup timeout + pub service_startup_timeout: Duration, + /// Service health check timeout + pub health_check_timeout: Duration, + /// Database connection timeout + pub database_timeout: Duration, + /// Kill switch activation timeout + pub kill_switch_timeout: Duration, + /// Performance threshold validation + pub performance_thresholds: PerformanceThresholds, + /// Test environment configuration + pub test_environment: TestEnvironment, +} + +impl Default for TestFrameworkConfig { + fn default() -> Self { + Self { + max_test_timeout: Duration::from_secs(300), + service_startup_timeout: Duration::from_secs(30), + health_check_timeout: Duration::from_secs(10), + database_timeout: Duration::from_secs(15), + kill_switch_timeout: Duration::from_secs(5), + performance_thresholds: PerformanceThresholds::hft_defaults(), + test_environment: TestEnvironment::Development, + } + } +} + +/// Performance thresholds for validation +#[derive(Debug, Clone)] +pub struct PerformanceThresholds { + /// Maximum end-to-end latency (microseconds) + pub max_e2e_latency_us: u64, + /// Maximum order processing latency (microseconds) + pub max_order_latency_us: u64, + /// Maximum risk validation latency (microseconds) + pub max_risk_latency_us: u64, + /// Maximum ML inference latency (milliseconds) + pub max_ml_latency_ms: u64, + /// Maximum database hot-reload latency (milliseconds) + pub max_config_reload_ms: u64, + /// Minimum throughput (operations per second) + pub min_throughput_ops_sec: u64, +} + +impl PerformanceThresholds { + pub fn hft_defaults() -> Self { + Self { + max_e2e_latency_us: 50, // 50μs end-to-end + max_order_latency_us: 20, // 20μs order processing + max_risk_latency_us: 10, // 10μs risk validation + max_ml_latency_ms: 50, // 50ms ML inference + max_config_reload_ms: 100, // 100ms config reload + min_throughput_ops_sec: 10000, // 10k ops/sec minimum + } + } +} + +/// Test environment types +#[derive(Debug, Clone, PartialEq)] +pub enum TestEnvironment { + Development, + CI, + Staging, + Performance, +} + +/// Comprehensive test result +#[derive(Debug, Clone)] +pub struct IntegrationTestResult { + pub test_name: String, + pub success: bool, + pub duration: Duration, + pub metrics: TestMetrics, + pub errors: Vec, + pub warnings: Vec, +} + +/// Test execution metrics +#[derive(Debug, Clone, Default)] +pub struct TestMetrics { + /// Service startup times + pub service_startup_times: HashMap, + /// gRPC communication latencies + pub grpc_latencies: HashMap>, + /// Database operation latencies + pub database_latencies: Vec, + /// Kill switch activation times + pub kill_switch_times: Vec, + /// Memory usage measurements + pub memory_usage: Vec, + /// Throughput measurements (ops/sec) + pub throughput_measurements: Vec, +} + +/// Test validation errors +#[derive(Debug, thiserror::Error)] +pub enum TestFrameworkError { + #[error("Service startup timeout: {service}")] + ServiceStartupTimeout { service: String }, + + #[error("Health check failed for service: {service}")] + HealthCheckFailed { service: String }, + + #[error("Performance threshold exceeded: {metric} = {value:?}, limit = {limit:?}")] + PerformanceThresholdExceeded { + metric: String, + value: Duration, + limit: Duration, + }, + + #[error("Kill switch activation failed: {reason}")] + KillSwitchFailed { reason: String }, + + #[error("Database hot-reload failed: {reason}")] + DatabaseHotReloadFailed { reason: String }, + + #[error("Cross-service integration failed: {reason}")] + CrossServiceIntegrationFailed { reason: String }, + + #[error("Test timeout exceeded: {test_name}")] + TestTimeout { test_name: String }, +} + +pub type TestResult = std::result::Result; \ No newline at end of file diff --git a/tests/gpu/mod.rs b/tests/gpu/mod.rs index 85883bd6b..9aa9159c5 100644 --- a/tests/gpu/mod.rs +++ b/tests/gpu/mod.rs @@ -16,12 +16,6 @@ pub mod gpu_performance_bench; pub mod production_gpu_integration_test; // Re-export commonly used test utilities -pub use cuda_initialization_test::*; -pub use gpu_memory_management_test::*; -pub use ml_gpu_inference_test::*; -pub use cuda_kernel_test::*; -pub use gpu_performance_bench::*; -pub use production_gpu_integration_test::*; /// Common GPU test utilities pub mod utils { diff --git a/tests/gpu/mod.rs.bak b/tests/gpu/mod.rs.bak new file mode 100644 index 000000000..85883bd6b --- /dev/null +++ b/tests/gpu/mod.rs.bak @@ -0,0 +1,64 @@ +//! GPU Testing Infrastructure for Foxhunt HFT System +//! +//! This module provides comprehensive GPU testing coverage including: +//! - CUDA device initialization and detection +//! - GPU memory management validation +//! - ML model GPU inference testing +//! - Performance benchmarks (GPU vs CPU) +//! - CUDA kernel validation +//! - Production GPU path testing + +pub mod cuda_initialization_test; +pub mod gpu_memory_management_test; +pub mod ml_gpu_inference_test; +pub mod cuda_kernel_test; +pub mod gpu_performance_bench; +pub mod production_gpu_integration_test; + +// Re-export commonly used test utilities +pub use cuda_initialization_test::*; +pub use gpu_memory_management_test::*; +pub use ml_gpu_inference_test::*; +pub use cuda_kernel_test::*; +pub use gpu_performance_bench::*; +pub use production_gpu_integration_test::*; + +/// Common GPU test utilities +pub mod utils { + use std::sync::Once; + + static INIT: Once = Once::new(); + + /// Initialize GPU test environment once per test run + pub fn init_gpu_test_env() { + INIT.call_once(|| { + env_logger::init(); + log::info!("🚀 Initializing GPU test environment"); + }); + } + + /// Check if CUDA is available for testing + pub fn cuda_available() -> bool { + #[cfg(feature = "cuda")] + { + cudarc::driver::CudaDevice::new(0).is_ok() + } + #[cfg(not(feature = "cuda"))] + { + false + } + } + + /// Get test device (CUDA if available, CPU otherwise) + pub fn get_test_device() -> candle_core::Device { + candle_core::Device::cuda_if_available(0).unwrap_or(candle_core::Device::Cpu) + } + + /// Skip test if CUDA not available + pub fn require_cuda() { + if !cuda_available() { + eprintln!("⚠️ Skipping CUDA test - CUDA not available"); + std::process::exit(0); + } + } +} \ No newline at end of file diff --git a/tests/harness/grpc_clients.rs b/tests/harness/grpc_clients.rs index ebecc8048..a1b38c3ef 100644 --- a/tests/harness/grpc_clients.rs +++ b/tests/harness/grpc_clients.rs @@ -12,17 +12,17 @@ use tokio::time::timeout; use tonic::transport::{Channel, Endpoint}; use super::TestConfig; -// Import generated gRPC clients (assuming they exist) -// These would be generated from the proto files -pub use foxhunt_ml::ml_service_client::MlServiceClient; -pub use foxhunt_ml::ml_training_service_client::MlTrainingServiceClient; +// REMOVED: All pub use statements eliminated per cleanup requirements +// Tests must import from canonical sources: +// use foxhunt_ml::ml_service_client::MlServiceClient; +// use foxhunt_ml::ml_training_service_client::MlTrainingServiceClient; /// Container for all gRPC service clients #[derive(Clone)] pub struct GrpcClients { pub tli_client: TliClient, - pub ml_training_client: MlTrainingServiceClient, - pub ml_service_client: MlServiceClient, + pub ml_training_client: foxhunt_ml::ml_training_service_client::MlTrainingServiceClient, + pub ml_service_client: foxhunt_ml::ml_service_client::MlServiceClient, pub trading_client: TradingServiceClient, config: TestConfig, } @@ -38,8 +38,8 @@ impl GrpcClients { let trading_channel = Self::create_channel(&config.trading_service_endpoint).await?; let tli_client = TliClient::new(tli_channel)?; - let ml_training_client = MlTrainingServiceClient::new(ml_training_channel); - let ml_service_client = MlServiceClient::new(ml_training_channel.clone()); + let ml_training_client = foxhunt_ml::ml_training_service_client::MlTrainingServiceClient::new(ml_training_channel); + let ml_service_client = foxhunt_ml::ml_service_client::MlServiceClient::new(ml_training_channel.clone()); let trading_client = TradingServiceClient::new(trading_channel)?; Ok(Self { diff --git a/tests/integration/mod.rs b/tests/integration/mod.rs index b33cd3faf..00d7ac5d2 100644 --- a/tests/integration/mod.rs +++ b/tests/integration/mod.rs @@ -32,11 +32,7 @@ pub mod tli_client_tests; pub mod service_tests; // Re-export test suites for easy access -pub use trading_service_tests::TradingServiceTests; -pub use backtesting_service_tests::BacktestingServiceTests; -pub use ml_training_service_tests::MLTrainingServiceTests; -pub use tli_client_tests::TLIClientTests; -pub use service_tests::ComprehensiveServiceTests; +// DO NOT RE-EXPORT - Use explicit imports at usage sites /// Master Integration Test Runner /// diff --git a/tests/integration/mod.rs.bak b/tests/integration/mod.rs.bak new file mode 100644 index 000000000..b33cd3faf --- /dev/null +++ b/tests/integration/mod.rs.bak @@ -0,0 +1,542 @@ +//! Integration tests across modules + +use std::time::{Duration, Instant}; + +use crate::framework::{TestOrchestrator, IntegrationTestResult}; + +// Existing integration tests +pub mod broker_integration_tests; +pub mod broker_failover; +pub mod icmarkets_validation; +pub mod interactive_brokers_validation; +pub mod broker_risk_integration; +pub mod database_integration; +pub mod end_to_end_trading; +pub mod order_lifecycle; +pub mod module_integration_test; +pub mod network_failure_simulation; +pub mod run_integration_tests; +pub mod run_broker_validation; + +// New comprehensive integration tests (Layer 1: Service Pairs) +pub mod tli_trading_integration; +pub mod ml_trading_integration; +pub mod trading_risk_integration; +pub mod dual_provider_test; + +// Enhanced comprehensive integration test framework +pub mod trading_service_tests; +pub mod backtesting_service_tests; +pub mod ml_training_service_tests; +pub mod tli_client_tests; +pub mod service_tests; + +// Re-export test suites for easy access +pub use trading_service_tests::TradingServiceTests; +pub use backtesting_service_tests::BacktestingServiceTests; +pub use ml_training_service_tests::MLTrainingServiceTests; +pub use tli_client_tests::TLIClientTests; +pub use service_tests::ComprehensiveServiceTests; + +/// Master Integration Test Runner +/// +/// Orchestrates execution of all integration test suites with proper +/// service lifecycle management, dependency handling, and result aggregation. +pub struct MasterIntegrationTestRunner { + orchestrator: TestOrchestrator, +} + +impl MasterIntegrationTestRunner { + /// Initialize the master test runner + pub async fn new() -> Result> { + let orchestrator = TestOrchestrator::new_with_defaults().await?; + + Ok(Self { + orchestrator, + }) + } + + /// Run all integration test suites in optimal order + /// + /// This method executes all integration tests with proper dependency management: + /// 1. Framework validation tests + /// 2. Individual service tests (parallel where possible) + /// 3. TLI client tests (requires all services) + /// 4. Comprehensive end-to-end tests + pub async fn run_all_integration_tests(&self) -> Result> { + println!("🚀 Starting Foxhunt HFT System - Master Integration Test Suite"); + println!(" Testing complete system with all services and components"); + + let master_start = Instant::now(); + let mut all_results = Vec::new(); + let mut test_summary = TestSummary::new(); + + // Phase 1: Framework Validation + println!("\n📋 Phase 1: Framework Validation Tests"); + match self.run_framework_validation_tests().await { + Ok(framework_results) => { + test_summary.add_results(&framework_results); + all_results.extend(framework_results); + println!("✅ Framework validation completed"); + } + Err(e) => { + println!("❌ Framework validation failed: {}", e); + let mut failed_result = IntegrationTestResult::new("Framework Validation"); + failed_result.add_failure(&format!("Framework validation failed: {}", e)); + failed_result.finalize(); + all_results.push(failed_result); + test_summary.framework_failed = true; + } + } + + // Phase 2: Individual Service Tests (Parallel Execution) + println!("\n🔧 Phase 2: Individual Service Integration Tests"); + if !test_summary.framework_failed { + match self.run_service_tests_parallel().await { + Ok(service_results) => { + test_summary.add_results(&service_results); + all_results.extend(service_results); + println!("✅ All service tests completed"); + } + Err(e) => { + println!("❌ Service tests failed: {}", e); + let mut failed_result = IntegrationTestResult::new("Service Tests"); + failed_result.add_failure(&format!("Service tests failed: {}", e)); + failed_result.finalize(); + all_results.push(failed_result); + test_summary.services_failed = true; + } + } + } else { + println!("⏭️ Skipping service tests due to framework validation failure"); + } + + // Phase 3: TLI Client Tests (Requires All Services) + println!("\n💻 Phase 3: TLI Client Integration Tests"); + if !test_summary.framework_failed && !test_summary.services_failed { + match self.run_tli_client_tests().await { + Ok(tli_results) => { + test_summary.add_results(&tli_results); + all_results.extend(tli_results); + println!("✅ TLI client tests completed"); + } + Err(e) => { + println!("❌ TLI client tests failed: {}", e); + let mut failed_result = IntegrationTestResult::new("TLI Client Tests"); + failed_result.add_failure(&format!("TLI client tests failed: {}", e)); + failed_result.finalize(); + all_results.push(failed_result); + test_summary.tli_failed = true; + } + } + } else { + println!("⏭️ Skipping TLI client tests due to prerequisite failures"); + } + + // Phase 4: Comprehensive End-to-End Tests + println!("\n🔄 Phase 4: Comprehensive End-to-End Tests"); + if !test_summary.has_critical_failures() { + match self.run_comprehensive_e2e_tests().await { + Ok(e2e_results) => { + test_summary.add_results(&e2e_results); + all_results.extend(e2e_results); + println!("✅ End-to-end tests completed"); + } + Err(e) => { + println!("❌ End-to-end tests failed: {}", e); + let mut failed_result = IntegrationTestResult::new("End-to-End Tests"); + failed_result.add_failure(&format!("End-to-end tests failed: {}", e)); + failed_result.finalize(); + all_results.push(failed_result); + test_summary.e2e_failed = true; + } + } + } else { + println!("⏭️ Skipping end-to-end tests due to critical failures in previous phases"); + } + + let master_duration = master_start.elapsed(); + + // Generate comprehensive report + let master_results = MasterTestResults { + total_duration: master_duration, + all_results, + summary: test_summary, + system_validated: !test_summary.has_critical_failures(), + }; + + self.print_master_summary(&master_results); + + Ok(master_results) + } + + /// Run framework validation tests + async fn run_framework_validation_tests(&self) -> Result, Box> { + let comprehensive_tests = ComprehensiveServiceTests::new().await?; + + // Run only the framework validation portion + let framework_result = comprehensive_tests.test_framework_initialization().await?; + + Ok(vec![framework_result]) + } + + /// Run individual service tests in parallel + async fn run_service_tests_parallel(&self) -> Result, Box> { + println!(" Running Trading, Backtesting, and ML Training service tests in parallel..."); + + // Create test suites + let trading_tests = TradingServiceTests::new().await?; + let backtesting_tests = BacktestingServiceTests::new().await?; + let ml_training_tests = MLTrainingServiceTests::new().await?; + + // Run service tests in parallel + let (trading_results, backtesting_results, ml_training_results) = tokio::join!( + trading_tests.run_all_tests(), + backtesting_tests.run_all_tests(), + ml_training_tests.run_all_tests() + ); + + let mut all_service_results = Vec::new(); + + // Collect Trading Service results + match trading_results { + Ok(results) => { + println!(" ✅ Trading Service: {}/{} test suites passed", + results.iter().filter(|r| r.passed).count(), + results.len()); + all_service_results.extend(results); + } + Err(e) => { + println!(" ❌ Trading Service tests failed: {}", e); + let mut failed_result = IntegrationTestResult::new("Trading Service Tests"); + failed_result.add_failure(&format!("Trading service tests failed: {}", e)); + failed_result.finalize(); + all_service_results.push(failed_result); + } + } + + // Collect Backtesting Service results + match backtesting_results { + Ok(results) => { + println!(" ✅ Backtesting Service: {}/{} test suites passed", + results.iter().filter(|r| r.passed).count(), + results.len()); + all_service_results.extend(results); + } + Err(e) => { + println!(" ❌ Backtesting Service tests failed: {}", e); + let mut failed_result = IntegrationTestResult::new("Backtesting Service Tests"); + failed_result.add_failure(&format!("Backtesting service tests failed: {}", e)); + failed_result.finalize(); + all_service_results.push(failed_result); + } + } + + // Collect ML Training Service results + match ml_training_results { + Ok(results) => { + println!(" ✅ ML Training Service: {}/{} test suites passed", + results.iter().filter(|r| r.passed).count(), + results.len()); + all_service_results.extend(results); + } + Err(e) => { + println!(" ❌ ML Training Service tests failed: {}", e); + let mut failed_result = IntegrationTestResult::new("ML Training Service Tests"); + failed_result.add_failure(&format!("ML training service tests failed: {}", e)); + failed_result.finalize(); + all_service_results.push(failed_result); + } + } + + Ok(all_service_results) + } + + /// Run TLI client tests + async fn run_tli_client_tests(&self) -> Result, Box> { + let tli_tests = TLIClientTests::new().await?; + let tli_results = tli_tests.run_all_tests().await?; + + println!(" ✅ TLI Client: {}/{} test suites passed", + tli_results.iter().filter(|r| r.passed).count(), + tli_results.len()); + + Ok(tli_results) + } + + /// Run comprehensive end-to-end tests + async fn run_comprehensive_e2e_tests(&self) -> Result, Box> { + let comprehensive_tests = ComprehensiveServiceTests::new().await?; + let e2e_results = comprehensive_tests.run_all_tests().await?; + + println!(" ✅ End-to-End Tests: {}/{} test suites passed", + e2e_results.iter().filter(|r| r.passed).count(), + e2e_results.len()); + + Ok(e2e_results) + } + + /// Print comprehensive test summary + fn print_master_summary(&self, results: &MasterTestResults) { + println!("\n" + "=".repeat(80).as_str()); + println!("🎯 FOXHUNT HFT SYSTEM - MASTER INTEGRATION TEST RESULTS"); + println!("=".repeat(80)); + + // Overall status + if results.system_validated { + println!("🎉 SYSTEM STATUS: ✅ VALIDATED - All critical tests passed"); + } else { + println!("⚠️ SYSTEM STATUS: ❌ VALIDATION FAILED - Critical issues detected"); + } + + println!("⏱️ Total Test Duration: {:.1} minutes", results.total_duration.as_secs_f64() / 60.0); + + // Test suite breakdown + println!("\n📊 Test Suite Breakdown:"); + println!(" Total Test Suites: {}", results.all_results.len()); + println!(" Passed: {} ✅", results.summary.total_passed); + println!(" Failed: {} ❌", results.summary.total_failed); + println!(" Success Rate: {:.1}%", + if results.all_results.is_empty() { 0.0 } else { + (results.summary.total_passed as f64 / results.all_results.len() as f64) * 100.0 + } + ); + + // Phase-by-phase results + println!("\n🔍 Phase-by-Phase Results:"); + + if !results.summary.framework_failed { + println!(" 📋 Framework Validation: ✅ PASSED"); + } else { + println!(" 📋 Framework Validation: ❌ FAILED"); + } + + if !results.summary.services_failed { + println!(" 🔧 Service Integration: ✅ PASSED"); + } else { + println!(" 🔧 Service Integration: ❌ FAILED"); + } + + if !results.summary.tli_failed { + println!(" 💻 TLI Client: ✅ PASSED"); + } else { + println!(" 💻 TLI Client: ❌ FAILED"); + } + + if !results.summary.e2e_failed { + println!(" 🔄 End-to-End: ✅ PASSED"); + } else { + println!(" 🔄 End-to-End: ❌ FAILED"); + } + + // Performance summary + if let Some(performance_summary) = &results.summary.performance_summary { + println!("\n⚡ Performance Summary:"); + println!(" Average Latency: {:.1}μs", performance_summary.avg_latency_us); + println!(" P99 Latency: {:.1}μs", performance_summary.p99_latency_us); + println!(" Throughput: {:.0} ops/sec", performance_summary.avg_throughput_ops_sec); + + if performance_summary.meets_hft_requirements { + println!(" HFT Requirements: ✅ MET"); + } else { + println!(" HFT Requirements: ❌ NOT MET"); + } + } + + // Failed tests detail + if results.summary.total_failed > 0 { + println!("\n❌ Failed Test Details:"); + for (i, result) in results.all_results.iter().enumerate() { + if !result.passed { + println!(" {}: {} ({} failures)", + i + 1, result.test_name, result.failures.len()); + for failure in &result.failures { + println!(" - {}", failure); + } + } + } + } + + // Next steps + println!("\n🎯 Next Steps:"); + if results.system_validated { + println!(" ✅ System ready for production deployment"); + println!(" ✅ All HFT performance requirements validated"); + println!(" ✅ All service integrations working correctly"); + } else { + println!(" ❌ Address critical test failures before deployment"); + println!(" ❌ Review failed test details above"); + println!(" ❌ Re-run integration tests after fixes"); + } + + println!("\n" + "=".repeat(80).as_str()); + } + + /// Run a subset of tests for quick validation + pub async fn run_smoke_tests(&self) -> Result> { + println!("💨 Running Smoke Tests - Quick System Validation"); + + let smoke_start = Instant::now(); + let mut smoke_results = Vec::new(); + let mut test_summary = TestSummary::new(); + + // Smoke test: Basic service connectivity + let comprehensive_tests = ComprehensiveServiceTests::new().await?; + + match comprehensive_tests.test_service_communication().await { + Ok(result) => { + test_summary.add_result(&result); + smoke_results.push(result); + } + Err(e) => { + let mut failed_result = IntegrationTestResult::new("Smoke Test - Service Communication"); + failed_result.add_failure(&format!("Smoke test failed: {}", e)); + failed_result.finalize(); + smoke_results.push(failed_result); + } + } + + // Smoke test: Basic TLI connectivity + let tli_tests = TLIClientTests::new().await?; + match tli_tests.test_service_connection_management().await { + Ok(result) => { + test_summary.add_result(&result); + smoke_results.push(result); + } + Err(e) => { + let mut failed_result = IntegrationTestResult::new("Smoke Test - TLI Connection"); + failed_result.add_failure(&format!("TLI smoke test failed: {}", e)); + failed_result.finalize(); + smoke_results.push(failed_result); + } + } + + let smoke_duration = smoke_start.elapsed(); + + let smoke_test_results = MasterTestResults { + total_duration: smoke_duration, + all_results: smoke_results, + summary: test_summary, + system_validated: test_summary.total_failed == 0, + }; + + println!("💨 Smoke Tests Completed in {:.1}s: {} passed, {} failed", + smoke_duration.as_secs_f64(), + smoke_test_results.summary.total_passed, + smoke_test_results.summary.total_failed); + + Ok(smoke_test_results) + } +} + +/// Aggregated results from all integration tests +#[derive(Debug)] +pub struct MasterTestResults { + pub total_duration: Duration, + pub all_results: Vec, + pub summary: TestSummary, + pub system_validated: bool, +} + +/// Test execution summary +#[derive(Debug)] +pub struct TestSummary { + pub total_passed: usize, + pub total_failed: usize, + pub framework_failed: bool, + pub services_failed: bool, + pub tli_failed: bool, + pub e2e_failed: bool, + pub performance_summary: Option, +} + +impl TestSummary { + pub fn new() -> Self { + Self { + total_passed: 0, + total_failed: 0, + framework_failed: false, + services_failed: false, + tli_failed: false, + e2e_failed: false, + performance_summary: None, + } + } + + pub fn add_result(&mut self, result: &IntegrationTestResult) { + if result.passed { + self.total_passed += 1; + } else { + self.total_failed += 1; + } + } + + pub fn add_results(&mut self, results: &[IntegrationTestResult]) { + for result in results { + self.add_result(result); + } + } + + pub fn has_critical_failures(&self) -> bool { + self.framework_failed || self.services_failed + } +} + +/// Performance metrics summary +#[derive(Debug)] +pub struct PerformanceSummary { + pub avg_latency_us: f64, + pub p99_latency_us: f64, + pub avg_throughput_ops_sec: f64, + pub meets_hft_requirements: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio; + + #[tokio::test] + async fn test_master_integration_runner_smoke_tests() { + let runner = MasterIntegrationTestRunner::new().await + .expect("Failed to create master test runner"); + + let smoke_results = runner.run_smoke_tests().await + .expect("Failed to run smoke tests"); + + // Smoke tests should complete quickly + assert!(smoke_results.total_duration.as_secs() <= 30, + "Smoke tests took too long: {}s", smoke_results.total_duration.as_secs()); + + // At least some tests should have run + assert!(!smoke_results.all_results.is_empty(), "No smoke tests executed"); + } + + #[tokio::test] + #[ignore] // This is a long-running test + async fn test_master_integration_runner_full_suite() { + let runner = MasterIntegrationTestRunner::new().await + .expect("Failed to create master test runner"); + + let full_results = runner.run_all_integration_tests().await + .expect("Failed to run full integration test suite"); + + // Full test suite should complete within reasonable time + assert!(full_results.total_duration.as_secs() <= 1800, // 30 minutes + "Full test suite took too long: {} minutes", full_results.total_duration.as_secs() / 60); + + // Should have comprehensive coverage + assert!(full_results.all_results.len() >= 10, + "Not enough test suites executed: {}", full_results.all_results.len()); + + // For a properly functioning system, most tests should pass + let success_rate = full_results.summary.total_passed as f64 / + (full_results.summary.total_passed + full_results.summary.total_failed) as f64; + + assert!(success_rate >= 0.8, + "Success rate too low: {:.1}% ({} passed, {} failed)", + success_rate * 100.0, + full_results.summary.total_passed, + full_results.summary.total_failed); + } +} diff --git a/tests/integration/trading_service_tests.rs b/tests/integration/trading_service_tests.rs index 20d2f8ed5..0ab6057c2 100644 --- a/tests/integration/trading_service_tests.rs +++ b/tests/integration/trading_service_tests.rs @@ -15,7 +15,7 @@ use common::types::MarketData; use common::types::Tick; use trading_engine::services::trading::{TradingService, OrderExecutor, PositionManager}; use risk::safety::KillSwitchController; -use trading_engine::types::events::{OrderEvent, PositionEvent, RiskEvent}; +use trading_engine::events::{OrderEvent, PositionEvent, RiskEvent}; use trading_engine::events::EventBus; /// Comprehensive Trading Service Integration Tests diff --git a/tests/mocks/mod.rs b/tests/mocks/mod.rs index bf95f8995..31f3e9ae2 100644 --- a/tests/mocks/mod.rs +++ b/tests/mocks/mod.rs @@ -21,10 +21,6 @@ pub mod mock_backtesting_service; pub mod mock_database; pub mod mock_ml_infrastructure; -pub use mock_trading_service::*; -pub use mock_backtesting_service::*; -pub use mock_database::*; -pub use mock_ml_infrastructure::*; /// Mock trading service for integration testing pub struct MockTradingService { diff --git a/tests/mocks/mod.rs.bak b/tests/mocks/mod.rs.bak new file mode 100644 index 000000000..bf95f8995 --- /dev/null +++ b/tests/mocks/mod.rs.bak @@ -0,0 +1,659 @@ +//! Mock Services for Integration Testing +//! +//! This module provides comprehensive mock implementations of all trading system +//! services for integration testing, including realistic latency simulation, +//! configurable failure rates, and comprehensive response patterns. + +use std::collections::HashMap; +use std::sync::{Arc, atomic::{AtomicU16, AtomicU64, Ordering}}; +use std::time::{Duration, Instant}; + +use tokio::sync::{mpsc, RwLock, Mutex}; +use uuid::Uuid; +use serde_json::json; +use chrono::{DateTime, Utc}; + +use tli::prelude::*; +use crate::fixtures::*; + +pub mod mock_trading_service; +pub mod mock_backtesting_service; +pub mod mock_database; +pub mod mock_ml_infrastructure; + +pub use mock_trading_service::*; +pub use mock_backtesting_service::*; +pub use mock_database::*; +pub use mock_ml_infrastructure::*; + +/// Mock trading service for integration testing +pub struct MockTradingService { + port: u16, + server_handle: Option>, + order_responses: Arc>>, + risk_rejections: Arc>>, + failure_sequence_count: Arc, + lifecycle_simulations: Arc>>>, + circuit_breaker_status: Arc>, + trading_status: Arc>, + config: MockServiceConfig, +} + +/// Mock service configuration +#[derive(Debug, Clone)] +pub struct MockServiceConfig { + pub latency_ms: u64, + pub failure_rate: f64, + pub enable_chaos: bool, + pub max_connections: usize, +} + +impl Default for MockServiceConfig { + fn default() -> Self { + Self { + latency_ms: 10, + failure_rate: 0.01, // 1% + enable_chaos: false, + max_connections: 100, + } + } +} + +// OrderStatus now imported from canonical source +use common::types::OrderStatus; + +/// Circuit breaker status +#[derive(Debug, Clone)] +pub struct CircuitBreakerStatus { + pub is_open: bool, + pub failure_count: u32, + pub last_failure_time: Option>, +} + +impl Default for CircuitBreakerStatus { + fn default() -> Self { + Self { + is_open: false, + failure_count: 0, + last_failure_time: None, + } + } +} + +/// Trading system status +#[derive(Debug, Clone)] +pub struct TradingStatus { + pub is_active: bool, + pub emergency_stop_active: bool, + pub last_heartbeat: DateTime, + pub active_orders: u64, + pub total_volume: Decimal, +} + +impl Default for TradingStatus { + fn default() -> Self { + Self { + is_active: true, + emergency_stop_active: false, + last_heartbeat: Utc::now(), + active_orders: 0, + total_volume: Decimal::ZERO, + } + } +} + +/// Submit order request structure +#[derive(Debug, Clone)] +pub struct SubmitOrderRequest { + pub symbol: String, + pub side: i32, // OrderSide enum as i32 + pub order_type: i32, // OrderType enum as i32 + pub quantity: f64, + pub price: Option, + pub client_order_id: String, + pub metadata: HashMap, +} + +/// Submit order response structure +#[derive(Debug, Clone)] +pub struct SubmitOrderResponse { + pub success: bool, + pub order_id: String, + pub message: String, + pub execution_time_ns: u64, +} + +/// Start backtest request structure +#[derive(Debug, Clone)] +pub struct StartBacktestRequest { + pub backtest_id: String, + pub enable_monitoring: bool, +} + +/// Start backtest response structure +#[derive(Debug, Clone)] +pub struct StartBacktestResponse { + pub success: bool, + pub message: String, +} + +impl MockTradingService { + /// Create new mock trading service + pub async fn new() -> TliResult { + Self::new_with_config(MockServiceConfig::default()).await + } + + /// Create new mock trading service with custom configuration + pub async fn new_with_config(config: MockServiceConfig) -> TliResult { + let port = TEST_PORT_MANAGER.allocate_port().await; + + Ok(Self { + port, + server_handle: None, + order_responses: Arc::new(RwLock::new(HashMap::new())), + risk_rejections: Arc::new(RwLock::new(HashMap::new())), + failure_sequence_count: Arc::new(AtomicU64::new(0)), + lifecycle_simulations: Arc::new(RwLock::new(HashMap::new())), + circuit_breaker_status: Arc::new(RwLock::new(CircuitBreakerStatus::default())), + trading_status: Arc::new(RwLock::new(TradingStatus::default())), + config, + }) + } + + /// Get the port the mock service is running on + pub fn port(&self) -> u16 { + self.port + } + + /// Configure a specific order response + pub async fn configure_order_response(&self, client_order_id: &str, response: SubmitOrderResponse) { + let mut responses = self.order_responses.write().await; + responses.insert(client_order_id.to_string(), response); + } + + /// Configure risk rejection for an order + pub async fn configure_risk_rejection(&self, client_order_id: &str, reason: String) { + let mut rejections = self.risk_rejections.write().await; + rejections.insert(client_order_id.to_string(), reason); + } + + /// Configure failure sequence for circuit breaker testing + pub async fn configure_failure_sequence(&self, count: u64) { + self.failure_sequence_count.store(count, Ordering::Relaxed); + } + + /// Configure order lifecycle simulation + pub async fn configure_lifecycle_simulation(&self, order_id: &str, statuses: Vec) { + let mut simulations = self.lifecycle_simulations.write().await; + simulations.insert(order_id.to_string(), statuses); + } + + /// Start the mock service + pub async fn start(&mut self) -> TliResult<()> { + let port = self.port; + let config = self.config.clone(); + let order_responses = Arc::clone(&self.order_responses); + let risk_rejections = Arc::clone(&self.risk_rejections); + let failure_sequence = Arc::clone(&self.failure_sequence_count); + let circuit_breaker = Arc::clone(&self.circuit_breaker_status); + let trading_status = Arc::clone(&self.trading_status); + + let handle = tokio::spawn(async move { + let listener = tokio::net::TcpListener::bind(format!("127.0.0.1:{}", port)) + .await + .expect("Failed to bind mock trading service"); + + while let Ok((stream, _)) = listener.accept().await { + let responses = Arc::clone(&order_responses); + let rejections = Arc::clone(&risk_rejections); + let failure_seq = Arc::clone(&failure_sequence); + let cb_status = Arc::clone(&circuit_breaker); + let trade_status = Arc::clone(&trading_status); + let service_config = config.clone(); + + tokio::spawn(async move { + Self::handle_connection(stream, responses, rejections, failure_seq, cb_status, trade_status, service_config).await; + }); + } + }); + + self.server_handle = Some(handle); + Ok(()) + } + + /// Handle individual client connection + async fn handle_connection( + stream: tokio::net::TcpStream, + order_responses: Arc>>, + risk_rejections: Arc>>, + failure_sequence: Arc, + circuit_breaker: Arc>, + trading_status: Arc>, + config: MockServiceConfig, + ) { + // Simulate service latency + if config.latency_ms > 0 { + tokio::time::sleep(Duration::from_millis(config.latency_ms)).await; + } + + // Simulate random failures if configured + if config.enable_chaos && fastrand::f64() < config.failure_rate { + return; // Drop connection to simulate failure + } + + // Handle gRPC-style protocol simulation + // In a real implementation, this would use tonic/gRPC + // For testing, we'll simulate the essential behavior + } + + /// Stop the mock service + pub async fn stop(&mut self) { + if let Some(handle) = self.server_handle.take() { + handle.abort(); + } + TEST_PORT_MANAGER.release_port(self.port).await; + } +} + +impl Drop for MockTradingService { + fn drop(&mut self) { + if let Some(handle) = self.server_handle.take() { + handle.abort(); + } + } +} + +/// Mock backtesting service for integration testing +pub struct MockBacktestingService { + port: u16, + server_handle: Option>, + backtest_responses: Arc>>, + ml_responses: Arc>>, + ensemble_responses: Arc>>, + config: MockServiceConfig, +} + +/// Backtest result structure +#[derive(Debug, Clone)] +pub struct BacktestResult { + pub backtest_id: String, + pub strategy_name: String, + pub total_return: f64, + pub annualized_return: f64, + pub max_drawdown: f64, + pub sharpe_ratio: f64, + pub total_trades: u64, + pub win_rate: f64, + pub avg_trade_return: f64, + pub final_value: f64, + pub execution_time_ms: u64, + pub events_processed: u64, +} + +/// ML backtest configuration +#[derive(Debug, Clone)] +pub struct MLBacktestConfig { + pub model_name: String, + pub expected_return: f64, + pub expected_sharpe: f64, + pub expected_trades: u64, +} + +/// Ensemble results structure +#[derive(Debug, Clone)] +pub struct EnsembleResults { + pub ensemble_return: f64, + pub ensemble_sharpe: f64, + pub individual_returns: Vec, + pub individual_sharpes: Vec, + pub diversification_benefit: f64, + pub model_weights_final: Vec, + pub rebalance_count: u32, +} + +/// Create backtest request structure +#[derive(Debug, Clone)] +pub struct CreateBacktestRequest { + pub name: String, + pub strategy_type: String, + pub symbol: String, + pub start_date: i64, + pub end_date: i64, + pub initial_capital: f64, + pub parameters: serde_json::Value, + pub enable_real_time_monitoring: bool, +} + +/// Create backtest response structure +#[derive(Debug, Clone)] +pub struct CreateBacktestResponse { + pub success: bool, + pub backtest_id: String, + pub message: String, +} + +impl MockBacktestingService { + /// Create new mock backtesting service + pub async fn new() -> TliResult { + let port = TEST_PORT_MANAGER.allocate_port().await; + + Ok(Self { + port, + server_handle: None, + backtest_responses: Arc::new(RwLock::new(HashMap::new())), + ml_responses: Arc::new(RwLock::new(HashMap::new())), + ensemble_responses: Arc::new(RwLock::new(HashMap::new())), + config: MockServiceConfig::default(), + }) + } + + /// Get the port the mock service is running on + pub fn port(&self) -> u16 { + self.port + } + + /// Configure backtest response + pub async fn configure_backtest_response(&self, name: &str, result: BacktestResult) { + let mut responses = self.backtest_responses.write().await; + responses.insert(name.to_string(), result); + } + + /// Configure ML backtest response + pub async fn configure_ml_backtest_response( + &self, + name: &str, + model_name: &str, + expected_return: f64, + expected_sharpe: f64, + expected_trades: u64, + ) { + let mut responses = self.ml_responses.write().await; + responses.insert(name.to_string(), MLBacktestConfig { + model_name: model_name.to_string(), + expected_return, + expected_sharpe, + expected_trades, + }); + } + + /// Configure ensemble response + pub async fn configure_ensemble_response(&self, name: &str, results: EnsembleResults) { + let mut responses = self.ensemble_responses.write().await; + responses.insert(name.to_string(), results); + } + + /// Stop the mock service + pub async fn stop(&mut self) { + if let Some(handle) = self.server_handle.take() { + handle.abort(); + } + TEST_PORT_MANAGER.release_port(self.port).await; + } +} + +/// Test database manager for PostgreSQL integration testing +pub struct TestDatabaseManager { + pool: sqlx::PgPool, + config: TestDatabaseConfig, +} + +/// Test database configuration +#[derive(Debug, Clone)] +pub struct TestDatabaseConfig { + pub database_url: String, + pub max_connections: u32, + pub enable_cleanup: bool, + pub test_schema: String, +} + +impl Default for TestDatabaseConfig { + fn default() -> Self { + Self { + database_url: std::env::var("TEST_DATABASE_URL") + .unwrap_or_else(|_| "postgresql://foxhunt_test:test_password@localhost:5432/foxhunt_test".to_string()), + max_connections: 20, + enable_cleanup: true, + test_schema: "test_".to_string(), + } + } +} + +impl TestDatabaseManager { + /// Create new test database manager + pub async fn new() -> TliResult { + Self::new_with_config(TestDatabaseConfig::default()).await + } + + /// Create new test database manager with custom configuration + pub async fn new_with_config(config: TestDatabaseConfig) -> TliResult { + let pool = sqlx::PgPool::connect(&config.database_url) + .await + .map_err(|e| TliError::DatabaseError(format!("Failed to connect to test database: {}", e)))?; + + Ok(Self { pool, config }) + } + + /// Get connection pool + pub fn get_pool(&self) -> &sqlx::PgPool { + &self.pool + } + + /// Verify order was persisted + pub async fn verify_order_persisted(&self, order_id: &str) -> TliResult { + let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM orders WHERE order_id = $1") + .bind(order_id) + .fetch_one(&self.pool) + .await + .map_err(|e| TliError::DatabaseError(format!("Failed to verify order persistence: {}", e)))?; + + Ok(count > 0) + } + + /// Clean up test data + pub async fn cleanup(&self) -> TliResult<()> { + if self.config.enable_cleanup { + let cleanup_queries = vec![ + "DELETE FROM trading_events WHERE source LIKE '%test%'", + "DELETE FROM orders WHERE client_order_id LIKE '%test%'", + "DELETE FROM positions WHERE account_id LIKE '%test%'", + "DELETE FROM risk_events WHERE account_id LIKE '%test%'", + "DELETE FROM market_data WHERE symbol LIKE '%TEST%'", + "DELETE FROM performance_metrics WHERE metric_name LIKE '%test%'", + ]; + + for query in cleanup_queries { + sqlx::query(query) + .execute(&self.pool) + .await + .map_err(|e| TliError::DatabaseError(format!("Cleanup failed: {}", e)))?; + } + } + + Ok(()) + } +} + +/// Test database structure +pub struct TestDatabase { + manager: TestDatabaseManager, +} + +impl TestDatabase { + /// Create new test database + pub async fn new() -> TliResult { + let manager = TestDatabaseManager::new().await?; + Ok(Self { manager }) + } + + /// Verify order was persisted + pub async fn verify_order_persisted(&self, order_id: &str) -> TliResult { + self.manager.verify_order_persisted(order_id).await + } +} + +/// Test data provider for market data simulation +pub struct TestDataProvider { + historical_data: HashMap>, +} + +/// Market data point structure +#[derive(Debug, Clone)] +pub struct MarketDataPoint { + pub timestamp: DateTime, + pub symbol: String, + pub price: f64, + pub volume: u64, + pub bid: Option, + pub ask: Option, +} + +impl TestDataProvider { + /// Create new test data provider + pub async fn new() -> TliResult { + let mut provider = Self { + historical_data: HashMap::new(), + }; + + // Generate sample data for common symbols + provider.generate_sample_data("AAPL", 1000).await; + provider.generate_sample_data("MSFT", 1000).await; + provider.generate_sample_data("GOOGL", 1000).await; + + Ok(provider) + } + + /// Generate sample market data + async fn generate_sample_data(&mut self, symbol: &str, count: usize) { + let mut data = Vec::new(); + let mut price = 150.0; + let start_time = Utc::now() - chrono::Duration::days(30); + + for i in 0..count { + let timestamp = start_time + chrono::Duration::seconds(i as i64 * 60); + + // Simple random walk + price += (fastrand::f64() - 0.5) * 2.0; + price = price.max(100.0).min(200.0); // Keep price in reasonable range + + data.push(MarketDataPoint { + timestamp, + symbol: symbol.to_string(), + price, + volume: 1000 + fastrand::u64(0..10000), + bid: Some(price - 0.01), + ask: Some(price + 0.01), + }); + } + + self.historical_data.insert(symbol.to_string(), data); + } + + /// Get historical data for symbol + pub fn get_historical_data(&self, symbol: &str) -> Option<&Vec> { + self.historical_data.get(symbol) + } +} + +/// ML testing infrastructure +pub struct MLTestInfrastructure { + mock_models: HashMap, +} + +/// Mock ML model +#[derive(Debug, Clone)] +pub struct MockMLModel { + pub name: String, + pub inference_latency_ns: u64, + pub accuracy: f64, + pub memory_usage_mb: u64, +} + +impl MLTestInfrastructure { + /// Create new ML testing infrastructure + pub async fn new() -> TliResult { + let mut models = HashMap::new(); + + // Add mock models with realistic characteristics + models.insert("TLOB".to_string(), MockMLModel { + name: "TLOB".to_string(), + inference_latency_ns: 15_000, // 15µs + accuracy: 0.89, + memory_usage_mb: 256, + }); + + models.insert("MAMBA".to_string(), MockMLModel { + name: "MAMBA".to_string(), + inference_latency_ns: 25_000, // 25µs + accuracy: 0.91, + memory_usage_mb: 512, + }); + + models.insert("TFT".to_string(), MockMLModel { + name: "TFT".to_string(), + inference_latency_ns: 35_000, // 35µs + accuracy: 0.87, + memory_usage_mb: 384, + }); + + models.insert("DQN".to_string(), MockMLModel { + name: "DQN".to_string(), + inference_latency_ns: 20_000, // 20µs + accuracy: 0.84, + memory_usage_mb: 128, + }); + + Ok(Self { + mock_models: models, + }) + } + + /// Get mock model + pub fn get_model(&self, name: &str) -> Option<&MockMLModel> { + self.mock_models.get(name) + } + + /// List available models + pub fn list_models(&self) -> Vec<&str> { + self.mock_models.keys().map(|s| s.as_str()).collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_mock_trading_service() { + let mut service = MockTradingService::new().await.unwrap(); + assert!(service.port() > 0); + + service.configure_order_response("test_order", SubmitOrderResponse { + success: true, + order_id: "order_123".to_string(), + message: "Success".to_string(), + execution_time_ns: 15_000, + }).await; + + service.stop().await; + } + + #[tokio::test] + async fn test_test_data_provider() { + let provider = TestDataProvider::new().await.unwrap(); + let aapl_data = provider.get_historical_data("AAPL").unwrap(); + assert_eq!(aapl_data.len(), 1000); + assert!(aapl_data[0].price > 0.0); + } + + #[tokio::test] + async fn test_ml_infrastructure() { + let ml_infra = MLTestInfrastructure::new().await.unwrap(); + let models = ml_infra.list_models(); + assert!(models.contains(&"TLOB")); + assert!(models.contains(&"MAMBA")); + + let tlob = ml_infra.get_model("TLOB").unwrap(); + assert_eq!(tlob.name, "TLOB"); + assert!(tlob.inference_latency_ns > 0); + } +} \ No newline at end of file diff --git a/tests/test_common/lib.rs b/tests/test_common/lib.rs index 62ea6aeb1..a74eb3b07 100644 --- a/tests/test_common/lib.rs +++ b/tests/test_common/lib.rs @@ -195,26 +195,12 @@ pub mod async_patterns { } } -// Re-export commonly used items for convenience -pub use database_helper::{ - get_test_database_pool, - get_test_database_pool_with_config, - setup_test_database, - teardown_test_database, - cleanup_all_test_data, - create_test_user, - create_test_order, - create_test_position, - create_test_execution, - benchmark_database_operations, - DatabaseTestConfig, - DatabaseTestPool, - DatabaseBenchmarkResult, -}; - -pub use test_config::UnifiedTestConfig; -pub use mock_data::{create_mock_order, create_mock_market_tick, MockMarketTick}; -pub use test_utils::{run_with_timeout, setup_test_tracing, assertions, test_symbol, test_price, test_quantity}; -pub use async_patterns::TestBroadcastReceiver; +// REMOVED: All pub use statements eliminated per cleanup requirements +// Tests must import from canonical sources: +// - tests::test_common::database_helper::* +// - tests::test_common::test_config::UnifiedTestConfig +// - tests::test_common::mock_data::* +// - tests::test_common::test_utils::* +// - tests::test_common::async_patterns::TestBroadcastReceiver // Re-export canonical types for test convenience diff --git a/tests/test_common/mod.rs b/tests/test_common/mod.rs index 1181c6c7d..6dbb53688 100644 --- a/tests/test_common/mod.rs +++ b/tests/test_common/mod.rs @@ -209,14 +209,10 @@ pub mod async_patterns { } // Re-export commonly used items for convenience -pub use database_helper::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites benchmark_database_operations, cleanup_all_test_data, create_test_execution, create_test_order, create_test_position, create_test_user, get_test_database_pool, get_test_database_pool_with_config, setup_test_database, teardown_test_database, DatabaseBenchmarkResult, DatabaseTestConfig, DatabaseTestPool, }; -pub use async_patterns::TestBroadcastReceiver; -pub use mock_data::{create_mock_market_tick, create_mock_order, MockMarketTick, MockOrder}; -pub use test_config::UnifiedTestConfig; -pub use test_utils::{assertions, run_with_timeout, setup_test_tracing}; diff --git a/tests/test_common/mod.rs.bak b/tests/test_common/mod.rs.bak new file mode 100644 index 000000000..1181c6c7d --- /dev/null +++ b/tests/test_common/mod.rs.bak @@ -0,0 +1,222 @@ +//! Common test utilities for Foxhunt HFT System +//! +//! This module provides shared testing infrastructure to eliminate duplication +//! across the 80+ test files in the project. +//! +//! # Usage +//! ```rust +//! use common::types::*; +//! use common::types::test_config::*; +//! use common::types::mock_data::*; +//! ``` + +pub mod database_helper; + +// Test Configuration Module +pub mod test_config { + use std::time::Duration; + + /// Unified test configuration for all test types + #[derive(Debug, Clone)] + pub struct UnifiedTestConfig { + pub environment_name: String, + pub docker_compose_file: Option, + pub cleanup_on_exit: bool, + pub persist_data: bool, + pub log_level: String, + pub test_database_url: String, + pub test_redis_url: String, + pub test_influxdb_url: String, + pub parallel_tests: bool, + pub timeout_seconds: u64, + pub max_retries: u32, + } + + impl Default for UnifiedTestConfig { + fn default() -> Self { + Self { + environment_name: "test".to_string(), + docker_compose_file: Some("docker-compose.test.yml".to_string()), + cleanup_on_exit: true, + persist_data: false, + log_level: "debug".to_string(), + test_database_url: std::env::var("TEST_DATABASE_URL").unwrap_or_else(|_| { + std::env::var("FOXHUNT_TEST_POSTGRES_URL").unwrap_or_else(|_| { + let db_host = std::env::var("DATABASE_HOST") + .or_else(|_| std::env::var("POSTGRES_HOST")) + .unwrap_or_else(|_| "localhost".to_string()); + format!("postgresql://{}:5432/hft_testing", db_host) + }) + }), + test_redis_url: std::env::var("TEST_REDIS_URL").unwrap_or_else(|_| { + std::env::var("FOXHUNT_TEST_REDIS_URL").unwrap_or_else(|_| { + let redis_host = + std::env::var("REDIS_HOST").unwrap_or_else(|_| "localhost".to_string()); + format!( + "redis://:{}@{}:6379/0", + std::env::var("REDIS_TEST_PASSWORD") + .unwrap_or_else(|_| "test_password".to_string()), + redis_host + ) + }) + }), + test_influxdb_url: std::env::var("TEST_INFLUXDB_URL").unwrap_or_else(|_| { + let influx_host = + std::env::var("INFLUXDB_HOST").unwrap_or_else(|_| "localhost".to_string()); + format!("http://{}:8086", influx_host) + }), + parallel_tests: true, + timeout_seconds: 30, + max_retries: 3, + } + } + } +} + +// Mock Data Generation Module +pub mod mock_data { + use uuid::Uuid; + + /// Generate mock order data for testing + pub fn create_mock_order() -> MockOrder { + MockOrder { + id: Uuid::new_v4().to_string(), + symbol: "BTCUSD".to_string(), + side: "Buy".to_string(), + quantity: 1.0, + price: 50000.0, + status: "Pending".to_string(), + } + } + + /// Generate mock market data + pub fn create_mock_market_tick(symbol: &str) -> MockMarketTick { + MockMarketTick { + symbol: symbol.to_string(), + price: 50000.0, + volume: 100.0, + timestamp: chrono::Utc::now().timestamp_millis(), + } + } + + /// Mock order structure for tests + #[derive(Debug, Clone)] + pub struct MockOrder { + pub id: String, + pub symbol: String, + pub side: String, + pub quantity: f64, + pub price: f64, + pub status: String, + } + + /// Mock market tick for tests + #[derive(Debug, Clone)] + pub struct MockMarketTick { + pub symbol: String, + pub price: f64, + pub volume: f64, + pub timestamp: i64, + } +} + +// Test Utilities Module +pub mod test_utils { + use std::time::Duration; + use tokio::time::timeout; + + /// Async test helper with timeout + pub async fn run_with_timeout(future: F, timeout_secs: u64) -> Result + where + F: std::future::Future, + { + timeout(Duration::from_secs(timeout_secs), future) + .await + .map_err(|_| "Test timed out") + } + + /// Setup tracing for tests + pub fn setup_test_tracing() { + use tracing_subscriber::{EnvFilter, FmtSubscriber}; + + let _ = tracing_subscriber::fmt() + .with_test_writer() + .with_env_filter(EnvFilter::from_default_env()) + .try_init(); + } + + /// Common test assertions + pub mod assertions { + use std::time::Duration; + + /// Assert that a value is within a percentage tolerance + pub fn assert_within_percent(actual: f64, expected: f64, percent: f64) { + let tolerance = expected * (percent / 100.0); + let diff = (actual - expected).abs(); + assert!( + diff <= tolerance, + "Value {} is not within {}% of expected {}, difference: {}", + actual, + percent, + expected, + diff + ); + } + + /// Assert that latency is within HFT requirements + pub fn assert_hft_latency(duration: Duration, max_microseconds: u64) { + let micros = duration.as_micros() as u64; + assert!( + micros <= max_microseconds, + "Latency {}μs exceeds HFT requirement of {}μs", + micros, + max_microseconds + ); + } + } +} + +// Async Test Patterns Module +pub mod async_patterns { + use tokio::sync::broadcast; + + /// Proper broadcast receiver pattern for tests + pub struct TestBroadcastReceiver { + receiver: broadcast::Receiver, + } + + impl TestBroadcastReceiver + where + T: Clone + Send + 'static, + { + pub fn new(receiver: broadcast::Receiver) -> Self { + Self { receiver } + } + + pub async fn wait_for_shutdown(mut self) -> Result<(), broadcast::error::RecvError> { + loop { + tokio::select! { + msg = self.receiver.recv() => { + match msg { + Ok(_) => return Ok(()), + Err(e) => return Err(e), + } + } + } + } + } + } +} + +// Re-export commonly used items for convenience +pub use database_helper::{ + benchmark_database_operations, cleanup_all_test_data, create_test_execution, create_test_order, + create_test_position, create_test_user, get_test_database_pool, + get_test_database_pool_with_config, setup_test_database, teardown_test_database, + DatabaseBenchmarkResult, DatabaseTestConfig, DatabaseTestPool, +}; + +pub use async_patterns::TestBroadcastReceiver; +pub use mock_data::{create_mock_market_tick, create_mock_order, MockMarketTick, MockOrder}; +pub use test_config::UnifiedTestConfig; +pub use test_utils::{assertions, run_with_timeout, setup_test_tracing}; diff --git a/tests/utils/hft_utils.rs b/tests/utils/hft_utils.rs index 5892bbcea..069a84433 100644 --- a/tests/utils/hft_utils.rs +++ b/tests/utils/hft_utils.rs @@ -327,8 +327,8 @@ pub mod orders { pub avg_fill_price: Option, } - // OrderSide, OrderType, and OrderStatus now imported from canonical source - pub use common::types::OrderSide; + // REMOVED: All pub use statements eliminated per cleanup requirements + // Use direct import: common::types::OrderSide use common::types::OrderType; use common::types::OrderStatus; diff --git a/tests/utils/mod.rs b/tests/utils/mod.rs index ef5688e2c..fd17bcef6 100644 --- a/tests/utils/mod.rs +++ b/tests/utils/mod.rs @@ -7,13 +7,13 @@ pub mod hft_utils; pub mod test_safety; // Re-export commonly used items for convenience (macros are exported at crate root) -pub use test_safety::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites with_test_timeout, with_test_timeout_result, SafeTestUnwrap, TestError, TestFixture, TestResult, }; // Note: test_assert and test_assert_eq macros are exported at crate root automatically -pub use hft_utils::{financial, market_data, orders, performance}; +// DO NOT RE-EXPORT - Use explicit imports at usage sites /// Macro to create a test with automatic error handling and context #[macro_export] diff --git a/tests/utils/mod.rs.bak b/tests/utils/mod.rs.bak new file mode 100644 index 000000000..ef5688e2c --- /dev/null +++ b/tests/utils/mod.rs.bak @@ -0,0 +1,174 @@ +//! Test Utilities Module +//! +//! Comprehensive test utilities for safe, reliable testing patterns +//! across the Foxhunt HFT trading system. + +pub mod hft_utils; +pub mod test_safety; + +// Re-export commonly used items for convenience (macros are exported at crate root) +pub use test_safety::{ + with_test_timeout, with_test_timeout_result, SafeTestUnwrap, TestError, TestFixture, TestResult, +}; + +// Note: test_assert and test_assert_eq macros are exported at crate root automatically + +pub use hft_utils::{financial, market_data, orders, performance}; + +/// Macro to create a test with automatic error handling and context +#[macro_export] +macro_rules! safe_test { + ($test_name:ident, $test_fn:expr) => { + #[tokio::test] + async fn $test_name() { + match $test_fn().await { + Ok(()) => {} + Err(e) => { + panic!("Test {} failed: {}", stringify!($test_name), e); + } + } + } + }; +} + +/// Macro to create a property based test +#[macro_export] +macro_rules! property_test { + ($test_name:ident, $iterations:expr, $test_fn:expr) => { + #[tokio::test] + async fn $test_name() { + use crate::utils::test_safety::property; + + match property::run_property_test(stringify!($test_name), $iterations, $test_fn) { + Ok(()) => {} + Err(e) => { + panic!("Property test {} failed: {}", stringify!($test_name), e); + } + } + } + }; +} + +/// Macro to create a performance benchmark test +#[macro_export] +macro_rules! benchmark_test { + ($test_name:ident, $operation:expr, $max_latency:expr) => { + #[tokio::test] + async fn $test_name() { + use crate::utils::hft_utils::performance; + use std::time::Duration; + + let measurements = performance::benchmark_operation( + $operation, + 10, // warmup iterations + 100, // measurement iterations + stringify!($test_name), + ) + .await + .expect("Benchmark should complete"); + + let avg_latency = measurements.average().expect("Should have measurements"); + let p99_latency = measurements.percentile(0.99).expect("Should have p99"); + + println!( + "Benchmark {}: avg={:?}, p99={:?}, max={:?}", + stringify!($test_name), + avg_latency, + p99_latency, + measurements.max().unwrap_or_default() + ); + + if avg_latency > $max_latency { + panic!( + "Benchmark {} failed: average latency {:?} exceeds maximum {:?}", + stringify!($test_name), + avg_latency, + $max_latency + ); + } + } + }; +} + +/// Test configuration for different environments +#[derive(Debug, Clone)] +pub struct TestConfig { + pub enable_chaos: bool, + pub chaos_failure_rate: f64, + pub default_timeout: std::time::Duration, + pub hft_latency_requirement: std::time::Duration, + pub enable_performance_validation: bool, +} + +impl Default for TestConfig { + fn default() -> Self { + Self { + enable_chaos: false, + chaos_failure_rate: 0.1, + default_timeout: std::time::Duration::from_secs(30), + hft_latency_requirement: std::time::Duration::from_micros(50), + enable_performance_validation: true, + } + } +} + +impl TestConfig { + pub fn for_unit_tests() -> Self { + Self { + enable_chaos: false, + default_timeout: std::time::Duration::from_secs(5), + ..Default::default() + } + } + + pub fn for_integration_tests() -> Self { + Self { + enable_chaos: false, + default_timeout: std::time::Duration::from_secs(30), + ..Default::default() + } + } + + pub fn for_chaos_tests() -> Self { + Self { + enable_chaos: true, + chaos_failure_rate: 0.2, + default_timeout: std::time::Duration::from_secs(60), + ..Default::default() + } + } +} + +/// Global test configuration accessor +static TEST_CONFIG: std::sync::OnceLock = std::sync::OnceLock::new(); + +pub fn get_test_config() -> &'static TestConfig { + TEST_CONFIG.get_or_init(|| { + std::env::var("TEST_MODE") + .map(|mode| match mode.as_str() { + "unit" => TestConfig::for_unit_tests(), + "integration" => TestConfig::for_integration_tests(), + "chaos" => TestConfig::for_chaos_tests(), + _ => TestConfig::default(), + }) + .unwrap_or_default() + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_config_default() { + let config = TestConfig::default(); + assert!(!config.enable_chaos); + assert_eq!(config.chaos_failure_rate, 0.1); + } + + #[tokio::test] + async fn test_config_access() { + let config = get_test_config(); + assert!(config.default_timeout.as_secs() > 0); + } +} diff --git a/tli/src/client/mod.rs b/tli/src/client/mod.rs index 9dcfa30d8..71675e264 100644 --- a/tli/src/client/mod.rs +++ b/tli/src/client/mod.rs @@ -16,32 +16,7 @@ pub mod stream_manager; pub mod trading_client; // Re-export main components -pub use connection_manager::{ - AuthConfig, CircuitBreaker, ConnectionConfig, ConnectionManager, ConnectionStats, - ConnectionStatus, ManagedConnection, ReconnectionConfig, TlsConfig, -}; - -pub use event_stream::{ - EventFilters, EventStreamConfig, EventStreamManager, EventType, ReconnectConfig, StreamStats, - TliEvent, -}; - -pub use trading_client::{ - ClientStats, MarketDataConfig, MarketDataSnapshot, MonitoringConfig, OrderContext, - OrderValidationConfig, OrderValidationResult, PreTradeCheckResult, RiskManagementConfig, - RiskValidationResult, TradingClient, TradingClientConfig, -}; - -pub use backtesting_client::{ - BacktestContext, BacktestPerformanceSummary, BacktestProgressSnapshot, BacktestQuery, - BacktestingClient, BacktestingClientConfig, -}; - -pub use ml_training_client::{ - MLTrainingClient, MLTrainingClientConfig, MLTrainingStats, ResourceMonitoringEvent, - TrainingJobContext, TrainingProgressEvent, -}; -pub use stream_manager::DataStreamManager; +// DO NOT RE-EXPORT - Use explicit imports at usage sites /// Service endpoints configuration #[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] diff --git a/tli/src/client/mod.rs.bak b/tli/src/client/mod.rs.bak new file mode 100644 index 000000000..9dcfa30d8 --- /dev/null +++ b/tli/src/client/mod.rs.bak @@ -0,0 +1,299 @@ +//! TLI gRPC client modules +//! +//! This module contains comprehensive gRPC client implementations for all +//! core trading system services with advanced features including: +//! - Connection pooling and health monitoring +//! - Real-time streaming support +//! - Automatic reconnection and circuit breakers +//! - Comprehensive error handling +//! - Metrics collection and alerting + +pub mod backtesting_client; +pub mod connection_manager; +pub mod event_stream; +pub mod ml_training_client; +pub mod stream_manager; +pub mod trading_client; + +// Re-export main components +pub use connection_manager::{ + AuthConfig, CircuitBreaker, ConnectionConfig, ConnectionManager, ConnectionStats, + ConnectionStatus, ManagedConnection, ReconnectionConfig, TlsConfig, +}; + +pub use event_stream::{ + EventFilters, EventStreamConfig, EventStreamManager, EventType, ReconnectConfig, StreamStats, + TliEvent, +}; + +pub use trading_client::{ + ClientStats, MarketDataConfig, MarketDataSnapshot, MonitoringConfig, OrderContext, + OrderValidationConfig, OrderValidationResult, PreTradeCheckResult, RiskManagementConfig, + RiskValidationResult, TradingClient, TradingClientConfig, +}; + +pub use backtesting_client::{ + BacktestContext, BacktestPerformanceSummary, BacktestProgressSnapshot, BacktestQuery, + BacktestingClient, BacktestingClientConfig, +}; + +pub use ml_training_client::{ + MLTrainingClient, MLTrainingClientConfig, MLTrainingStats, ResourceMonitoringEvent, + TrainingJobContext, TrainingProgressEvent, +}; +pub use stream_manager::DataStreamManager; + +/// Service endpoints configuration +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +pub struct ServiceEndpoints { + /// Trading engine endpoint + pub trading_engine: String, + /// Market data endpoint + pub market_data: String, + /// Backtesting service endpoint + pub backtesting_service: String, + /// ML training service endpoint + pub ml_training_service: String, +} + +impl ServiceEndpoints { + /// Create default endpoints for local development + pub fn localhost() -> Self { + Self { + trading_engine: "http://localhost:50051".to_string(), + market_data: "http://localhost:50052".to_string(), + backtesting_service: "http://localhost:50053".to_string(), + ml_training_service: "http://localhost:50054".to_string(), + } + } +} + +/// Client factory for creating and managing all service clients +#[derive(Debug)] +pub struct ClientFactory { + /// Connection manager shared across all clients + connection_manager: std::sync::Arc, + /// Global connection configuration + connection_config: ConnectionConfig, +} + +impl ClientFactory { + /// Create a new client factory + pub fn new(connection_config: ConnectionConfig) -> Self { + let connection_manager = + std::sync::Arc::new(ConnectionManager::new(connection_config.clone())); + + Self { + connection_manager, + connection_config, + } + } + + /// Create a trading client + pub fn create_trading_client(&self, config: TradingClientConfig) -> TradingClient { + TradingClient::new(self.connection_manager.clone(), config) + } + + /// Create a backtesting client + pub fn create_backtesting_client(&self, config: BacktestingClientConfig) -> BacktestingClient { + BacktestingClient::new(self.connection_manager.clone(), config) + } + + /// Create an ML training client + pub fn create_ml_training_client(&self, config: MLTrainingClientConfig) -> MLTrainingClient { + MLTrainingClient::new(self.connection_manager.clone(), config) + } + + /// Add a service connection to the pool + pub async fn add_service( + &self, + service_name: String, + config: ConnectionConfig, + ) -> crate::error::TliResult<()> { + self.connection_manager + .add_service(service_name, config) + .await + } + + /// Get connection statistics for all services + pub async fn get_connection_stats( + &self, + ) -> std::collections::HashMap> { + self.connection_manager.get_pool_stats().await + } + + /// Shutdown all connections and clients + pub async fn shutdown(&self) { + self.connection_manager.shutdown().await; + } +} + +/// Convenience builder for creating a complete TLI client setup +#[derive(Debug)] +pub struct TliClientBuilder { + /// Connection configuration + connection_config: ConnectionConfig, + /// Service endpoints + service_endpoints: std::collections::HashMap, + /// Client configurations + trading_config: Option, + backtesting_config: Option, + ml_training_config: Option, +} + +impl Default for TliClientBuilder { + fn default() -> Self { + Self::new() + } +} + +impl TliClientBuilder { + /// Create a new builder + pub fn new() -> Self { + Self { + connection_config: ConnectionConfig::default(), + service_endpoints: std::collections::HashMap::new(), + trading_config: None, + backtesting_config: None, + ml_training_config: None, + } + } + + /// Set connection configuration + pub fn with_connection_config(mut self, config: ConnectionConfig) -> Self { + self.connection_config = config; + self + } + + /// Add a service endpoint + pub fn with_service_endpoint(mut self, service_name: String, endpoint: String) -> Self { + self.service_endpoints.insert(service_name, endpoint); + self + } + + /// Set trading client configuration + pub fn with_trading_config(mut self, config: TradingClientConfig) -> Self { + self.trading_config = Some(config); + self + } + + /// Set backtesting client configuration + pub fn with_backtesting_config(mut self, config: BacktestingClientConfig) -> Self { + self.backtesting_config = Some(config); + self + } + + /// Set ML training client configuration + pub fn with_ml_training_config(mut self, config: MLTrainingClientConfig) -> Self { + self.ml_training_config = Some(config); + self + } + + /// Build the complete TLI client setup + pub async fn build(self) -> crate::error::TliResult { + let factory = ClientFactory::new(self.connection_config.clone()); + + // Add service connections + for (service_name, endpoint) in self.service_endpoints { + let mut service_config = self.connection_config.clone(); + service_config.endpoint = endpoint; + factory.add_service(service_name, service_config).await?; + } + + // Create clients + let trading_client = if let Some(config) = self.trading_config { + Some(factory.create_trading_client(config)) + } else { + None + }; + + let backtesting_client = if let Some(config) = self.backtesting_config { + Some(factory.create_backtesting_client(config)) + } else { + None + }; + + let ml_training_client = if let Some(config) = self.ml_training_config { + Some(factory.create_ml_training_client(config)) + } else { + None + }; + + Ok(TliClientSuite { + factory, + trading_client, + backtesting_client, + ml_training_client, + }) + } +} + +/// Complete TLI client suite with all service clients +#[derive(Debug)] +pub struct TliClientSuite { + /// Client factory + pub factory: ClientFactory, + /// Trading client (includes all operations: trading, risk, monitoring, config, system status) + pub trading_client: Option, + /// Backtesting client + pub backtesting_client: Option, + /// ML training client + pub ml_training_client: Option, +} + +impl TliClientSuite { + /// Get connection statistics for all services + pub async fn get_connection_stats( + &self, + ) -> std::collections::HashMap> { + self.factory.get_connection_stats().await + } + + /// Shutdown all clients and connections + pub async fn shutdown(self) { + if let Some(client) = self.trading_client { + client.shutdown().await; + } + if let Some(client) = self.backtesting_client { + client.shutdown().await; + } + if let Some(client) = self.ml_training_client { + client.shutdown().await; + } + + self.factory.shutdown().await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_client_factory_creation() { + let config = ConnectionConfig::default(); + let factory = ClientFactory::new(config); + + // Test that factory can create clients + let trading_config = TradingClientConfig::default(); + let _trading_client = factory.create_trading_client(trading_config); + } + + #[test] + fn test_builder_pattern() { + let builder = TliClientBuilder::new() + .with_service_endpoint( + "trading_service".to_string(), + "http://localhost:50051".to_string(), + ) + .with_trading_config(TradingClientConfig::default()) + .with_backtesting_config(BacktestingClientConfig::default()) + .with_ml_training_config(MLTrainingClientConfig::default()); + + // Builder should have the configuration set + assert!(builder.trading_config.is_some()); + assert!(builder.backtesting_config.is_some()); + assert!(builder.ml_training_config.is_some()); + assert!(builder.service_endpoints.contains_key("trading_service")); + } +} diff --git a/tli/src/dashboard/config.rs b/tli/src/dashboard/config.rs index a1855adde..d653890dd 100644 --- a/tli/src/dashboard/config.rs +++ b/tli/src/dashboard/config.rs @@ -2,7 +2,8 @@ //! //! Re-exports the comprehensive `ConfigurationDashboard` from dashboards/configuration.rs -pub use crate::dashboards::configuration::ConfigurationDashboard as ConfigDashboard; +// REMOVED: All pub use statements eliminated per cleanup requirements +// Use direct import: crate::dashboards::configuration::ConfigurationDashboard // Legacy compatibility - keeping the same interface use super::{Dashboard, DashboardEvent}; @@ -10,5 +11,5 @@ use tokio::sync::mpsc; /// Create a new configuration dashboard pub fn create_config_dashboard(_event_sender: mpsc::Sender) -> Box { - Box::new(ConfigDashboard::new(_event_sender)) + Box::new(crate::dashboards::configuration::ConfigurationDashboard::new(_event_sender)) } diff --git a/tli/src/dashboard/mod.rs b/tli/src/dashboard/mod.rs index 154b1edaa..a02487f4c 100644 --- a/tli/src/dashboard/mod.rs +++ b/tli/src/dashboard/mod.rs @@ -28,16 +28,8 @@ pub mod risk; pub mod trading; pub mod vault_status; -pub use backtesting::BacktestingDashboard; +// DO NOT RE-EXPORT - Use explicit imports at usage sites -pub use crate::dashboards::config_manager::ConfigManagerDashboard as ConfigDashboard; -pub use events::*; -pub use layout::LayoutManager; -pub use ml::MLDashboard; -pub use performance::PerformanceDashboard; -pub use risk::RiskDashboard; -pub use trading::TradingDashboard; -pub use vault_status::VaultStatusWidget; /// Main dashboard manager that coordinates all dashboards pub struct DashboardManager { diff --git a/tli/src/dashboard/mod.rs.bak b/tli/src/dashboard/mod.rs.bak new file mode 100644 index 000000000..154b1edaa --- /dev/null +++ b/tli/src/dashboard/mod.rs.bak @@ -0,0 +1,317 @@ +//! Dashboard Framework for TLI Terminal Interface +//! +//! This module provides a comprehensive dashboard system for the Foxhunt HFT trading system. +//! It implements a multi-dashboard architecture with real-time data streaming and interactive +//! controls using Ratatui for terminal-based visualization. +//! +//! ## Architecture +//! - **`DashboardManager`**: Central coordinator for all dashboards +//! - **Dashboard Trait**: Common interface for all dashboard implementations +//! - **Real-time Updates**: Event-driven data streaming from gRPC services +//! - **Navigation**: Keyboard shortcuts for dashboard switching +//! - **Layout Management**: Consistent UI layout across all dashboards + +use anyhow::Result; +use crossterm::event::KeyEvent; +use ratatui::prelude::*; +use std::collections::HashMap; + +use tokio::sync::mpsc; + +pub mod backtesting; +pub mod config; +pub mod events; +pub mod layout; +pub mod ml; +pub mod performance; +pub mod risk; +pub mod trading; +pub mod vault_status; + +pub use backtesting::BacktestingDashboard; + +pub use crate::dashboards::config_manager::ConfigManagerDashboard as ConfigDashboard; +pub use events::*; +pub use layout::LayoutManager; +pub use ml::MLDashboard; +pub use performance::PerformanceDashboard; +pub use risk::RiskDashboard; +pub use trading::TradingDashboard; +pub use vault_status::VaultStatusWidget; + +/// Main dashboard manager that coordinates all dashboards +pub struct DashboardManager { + pub active_dashboard: DashboardType, + pub dashboards: HashMap>, + pub layout_manager: LayoutManager, + pub event_receiver: mpsc::Receiver, + pub _event_sender: mpsc::Sender, + // Vault service removed - TLI is pure client, uses shared config crate +} + +/// Available dashboard types +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum DashboardType { + Trading, // Live positions, orders, executions, market data + Risk, // VaR, drawdown, position limits, safety controls + ML, // Model predictions, signal strength, confidence + Performance, // PnL, Sharpe ratios, strategy performance + Config, // System configuration management + Backtesting, // Strategy testing, historical analysis, results + Vault, // Vault status, credentials, service discovery +} + +impl DashboardType { + pub fn all() -> Vec { + vec![ + DashboardType::Trading, + DashboardType::Risk, + DashboardType::ML, + DashboardType::Performance, + DashboardType::Config, + DashboardType::Backtesting, + DashboardType::Vault, + ] + } + + pub const fn shortcut_key(&self) -> char { + match self { + DashboardType::Trading => 't', + DashboardType::Risk => 'r', + DashboardType::ML => 'm', + DashboardType::Performance => 'p', + DashboardType::Config => 'c', + DashboardType::Backtesting => 'b', + DashboardType::Vault => 'v', + } + } + + pub const fn title(&self) -> &'static str { + match self { + DashboardType::Trading => "Trading", + DashboardType::Risk => "Risk", + DashboardType::ML => "ML", + DashboardType::Performance => "Performance", + DashboardType::Config => "Configuration", + DashboardType::Backtesting => "Backtesting", + DashboardType::Vault => "Vault Status", + } + } +} + +/// Common interface for all dashboard implementations +pub trait Dashboard: Send + Sync { + /// Render the dashboard to the given frame area + fn render(&mut self, frame: &mut Frame, area: Rect) -> Result<()>; + + /// Handle keyboard input and return optional dashboard events + fn handle_input(&mut self, key: KeyEvent) -> Result>; + + /// Update dashboard with new data/events + fn update(&mut self, event: DashboardEvent) -> Result<()>; + + /// Get dashboard title for display + fn title(&self) -> &str; + + /// Get keyboard shortcut for this dashboard + fn shortcut_key(&self) -> char; + + /// Check if dashboard needs redraw + fn needs_redraw(&self) -> bool; + + /// Mark dashboard as drawn + fn mark_drawn(&mut self); +} + +impl DashboardManager { + pub fn new() -> (Self, mpsc::Sender) { + let (_event_sender, event_receiver) = mpsc::channel(1000); + + let mut dashboards: HashMap> = HashMap::new(); + + // Initialize all dashboards + dashboards.insert( + DashboardType::Trading, + Box::new(TradingDashboard::new(_event_sender.clone())), + ); + dashboards.insert( + DashboardType::Risk, + Box::new(RiskDashboard::new(_event_sender.clone())), + ); + dashboards.insert( + DashboardType::ML, + Box::new(MLDashboard::new(_event_sender.clone())), + ); + dashboards.insert( + DashboardType::Performance, + Box::new(PerformanceDashboard::new(_event_sender.clone())), + ); + dashboards.insert( + DashboardType::Config, + Box::new(ConfigDashboard::new(_event_sender.clone())), + ); + dashboards.insert( + DashboardType::Backtesting, + Box::new(BacktestingDashboard::new(_event_sender.clone())), + ); + dashboards.insert( + DashboardType::Vault, + Box::new(VaultStatusWidget::new(_event_sender.clone())), + ); + + let manager = Self { + active_dashboard: DashboardType::Trading, + dashboards, + layout_manager: LayoutManager::new(), + event_receiver, + _event_sender: _event_sender.clone(), + }; + + (manager, _event_sender) + } + + pub fn render(&mut self, frame: &mut Frame) -> Result<()> { + let area = frame.area(); + + // Create main layout + let (header_area, content_area, sidebar_area, footer_area) = + self.layout_manager.create_layout(area); + + // Render header with navigation tabs + self.render_header(frame, header_area)?; + + // Render active dashboard + if let Some(dashboard) = self.dashboards.get_mut(&self.active_dashboard) { + dashboard.render(frame, content_area)?; + } + + // Render sidebar with quick stats + self.render_sidebar(frame, sidebar_area)?; + + // Render footer with help and status + self.render_footer(frame, footer_area)?; + + Ok(()) + } + + pub fn handle_input(&mut self, key: KeyEvent) -> Result> { + // Check for dashboard switching shortcuts first + for dashboard_type in DashboardType::all() { + if key.code == crossterm::event::KeyCode::Char(dashboard_type.shortcut_key()) { + self.active_dashboard = dashboard_type; + return Ok(Some(DashboardEvent::SwitchDashboard(dashboard_type))); + } + } + + // Handle ESC for exit + if key.code == crossterm::event::KeyCode::Esc { + return Ok(Some(DashboardEvent::Exit)); + } + + // Pass input to active dashboard + if let Some(dashboard) = self.dashboards.get_mut(&self.active_dashboard) { + dashboard.handle_input(key) + } else { + Ok(None) + } + } + + pub async fn handle_event(&mut self, event: DashboardEvent) -> Result { + match event { + DashboardEvent::SwitchDashboard(dashboard_type) => { + self.active_dashboard = dashboard_type; + Ok(false) + } + DashboardEvent::Exit => { + Ok(true) // Signal to exit + } + _ => { + // Forward event to all dashboards that might be interested + for dashboard in self.dashboards.values_mut() { + let _ = dashboard.update(event.clone()); + } + Ok(false) + } + } + } + + // Vault service functionality removed - TLI is pure client, uses shared config crate + + /// All vault-related methods removed - TLI uses shared config crate instead + // Vault service functionality completely removed from TLI + // TLI is a pure client - no vault service management + + fn render_header(&self, frame: &mut Frame, area: Rect) -> Result<()> { + let titles: Vec = DashboardType::all() + .iter() + .map(|dt| { + let _prefix = if *dt == self.active_dashboard { + "\u{25cf}" + } else { + "\u{25cb}" + }; + format!("[{}]{}", dt.shortcut_key().to_uppercase(), dt.title()) + }) + .collect(); + + let tabs = ratatui::widgets::Tabs::new(titles) + .block( + ratatui::widgets::Block::default() + .borders(ratatui::widgets::Borders::ALL) + .title("Foxhunt HFT Trading System - TLI Terminal"), + ) + .style(Style::default().fg(Color::White)) + .highlight_style( + Style::default() + .fg(Color::Yellow) + .add_modifier(Modifier::BOLD), + ) + .select(self.active_dashboard as usize); + + frame.render_widget(tabs, area); + Ok(()) + } + + fn render_sidebar(&self, frame: &mut Frame, area: Rect) -> Result<()> { + let block = ratatui::widgets::Block::default() + .borders(ratatui::widgets::Borders::ALL) + .title("Quick Stats"); + + // Get actual Vault status (placeholder - TLI uses shared config crate) + let vault_status = "\u{25cb}"; // Empty circle for not available - use config crate integration + + let content = ratatui::widgets::Paragraph::new( + format!( + "Connection: \u{25cf}\u{25cf}\u{25cf}\nVault: {}\nLatency: 12ms\nOrders: 15\nPositions: 5\nPnL: +$2,500", + vault_status + ), + ) + .block(block) + .wrap(ratatui::widgets::Wrap { trim: true }); + + frame.render_widget(content, area); + Ok(()) + } + + fn render_footer(&self, frame: &mut Frame, area: Rect) -> Result<()> { + let help_text = format!( + "[{}] Dashboards | [ESC] Exit | Status: Connected", + DashboardType::all() + .iter() + .map(|dt| format!( + "[{}]{}", + dt.shortcut_key().to_uppercase(), + dt.title().chars().next().unwrap() + )) + .collect::>() + .join(" ") + ); + + let footer = ratatui::widgets::Paragraph::new(help_text) + .block(ratatui::widgets::Block::default().borders(ratatui::widgets::Borders::ALL)) + .style(Style::default().fg(Color::Gray)); + + frame.render_widget(footer, area); + Ok(()) + } +} diff --git a/tli/src/dashboards/mod.rs b/tli/src/dashboards/mod.rs index 1d29a9847..951403664 100644 --- a/tli/src/dashboards/mod.rs +++ b/tli/src/dashboards/mod.rs @@ -6,5 +6,3 @@ pub mod config_manager; pub mod configuration; -pub use config_manager::{CategoryConfigDashboard, ConfigManagerDashboard}; -pub use configuration::ConfigurationDashboard; diff --git a/tli/src/dashboards/mod.rs.bak b/tli/src/dashboards/mod.rs.bak new file mode 100644 index 000000000..1d29a9847 --- /dev/null +++ b/tli/src/dashboards/mod.rs.bak @@ -0,0 +1,10 @@ +//! Dashboard implementations for TLI +//! +//! This module contains the actual dashboard implementations that are used +//! by the dashboard framework. + +pub mod config_manager; +pub mod configuration; + +pub use config_manager::{CategoryConfigDashboard, ConfigManagerDashboard}; +pub use configuration::ConfigurationDashboard; diff --git a/tli/src/error_consolidated.rs b/tli/src/error_consolidated.rs index b75b37196..42e9800ae 100644 --- a/tli/src/error_consolidated.rs +++ b/tli/src/error_consolidated.rs @@ -3,12 +3,12 @@ //! This module demonstrates the consolidated error handling pattern //! using the common error system across all Foxhunt TLI services. -// Re-export shared error types and utilities -pub use common::error::{CommonError, CommonResult, ErrorCategory, RetryStrategy, ErrorSeverity}; +// REMOVED: All pub use statements eliminated per cleanup requirements +// Use direct imports: common::error::{CommonError, CommonResult, ErrorCategory, RetryStrategy, ErrorSeverity} use tonic::{Code, Status}; /// Result type for TLI operations using CommonError -pub type TliResult = CommonResult; +pub type TliResult = common::error::CommonResult; /// TLI module specific error extensions /// For cases where we need domain-specific error information beyond CommonError @@ -16,7 +16,7 @@ pub type TliResult = CommonResult; pub enum TliServiceError { /// Common error with context #[error("TLI service error: {0}")] - Common(#[from] CommonError), + Common(#[from] common::error::CommonError), /// gRPC connection specific error with service context #[error("gRPC connection error: {service} at {endpoint} - {message}")] @@ -86,50 +86,50 @@ pub enum TliServiceError { impl TliServiceError { /// Convert to CommonError for metrics and monitoring - pub fn to_common_error(self) -> CommonError { + pub fn to_common_error(self) -> common::error::CommonError { match self { TliServiceError::Common(err) => err, TliServiceError::GrpcConnection { service, endpoint, message } => { - CommonError::connection( + common::error::CommonError::connection( format!("grpc://{}:{}", service, endpoint), message ) } TliServiceError::OrderValidation { order_id, field, message } => { - CommonError::validation( + common::error::CommonError::validation( format!("order[{}].{}", order_id, field), message ) } TliServiceError::DashboardRendering { widget, message } => { - CommonError::internal(format!("Dashboard widget {}: {}", widget, message)) + common::error::CommonError::internal(format!("Dashboard widget {}: {}", widget, message)) } TliServiceError::EventBufferOverflow { buffer_name, capacity } => { - CommonError::resource_exhausted( + common::error::CommonError::resource_exhausted( format!("Event buffer {} (capacity: {})", buffer_name, capacity) ) } TliServiceError::ConfigHotReload { config_key, message } => { - CommonError::config(format!("Hot-reload {} failed: {}", config_key, message)) + common::error::CommonError::config(format!("Hot-reload {} failed: {}", config_key, message)) } TliServiceError::CertificateValidation { cert_type, message } => { - CommonError::authentication(format!("Certificate {}: {}", cert_type, message)) + common::error::CommonError::authentication(format!("Certificate {}: {}", cert_type, message)) } TliServiceError::TradingService { operation, message } => { - CommonError::service( - ErrorCategory::Trading, + common::error::CommonError::service( + common::error::ErrorCategory::Trading, format!("Trading service {}: {}", operation, message) ) } TliServiceError::MLService { operation, message } => { - CommonError::service( - ErrorCategory::ML, + common::error::CommonError::service( + common::error::ErrorCategory::ML, format!("ML service {}: {}", operation, message) ) } TliServiceError::BacktestingService { operation, message } => { - CommonError::service( - ErrorCategory::System, + common::error::CommonError::service( + common::error::ErrorCategory::System, format!("Backtesting service {}: {}", operation, message) ) } @@ -137,74 +137,74 @@ impl TliServiceError { } /// Get error category for metrics - pub fn category(&self) -> ErrorCategory { + pub fn category(&self) -> common::error::ErrorCategory { match self { TliServiceError::Common(_) => self.to_common_error().category(), - TliServiceError::TradingService { .. } => ErrorCategory::Trading, - TliServiceError::MLService { .. } => ErrorCategory::ML, - TliServiceError::BacktestingService { .. } => ErrorCategory::System, - TliServiceError::GrpcConnection { .. } => ErrorCategory::Network, - TliServiceError::OrderValidation { .. } => ErrorCategory::Validation, - TliServiceError::CertificateValidation { .. } => ErrorCategory::Security, - _ => ErrorCategory::System, + TliServiceError::TradingService { .. } => common::error::ErrorCategory::Trading, + TliServiceError::MLService { .. } => common::error::ErrorCategory::ML, + TliServiceError::BacktestingService { .. } => common::error::ErrorCategory::System, + TliServiceError::GrpcConnection { .. } => common::error::ErrorCategory::Network, + TliServiceError::OrderValidation { .. } => common::error::ErrorCategory::Validation, + TliServiceError::CertificateValidation { .. } => common::error::ErrorCategory::Security, + _ => common::error::ErrorCategory::System, } } /// Get error severity - pub fn severity(&self) -> ErrorSeverity { + pub fn severity(&self) -> common::error::ErrorSeverity { match self { - TliServiceError::CertificateValidation { .. } => ErrorSeverity::Critical, - TliServiceError::ConfigHotReload { .. } => ErrorSeverity::Error, - TliServiceError::TradingService { .. } => ErrorSeverity::Error, - TliServiceError::MLService { .. } => ErrorSeverity::Error, - TliServiceError::BacktestingService { .. } => ErrorSeverity::Error, - TliServiceError::GrpcConnection { .. } => ErrorSeverity::Warn, - TliServiceError::EventBufferOverflow { .. } => ErrorSeverity::Warn, - TliServiceError::OrderValidation { .. } => ErrorSeverity::Info, - TliServiceError::DashboardRendering { .. } => ErrorSeverity::Info, + TliServiceError::CertificateValidation { .. } => common::error::ErrorSeverity::Critical, + TliServiceError::ConfigHotReload { .. } => common::error::ErrorSeverity::Error, + TliServiceError::TradingService { .. } => common::error::ErrorSeverity::Error, + TliServiceError::MLService { .. } => common::error::ErrorSeverity::Error, + TliServiceError::BacktestingService { .. } => common::error::ErrorSeverity::Error, + TliServiceError::GrpcConnection { .. } => common::error::ErrorSeverity::Warn, + TliServiceError::EventBufferOverflow { .. } => common::error::ErrorSeverity::Warn, + TliServiceError::OrderValidation { .. } => common::error::ErrorSeverity::Info, + TliServiceError::DashboardRendering { .. } => common::error::ErrorSeverity::Info, TliServiceError::Common(_) => self.to_common_error().severity(), } } /// Get retry strategy - pub fn retry_strategy(&self) -> RetryStrategy { + pub fn retry_strategy(&self) -> common::error::RetryStrategy { match self { // Authentication/security errors should not be retried - TliServiceError::CertificateValidation { .. } => RetryStrategy::NoRetry, - TliServiceError::OrderValidation { .. } => RetryStrategy::NoRetry, - + TliServiceError::CertificateValidation { .. } => common::error::RetryStrategy::NoRetry, + TliServiceError::OrderValidation { .. } => common::error::RetryStrategy::NoRetry, + // Network/connection errors can be retried with backoff - TliServiceError::GrpcConnection { .. } => RetryStrategy::Exponential { + TliServiceError::GrpcConnection { .. } => common::error::RetryStrategy::Exponential { base_delay_ms: 500, max_delay_ms: 5000, }, - TliServiceError::TradingService { .. } => RetryStrategy::Exponential { + TliServiceError::TradingService { .. } => common::error::RetryStrategy::Exponential { base_delay_ms: 100, max_delay_ms: 2000, }, - TliServiceError::MLService { .. } => RetryStrategy::Linear { + TliServiceError::MLService { .. } => common::error::RetryStrategy::Linear { base_delay_ms: 1000, }, - TliServiceError::BacktestingService { .. } => RetryStrategy::Linear { + TliServiceError::BacktestingService { .. } => common::error::RetryStrategy::Linear { base_delay_ms: 2000, }, - + // System errors can retry with delay - TliServiceError::ConfigHotReload { .. } => RetryStrategy::Linear { + TliServiceError::ConfigHotReload { .. } => common::error::RetryStrategy::Linear { base_delay_ms: 5000, }, - TliServiceError::EventBufferOverflow { .. } => RetryStrategy::Linear { + TliServiceError::EventBufferOverflow { .. } => common::error::RetryStrategy::Linear { base_delay_ms: 1000, }, - TliServiceError::DashboardRendering { .. } => RetryStrategy::Immediate, - + TliServiceError::DashboardRendering { .. } => common::error::RetryStrategy::Immediate, + TliServiceError::Common(_) => self.to_common_error().retry_strategy(), } } /// Check if error is retryable pub fn is_retryable(&self) -> bool { - !matches!(self.retry_strategy(), RetryStrategy::NoRetry) + !matches!(self.retry_strategy(), common::error::RetryStrategy::NoRetry) } /// Get error code for monitoring @@ -227,19 +227,19 @@ impl TliServiceError { /// Convert standard errors to CommonError for consistent handling impl From for TliServiceError { fn from(err: std::io::Error) -> Self { - TliServiceError::Common(CommonError::network(format!("IO error: {}", err))) + TliServiceError::Common(common::error::CommonError::network(format!("IO error: {}", err))) } } impl From for TliServiceError { fn from(err: serde_json::Error) -> Self { - TliServiceError::Common(CommonError::serialization(format!("JSON error: {}", err))) + TliServiceError::Common(common::error::CommonError::serialization(format!("JSON error: {}", err))) } } impl From for TliServiceError { fn from(err: anyhow::Error) -> Self { - TliServiceError::Common(CommonError::internal(format!("Anyhow error: {}", err))) + TliServiceError::Common(common::error::CommonError::internal(format!("Anyhow error: {}", err))) } } @@ -247,16 +247,16 @@ impl From for TliServiceError { fn from(status: tonic::Status) -> Self { let message = status.message().to_string(); match status.code() { - Code::InvalidArgument => TliServiceError::Common(CommonError::validation("request", message)), - Code::NotFound => TliServiceError::Common(CommonError::not_found("resource", message)), - Code::PermissionDenied => TliServiceError::Common(CommonError::authorization(message)), - Code::Unauthenticated => TliServiceError::Common(CommonError::authentication(message)), - Code::ResourceExhausted => TliServiceError::Common(CommonError::rate_limited(message)), - Code::FailedPrecondition => TliServiceError::Common(CommonError::validation("precondition", message)), - Code::Unavailable => TliServiceError::Common(CommonError::service_unavailable("grpc_service", message)), - Code::DeadlineExceeded => TliServiceError::Common(CommonError::timeout(5000, 2000)), - Code::Internal => TliServiceError::Common(CommonError::internal(message)), - _ => TliServiceError::Common(CommonError::internal(format!("gRPC error: {}", message))), + Code::InvalidArgument => TliServiceError::Common(common::error::CommonError::validation("request", message)), + Code::NotFound => TliServiceError::Common(common::error::CommonError::not_found("resource", message)), + Code::PermissionDenied => TliServiceError::Common(common::error::CommonError::authorization(message)), + Code::Unauthenticated => TliServiceError::Common(common::error::CommonError::authentication(message)), + Code::ResourceExhausted => TliServiceError::Common(common::error::CommonError::rate_limited(message)), + Code::FailedPrecondition => TliServiceError::Common(common::error::CommonError::validation("precondition", message)), + Code::Unavailable => TliServiceError::Common(common::error::CommonError::service_unavailable("grpc_service", message)), + Code::DeadlineExceeded => TliServiceError::Common(common::error::CommonError::timeout(5000, 2000)), + Code::Internal => TliServiceError::Common(common::error::CommonError::internal(message)), + _ => TliServiceError::Common(common::error::CommonError::internal(format!("gRPC error: {}", message))), } } } @@ -383,42 +383,42 @@ impl TliServiceError { /// Create network error using CommonError pub fn network>(message: M) -> Self { - Self::Common(CommonError::network(message)) + Self::Common(common::error::CommonError::network(message)) } /// Create authentication error using CommonError pub fn authentication>(message: M) -> Self { - Self::Common(CommonError::authentication(message)) + Self::Common(common::error::CommonError::authentication(message)) } /// Create configuration error using CommonError pub fn configuration>(message: M) -> Self { - Self::Common(CommonError::config(message)) + Self::Common(common::error::CommonError::config(message)) } /// Create validation error using CommonError pub fn validation, M: Into>(field: F, message: M) -> Self { - Self::Common(CommonError::validation(field, message)) + Self::Common(common::error::CommonError::validation(field, message)) } /// Create timeout error using CommonError pub fn timeout(actual_ms: u64, max_ms: u64) -> Self { - Self::Common(CommonError::timeout(actual_ms, max_ms)) + Self::Common(common::error::CommonError::timeout(actual_ms, max_ms)) } /// Create internal error using CommonError pub fn internal>(message: M) -> Self { - Self::Common(CommonError::internal(message)) + Self::Common(common::error::CommonError::internal(message)) } /// Create not found error using CommonError pub fn not_found, I: Into>(resource: R, identifier: I) -> Self { - Self::Common(CommonError::not_found(resource, identifier)) + Self::Common(common::error::CommonError::not_found(resource, identifier)) } } /// Convert to CommonError automatically for interop -impl From for CommonError { +impl From for common::error::CommonError { fn from(err: TliServiceError) -> Self { err.to_common_error() } @@ -431,24 +431,24 @@ mod tests { #[test] fn test_tli_service_error_categorization() { let grpc_error = TliServiceError::grpc_connection("trading", "localhost:50051", "Connection refused"); - assert_eq!(grpc_error.category(), ErrorCategory::Network); + assert_eq!(grpc_error.category(), common::error::ErrorCategory::Network); assert_eq!(grpc_error.error_code(), "TLI_GRPC_CONNECTION_ERROR"); assert!(grpc_error.is_retryable()); let order_error = TliServiceError::order_validation("ORD123", "quantity", "Must be positive"); - assert_eq!(order_error.category(), ErrorCategory::Validation); + assert_eq!(order_error.category(), common::error::ErrorCategory::Validation); assert!(!order_error.is_retryable()); } #[test] fn test_service_specific_errors() { let trading_error = TliServiceError::trading_service("submit_order", "Service unavailable"); - assert_eq!(trading_error.category(), ErrorCategory::Trading); - assert_eq!(trading_error.severity(), ErrorSeverity::Error); + assert_eq!(trading_error.category(), common::error::ErrorCategory::Trading); + assert_eq!(trading_error.severity(), common::error::ErrorSeverity::Error); assert!(trading_error.is_retryable()); let ml_error = TliServiceError::ml_service("train_model", "GPU memory exhausted"); - assert_eq!(ml_error.category(), ErrorCategory::ML); + assert_eq!(ml_error.category(), common::error::ErrorCategory::ML); assert!(ml_error.is_retryable()); } @@ -456,12 +456,12 @@ mod tests { fn test_retry_strategies() { let cert_error = TliServiceError::certificate_validation("TLS", "Certificate expired"); assert!(!cert_error.is_retryable()); - assert_eq!(cert_error.retry_strategy(), RetryStrategy::NoRetry); + assert_eq!(cert_error.retry_strategy(), common::error::RetryStrategy::NoRetry); let grpc_error = TliServiceError::grpc_connection("ml", "localhost:50052", "Connection timeout"); assert!(grpc_error.is_retryable()); match grpc_error.retry_strategy() { - RetryStrategy::Exponential { base_delay_ms, max_delay_ms } => { + common::error::RetryStrategy::Exponential { base_delay_ms, max_delay_ms } => { assert_eq!(base_delay_ms, 500); assert_eq!(max_delay_ms, 5000); } @@ -486,10 +486,10 @@ mod tests { #[test] fn test_buffer_overflow_error() { let buffer_error = TliServiceError::event_buffer_overflow("order_events", 10000); - assert_eq!(buffer_error.category(), ErrorCategory::System); - assert_eq!(buffer_error.severity(), ErrorSeverity::Warn); + assert_eq!(buffer_error.category(), common::error::ErrorCategory::System); + assert_eq!(buffer_error.severity(), common::error::ErrorSeverity::Warn); assert!(buffer_error.is_retryable()); - + let status: tonic::Status = buffer_error.into(); assert_eq!(status.code(), Code::ResourceExhausted); assert!(status.message().contains("10000")); @@ -498,10 +498,10 @@ mod tests { #[test] fn test_common_error_integration() { let config_error = TliServiceError::configuration("Missing gRPC endpoint"); - let common_error: CommonError = config_error.into(); - - assert_eq!(common_error.category(), ErrorCategory::Configuration); - assert_eq!(common_error.severity(), ErrorSeverity::Critical); + let common_error: common::error::CommonError = config_error.into(); + + assert_eq!(common_error.category(), common::error::ErrorCategory::Configuration); + assert_eq!(common_error.severity(), common::error::ErrorSeverity::Critical); assert!(!common_error.is_retryable()); } @@ -509,12 +509,12 @@ mod tests { fn test_error_conversion_chain() { let status = tonic::Status::deadline_exceeded("Request timeout"); let tli_error: TliServiceError = status.into(); - let common_error: CommonError = tli_error.into(); - - assert_eq!(common_error.category(), ErrorCategory::System); + let common_error: common::error::CommonError = tli_error.into(); + + assert_eq!(common_error.category(), common::error::ErrorCategory::System); assert!(common_error.is_retryable()); match common_error.retry_strategy() { - RetryStrategy::Linear { .. } => (), + common::error::RetryStrategy::Linear { .. } => (), _ => panic!("Expected linear backoff for timeout errors"), } } diff --git a/tli/src/events/mod.rs b/tli/src/events/mod.rs index c06cd3d93..d59c69218 100644 --- a/tli/src/events/mod.rs +++ b/tli/src/events/mod.rs @@ -34,9 +34,7 @@ use tracing::{debug, error, info, warn}; use uuid::Uuid; // Re-export main components -pub use aggregator::{AggregationConfig, AggregationRule, EventAggregator}; -pub use event_buffer::{EventBuffer, EventBufferConfig, EventBufferMetrics}; -pub use stream_manager::{StreamConfig, StreamHealth, StreamManager}; +// DO NOT RE-EXPORT - Use explicit imports at usage sites // pub use replay_system::{ReplaySystem, ReplayConfig, ReplayFilter}; // Disabled // WebSocketServer removed - TLI is pure client, no server components diff --git a/tli/src/events/mod.rs.bak b/tli/src/events/mod.rs.bak new file mode 100644 index 000000000..c06cd3d93 --- /dev/null +++ b/tli/src/events/mod.rs.bak @@ -0,0 +1,595 @@ +//! Real-time event streaming system for TLI +//! +//! This module provides comprehensive event handling for live data including: +//! - gRPC streaming client management with automatic reconnection +//! - Event aggregation and buffering with back-pressure handling +//! - Event replay capabilities for historical analysis +//! - WebSocket support for browser clients +//! - Memory-efficient event storage and deduplication +//! - Performance metrics and monitoring +//! +//! Architecture: +//! ```text +//! gRPC Services → StreamManager → EventBuffer → Aggregator → [WebSocket|Replay] +//! ↓ ↓ ↓ +//! Reconnection Back-pressure Deduplication +//! Exponential Memory Mgmt Ordering +//! Backoff Flow Control Metrics +//! ``` + +pub mod aggregator; +pub mod event_buffer; +pub mod stream_manager; +// pub mod replay_system; // Disabled - client should not have database dependencies +// websocket_server module removed - TLI is pure client + +use crate::error::TliResult; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::Arc; +use tokio::sync::{broadcast, mpsc, RwLock}; +// BroadcastStream is now available with tokio-stream sync feature enabled +use tracing::{debug, error, info, warn}; +use uuid::Uuid; + +// Re-export main components +pub use aggregator::{AggregationConfig, AggregationRule, EventAggregator}; +pub use event_buffer::{EventBuffer, EventBufferConfig, EventBufferMetrics}; +pub use stream_manager::{StreamConfig, StreamHealth, StreamManager}; +// pub use replay_system::{ReplaySystem, ReplayConfig, ReplayFilter}; // Disabled +// WebSocketServer removed - TLI is pure client, no server components + +/// Event types supported by the streaming system +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub enum EventType { + /// Market data events (quotes, trades, order book) + MarketData, + /// Trading events (orders, executions, positions) + Trading, + /// Risk management events (limits, breaches, alerts) + Risk, + /// ML signals and predictions + MlSignal, + /// System health and monitoring + System, + /// Configuration changes + Config, + /// Custom user-defined events + Custom(String), +} + +impl EventType { + /// Convert to string for serialization + pub fn as_str(&self) -> &str { + match self { + EventType::MarketData => "market_data", + EventType::Trading => "trading", + EventType::Risk => "risk", + EventType::MlSignal => "ml_signal", + EventType::System => "system", + EventType::Config => "config", + EventType::Custom(name) => name, + } + } + + /// Parse from string + pub fn from_str(s: &str) -> Self { + match s { + "market_data" => EventType::MarketData, + "trading" => EventType::Trading, + "risk" => EventType::Risk, + "ml_signal" => EventType::MlSignal, + "system" => EventType::System, + "config" => EventType::Config, + name => EventType::Custom(name.to_string()), + } + } +} + +/// Event severity levels for filtering and prioritization +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub enum EventSeverity { + /// Low priority informational events + Info, + /// Warning events that may require attention + Warning, + /// Error events that require immediate attention + Error, + /// Critical events that require urgent action + Critical, +} + +/// Core event structure for all streaming data +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Event { + /// Unique event identifier + pub id: Uuid, + /// Event type classification + pub event_type: EventType, + /// Event severity level + pub severity: EventSeverity, + /// Source service that generated the event + pub source: String, + /// Event timestamp (nanoseconds since Unix epoch) + pub timestamp_nanos: i64, + /// Sequence number for ordering within source + pub sequence: u64, + /// Event payload as JSON value + pub payload: serde_json::Value, + /// Optional correlation ID for related events + pub correlation_id: Option, + /// Event metadata and labels + pub metadata: HashMap, + /// TTL in seconds (0 = no expiry) + pub ttl_seconds: u64, +} + +impl Event { + /// Create a new event with required fields + pub fn new( + event_type: EventType, + severity: EventSeverity, + source: String, + payload: serde_json::Value, + ) -> Self { + Self { + id: Uuid::new_v4(), + event_type, + severity, + source, + timestamp_nanos: crate::types::current_unix_nanos(), + sequence: 0, // Set by stream manager + payload, + correlation_id: None, + metadata: HashMap::new(), + ttl_seconds: 3600, // 1 hour default TTL + } + } + + /// Create a new event with correlation ID + pub fn with_correlation( + event_type: EventType, + severity: EventSeverity, + source: String, + payload: serde_json::Value, + correlation_id: Uuid, + ) -> Self { + let mut event = Self::new(event_type, severity, source, payload); + event.correlation_id = Some(correlation_id); + event + } + + /// Set sequence number (called by stream manager) + pub fn set_sequence(&mut self, sequence: u64) { + self.sequence = sequence; + } + + /// Add metadata label + pub fn add_metadata(&mut self, key: String, value: String) { + self.metadata.insert(key, value); + } + + /// Set TTL in seconds + pub fn set_ttl(&mut self, ttl_seconds: u64) { + self.ttl_seconds = ttl_seconds; + } + + /// Check if event has expired + pub fn is_expired(&self) -> bool { + if self.ttl_seconds == 0 { + return false; // No expiry + } + + let current_nanos = crate::types::current_unix_nanos(); + let expiry_nanos = self.timestamp_nanos + (self.ttl_seconds as i64 * 1_000_000_000); + current_nanos > expiry_nanos + } + + /// Get event age in milliseconds + pub fn age_millis(&self) -> i64 { + let current_nanos = crate::types::current_unix_nanos(); + (current_nanos - self.timestamp_nanos) / 1_000_000 + } + + /// Convert to DateTime for display + pub fn timestamp_utc(&self) -> DateTime { + let secs = self.timestamp_nanos / 1_000_000_000; + let nanos = (self.timestamp_nanos % 1_000_000_000) as u32; + DateTime::from_timestamp(secs, nanos).unwrap_or_else(Utc::now) + } +} + +/// Event stream subscription filter +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EventFilter { + /// Event types to include (empty = all types) + pub event_types: Vec, + /// Minimum severity level + pub min_severity: EventSeverity, + /// Source services to include (empty = all sources) + pub sources: Vec, + /// Metadata filters (key-value pairs that must match) + pub metadata_filters: HashMap, + /// Correlation ID filter + pub correlation_id: Option, + /// Time range filter (start timestamp in nanos) + pub start_time_nanos: Option, + /// Time range filter (end timestamp in nanos) + pub end_time_nanos: Option, +} + +impl EventFilter { + /// Create a filter for all events + pub fn all() -> Self { + Self { + event_types: Vec::new(), + min_severity: EventSeverity::Info, + sources: Vec::new(), + metadata_filters: HashMap::new(), + correlation_id: None, + start_time_nanos: None, + end_time_nanos: None, + } + } + + /// Create a filter for specific event types + pub fn for_types(event_types: Vec) -> Self { + Self { + event_types, + ..Self::all() + } + } + + /// Create a filter for specific sources + pub fn for_sources(sources: Vec) -> Self { + Self { + sources, + ..Self::all() + } + } + + /// Create a filter for minimum severity + pub fn with_min_severity(min_severity: EventSeverity) -> Self { + Self { + min_severity, + ..Self::all() + } + } + + /// Check if event matches this filter + pub fn matches(&self, event: &Event) -> bool { + // Check event types + if !self.event_types.is_empty() && !self.event_types.contains(&event.event_type) { + return false; + } + + // Check severity + if event.severity < self.min_severity { + return false; + } + + // Check sources + if !self.sources.is_empty() && !self.sources.contains(&event.source) { + return false; + } + + // Check correlation ID + if let Some(filter_correlation_id) = &self.correlation_id { + if event.correlation_id.as_ref() != Some(filter_correlation_id) { + return false; + } + } + + // Check metadata filters + for (key, value) in &self.metadata_filters { + if event.metadata.get(key) != Some(value) { + return false; + } + } + + // Check time range + if let Some(start_time) = self.start_time_nanos { + if event.timestamp_nanos < start_time { + return false; + } + } + + if let Some(end_time) = self.end_time_nanos { + if event.timestamp_nanos > end_time { + return false; + } + } + + true + } +} + +/// Event subscription handle for managing live event streams +pub struct EventSubscription { + /// Subscription ID + pub id: Uuid, + /// Event filter + pub filter: EventFilter, + /// Event receiver + pub receiver: mpsc::UnboundedReceiver, + /// Subscription metadata + pub metadata: HashMap, +} + +impl EventSubscription { + /// Create a new subscription + pub fn new(filter: EventFilter, receiver: mpsc::UnboundedReceiver) -> Self { + Self { + id: Uuid::new_v4(), + filter, + receiver, + metadata: HashMap::new(), + } + } + + /// Add subscription metadata + pub fn add_metadata(&mut self, key: String, value: String) { + self.metadata.insert(key, value); + } +} + +/// Core event streaming system that coordinates all components +pub struct EventStreamingSystem { + /// Stream manager for gRPC connections + stream_manager: Arc, + /// Event buffer for aggregation and storage + event_buffer: Arc, + /// Event aggregator for processing + aggregator: Arc, + // replay_system: Arc, // Disabled - client should not have database dependencies + /// WebSocket server removed - TLI is pure client, no server components + /// Event broadcast channel for live subscriptions + _event_sender: broadcast::Sender, + /// System shutdown signal + shutdown_sender: tokio::sync::watch::Sender, + shutdown_receiver: tokio::sync::watch::Receiver, + /// System metrics + metrics: Arc>, +} + +/// System-wide event streaming metrics +#[derive(Debug, Default, Clone)] +pub struct EventSystemMetrics { + /// Total events processed + pub events_processed: u64, + /// Events processed per second + pub events_per_second: f64, + /// Total active subscriptions + pub active_subscriptions: u64, + /// Stream connection health + pub stream_health: HashMap, + /// Memory usage in bytes + pub memory_usage_bytes: u64, + /// Last update timestamp + pub last_updated: DateTime, +} + +impl EventStreamingSystem { + /// Create a new event streaming system + pub async fn new( + stream_config: StreamConfig, + buffer_config: EventBufferConfig, + aggregation_config: AggregationConfig, + // replay_config: ReplayConfig, // Disabled + // websocket_config removed - TLI is pure client + ) -> TliResult { + info!("Initializing event streaming system"); + + // Create broadcast channel for live events + let (_event_sender, _) = broadcast::channel(10000); + + // Create shutdown channel + let (shutdown_sender, shutdown_receiver) = tokio::sync::watch::channel(false); + + // Initialize components + let stream_manager = Arc::new(StreamManager::new(stream_config).await?); + let event_buffer = Arc::new(EventBuffer::new(buffer_config)); + let aggregator = Arc::new(EventAggregator::new(aggregation_config)); + // let replay_system = Arc::new(ReplaySystem::new(replay_config).await?); // Disabled + + // WebSocket server initialization removed - TLI is pure client + + let metrics = Arc::new(RwLock::new(EventSystemMetrics::default())); + + Ok(Self { + stream_manager, + event_buffer, + aggregator, + // replay_system, // Disabled + // websocket_server removed - TLI is pure client + _event_sender, + shutdown_sender, + shutdown_receiver, + metrics, + }) + } + + /// Start the event streaming system + pub async fn start(&self) -> TliResult<()> { + info!("Starting event streaming system"); + + // Start stream manager + let stream_manager = self.stream_manager.clone(); + let _event_sender = self._event_sender.clone(); + let shutdown_receiver = self.shutdown_receiver.clone(); + + tokio::spawn(async move { + if let Err(e) = stream_manager.start(_event_sender, shutdown_receiver).await { + error!("Stream manager error: {}", e); + } + }); + + // Start event buffer processing + let buffer = self.event_buffer.clone(); + let aggregator = self.aggregator.clone(); + let mut event_receiver = self._event_sender.subscribe(); + let shutdown_receiver = self.shutdown_receiver.clone(); + + tokio::spawn(async move { + let mut shutdown = shutdown_receiver.clone(); + loop { + tokio::select! { + event_result = event_receiver.recv() => { + match event_result { + Ok(event) => { + if let Err(e) = buffer.add_event(event.clone()).await { + error!("Failed to add event to buffer: {}", e); + continue; + } + + if let Err(e) = aggregator.process_event(event).await { + error!("Failed to process event in aggregator: {}", e); + } + } + Err(broadcast::error::RecvError::Lagged(skipped)) => { + warn!("Event receiver lagged, skipped {} events", skipped); + } + Err(broadcast::error::RecvError::Closed) => { + debug!("Event receiver closed"); + break; + } + } + } + _ = shutdown.changed() => { + if *shutdown.borrow() { + debug!("Event buffer processing shutdown"); + break; + } + } + } + } + }); + + // WebSocket server startup removed - TLI is pure client, no server components + + // Start metrics collection + let metrics = self.metrics.clone(); + let shutdown_receiver = self.shutdown_receiver.clone(); + + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(5)); + let mut shutdown = shutdown_receiver.clone(); + + loop { + tokio::select! { + _ = interval.tick() => { + let mut metrics_guard = metrics.write().await; + metrics_guard.last_updated = Utc::now(); + // Update other metrics here + } + _ = shutdown.changed() => { + if *shutdown.borrow() { + debug!("Metrics collection shutdown"); + break; + } + } + } + } + }); + + info!("Event streaming system started successfully"); + Ok(()) + } + + /// Subscribe to events with a filter + pub async fn subscribe(&self, filter: EventFilter) -> TliResult { + let (sender, receiver) = mpsc::unbounded_channel(); + let mut event_receiver = self._event_sender.subscribe(); + let filter_clone = filter.clone(); + + tokio::spawn(async move { + while let Ok(event) = event_receiver.recv().await { + if filter_clone.matches(&event) { + if sender.send(event).is_err() { + debug!("Event subscription receiver dropped"); + break; + } + } + } + }); + + // Update subscription count + { + let mut metrics = self.metrics.write().await; + metrics.active_subscriptions += 1; + } + + Ok(EventSubscription::new(filter, receiver)) + } + + /// Get system metrics + pub async fn get_metrics(&self) -> EventSystemMetrics { + (*self.metrics.read().await).clone() + } + + /// Shutdown the event streaming system + pub async fn shutdown(&self) -> TliResult<()> { + info!("Shutting down event streaming system"); + + if let Err(e) = self.shutdown_sender.send(true) { + warn!("Failed to send shutdown signal: {}", e); + } + + // Give components time to shutdown gracefully + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + + info!("Event streaming system shutdown complete"); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_event_creation() { + let payload = serde_json::json!({"test": "data"}); + let event = Event::new( + EventType::Trading, + EventSeverity::Info, + "test_service".to_string(), + payload, + ); + + assert_eq!(event.event_type, EventType::Trading); + assert_eq!(event.severity, EventSeverity::Info); + assert_eq!(event.source, "test_service"); + assert!(!event.is_expired()); + } + + #[test] + fn test_event_filter() { + let filter = EventFilter::for_types(vec![EventType::Trading]); + + let trading_event = Event::new( + EventType::Trading, + EventSeverity::Info, + "service".to_string(), + serde_json::json!({}), + ); + + let market_event = Event::new( + EventType::MarketData, + EventSeverity::Info, + "service".to_string(), + serde_json::json!({}), + ); + + assert!(filter.matches(&trading_event)); + assert!(!filter.matches(&market_event)); + } + + #[test] + fn test_event_severity_ordering() { + assert!(EventSeverity::Critical > EventSeverity::Error); + assert!(EventSeverity::Error > EventSeverity::Warning); + assert!(EventSeverity::Warning > EventSeverity::Info); + } +} diff --git a/tli/src/lib.rs b/tli/src/lib.rs index 655f45199..8dc48c87a 100644 --- a/tli/src/lib.rs +++ b/tli/src/lib.rs @@ -151,6 +151,3 @@ pub mod proto { } -/// Common imports for TLI module consumers -pub mod prelude { -} diff --git a/tli/src/ui/widgets/mod.rs b/tli/src/ui/widgets/mod.rs index 0ed79dd5c..c0b14d2ba 100644 --- a/tli/src/ui/widgets/mod.rs +++ b/tli/src/ui/widgets/mod.rs @@ -26,12 +26,6 @@ pub mod risk_gauge; pub mod config_form; pub mod sparkline; -pub use candlestick_chart::CandlestickChart; -pub use order_book::OrderBookWidget; -pub use pnl_heatmap::PnlHeatmap; -pub use risk_gauge::RiskGauge; -pub use config_form::ConfigForm; -pub use sparkline::Sparkline; /// Common color scheme for financial widgets #[derive(Debug, Clone)] diff --git a/tli/src/ui/widgets/mod.rs.bak b/tli/src/ui/widgets/mod.rs.bak new file mode 100644 index 000000000..0ed79dd5c --- /dev/null +++ b/tli/src/ui/widgets/mod.rs.bak @@ -0,0 +1,283 @@ +//! Custom Ratatui widgets for financial data visualization +//! +//! This module provides specialized widgets for the Foxhunt HFT trading terminal: +//! - Real-time candlestick charts with OHLC data +//! - Order book visualization with bid/ask spreads +//! - P&L heatmaps and sparklines for performance tracking +//! - Risk gauge widgets with color-coded status indicators +//! - Configuration forms with input validation +//! +//! All widgets are optimized for high-frequency updates and minimal screen flicker, +//! supporting mouse and keyboard interactions where appropriate. + +use ratatui::{ + prelude::*, + widgets::{Block, Borders, Widget}, + symbols::DOT, +}; +use std::collections::VecDeque; +use chrono::{DateTime, Utc}; +use adaptive_strategy::microstructure::OrderLevel; + +pub mod candlestick_chart; +pub mod order_book; +pub mod pnl_heatmap; +pub mod risk_gauge; +pub mod config_form; +pub mod sparkline; + +pub use candlestick_chart::CandlestickChart; +pub use order_book::OrderBookWidget; +pub use pnl_heatmap::PnlHeatmap; +pub use risk_gauge::RiskGauge; +pub use config_form::ConfigForm; +pub use sparkline::Sparkline; + +/// Common color scheme for financial widgets +#[derive(Debug, Clone)] +pub struct FinancialColors { + pub profit: Color, + pub loss: Color, + pub neutral: Color, + pub bid: Color, + pub ask: Color, + pub warning: Color, + pub critical: Color, + pub background: Color, + pub text: Color, + pub border: Color, +} + +impl Default for FinancialColors { + fn default() -> Self { + Self { + profit: Color::Green, + loss: Color::Red, + neutral: Color::Yellow, + bid: Color::Cyan, + ask: Color::Magenta, + warning: Color::Yellow, + critical: Color::Red, + background: Color::Black, + text: Color::White, + border: Color::Gray, + } + } +} + +/// Base trait for all financial widgets with real-time data updates +pub trait FinancialWidget { + type Data; + + /// Update widget with new data + fn update_data(&mut self, data: Self::Data); + + /// Clear all data from the widget + fn clear(&mut self); + + /// Get the widget's title + fn title(&self) -> &str; + + /// Check if widget has data to display + fn has_data(&self) -> bool; +} + +/// Common data structures for financial widgets + +/// OHLC (Open, High, Low, Close) candle data +#[derive(Debug, Clone)] +pub struct Candle { + pub timestamp: DateTime, + pub open: Decimal, + pub high: Decimal, + pub low: Decimal, + pub close: Decimal, + pub volume: Decimal, +} + +/// Order book snapshot with bids and asks +#[derive(Debug, Clone)] +pub struct OrderBookSnapshot { + pub timestamp: DateTime, + pub bids: Vec, + pub asks: Vec, + pub spread: Decimal, +} + +/// P&L data point for performance tracking +#[derive(Debug, Clone)] +pub struct PnlData { + pub timestamp: DateTime, + pub realized_pnl: Decimal, + pub unrealized_pnl: Decimal, + pub total_pnl: Decimal, + pub strategy: String, +} + +/// Risk metrics for gauge display +#[derive(Debug, Clone)] +pub struct RiskMetrics { + pub var_utilization: f64, // 0.0 to 1.0 + pub position_utilization: f64, // 0.0 to 1.0 + pub drawdown: Decimal, + pub sharpe_ratio: f64, + pub risk_level: RiskLevel, +} + +/// Risk level classification +#[derive(Debug, Clone, PartialEq)] +pub enum RiskLevel { + Low, + Medium, + High, + Critical, +} + +impl RiskLevel { + pub fn color(&self, colors: &FinancialColors) -> Color { + match self { + RiskLevel::Low => colors.profit, + RiskLevel::Medium => colors.neutral, + RiskLevel::High => colors.warning, + RiskLevel::Critical => colors.critical, + } + } +} + +/// Configuration field types for forms +#[derive(Debug, Clone)] +pub enum ConfigField { + Text { value: String, placeholder: String }, + Number { value: f64, min: f64, max: f64 }, + Boolean { value: bool }, + Select { value: String, options: Vec }, +} + +/// Configuration form field definition +#[derive(Debug, Clone)] +pub struct FormField { + pub name: String, + pub label: String, + pub field_type: ConfigField, + pub required: bool, + pub validation_error: Option, +} + +/// Helper functions for common widget operations + +/// Format decimal for display with appropriate precision +pub fn format_price(price: Decimal, precision: u32) -> String { + format!("{:.precision$}", price, precision = precision as usize) +} + +/// Format percentage with sign +pub fn format_percentage(value: f64) -> String { + if value >= 0.0 { + format!("+{:.2}%", value * 100.0) + } else { + format!("{:.2}%", value * 100.0) + } +} + +/// Get color for price change +pub fn price_change_color(change: Decimal, colors: &FinancialColors) -> Color { + if change > Decimal::ZERO { + colors.profit + } else if change < Decimal::ZERO { + colors.loss + } else { + colors.neutral + } +} + +/// Create bordered block for widgets +pub fn create_block(title: &str, colors: &FinancialColors) -> Block { + Block::default() + .title(title) + .borders(Borders::ALL) + .border_style(Style::default().fg(colors.border)) + .title_style(Style::default().fg(colors.text).add_modifier(Modifier::BOLD)) +} + +/// Utility for maintaining fixed-size data buffers +pub struct CircularBuffer { + data: VecDeque, + capacity: usize, +} + +impl CircularBuffer { + pub fn new(capacity: usize) -> Self { + Self { + data: VecDeque::with_capacity(capacity), + capacity, + } + } + + pub fn push(&mut self, item: T) { + if self.data.len() >= self.capacity { + self.data.pop_front(); + } + self.data.push_back(item); + } + + pub fn iter(&self) -> impl Iterator { + self.data.iter() + } + + pub fn len(&self) -> usize { + self.data.len() + } + + pub fn is_empty(&self) -> bool { + self.data.is_empty() + } + + pub fn clear(&mut self) { + self.data.clear(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_circular_buffer() { + let mut buffer: CircularBuffer = CircularBuffer::new(3); + + buffer.push(1); + buffer.push(2); + buffer.push(3); + assert_eq!(buffer.len(), 3); + + buffer.push(4); + assert_eq!(buffer.len(), 3); + + let values: Vec<&i32> = buffer.iter().collect(); + assert_eq!(values, vec![&2, &3, &4]); + } + + #[test] + fn test_format_price() { + let price = Decimal::new(12345, 2); // 123.45 + assert_eq!(format_price(price, 2), "123.45"); + assert_eq!(format_price(price, 4), "123.4500"); + } + + #[test] + fn test_format_percentage() { + assert_eq!(format_percentage(0.1234), "+12.34%"); + assert_eq!(format_percentage(-0.0567), "-5.67%"); + assert_eq!(format_percentage(0.0), "+0.00%"); + } + + #[test] + fn test_risk_level_color() { + let colors = FinancialColors::default(); + + assert_eq!(RiskLevel::Low.color(&colors), Color::Green); + assert_eq!(RiskLevel::Medium.color(&colors), Color::Yellow); + assert_eq!(RiskLevel::High.color(&colors), Color::Yellow); + assert_eq!(RiskLevel::Critical.color(&colors), Color::Red); + } +} \ No newline at end of file diff --git a/tli/tests/integration/mod.rs b/tli/tests/integration/mod.rs index d8b6e439e..93d1d7afa 100644 --- a/tli/tests/integration/mod.rs +++ b/tli/tests/integration/mod.rs @@ -10,6 +10,6 @@ pub mod performance_tests; pub mod service_integration_tests; // Re-export existing integration module -pub use super::integration::*; +// DO NOT RE-EXPORT - Use explicit imports at usage sites // Additional integration test utilities and shared code can be added here diff --git a/tli/tests/integration/mod.rs.bak b/tli/tests/integration/mod.rs.bak new file mode 100644 index 000000000..d8b6e439e --- /dev/null +++ b/tli/tests/integration/mod.rs.bak @@ -0,0 +1,15 @@ +//! Integration test module declarations and shared utilities +//! +//! This module provides common test infrastructure, utilities, and configurations +//! used across all integration test modules. + +// Test module declarations - DATABASE TESTS REMOVED (TLI is pure client) +pub mod end_to_end_tests; +pub mod error_handling_tests; +pub mod performance_tests; +pub mod service_integration_tests; + +// Re-export existing integration module +pub use super::integration::*; + +// Additional integration test utilities and shared code can be added here diff --git a/tli/tests/lib.rs b/tli/tests/lib.rs index 3f60f57a5..fdc222a11 100644 --- a/tli/tests/lib.rs +++ b/tli/tests/lib.rs @@ -8,9 +8,10 @@ pub mod integration; pub mod mocks; -// Test utilities and shared components -pub use integration::{integration_test, TestConfig, TestUtilities}; -pub use mocks::*; +// REMOVED: All pub use statements eliminated per cleanup requirements +// Tests must import from canonical sources: +// use integration::{integration_test, TestConfig, TestUtilities}; +// use mocks::*; // Re-export the main test runner from mod.rs if needed // This allows running tests with: cargo test --lib diff --git a/tli/tests/mocks/mod.rs b/tli/tests/mocks/mod.rs index 0de0a6507..142ed9854 100644 --- a/tli/tests/mocks/mod.rs +++ b/tli/tests/mocks/mod.rs @@ -6,6 +6,6 @@ pub mod grpc_server; // Re-export commonly used mock components -pub use grpc_server::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites MockConfigService, MockGrpcServer, MockHealthService, MockMonitoringService, MockTradingService, }; diff --git a/tli/tests/mocks/mod.rs.bak b/tli/tests/mocks/mod.rs.bak new file mode 100644 index 000000000..0de0a6507 --- /dev/null +++ b/tli/tests/mocks/mod.rs.bak @@ -0,0 +1,11 @@ +//! Mock implementations for TLI testing +//! +//! This module provides mock implementations of various TLI services +//! for comprehensive testing scenarios. + +pub mod grpc_server; + +// Re-export commonly used mock components +pub use grpc_server::{ + MockConfigService, MockGrpcServer, MockHealthService, MockMonitoringService, MockTradingService, +}; diff --git a/tli/tests/mod.rs b/tli/tests/mod.rs index 1e937cc0e..0a94a6ece 100644 --- a/tli/tests/mod.rs +++ b/tli/tests/mod.rs @@ -17,7 +17,7 @@ pub mod unit_tests; pub mod integration; // Re-export test utilities and monitoring tools -pub use test_monitoring::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites OutputFormat, TestCategory, TestEnvironment, TestMonitor, TestMonitorConfig, TestResult, TestStatus, TestSuiteSummary, }; @@ -421,7 +421,7 @@ impl TestRunner { } // Re-export test monitoring types -pub use test_monitoring::{CategoryStatistics, TestStatistics}; +// DO NOT RE-EXPORT - Use explicit imports at usage sites /// Convenience function to run all tests with default configuration pub async fn run_comprehensive_tests() -> TliResult { diff --git a/tli/tests/mod.rs.bak b/tli/tests/mod.rs.bak new file mode 100644 index 000000000..1e937cc0e --- /dev/null +++ b/tli/tests/mod.rs.bak @@ -0,0 +1,539 @@ +//! Comprehensive test suite for TLI system +//! +//! This module organizes and provides access to all test suites including: +//! - Unit tests for individual components +//! - Integration tests for end-to-end workflows +//! - Performance tests for latency and throughput validation +//! - Property-based tests for comprehensive edge case coverage +//! - Continuous monitoring infrastructure + +pub mod integration_tests; +pub mod performance_tests; +pub mod property_tests; +pub mod test_monitoring; +pub mod unit_tests; + +// Re-export integration test modules +pub mod integration; + +// Re-export test utilities and monitoring tools +pub use test_monitoring::{ + OutputFormat, TestCategory, TestEnvironment, TestMonitor, TestMonitorConfig, TestResult, + TestStatus, TestSuiteSummary, +}; + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::RwLock; + +use tli::error::{TliError, TliResult}; +use tli::types::current_unix_nanos; + +/// Test suite configuration +#[derive(Debug, Clone)] +pub struct TestSuiteConfig { + /// Enable unit tests + pub enable_unit_tests: bool, + /// Enable integration tests + pub enable_integration_tests: bool, + /// Enable performance tests + pub enable_performance_tests: bool, + /// Enable property-based tests + pub enable_property_tests: bool, + /// Enable test monitoring + pub enable_monitoring: bool, + /// Performance test timeout + pub performance_timeout: Duration, + /// Integration test timeout + pub integration_timeout: Duration, + /// Property test case count + pub property_test_cases: u32, + /// Test parallelism level + pub parallelism: usize, +} + +impl Default for TestSuiteConfig { + fn default() -> Self { + Self { + enable_unit_tests: true, + enable_integration_tests: true, + enable_performance_tests: true, + enable_property_tests: true, + enable_monitoring: true, + performance_timeout: Duration::from_secs(30), + integration_timeout: Duration::from_secs(60), + property_test_cases: 1000, + parallelism: num_cpus::get(), + } + } +} + +/// Comprehensive test runner for the TLI system +pub struct TestRunner { + /// Test configuration + config: TestSuiteConfig, + /// Test monitor for tracking results + monitor: Option>, + /// Test results + results: Arc>>, +} + +impl TestRunner { + /// Create a new test runner + pub fn new(config: TestSuiteConfig) -> Self { + Self { + config, + monitor: None, + results: Arc::new(RwLock::new(Vec::new())), + } + } + + /// Create test runner with monitoring + pub fn with_monitoring>( + config: TestSuiteConfig, + output_dir: P, + monitor_config: test_monitoring::TestMonitorConfig, + ) -> TliResult { + let monitor = Arc::new(TestMonitor::new(output_dir, monitor_config)?); + + Ok(Self { + config, + monitor: Some(monitor), + results: Arc::new(RwLock::new(Vec::new())), + }) + } + + /// Run all enabled test suites + pub async fn run_all_tests(&self) -> TliResult { + let start_time = Instant::now(); + let environment = TestEnvironment::current(); + + // Start test suite in monitor if available + let execution_id = if let Some(monitor) = &self.monitor { + monitor.load_baselines().await?; + Some(monitor.start_test_suite(environment.clone()).await) + } else { + None + }; + + println!("🚀 Starting comprehensive TLI test suite execution"); + println!("Configuration: {:?}", self.config); + + // Run test suites in order + if self.config.enable_unit_tests { + self.run_unit_tests().await?; + } + + if self.config.enable_integration_tests { + self.run_integration_tests().await?; + } + + if self.config.enable_performance_tests { + self.run_performance_tests().await?; + } + + if self.config.enable_property_tests { + self.run_property_tests().await?; + } + + // Finish test suite and generate report + let summary = if let Some(monitor) = &self.monitor { + monitor.finish_test_suite().await? + } else { + self.create_summary( + execution_id.unwrap_or_else(|| "manual".to_string()), + start_time, + environment, + ) + .await + }; + + println!("✅ Test suite execution completed"); + println!( + "Results: {} passed, {} failed, {} skipped", + summary.passed_tests, summary.failed_tests, summary.skipped_tests + ); + + if summary.performance_regression { + println!("⚠️ Performance regression detected!"); + } + + Ok(summary) + } + + /// Run unit tests + async fn run_unit_tests(&self) -> TliResult<()> { + println!("🧪 Running unit tests..."); + + // Unit tests are typically run via `cargo test` but we can track results here + let test_categories = vec![ + "client_tests", + "types_tests", + "error_tests", + "validation_tests", + "database_tests", + "encryption_tests", + ]; + + for category in test_categories { + let result = self + .simulate_test_execution( + &format!("unit::{}", category), + TestCategory::Unit, + Duration::from_millis(50 + rand::random::() % 200), + 0.95, // 95% pass rate + ) + .await; + + self.record_result(result).await?; + } + + println!("✅ Unit tests completed"); + Ok(()) + } + + /// Run integration tests + async fn run_integration_tests(&self) -> TliResult<()> { + println!("🔄 Running integration tests..."); + + let integration_tests = vec![ + "grpc_communication", + "database_transactions", + "event_processing", + "configuration_hot_reload", + "security_authentication", + ]; + + for test_name in integration_tests { + let result = self + .simulate_test_execution( + &format!("integration::{}", test_name), + TestCategory::Integration, + Duration::from_millis(500 + rand::random::() % 2000), + 0.90, // 90% pass rate (integration tests are more complex) + ) + .await; + + self.record_result(result).await?; + } + + println!("✅ Integration tests completed"); + Ok(()) + } + + /// Run performance tests + async fn run_performance_tests(&self) -> TliResult<()> { + println!("⚡ Running performance tests..."); + + let performance_tests = vec![ + ("latency::order_submission", "latency_us", 25.0), + ("latency::timestamp_conversion", "latency_ns", 500.0), + ("throughput::order_processing", "orders_per_sec", 15000.0), + ("throughput::event_processing", "events_per_sec", 150000.0), + ("memory::allocation_patterns", "allocation_ns", 5000.0), + ]; + + for (test_name, metric_name, target_value) in performance_tests { + let mut metrics = HashMap::new(); + + // Simulate performance measurement with some variance + let actual_value = target_value * (0.8 + rand::random::() * 0.4); // ±20% variance + metrics.insert(metric_name.to_string(), actual_value); + + let result = TestResult { + test_name: test_name.to_string(), + test_category: TestCategory::Performance, + status: if actual_value <= target_value * 1.2 { + TestStatus::Passed + } else { + TestStatus::Failed + }, + duration: Duration::from_millis(100 + rand::random::() % 500), + error_message: if actual_value > target_value * 1.2 { + Some(format!( + "Performance target missed: {} > {}", + actual_value, target_value + )) + } else { + None + }, + metrics, + timestamp: current_unix_nanos(), + environment: TestEnvironment::current(), + }; + + self.record_result(result).await?; + } + + println!("✅ Performance tests completed"); + Ok(()) + } + + /// Run property-based tests + async fn run_property_tests(&self) -> TliResult<()> { + println!("🎲 Running property-based tests..."); + + let property_tests = vec![ + "prop_order_validation", + "prop_timestamp_conversion", + "prop_type_conversions", + "prop_position_calculations", + "prop_encryption_reversible", + "prop_database_consistency", + ]; + + for test_name in property_tests { + let result = self + .simulate_test_execution( + &format!("property::{}", test_name), + TestCategory::Property, + Duration::from_millis(200 + rand::random::() % 800), + 0.98, // 98% pass rate (property tests are thorough) + ) + .await; + + self.record_result(result).await?; + } + + println!("✅ Property-based tests completed"); + Ok(()) + } + + /// Simulate test execution (in real implementation, would run actual tests) + async fn simulate_test_execution( + &self, + test_name: &str, + category: TestCategory, + duration: Duration, + pass_rate: f64, + ) -> TestResult { + // Simulate test execution time + tokio::time::sleep(Duration::from_millis(10)).await; + + let passed = rand::random::() < pass_rate; + + TestResult { + test_name: test_name.to_string(), + test_category: category, + status: if passed { + TestStatus::Passed + } else { + TestStatus::Failed + }, + duration, + error_message: if !passed { + Some(format!("Simulated test failure for {}", test_name)) + } else { + None + }, + metrics: HashMap::new(), + timestamp: current_unix_nanos(), + environment: TestEnvironment::current(), + } + } + + /// Record test result + async fn record_result(&self, result: TestResult) -> TliResult<()> { + // Record in monitor if available + if let Some(monitor) = &self.monitor { + monitor.record_test_result(result.clone()).await?; + } + + // Store in local results + self.results.write().await.push(result); + + Ok(()) + } + + /// Create test suite summary + async fn create_summary( + &self, + execution_id: String, + start_time: Instant, + environment: TestEnvironment, + ) -> TestSuiteSummary { + let results = self.results.read().await; + let total_duration = start_time.elapsed(); + + let total_tests = results.len(); + let passed_tests = results + .iter() + .filter(|r| r.status == TestStatus::Passed) + .count(); + let failed_tests = results + .iter() + .filter(|r| r.status == TestStatus::Failed) + .count(); + let skipped_tests = results + .iter() + .filter(|r| r.status == TestStatus::Skipped) + .count(); + + // Check for performance regressions (simplified) + let performance_regression = results + .iter() + .filter(|r| r.test_category == TestCategory::Performance) + .any(|r| r.status == TestStatus::Failed); + + TestSuiteSummary { + execution_id, + total_tests, + passed_tests, + failed_tests, + skipped_tests, + total_duration, + coverage_percentage: Some(95.2), // Simulated coverage + performance_regression, + start_timestamp: current_unix_nanos() - total_duration.as_nanos() as i64, + environment, + test_results: results.clone(), + } + } + + /// Get test statistics + pub async fn get_statistics(&self) -> TestStatistics { + if let Some(monitor) = &self.monitor { + monitor.get_test_statistics().await + } else { + let results = self.results.read().await; + let mut stats = TestStatistics::default(); + + for result in results.iter() { + stats.total_tests += 1; + match result.status { + TestStatus::Passed => stats.passed_tests += 1, + TestStatus::Failed => stats.failed_tests += 1, + TestStatus::Skipped => stats.skipped_tests += 1, + _ => {} + } + stats.total_duration += result.duration; + } + + if stats.total_tests > 0 { + stats.average_duration = stats.total_duration / stats.total_tests as u32; + stats.pass_rate = stats.passed_tests as f64 / stats.total_tests as f64; + } + + stats + } + } +} + +// Re-export test monitoring types +pub use test_monitoring::{CategoryStatistics, TestStatistics}; + +/// Convenience function to run all tests with default configuration +pub async fn run_comprehensive_tests() -> TliResult { + let config = TestSuiteConfig::default(); + let runner = TestRunner::new(config); + runner.run_all_tests().await +} + +/// Convenience function to run tests with monitoring +pub async fn run_tests_with_monitoring>( + output_dir: P, +) -> TliResult { + let config = TestSuiteConfig::default(); + let monitor_config = test_monitoring::TestMonitorConfig::default(); + + let runner = TestRunner::with_monitoring(config, output_dir, monitor_config)?; + runner.run_all_tests().await +} + +/// Macro for running a specific test category +#[macro_export] +macro_rules! run_test_category { + ($category:expr, $config:expr) => {{ + let mut test_config = $config; + test_config.enable_unit_tests = false; + test_config.enable_integration_tests = false; + test_config.enable_performance_tests = false; + test_config.enable_property_tests = false; + + match $category { + TestCategory::Unit => test_config.enable_unit_tests = true, + TestCategory::Integration => test_config.enable_integration_tests = true, + TestCategory::Performance => test_config.enable_performance_tests = true, + TestCategory::Property => test_config.enable_property_tests = true, + _ => {} + } + + let runner = TestRunner::new(test_config); + runner.run_all_tests().await + }}; +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[tokio::test] + async fn test_runner_creation() { + let config = TestSuiteConfig::default(); + let runner = TestRunner::new(config); + + // Should create successfully + assert!(runner.results.read().await.is_empty()); + } + + #[tokio::test] + async fn test_runner_with_monitoring() { + let temp_dir = TempDir::new().expect("Failed to create temp directory"); + let config = TestSuiteConfig::default(); + let monitor_config = test_monitoring::TestMonitorConfig::default(); + + let runner = TestRunner::with_monitoring(config, temp_dir.path(), monitor_config); + assert!(runner.is_ok()); + } + + #[tokio::test] + async fn test_comprehensive_test_execution() { + let temp_dir = TempDir::new().expect("Failed to create temp directory"); + let config = TestSuiteConfig { + enable_unit_tests: true, + enable_integration_tests: false, // Disable to speed up test + enable_performance_tests: false, // Disable to speed up test + enable_property_tests: false, // Disable to speed up test + ..Default::default() + }; + + let monitor_config = test_monitoring::TestMonitorConfig { + enable_detailed_logging: false, + enable_real_time_monitoring: false, + ..Default::default() + }; + + let runner = TestRunner::with_monitoring(config, temp_dir.path(), monitor_config).unwrap(); + let summary = runner.run_all_tests().await.unwrap(); + + assert!(summary.total_tests > 0); + assert!(summary.passed_tests > 0); + } + + #[tokio::test] + async fn test_statistics_collection() { + let config = TestSuiteConfig::default(); + let runner = TestRunner::new(config); + + // Simulate some test results + let test_result = TestResult { + test_name: "test_example".to_string(), + test_category: TestCategory::Unit, + status: TestStatus::Passed, + duration: Duration::from_millis(50), + error_message: None, + metrics: HashMap::new(), + timestamp: current_unix_nanos(), + environment: TestEnvironment::current(), + }; + + runner.record_result(test_result).await.unwrap(); + + let stats = runner.get_statistics().await; + assert_eq!(stats.total_tests, 1); + assert_eq!(stats.passed_tests, 1); + assert_eq!(stats.failed_tests, 0); + } +} diff --git a/trading-data/src/models.rs b/trading-data/src/models.rs index 2de95b8b9..cb492efd0 100644 --- a/trading-data/src/models.rs +++ b/trading-data/src/models.rs @@ -7,8 +7,8 @@ // Removed direct rust_decimal imports - using common::Decimal via prelude // use rust_decimal_macros::dec; // Use common::dec! macro instead -// Re-export canonical types from common crate -pub use common::types::Order; +// REMOVED: All pub use statements eliminated per cleanup requirements +// Use direct import: common::types::Order // Order, Position, and Execution are now imported from common::prelude // All implementations are maintained in the canonical location: common/src/types.rs diff --git a/trading_engine/src/brokers/icmarkets.rs b/trading_engine/src/brokers/icmarkets.rs index 2397b3484..30697e16d 100644 --- a/trading_engine/src/brokers/icmarkets.rs +++ b/trading_engine/src/brokers/icmarkets.rs @@ -3,8 +3,9 @@ //! Production-ready FIX connector for `ICMarkets` cTrader with real trading capabilities. use crate::trading::data_interface::{ - BrokerConnectionStatus, BrokerError, BrokerInterface, ExecutionReport, Position, + BrokerConnectionStatus, BrokerError, BrokerInterface, }; +use common::types::{Execution as ExecutionReport, Position}; use crate::trading_operations::TradingOrder; use async_trait::async_trait; use serde::{Deserialize, Serialize}; diff --git a/trading_engine/src/brokers/interactive_brokers.rs b/trading_engine/src/brokers/interactive_brokers.rs index 5a0e58210..67731daa3 100644 --- a/trading_engine/src/brokers/interactive_brokers.rs +++ b/trading_engine/src/brokers/interactive_brokers.rs @@ -3,8 +3,9 @@ //! Simple stub implementation for compilation purposes. use crate::trading::data_interface::{ - BrokerConnectionStatus, BrokerError, BrokerInterface, ExecutionReport, Position, + BrokerConnectionStatus, BrokerError, BrokerInterface, }; +use common::types::{Execution as ExecutionReport, Position}; use crate::trading_operations::TradingOrder; use async_trait::async_trait; use serde::{Deserialize, Serialize}; diff --git a/trading_engine/src/brokers/mod.rs b/trading_engine/src/brokers/mod.rs index bf82c26a1..a159afb2b 100644 --- a/trading_engine/src/brokers/mod.rs +++ b/trading_engine/src/brokers/mod.rs @@ -17,12 +17,6 @@ pub mod routing; pub mod security; // Re-exports for convenience -pub use self::config::BrokerConnectorConfig; -pub use self::error::{BrokerError, Result}; -pub use self::fix::FixMessage; -pub use self::icmarkets::ICMarketsClient; -pub use self::interactive_brokers::InteractiveBrokersClient; -pub use self::routing::{OrderRouter, RoutingDecision}; /// Simple broker connector for benchmarking #[derive(Debug)] diff --git a/trading_engine/src/brokers/mod.rs.bak b/trading_engine/src/brokers/mod.rs.bak new file mode 100644 index 000000000..bf82c26a1 --- /dev/null +++ b/trading_engine/src/brokers/mod.rs.bak @@ -0,0 +1,75 @@ +//! # Broker Connector Service +//! +//! Simplified broker connectivity service for benchmark compilation. + +#![warn(missing_docs)] + +// Re-export core types + +// Public modules +pub mod config; +pub mod error; +pub mod fix; +pub mod icmarkets; +pub mod interactive_brokers; +pub mod monitoring; +pub mod routing; +pub mod security; + +// Re-exports for convenience +pub use self::config::BrokerConnectorConfig; +pub use self::error::{BrokerError, Result}; +pub use self::fix::FixMessage; +pub use self::icmarkets::ICMarketsClient; +pub use self::interactive_brokers::InteractiveBrokersClient; +pub use self::routing::{OrderRouter, RoutingDecision}; + +/// Simple broker connector for benchmarking +#[derive(Debug)] +pub struct BrokerConnector {} + +impl BrokerConnector { + /// Create a new broker connector + pub fn new(_config: BrokerConnectorConfig) -> Self { + Self {} + } + + /// Initialize broker connections (placeholder) + pub async fn initialize(&mut self) -> Result<()> { + Ok(()) + } + + /// Submit an order (placeholder) + pub async fn submit_order(&self, _order_id: &str) -> Result { + Ok("placeholder_broker_order_id".to_owned()) + } + + /// Cancel an order (placeholder) + pub async fn cancel_order(&self, _order_id: &str) -> Result<()> { + Ok(()) + } + + /// Get connected brokers (placeholder) + pub async fn get_connected_brokers(&self) -> Vec { + vec!["InteractiveBrokers".to_owned()] + } + + /// Shutdown broker connections (placeholder) + pub async fn shutdown(&mut self) -> Result<()> { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_broker_connector_creation() { + let config = BrokerConnectorConfig::default(); + let connector = BrokerConnector::new(config); + + let connected_brokers = connector.get_connected_brokers().await; + assert!(!connected_brokers.is_empty()); + } +} diff --git a/trading_engine/src/compliance/mod.rs.bak b/trading_engine/src/compliance/mod.rs.bak new file mode 100644 index 000000000..976f21696 --- /dev/null +++ b/trading_engine/src/compliance/mod.rs.bak @@ -0,0 +1,794 @@ +//! Comprehensive Regulatory Compliance Framework +//! +//! This module provides enterprise-grade compliance capabilities for financial trading +//! operations, ensuring full adherence to global regulatory requirements including: +//! - MiFID II (Markets in Financial Instruments Directive) +//! - SOX (Sarbanes-Oxley Act) +//! - MAR (Market Abuse Regulation) +//! - GDPR/CCPA (Data Protection) +//! - Basel III Capital Requirements +//! - Dodd-Frank Act +//! - EMIR (European Market Infrastructure Regulation) + +#![deny(clippy::unwrap_used, clippy::expect_used)] + +pub mod audit_trails; +pub mod best_execution; +pub mod transaction_reporting; +// TODO: Implement missing compliance modules +// pub mod market_surveillance; +// pub mod automated_reporting; // Temporarily disabled due to SOX/best_execution dependencies +pub mod regulatory_api; +// pub mod regulatory_reporting; +// pub mod audit_reports; +// pub mod sox_compliance; // Temporarily disabled due to SOXConfig conflicts +// pub mod mifid_compliance; +// pub mod mar_compliance; +pub mod compliance_reporting; +pub mod iso27001_compliance; + +use chrono::{DateTime, Duration, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use uuid::Uuid; + +// Import common trading types +use common::types::{OrderId, OrderSide, OrderType, Quantity, Price, Decimal}; + +/// Compliance framework configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComplianceConfig { + /// `MiFID` II configuration + pub mifid2: MiFIDConfig, + /// SOX compliance settings + pub sox: SOXConfig, + /// Market surveillance parameters + pub mar: MARConfig, + /// Data protection settings + pub data_protection: DataProtectionConfig, + /// Reporting intervals + pub reporting_intervals: HashMap, + /// Audit retention period (minimum 7 years for regulatory compliance) + pub audit_retention_days: u32, +} + +/// `MiFID` II specific configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MiFIDConfig { + /// Enable best execution analysis + pub best_execution_enabled: bool, + /// Transaction reporting endpoint + pub transaction_reporting_endpoint: Option, + /// Client categorization enabled + pub client_categorization_enabled: bool, + /// Product governance enabled + pub product_governance_enabled: bool, + /// Position limit monitoring + pub position_limit_monitoring: bool, +} + +/// SOX compliance configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SOXConfig { + /// Management certification required + pub management_certification_required: bool, + /// Internal controls testing + pub internal_controls_testing: bool, + /// Audit trail required + pub audit_trail_required: bool, + /// Section 404 compliance + pub section_404_enabled: bool, +} + +/// Market Abuse Regulation configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MARConfig { + /// Real-time surveillance enabled + pub real_time_surveillance: bool, + /// Insider trading detection + pub insider_trading_detection: bool, + /// Market manipulation detection + pub market_manipulation_detection: bool, + /// Suspicious activity reporting + pub suspicious_activity_reporting: bool, +} + +/// Data protection compliance configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DataProtectionConfig { + /// GDPR compliance enabled + pub gdpr_enabled: bool, + /// CCPA compliance enabled + pub ccpa_enabled: bool, + /// Data retention policies + pub data_retention_policies: HashMap, + /// Consent management + pub consent_management_enabled: bool, +} + +/// Comprehensive compliance status +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ComplianceStatus { + /// Fully compliant with all regulations + Compliant, + /// Minor issues requiring attention + Warning(Vec), + /// Serious violations requiring immediate action + Violation(Vec), + /// Under regulatory review + UnderReview, + /// Non-applicable for this context + NotApplicable, +} + +/// Regulatory compliance result +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComplianceResult { + /// Overall compliance status + pub status: ComplianceStatus, + /// `MiFID` II compliance details + pub mifid2_status: ComplianceStatus, + /// SOX compliance details + pub sox_status: ComplianceStatus, + /// MAR compliance details + pub mar_status: ComplianceStatus, + /// Data protection compliance + pub data_protection_status: ComplianceStatus, + /// Compliance score (0-100) + pub compliance_score: f64, + /// Detailed findings + pub findings: Vec, + /// Timestamp of assessment + pub assessment_timestamp: DateTime, +} + +/// Individual compliance finding +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComplianceFinding { + /// Finding ID + pub id: String, + /// Regulation category + pub regulation: String, + /// Finding severity + pub severity: ComplianceSeverity, + /// Description of the finding + pub description: String, + /// Recommended remediation action + pub remediation: String, + /// Due date for remediation + pub due_date: Option>, + /// Finding status + pub status: FindingStatus, +} + +/// Compliance finding severity levels +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum ComplianceSeverity { + /// Critical regulatory violation + Critical, + /// High priority issue + High, + /// Medium priority concern + Medium, + /// Low priority observation + Low, + /// Informational note + Info, +} + +/// Status of compliance findings +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum FindingStatus { + /// Newly identified finding + Open, + /// Being addressed + InProgress, + /// Resolved successfully + Resolved, + /// Accepted risk + Accepted, + /// False positive + Dismissed, +} + +impl Default for ComplianceConfig { + fn default() -> Self { + Self { + mifid2: MiFIDConfig { + best_execution_enabled: true, + transaction_reporting_endpoint: None, + client_categorization_enabled: true, + product_governance_enabled: true, + position_limit_monitoring: true, + }, + sox: SOXConfig { + management_certification_required: true, + internal_controls_testing: true, + audit_trail_required: true, + section_404_enabled: true, + }, + mar: MARConfig { + real_time_surveillance: true, + insider_trading_detection: true, + market_manipulation_detection: true, + suspicious_activity_reporting: true, + }, + data_protection: DataProtectionConfig { + gdpr_enabled: true, + ccpa_enabled: true, + data_retention_policies: HashMap::new(), + consent_management_enabled: true, + }, + reporting_intervals: HashMap::new(), + audit_retention_days: 2555, // 7 years minimum + } + } +} + +/// Master compliance engine that coordinates all regulatory requirements +#[derive(Debug)] +pub struct ComplianceEngine { + config: ComplianceConfig, + best_execution: best_execution::BestExecutionAnalyzer, + // TODO: Add when modules are implemented + // transaction_reporting: transaction_reporting::TransactionReporter, + // market_surveillance: market_surveillance::MarketSurveillanceEngine, + // regulatory_reporting: regulatory_reporting::RegulatoryReporter, + // audit_reports: audit_reports::AuditReportGenerator, +} + +impl ComplianceEngine { + /// Create new compliance engine with configuration + pub fn new(config: ComplianceConfig) -> Self { + Self { + best_execution: best_execution::BestExecutionAnalyzer::new(&config.mifid2), + // TODO: Initialize when modules are implemented + // transaction_reporting: transaction_reporting::TransactionReporter::new(&config.mifid2), + // market_surveillance: market_surveillance::MarketSurveillanceEngine::new(&config.mar), + // regulatory_reporting: regulatory_reporting::RegulatoryReporter::new(&config), + // audit_reports: audit_reports::AuditReportGenerator::new(&config), + config, + } + } + + /// Perform comprehensive compliance assessment + pub async fn assess_compliance( + &self, + context: &ComplianceContext, + ) -> Result { + let mut findings = Vec::new(); + let assessment_timestamp = Utc::now(); + + // MiFID II compliance assessment + let mifid2_status = self + .assess_mifid2_compliance(context, &mut findings) + .await?; + + // SOX compliance assessment + let sox_status = self.assess_sox_compliance(context, &mut findings).await?; + + // MAR compliance assessment + let mar_status = self.assess_mar_compliance(context, &mut findings).await?; + + // Data protection compliance + let data_protection_status = self + .assess_data_protection_compliance(context, &mut findings) + .await?; + + // Calculate overall compliance score + let compliance_score = self.calculate_compliance_score(&findings); + + // Determine overall status + let status = self.determine_overall_status(&[ + &mifid2_status, + &sox_status, + &mar_status, + &data_protection_status, + ]); + + Ok(ComplianceResult { + status, + mifid2_status, + sox_status, + mar_status, + data_protection_status, + compliance_score, + findings, + assessment_timestamp, + }) + } + + /// Assess `MiFID` II compliance + async fn assess_mifid2_compliance( + &self, + context: &ComplianceContext, + findings: &mut Vec, + ) -> Result { + if !self.config.mifid2.best_execution_enabled { + return Ok(ComplianceStatus::NotApplicable); + } + + // Best execution analysis + if let Some(order) = &context.order_info { + if let Ok(analysis) = self.best_execution.analyze_best_execution(order).await { + if !analysis.is_compliant { + findings.push(ComplianceFinding { + id: format!("MIFID2-BE-{}", Uuid::new_v4()), + regulation: "MiFID II Article 27".to_owned(), + severity: ComplianceSeverity::High, + description: "Best execution requirements not met".to_owned(), + remediation: "Review execution venue selection and cost analysis" + .to_owned(), + due_date: Some(Utc::now() + Duration::hours(24)), + status: FindingStatus::Open, + }); + return Ok(ComplianceStatus::Violation(vec![ + "Best execution failure".to_owned() + ])); + } + } else { + findings.push(ComplianceFinding { + id: format!("MIFID2-BE-ERROR-{}", Uuid::new_v4()), + regulation: "MiFID II Article 27".to_owned(), + severity: ComplianceSeverity::Critical, + description: "Best execution analysis failed".to_owned(), + remediation: "Fix best execution analysis system".to_owned(), + due_date: Some(Utc::now() + Duration::hours(1)), + status: FindingStatus::Open, + }); + return Ok(ComplianceStatus::Violation(vec![ + "Analysis system failure".to_owned() + ])); + } + } + + // Transaction reporting check + if self.config.mifid2.transaction_reporting_endpoint.is_none() { + findings.push(ComplianceFinding { + id: format!("MIFID2-TR-{}", Uuid::new_v4()), + regulation: "MiFID II Article 26".to_owned(), + severity: ComplianceSeverity::Medium, + description: "Transaction reporting endpoint not configured".to_owned(), + remediation: "Configure transaction reporting endpoint".to_owned(), + due_date: Some(Utc::now() + Duration::days(7)), + status: FindingStatus::Open, + }); + return Ok(ComplianceStatus::Warning(vec![ + "Missing reporting config".to_owned() + ])); + } + + Ok(ComplianceStatus::Compliant) + } + + /// Assess SOX compliance + async fn assess_sox_compliance( + &self, + _context: &ComplianceContext, + findings: &mut Vec, + ) -> Result { + if !self.config.sox.management_certification_required { + return Ok(ComplianceStatus::NotApplicable); + } + + // Check internal controls + if !self.config.sox.internal_controls_testing { + findings.push(ComplianceFinding { + id: format!("SOX-IC-{}", Uuid::new_v4()), + regulation: "SOX Section 404".to_owned(), + severity: ComplianceSeverity::High, + description: "Internal controls testing not enabled".to_owned(), + remediation: "Enable comprehensive internal controls testing".to_owned(), + due_date: Some(Utc::now() + Duration::days(30)), + status: FindingStatus::Open, + }); + return Ok(ComplianceStatus::Violation(vec![ + "Missing internal controls".to_owned(), + ])); + } + + // Check audit trail requirements + if !self.config.sox.audit_trail_required { + findings.push(ComplianceFinding { + id: format!("SOX-AT-{}", Uuid::new_v4()), + regulation: "SOX Section 302".to_owned(), + severity: ComplianceSeverity::Critical, + description: "Audit trail requirements not met".to_owned(), + remediation: "Implement comprehensive audit logging".to_owned(), + due_date: Some(Utc::now() + Duration::days(14)), + status: FindingStatus::Open, + }); + return Ok(ComplianceStatus::Violation(vec![ + "Missing audit trail".to_owned() + ])); + } + + Ok(ComplianceStatus::Compliant) + } + + /// Assess MAR compliance + async fn assess_mar_compliance( + &self, + context: &ComplianceContext, + findings: &mut Vec, + ) -> Result { + if !self.config.mar.real_time_surveillance { + return Ok(ComplianceStatus::NotApplicable); + } + + // TODO: Market surveillance analysis (when module is implemented) + // Market surveillance analysis + if let Some(_order) = &context.order_info { + // Placeholder - market surveillance module not yet implemented + findings.push(ComplianceFinding { + id: format!("MAR-TODO-{}", Uuid::new_v4()), + regulation: "Market Abuse Regulation".to_owned(), + severity: ComplianceSeverity::Info, + description: "Market surveillance module not yet implemented".to_owned(), + remediation: "Implement market surveillance analysis".to_owned(), + due_date: Some(Utc::now() + Duration::days(30)), + status: FindingStatus::Open, + }); + } + + Ok(ComplianceStatus::Compliant) + } + + /// Assess data protection compliance + async fn assess_data_protection_compliance( + &self, + _context: &ComplianceContext, + findings: &mut Vec, + ) -> Result { + if !self.config.data_protection.gdpr_enabled && !self.config.data_protection.ccpa_enabled { + return Ok(ComplianceStatus::NotApplicable); + } + + // Check consent management + if !self.config.data_protection.consent_management_enabled { + findings.push(ComplianceFinding { + id: format!("GDPR-CM-{}", Uuid::new_v4()), + regulation: "GDPR Article 7".to_owned(), + severity: ComplianceSeverity::High, + description: "Consent management not enabled".to_owned(), + remediation: "Implement consent management system".to_owned(), + due_date: Some(Utc::now() + Duration::days(30)), + status: FindingStatus::Open, + }); + return Ok(ComplianceStatus::Warning(vec![ + "Missing consent management".to_owned(), + ])); + } + + // Check data retention policies + if self + .config + .data_protection + .data_retention_policies + .is_empty() + { + findings.push(ComplianceFinding { + id: format!("GDPR-DRP-{}", Uuid::new_v4()), + regulation: "GDPR Article 5".to_owned(), + severity: ComplianceSeverity::Medium, + description: "Data retention policies not configured".to_owned(), + remediation: "Configure appropriate data retention policies".to_owned(), + due_date: Some(Utc::now() + Duration::days(60)), + status: FindingStatus::Open, + }); + return Ok(ComplianceStatus::Warning(vec![ + "Missing retention policies".to_owned(), + ])); + } + + Ok(ComplianceStatus::Compliant) + } + + /// Calculate overall compliance score + fn calculate_compliance_score(&self, findings: &[ComplianceFinding]) -> f64 { + if findings.is_empty() { + return 100.0; + } + + let total_deduction: f64 = findings + .iter() + .map(|f| match f.severity { + ComplianceSeverity::Critical => 25.0, + ComplianceSeverity::High => 15.0, + ComplianceSeverity::Medium => 8.0, + ComplianceSeverity::Low => 3.0, + ComplianceSeverity::Info => 0.0, + }) + .sum(); + + (100.0 - total_deduction).max(0.0) + } + + /// Determine overall compliance status from individual statuses + fn determine_overall_status(&self, statuses: &[&ComplianceStatus]) -> ComplianceStatus { + for status in statuses { + match status { + ComplianceStatus::Violation(_) => return (*status).clone(), + _ => {} + } + } + + for status in statuses { + match status { + ComplianceStatus::Warning(_) => return (*status).clone(), + _ => {} + } + } + + ComplianceStatus::Compliant + } +} + +/// Order information for compliance assessment +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OrderInfo { + /// Order ID + pub order_id: OrderId, + /// Order side (buy/sell) + pub side: OrderSide, + /// Order type + pub order_type: OrderType, + /// Quantity + pub quantity: Quantity, + /// Price (optional for market orders) + pub price: Option, + /// Instrument symbol + pub symbol: String, + /// Client ID + pub client_id: String, + /// Order timestamp + pub timestamp: DateTime, +} + +/// Context for compliance assessment +#[derive(Debug, Clone)] +pub struct ComplianceContext { + /// Order information for assessment + pub order_info: Option, + /// Client information + pub client_info: Option, + /// Market data context + pub market_context: Option, + /// Assessment timestamp + pub timestamp: DateTime, +} + +/// Client information for compliance +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ClientInfo { + /// Client ID + pub client_id: String, + /// Client classification + pub classification: ClientType, + /// Risk tolerance + pub risk_tolerance: RiskTolerance, + /// Jurisdiction + pub jurisdiction: String, +} + +/// Market context for compliance assessment +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MarketContext { + /// Market conditions + pub conditions: MarketConditions, + /// Trading session + pub session: TradingSession, + /// Volatility level + pub volatility: f64, +} + +/// Client classification types +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ClientType { + /// Retail client + Retail, + /// Professional client + Professional, + /// Eligible counterparty + EligibleCounterparty, +} + +/// Risk tolerance levels +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum RiskTolerance { + /// Conservative risk profile + Conservative, + /// Moderate risk profile + Moderate, + /// Aggressive risk profile + Aggressive, +} + +/// Market conditions +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum MarketConditions { + /// Normal market conditions + Normal, + /// High volatility + HighVolatility, + /// Market stress + Stress, + /// Market closure + Closed, +} + +/// Trading session types +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum TradingSession { + /// Pre-market session + PreMarket, + /// Regular trading hours + Regular, + /// After-hours session + AfterHours, + /// Closed session + Closed, +} + +/// Compliance-related errors +#[derive(Debug, thiserror::Error)] +pub enum ComplianceError { + /// Configuration error + #[error("Configuration error: {0}")] + Configuration(String), + /// Analysis error + #[error("Analysis error: {0}")] + Analysis(String), + /// Reporting error + #[error("Reporting error: {0}")] + Reporting(String), + /// Data access error + #[error("Data access error: {0}")] + DataAccess(String), +} + +/// Compliance violation record +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComplianceViolation { + /// Rule ID that was violated + pub rule_id: String, + /// Severity of the violation + pub severity: ComplianceSeverity, + /// Description of the violation + pub description: String, + /// Regulation that was violated + pub regulation: ComplianceRegulation, + /// When the violation was detected + pub detected_at: DateTime, + /// Entity involved (trader, client, etc.) + pub entity_id: Option, + /// Trade ID if applicable + pub trade_id: Option, + /// Symbol if applicable + pub symbol: Option, +} + +/// Regulatory frameworks +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum ComplianceRegulation { + /// Markets in Financial Instruments Directive II + MiFIDII, + /// Sarbanes-Oxley Act + SOX, + /// Market Abuse Regulation + MAR, + /// General Data Protection Regulation + GDPR, + /// California Consumer Privacy Act + CCPA, + /// Basel III + BaselIII, + /// Dodd-Frank Act + DoddFrank, + /// European Market Infrastructure Regulation + EMIR, +} + +/// SOX Compliance Manager +#[derive(Debug, Clone)] +pub struct SOXCompliance { + /// Configuration + pub config: SOXConfig, + /// Whether controls are enabled + pub enabled: bool, +} + +impl SOXCompliance { + pub const fn new(config: SOXConfig) -> Self { + Self { + config, + enabled: true, + } + } +} + +/// `MiFID` Compliance Manager +#[derive(Debug, Clone)] +pub struct MiFIDCompliance { + /// Configuration + pub config: MiFIDConfig, + /// Whether compliance is enabled + pub enabled: bool, +} + +impl MiFIDCompliance { + pub const fn new(config: MiFIDConfig) -> Self { + Self { + config, + enabled: true, + } + } +} + +/// Compliance rule definition +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComplianceRule { + /// Rule ID + pub id: String, + /// Rule name + pub name: String, + /// Rule description + pub description: String, + /// Regulation this rule belongs to + pub regulation: ComplianceRegulation, + /// Rule severity + pub severity: ComplianceSeverity, + /// Whether rule is active + pub active: bool, +} + +/// Compliance monitoring system +#[derive(Debug, Clone)] +pub struct ComplianceMonitor { + /// Rules being monitored + pub rules: Vec, + /// Configuration + pub config: ComplianceConfig, +} + +impl ComplianceMonitor { + pub const fn new(config: ComplianceConfig) -> Self { + Self { + rules: Vec::new(), + config, + } + } + + pub fn add_rule(&mut self, rule: ComplianceRule) { + self.rules.push(rule); + } +} +/// Risk level enumeration +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum RiskLevel { + /// Low risk + Low, + /// Medium risk + Medium, + /// High risk + High, + /// Critical risk + Critical, +} + +/// SOX audit event for compliance reporting +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SOXAuditEvent { + /// Event identifier + pub id: String, + /// Timestamp of the event + pub timestamp: DateTime, + /// Event type + pub event_type: String, + /// User or system that triggered the event + pub user: Option, + /// Description of the event + pub description: String, + /// Additional metadata + pub metadata: HashMap, +} diff --git a/trading_engine/src/comprehensive_performance_benchmarks.rs b/trading_engine/src/comprehensive_performance_benchmarks.rs index d39f69031..9ac96ba8e 100644 --- a/trading_engine/src/comprehensive_performance_benchmarks.rs +++ b/trading_engine/src/comprehensive_performance_benchmarks.rs @@ -19,7 +19,9 @@ use std::thread; use std::time::{Duration, Instant}; use crate::lockfree::{ - message_types, BatchMode, HftMessage, MPSCQueue, SharedMemoryChannel, SmallBatchRing, + message_types, HftMessage, SharedMemoryChannel, + mpsc_queue::MPSCQueue, + small_batch_ring::{BatchMode, SmallBatchRing}, }; use crate::simd::{AlignedPrices, AlignedVolumes, SimdMarketDataOps, SimdPriceOps, SimdRiskEngine}; use crate::timing::{calibrate_tsc, HardwareTimestamp, LatencyMeasurement}; diff --git a/trading_engine/src/events/event_processor_refactored.rs b/trading_engine/src/events/event_processor_refactored.rs index 1c3a2619b..fb9d8da98 100644 --- a/trading_engine/src/events/event_processor_refactored.rs +++ b/trading_engine/src/events/event_processor_refactored.rs @@ -391,8 +391,7 @@ impl RepositoryBackedWriter { } } -/// Re-use existing EventMetrics from the original module -pub use super::{EventMetrics, HealthMonitor, HealthStatus}; +// Removed pub use - import EventMetrics, HealthMonitor, HealthStatus directly where needed #[cfg(test)] mod tests { diff --git a/trading_engine/src/events/mod.rs b/trading_engine/src/events/mod.rs index b26402a17..f8edfc975 100644 --- a/trading_engine/src/events/mod.rs +++ b/trading_engine/src/events/mod.rs @@ -77,9 +77,7 @@ pub mod postgres_writer; pub mod ring_buffer; // Re-export key types for convenience -pub use event_types::{EventLevel, EventMetadata, EventSequence, SystemEventType, TradingEvent}; -pub use postgres_writer::{BatchProcessor, PostgresWriter, WriterConfig, WriterStats}; -pub use ring_buffer::{BufferManager, BufferStats, EventRingBuffer}; +// DO NOT RE-EXPORT - Use explicit imports at usage sites /// Configuration for the event processing pipeline #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/trading_engine/src/events/mod.rs.bak b/trading_engine/src/events/mod.rs.bak new file mode 100644 index 000000000..b26402a17 --- /dev/null +++ b/trading_engine/src/events/mod.rs.bak @@ -0,0 +1,775 @@ +#![allow(clippy::mod_module_files)] // Events module structure is more maintainable +//! High-Performance Event Processing Pipeline for Trading Service +//! +//! This module provides ultra-low latency event capture and reliable PostgreSQL persistence +//! for compliance logging while maintaining sub-microsecond event capture performance. +//! +//! ## Architecture Overview +//! +//! ```text +//! ┌─────────────────────────────────────────────────────────────────────┐ +//! │ Event Processing Pipeline Architecture │ +//! ├─────────────────────────────────────────────────────────────────────┤ +//! │ Producer Threads: Sub-μs Event Capture (Lock-Free Ring Buffers) │ +//! ├─────────────────────────────────────────────────────────────────────┤ +//! │ Buffer Management: Multiple Ring Buffers + Sequence Numbers │ +//! ├─────────────────────────────────────────────────────────────────────┤ +//! │ Async Writer Pool: Batched PostgreSQL Inserts + Error Recovery │ +//! ├─────────────────────────────────────────────────────────────────────┤ +//! │ Storage Layer: PostgreSQL with Write-Behind + WAL Persistence │ +//! └─────────────────────────────────────────────────────────────────────┘ +//! ``` +//! +//! ## Performance Characteristics +//! +//! - **Event Capture**: Sub-microsecond lock-free event recording +//! - **Memory Allocation**: Zero allocation in hot path +//! - **Batch Processing**: Configurable batch sizes (1-10000 events) +//! - **Recovery**: Guaranteed delivery with sequence number tracking +//! - **Monitoring**: Real-time metrics and health monitoring +//! +//! ## Core Components +//! +//! - `EventCapture`: Lock-free event recording with hardware timestamps +//! - `RingBufferManager`: Multiple ring buffers with load balancing +//! - `PostgresWriter`: Async batched database writer with error recovery +//! - `EventTypes`: Type-safe event definitions with serialization +//! +//! ## Usage Example +//! +//! ```rust +//! use core::events::{EventProcessor, EventProcessorConfig, TradingEvent}; +//! use core::timing::HardwareTimestamp; +//! +//! // Initialize event processor +//! let config = EventProcessorConfig::default(); +//! let processor = EventProcessor::new(config).await?; +//! +//! // Capture high-frequency trading events +//! let event = TradingEvent::OrderSubmitted { +//! order_id: "ORD-12345".to_string(), +//! symbol: "EURUSD".to_string(), +//! quantity: rust_decimal::Decimal::new(100000, 0), +//! price: rust_decimal::Decimal::new(10850, 4), +//! timestamp: HardwareTimestamp::now(), +//! }; +//! +//! // Sub-microsecond event capture +//! processor.capture_event(event).await?; +//! ``` + +use anyhow::{anyhow, Result}; +use serde::{Deserialize, Serialize}; +use sqlx::{postgres::PgPoolOptions, PgPool}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Duration; +use thiserror::Error; +use tokio::sync::RwLock; +use tokio::time::sleep; + +// Import timing infrastructure +use crate::timing::HardwareTimestamp; + +// Re-export core modules +pub mod event_types; +pub mod postgres_writer; +pub mod ring_buffer; + +// Re-export key types for convenience +pub use event_types::{EventLevel, EventMetadata, EventSequence, SystemEventType, TradingEvent}; +pub use postgres_writer::{BatchProcessor, PostgresWriter, WriterConfig, WriterStats}; +pub use ring_buffer::{BufferManager, BufferStats, EventRingBuffer}; + +/// Configuration for the event processing pipeline +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EventProcessorConfig { + /// `PostgreSQL` connection string + pub database_url: String, + /// Number of ring buffers for load balancing + pub buffer_count: usize, + /// Size of each ring buffer (must be power of 2) + pub buffer_size: usize, + /// Maximum batch size for database inserts + pub batch_size: usize, + /// Batch timeout in milliseconds + pub batch_timeout_ms: u64, + /// Number of writer threads + pub writer_threads: usize, + /// Maximum database connections + pub max_db_connections: u32, + /// Connection timeout in seconds + pub db_timeout_seconds: u64, + /// Enable compression for large events + pub enable_compression: bool, + /// Maximum memory usage before applying backpressure (bytes) + pub max_memory_usage: usize, + /// Enable detailed monitoring + pub enable_monitoring: bool, + /// Retry attempts for failed writes + pub max_retry_attempts: usize, + /// Retry delay base in milliseconds + pub retry_delay_ms: u64, +} + +impl Default for EventProcessorConfig { + fn default() -> Self { + Self { + database_url: "postgresql://foxhunt:foxhunt@localhost/trading_events".to_owned(), + buffer_count: num_cpus::get().max(4), + buffer_size: 8192, // 8K events per buffer + batch_size: 1000, + batch_timeout_ms: 10, + writer_threads: 2, + max_db_connections: 20, + db_timeout_seconds: 30, + enable_compression: true, + max_memory_usage: 100 * 1024 * 1024, // 100MB + enable_monitoring: true, + max_retry_attempts: 3, + retry_delay_ms: 100, + } + } +} + +/// High-performance event processor with guaranteed delivery +pub struct EventProcessor { + /// Buffer manager for load balancing across multiple ring buffers + buffer_manager: Arc, + /// `PostgreSQL` connection pool + db_pool: PgPool, + /// Async writer pool + writers: Vec>, + /// Global sequence number generator + sequence_generator: Arc, + /// Shutdown signal + shutdown: Arc, + /// Performance monitoring + metrics: Arc, + /// Health monitor + health_monitor: Arc, +} + +impl EventProcessor { + /// Create a new event processor with the given configuration + pub async fn new(config: EventProcessorConfig) -> Result { + tracing::info!("Initializing event processor with config: {:?}", config); + + // Create PostgreSQL connection pool + let db_pool = PgPoolOptions::new() + .max_connections(config.max_db_connections) + .min_connections(2) + .acquire_timeout(Duration::from_secs(config.db_timeout_seconds)) + .idle_timeout(Duration::from_secs(300)) + .max_lifetime(Duration::from_secs(1800)) + .test_before_acquire(true) + .connect(&config.database_url) + .await + .map_err(|e| anyhow!("Failed to connect to PostgreSQL: {}", e))?; + + // Initialize database schema + Self::initialize_schema(&db_pool).await?; + + // Create buffer manager + let buffer_manager = Arc::new(BufferManager::new(config.buffer_count, config.buffer_size)?); + + // Create metrics and monitoring + let metrics = Arc::new(EventMetrics::new()); + let health_monitor = Arc::new(HealthMonitor::new()); + + // Create PostgreSQL writers + let mut writers = Vec::with_capacity(config.writer_threads); + for i in 0..config.writer_threads { + let writer_config = WriterConfig { + batch_size: config.batch_size, + batch_timeout: Duration::from_millis(config.batch_timeout_ms), + max_retry_attempts: config.max_retry_attempts, + retry_delay: Duration::from_millis(config.retry_delay_ms), + enable_compression: config.enable_compression, + thread_id: i, + }; + + let writer = Arc::new( + PostgresWriter::new(writer_config, db_pool.clone(), metrics.clone()).await?, + ); + + writers.push(writer); + } + + let processor = Self { + buffer_manager, + db_pool, + writers, + sequence_generator: Arc::new(AtomicU64::new(1)), + shutdown: Arc::new(AtomicBool::new(false)), + metrics, + health_monitor, + }; + + // Start background processing tasks + processor.start_background_tasks().await?; + + tracing::info!("Event processor initialized successfully"); + Ok(processor) + } + + /// Capture a trading event with sub-microsecond latency + #[inline(always)] + pub async fn capture_event(&self, mut event: TradingEvent) -> Result { + let start_time = HardwareTimestamp::now(); + + // Generate global sequence number + let sequence_number = self.sequence_generator.fetch_add(1, Ordering::Relaxed); + + // Add metadata + event.set_sequence_number(sequence_number); + event.set_capture_timestamp(start_time); + + // Find optimal buffer (load balancing) + let buffer_index = self.buffer_manager.select_buffer(); + + // Attempt to store in ring buffer (lock-free) + let result = self.buffer_manager.try_push(buffer_index, event).await; + + // Update metrics + let capture_latency = HardwareTimestamp::now().latency_ns(&start_time); + self.metrics.record_capture_latency(capture_latency); + + match result { + Ok(seq) => { + self.metrics.increment_events_captured(); + Ok(seq) + } + Err(e) => { + self.metrics.increment_events_dropped(); + Err(anyhow!("Failed to capture event: {}", e)) + } + } + } + + /// Initialize the `PostgreSQL` database schema + async fn initialize_schema(pool: &PgPool) -> Result<()> { + tracing::info!("Initializing database schema"); + + // Create extension for better performance + sqlx::query( + " + CREATE EXTENSION IF NOT EXISTS pg_stat_statements; + ", + ) + .execute(pool) + .await + .map_err(|e| anyhow!("Failed to create extensions: {}", e))?; + + // Create trading events table with optimal indexing + sqlx::query( + " + CREATE TABLE IF NOT EXISTS trading_events ( + id BIGSERIAL PRIMARY KEY, + sequence_number BIGINT NOT NULL UNIQUE, + event_type VARCHAR(50) NOT NULL, + event_level VARCHAR(20) NOT NULL DEFAULT 'INFO', + timestamp_ns BIGINT NOT NULL, + capture_timestamp_ns BIGINT NOT NULL, + processing_timestamp_ns BIGINT, + symbol VARCHAR(20), + order_id VARCHAR(50), + trade_id VARCHAR(50), + price DECIMAL(20,8), + quantity DECIMAL(20,8), + side VARCHAR(10), + event_data JSONB NOT NULL, + compressed_data BYTEA, + metadata JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + INDEX (sequence_number), + INDEX (timestamp_ns), + INDEX (event_type, timestamp_ns), + INDEX (symbol, timestamp_ns), + INDEX (order_id) WHERE order_id IS NOT NULL, + INDEX (trade_id) WHERE trade_id IS NOT NULL + ); + ", + ) + .execute(pool) + .await + .map_err(|e| anyhow!("Failed to create trading_events table: {}", e))?; + + // Create sequence tracking table for recovery + sqlx::query( + " + CREATE TABLE IF NOT EXISTS event_sequence_tracking ( + partition_id INTEGER PRIMARY KEY, + last_processed_sequence BIGINT NOT NULL DEFAULT 0, + last_updated TIMESTAMPTZ NOT NULL DEFAULT NOW() + ); + ", + ) + .execute(pool) + .await + .map_err(|e| anyhow!("Failed to create sequence tracking table: {}", e))?; + + // Create performance monitoring table + sqlx::query( + " + CREATE TABLE IF NOT EXISTS event_processing_stats ( + timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW(), + events_per_second BIGINT NOT NULL, + avg_capture_latency_ns BIGINT NOT NULL, + avg_write_latency_ms DECIMAL(10,3) NOT NULL, + buffer_utilization DECIMAL(5,2) NOT NULL, + failed_writes BIGINT NOT NULL DEFAULT 0, + retried_writes BIGINT NOT NULL DEFAULT 0 + ); + ", + ) + .execute(pool) + .await + .map_err(|e| anyhow!("Failed to create stats table: {}", e))?; + + // Create indexes for optimal query performance + sqlx::query( + " + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_trading_events_timestamp_ns + ON trading_events (timestamp_ns DESC); + ", + ) + .execute(pool) + .await + .map_err(|e| anyhow!("Failed to create timestamp index: {}", e))?; + + sqlx::query( + " + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_trading_events_symbol_timestamp + ON trading_events (symbol, timestamp_ns DESC) + WHERE symbol IS NOT NULL; + ", + ) + .execute(pool) + .await + .map_err(|e| anyhow!("Failed to create symbol index: {}", e))?; + + tracing::info!("Database schema initialized successfully"); + Ok(()) + } + + /// Start background processing tasks + async fn start_background_tasks(&self) -> Result<()> { + let shutdown = self.shutdown.clone(); + let buffer_manager = self.buffer_manager.clone(); + let writers = self.writers.clone(); + let metrics = self.metrics.clone(); + let _health_monitor = self.health_monitor.clone(); + + // Start buffer-to-writer routing task + tokio::spawn(async move { + Self::buffer_router_task(shutdown, buffer_manager, writers, metrics).await; + }); + + // Start health monitoring task + let shutdown_monitor = self.shutdown.clone(); + let health_monitor_clone = self.health_monitor.clone(); + let metrics_clone = self.metrics.clone(); + tokio::spawn(async move { + Self::health_monitor_task(shutdown_monitor, health_monitor_clone, metrics_clone).await; + }); + + // Start metrics reporting task + let shutdown_metrics = self.shutdown.clone(); + let metrics_reporting = self.metrics.clone(); + let db_pool_metrics = self.db_pool.clone(); + tokio::spawn(async move { + Self::metrics_reporting_task(shutdown_metrics, metrics_reporting, db_pool_metrics) + .await; + }); + + Ok(()) + } + + /// Background task to route events from buffers to writers + async fn buffer_router_task( + shutdown: Arc, + buffer_manager: Arc, + writers: Vec>, + metrics: Arc, + ) { + let mut writer_index = 0; + + while !shutdown.load(Ordering::Relaxed) { + let mut events_routed = 0; + + // Check all buffers for events + for buffer_id in 0..buffer_manager.buffer_count() { + if let Some(events) = buffer_manager.drain_buffer(buffer_id, 100).await { + if !events.is_empty() { + // Round-robin distribution to writers + let writer = &writers[writer_index % writers.len()]; + + // Send batch to writer + if let Err(e) = writer.submit_batch(events).await { + tracing::error!( + "Failed to submit batch to writer {}: {}", + writer_index, + e + ); + metrics.increment_routing_errors(); + } else { + events_routed += 1; + } + + writer_index = (writer_index + 1) % writers.len(); + } + } + } + + // Short sleep to prevent busy waiting + if events_routed == 0 { + sleep(Duration::from_micros(100)).await; + } + } + } + + /// Background health monitoring task + async fn health_monitor_task( + shutdown: Arc, + health_monitor: Arc, + metrics: Arc, + ) { + while !shutdown.load(Ordering::Relaxed) { + // Update health status + health_monitor.update_health(metrics.get_snapshot()).await; + + // Sleep for 1 second between health checks + sleep(Duration::from_secs(1)).await; + } + } + + /// Background metrics reporting task + async fn metrics_reporting_task( + shutdown: Arc, + metrics: Arc, + db_pool: PgPool, + ) { + while !shutdown.load(Ordering::Relaxed) { + // Log metrics to database every 30 seconds + if let Err(e) = Self::persist_metrics(&metrics, &db_pool).await { + tracing::error!("Failed to persist metrics: {}", e); + } + + sleep(Duration::from_secs(30)).await; + } + } + + /// Persist metrics to database + async fn persist_metrics(metrics: &EventMetrics, db_pool: &PgPool) -> Result<()> { + let snapshot = metrics.get_snapshot(); + + sqlx::query( + " + INSERT INTO event_processing_stats ( + events_per_second, + avg_capture_latency_ns, + avg_write_latency_ms, + buffer_utilization, + failed_writes, + retried_writes + ) VALUES ($1, $2, $3, $4, $5, $6) + ", + ) + .bind(snapshot.events_per_second as i64) + .bind(snapshot.avg_capture_latency_ns as i64) + .bind(snapshot.avg_write_latency_ms) + .bind(snapshot.buffer_utilization) + .bind(snapshot.failed_writes as i64) + .bind(snapshot.retried_writes as i64) + .execute(db_pool) + .await + .map_err(|e| anyhow!("Failed to insert metrics: {}", e))?; + + Ok(()) + } + + /// Get current performance metrics + pub fn get_metrics(&self) -> EventMetricsSnapshot { + self.metrics.get_snapshot() + } + + /// Get health status + pub async fn get_health(&self) -> HealthStatus { + self.health_monitor.get_status().await + } + + /// Get buffer statistics + pub async fn get_buffer_stats(&self) -> Vec { + self.buffer_manager.get_all_stats().await + } + + /// Graceful shutdown + pub async fn shutdown(&self) -> Result<()> { + tracing::info!("Initiating graceful shutdown"); + + // Set shutdown flag + self.shutdown.store(true, Ordering::Relaxed); + + // Wait for writers to finish processing + for writer in &self.writers { + writer.shutdown().await?; + } + + // Drain remaining buffers + self.buffer_manager.drain_all_buffers().await?; + + // Close database connections + self.db_pool.close().await; + + tracing::info!("Event processor shutdown complete"); + Ok(()) + } +} + +/// Real-time performance metrics +#[derive(Debug)] +pub struct EventMetrics { + events_captured: AtomicU64, + events_dropped: AtomicU64, + events_written: AtomicU64, + routing_errors: AtomicU64, + capture_latency_sum: AtomicU64, + capture_latency_count: AtomicU64, + write_latency_sum: AtomicU64, + write_latency_count: AtomicU64, + failed_writes: AtomicU64, + retried_writes: AtomicU64, + start_time: std::time::Instant, +} + +impl EventMetrics { + pub fn new() -> Self { + Self { + events_captured: AtomicU64::new(0), + events_dropped: AtomicU64::new(0), + events_written: AtomicU64::new(0), + routing_errors: AtomicU64::new(0), + capture_latency_sum: AtomicU64::new(0), + capture_latency_count: AtomicU64::new(0), + write_latency_sum: AtomicU64::new(0), + write_latency_count: AtomicU64::new(0), + failed_writes: AtomicU64::new(0), + retried_writes: AtomicU64::new(0), + start_time: std::time::Instant::now(), + } + } + + pub fn increment_events_captured(&self) { + self.events_captured.fetch_add(1, Ordering::Relaxed); + } + + pub fn increment_events_dropped(&self) { + self.events_dropped.fetch_add(1, Ordering::Relaxed); + } + + pub fn increment_events_written(&self, count: u64) { + self.events_written.fetch_add(count, Ordering::Relaxed); + } + + pub fn increment_routing_errors(&self) { + self.routing_errors.fetch_add(1, Ordering::Relaxed); + } + + pub fn record_capture_latency(&self, latency_ns: u64) { + self.capture_latency_sum + .fetch_add(latency_ns, Ordering::Relaxed); + self.capture_latency_count.fetch_add(1, Ordering::Relaxed); + } + + pub fn record_write_latency(&self, latency_ms: f64) { + let latency_us = (latency_ms * 1000.0) as u64; + self.write_latency_sum + .fetch_add(latency_us, Ordering::Relaxed); + self.write_latency_count.fetch_add(1, Ordering::Relaxed); + } + + pub fn increment_failed_writes(&self) { + self.failed_writes.fetch_add(1, Ordering::Relaxed); + } + + pub fn increment_retried_writes(&self) { + self.retried_writes.fetch_add(1, Ordering::Relaxed); + } + + pub fn get_snapshot(&self) -> EventMetricsSnapshot { + let events_captured = self.events_captured.load(Ordering::Relaxed); + let elapsed_secs = self.start_time.elapsed().as_secs_f64(); + let events_per_second = if elapsed_secs > 0.0 { + (events_captured as f64 / elapsed_secs) as u64 + } else { + 0 + }; + + let capture_count = self.capture_latency_count.load(Ordering::Relaxed); + let avg_capture_latency_ns = if capture_count > 0 { + self.capture_latency_sum.load(Ordering::Relaxed) / capture_count + } else { + 0 + }; + + let write_count = self.write_latency_count.load(Ordering::Relaxed); + let avg_write_latency_ms = if write_count > 0 { + (self.write_latency_sum.load(Ordering::Relaxed) / write_count) as f64 / 1000.0 + } else { + 0.0 + }; + + EventMetricsSnapshot { + events_captured, + events_dropped: self.events_dropped.load(Ordering::Relaxed), + events_written: self.events_written.load(Ordering::Relaxed), + routing_errors: self.routing_errors.load(Ordering::Relaxed), + events_per_second, + avg_capture_latency_ns, + avg_write_latency_ms, + buffer_utilization: 0.0, // Updated by buffer manager + failed_writes: self.failed_writes.load(Ordering::Relaxed), + retried_writes: self.retried_writes.load(Ordering::Relaxed), + } + } +} + +/// Snapshot of event processing metrics +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EventMetricsSnapshot { + pub events_captured: u64, + pub events_dropped: u64, + pub events_written: u64, + pub routing_errors: u64, + pub events_per_second: u64, + pub avg_capture_latency_ns: u64, + pub avg_write_latency_ms: f64, + pub buffer_utilization: f64, + pub failed_writes: u64, + pub retried_writes: u64, +} + +/// Health monitoring for the event processing system +#[derive(Debug)] +pub struct HealthMonitor { + status: RwLock, +} + +impl HealthMonitor { + pub fn new() -> Self { + Self { + status: RwLock::new(HealthStatus::Healthy), + } + } + + pub async fn update_health(&self, metrics: EventMetricsSnapshot) { + let mut status = self.status.write().await; + + // Determine health based on metrics + *status = if metrics.events_dropped > metrics.events_captured / 10 { + HealthStatus::Degraded("High event drop rate".to_owned()) + } else if metrics.avg_capture_latency_ns > 10_000 { + HealthStatus::Degraded("High capture latency".to_owned()) + } else if metrics.avg_write_latency_ms > 100.0 { + HealthStatus::Degraded("High write latency".to_owned()) + } else if metrics.failed_writes > 0 { + HealthStatus::Warning("Database write failures detected".to_owned()) + } else { + HealthStatus::Healthy + }; + } + + pub async fn get_status(&self) -> HealthStatus { + self.status.read().await.clone() + } +} + +/// Health status of the event processing system +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum HealthStatus { + Healthy, + Warning(String), + Degraded(String), + Critical(String), +} + +/// Errors that can occur during event processing +#[derive(Debug, Error)] +pub enum EventProcessingError { + #[error("Database error: {0}")] + Database(#[from] sqlx::Error), + #[error("Buffer full: {0}")] + BufferFull(String), + #[error("Serialization error: {0}")] + Serialization(#[from] serde_json::Error), + #[error("Compression error: {0}")] + Compression(String), + #[error("Configuration error: {0}")] + Configuration(String), + #[error("Timeout error: {0}")] + Timeout(String), + #[error("Writer error: {0}")] + Writer(String), +} + +/// Type alias for event processing results + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_event_processor_creation() -> Result<()> { + // Create test configuration with in-memory database + let config = EventProcessorConfig { + database_url: "postgresql://test:test@localhost/test_db".to_string(), + buffer_count: 2, + buffer_size: 64, + batch_size: 10, + ..Default::default() + }; + + // This test would require a real PostgreSQL database + // In a real test environment, you would set up a test database + + Ok(()) + } + + #[tokio::test] + async fn test_event_metrics() { + let metrics = EventMetrics::new(); + + metrics.increment_events_captured(); + metrics.record_capture_latency(500); + + let snapshot = metrics.get_snapshot(); + assert_eq!(snapshot.events_captured, 1); + assert_eq!(snapshot.avg_capture_latency_ns, 500); + } + + #[tokio::test] + async fn test_health_monitor() { + let monitor = HealthMonitor::new(); + + let metrics = EventMetricsSnapshot { + events_captured: 1000, + events_dropped: 50, + events_written: 950, + routing_errors: 0, + events_per_second: 1000, + avg_capture_latency_ns: 500, + avg_write_latency_ms: 5.0, + buffer_utilization: 0.5, + failed_writes: 0, + retried_writes: 0, + }; + + monitor.update_health(metrics).await; + + match monitor.get_status().await { + HealthStatus::Healthy => {} + _ => panic!("Expected healthy status"), + } + } +} diff --git a/trading_engine/src/events/ring_buffer.rs b/trading_engine/src/events/ring_buffer.rs index e91a66666..e2b16e21c 100644 --- a/trading_engine/src/events/ring_buffer.rs +++ b/trading_engine/src/events/ring_buffer.rs @@ -5,7 +5,7 @@ use super::event_types::{EventSequence, TradingEvent}; use super::EventProcessingError; -use crate::lockfree::LockFreeRingBuffer; +use crate::lockfree::ring_buffer::LockFreeRingBuffer; use crate::timing::HardwareTimestamp; use anyhow::{anyhow, Result}; use serde::{Deserialize, Serialize}; diff --git a/trading_engine/src/features/mod.rs b/trading_engine/src/features/mod.rs index efb441fe7..4bf417012 100644 --- a/trading_engine/src/features/mod.rs +++ b/trading_engine/src/features/mod.rs @@ -68,7 +68,7 @@ pub mod unified_extractor; use async_trait::async_trait; // Re-export the main types for convenient access -pub use unified_extractor::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites AnalystRating, // Base feature components BaseMarketFeatures, diff --git a/trading_engine/src/features/mod.rs.bak b/trading_engine/src/features/mod.rs.bak new file mode 100644 index 000000000..efb441fe7 --- /dev/null +++ b/trading_engine/src/features/mod.rs.bak @@ -0,0 +1,410 @@ +//! Features Module - Core Feature Engineering +//! +//! This module provides the unified feature extraction system that ensures +//! zero training/serving skew across all ML models and trading stages. +//! +//! ## Key Components +//! +//! - `UnifiedFeatureExtractor`: Single source of truth for all feature calculations +//! - Model-specific feature sets: TLOB, MAMBA, DQN, PPO, Liquid, TFT +//! - Data provider integration: Databento (market data) + Benzinga (news/sentiment) +//! - High-performance SIMD optimizations for real-time processing +//! +//! ## Architecture Principles +//! +//! 1. **Single Source of Truth**: All features calculated identically across: +//! - Training: Historical data processing +//! - Backtesting: Strategy validation +//! - Live Trading: Real-time inference +//! +//! 2. **Data Provider Separation**: +//! - Databento: Market microstructure (trades, quotes, order books) +//! - Benzinga: News sentiment, analyst ratings, unusual options +//! +//! 3. **Model-Specific Features**: +//! - TLOB: Order book sequences for transformer analysis +//! - MAMBA: Long sequences for state space modeling +//! - DQN: State representation for reinforcement learning +//! - PPO: Policy-specific features with advantage estimation +//! - Liquid: Adaptive features for regime detection +//! - TFT: Multi-horizon sequences with attention inputs +//! +//! ## Usage Example +//! +//! ```rust +//! use core::features::{UnifiedFeatureExtractor, UnifiedConfig}; +//! +//! let config = UnifiedConfig::default(); +//! let mut extractor = UnifiedFeatureExtractor::new(config); +//! +//! // Extract TLOB features for transformer model +//! let tlob_features = extractor.extract_tlob_features( +//! &symbol, +//! &databento_data, +//! &benzinga_data, +//! &historical_data +//! ).await?; +//! +//! // Extract DQN features for reinforcement learning +//! let dqn_features = extractor.extract_dqn_features( +//! &symbol, +//! &databento_data, +//! &benzinga_data, +//! &historical_data, +//! current_position, +//! unrealized_pnl +//! ).await?; +//! ``` +//! +//! ## Performance Characteristics +//! +//! - **Latency**: Sub-millisecond feature extraction via SIMD +//! - **Throughput**: 10,000+ symbols processed per second +//! - **Memory**: Efficient caching with configurable TTL +//! - **Accuracy**: Identical calculations across all environments + +pub mod unified_extractor; + +use async_trait::async_trait; + +// Re-export the main types for convenient access +pub use unified_extractor::{ + AnalystRating, + // Base feature components + BaseMarketFeatures, + BenzingaNewsData, + BenzingaNewsFeatures, + + DQNFeatures, + // Data provider structures + DatabentoBuData, + DatabentoBuFeatures, + FeatureError, + + LiquidFeatures, + MAMBAFeatures, + NewsArticle, + PPOFeatures, + SentimentScore, + TFTFeatures, + + // Model-specific feature sets + TLOBFeatures, + UnifiedConfig, + UnifiedFeatureExtractor, + UnusualOptionsActivity, +}; + +/// Feature extraction result type for ergonomic error handling +pub type FeatureResult = Result; + +/// Trait for model-specific feature extraction +#[async_trait] +pub trait ModelFeatureExtractor { + /// Extract features specific to this model type + async fn extract_features( + &mut self, + extractor: &mut UnifiedFeatureExtractor, + symbol: &common::types::Symbol, + databento_data: &DatabentoBuData, + benzinga_data: &BenzingaNewsData, + historical_data: &[common::types::MarketTick], + ) -> FeatureResult; +} + +/// Convenience macros for feature extraction +#[macro_export] +macro_rules! extract_features { + ($extractor:expr, $model:ident, $symbol:expr, $databento:expr, $benzinga:expr, $historical:expr) => { + $extractor.paste::paste! { + [] + }($symbol, $databento, $benzinga, $historical).await + }; + + ($extractor:expr, $model:ident, $symbol:expr, $databento:expr, $benzinga:expr, $historical:expr, $($extra:expr),+) => { + $extractor.paste::paste! { + [] + }($symbol, $databento, $benzinga, $historical, $($extra),+).await + }; +} + +/// Feature validation utilities +pub mod validation { + use super::{ + BaseMarketFeatures, BenzingaNewsFeatures, DatabentoBuFeatures, FeatureError, FeatureResult, + }; + + /// Validate feature quality and completeness + pub fn validate_base_features(features: &BaseMarketFeatures) -> FeatureResult<()> { + // Check for NaN/Inf values + if !features.returns_1m.is_finite() { + return Err(FeatureError::MathematicalError { + feature: "returns_1m".to_owned(), + reason: "Non-finite value detected".to_owned(), + }); + } + + // Validate ranges + if features.rsi_14 < 0.0 || features.rsi_14 > 100.0 { + return Err(FeatureError::MathematicalError { + feature: "rsi_14".to_owned(), + reason: format!("RSI out of range: {}", features.rsi_14), + }); + } + + // Check bollinger position is within reasonable bounds + if features.bollinger_position < -5.0 || features.bollinger_position > 5.0 { + return Err(FeatureError::MathematicalError { + feature: "bollinger_position".to_owned(), + reason: format!( + "Bollinger position extreme: {}", + features.bollinger_position + ), + }); + } + + Ok(()) + } + + /// Validate Databento features + pub fn validate_databento_features(features: &DatabentoBuFeatures) -> FeatureResult<()> { + // Spread should be positive + if features.bid_ask_spread_bps < 0.0 { + return Err(FeatureError::MathematicalError { + feature: "bid_ask_spread_bps".to_owned(), + reason: "Negative spread detected".to_owned(), + }); + } + + // Order book imbalance should be in [-1, 1] + if features.order_book_imbalance < -1.0 || features.order_book_imbalance > 1.0 { + return Err(FeatureError::MathematicalError { + feature: "order_book_imbalance".to_owned(), + reason: format!("Imbalance out of range: {}", features.order_book_imbalance), + }); + } + + // Trade sign should be -1, 0, or 1 + if ![-1, 0, 1].contains(&features.trade_sign) { + return Err(FeatureError::MathematicalError { + feature: "trade_sign".to_owned(), + reason: format!("Invalid trade sign: {}", features.trade_sign), + }); + } + + Ok(()) + } + + /// Validate Benzinga sentiment features + pub fn validate_benzinga_features(features: &BenzingaNewsFeatures) -> FeatureResult<()> { + // Sentiment score should be in [-1, 1] + if features.sentiment_score < -1.0 || features.sentiment_score > 1.0 { + return Err(FeatureError::MathematicalError { + feature: "sentiment_score".to_owned(), + reason: format!("Sentiment out of range: {}", features.sentiment_score), + }); + } + + // Confidence should be in [0, 1] + if features.sentiment_confidence < 0.0 || features.sentiment_confidence > 1.0 { + return Err(FeatureError::MathematicalError { + feature: "sentiment_confidence".to_owned(), + reason: format!("Confidence out of range: {}", features.sentiment_confidence), + }); + } + + // News velocity should be non-negative + if features.news_velocity < 0.0 { + return Err(FeatureError::MathematicalError { + feature: "news_velocity".to_owned(), + reason: "Negative news velocity".to_owned(), + }); + } + + Ok(()) + } +} + +/// Performance monitoring for feature extraction +pub mod monitoring { + use std::collections::HashMap; + use std::time::{Duration, Instant}; + + /// Feature extraction performance metrics + #[derive(Debug, Clone)] + pub struct FeatureMetrics { + pub extraction_time: Duration, + pub feature_count: usize, + pub cache_hits: usize, + pub cache_misses: usize, + pub validation_time: Duration, + } + + /// Performance monitor for feature extraction + pub struct FeatureMonitor { + metrics: HashMap>, + start_times: HashMap, + } + + impl FeatureMonitor { + pub fn new() -> Self { + Self { + metrics: HashMap::new(), + start_times: HashMap::new(), + } + } + + /// Start timing a feature extraction operation + pub fn start_timing(&mut self, operation: &str) { + self.start_times + .insert(operation.to_owned(), Instant::now()); + } + + /// End timing and record metrics + pub fn end_timing( + &mut self, + operation: &str, + feature_count: usize, + cache_hits: usize, + cache_misses: usize, + ) { + if let Some(start_time) = self.start_times.remove(operation) { + let extraction_time = start_time.elapsed(); + let metrics = FeatureMetrics { + extraction_time, + feature_count, + cache_hits, + cache_misses, + validation_time: Duration::from_nanos(0), // Set by validation + }; + + self.metrics + .entry(operation.to_owned()) + .or_insert_with(Vec::new) + .push(metrics); + } + } + + /// Get average extraction time for an operation + pub fn average_extraction_time(&self, operation: &str) -> Option { + self.metrics.get(operation).and_then(|metrics| { + if metrics.is_empty() { + return None; + } + + let total: Duration = metrics.iter().map(|m| m.extraction_time).sum(); + Some(total / metrics.len() as u32) + }) + } + + /// Get cache hit rate for an operation + pub fn cache_hit_rate(&self, operation: &str) -> Option { + self.metrics.get(operation).and_then(|metrics| { + if metrics.is_empty() { + return None; + } + + let total_hits: usize = metrics.iter().map(|m| m.cache_hits).sum(); + let total_requests: usize = + metrics.iter().map(|m| m.cache_hits + m.cache_misses).sum(); + + if total_requests == 0 { + None + } else { + Some(total_hits as f64 / total_requests as f64) + } + }) + } + } + + impl Default for FeatureMonitor { + fn default() -> Self { + Self::new() + } + } +} + +/// Testing utilities for feature validation +#[cfg(test)] +pub mod test_utils { + use super::*; + use common::types::*; + use chrono::Utc; + + /// Create mock Databento data for testing + pub fn create_mock_databento_data() -> DatabentoBuData { + DatabentoBuData { + order_book: vec![ + OrderBookLevel { + price: Price::from_dollars(100.50), + size: Decimal::from(1000), + side: OrderSide::Bid, + }, + OrderBookLevel { + price: Price::from_dollars(100.51), + size: Decimal::from(800), + side: OrderSide::Ask, + }, + ], + trades: vec![Trade { + symbol: Symbol::new("AAPL"), + price: Price::from_dollars(100.505), + volume: Decimal::from(100), + timestamp: Utc::now(), + side: OrderSide::Buy, + trade_id: "T123".to_string(), + }], + quotes: vec![], + timestamp: Utc::now(), + } + } + + /// Create mock Benzinga data for testing + pub fn create_mock_benzinga_data() -> BenzingaNewsData { + BenzingaNewsData { + articles: vec![NewsArticle { + title: "Apple Reports Strong Q4 Earnings".to_string(), + content: "Apple exceeded expectations...".to_string(), + source: "Reuters".to_string(), + timestamp: Utc::now(), + symbols: vec![Symbol::new("AAPL")], + category: "earnings".to_string(), + importance: 0.8, + }], + sentiment_scores: vec![SentimentScore { + symbol: Symbol::new("AAPL"), + score: 0.6, + confidence: 0.9, + timestamp: Utc::now(), + }], + analyst_ratings: vec![], + unusual_options: vec![], + timestamp: Utc::now(), + } + } + + /// Create mock historical market data + pub fn create_mock_historical_data() -> Vec { + let base_price = 100.0; + let mut data = Vec::new(); + + for i in 0..1000 { + let price_change = (i as f64 / 100.0).sin() * 0.01; + let price = Price::from_dollars(base_price + price_change); + let volume = Decimal::from(1000 + (i % 500) as i64); + + data.push(MarketTick { + symbol: Symbol::new("AAPL"), + price, + volume, + timestamp: Utc::now() - chrono::Duration::seconds(1000 - i as i64), + bid: Some(price - Price::from_cents(1)), + ask: Some(price + Price::from_cents(1)), + bid_size: Some(volume), + ask_size: Some(volume), + }); + } + + data + } +} diff --git a/trading_engine/src/lockfree/atomic_ops.rs b/trading_engine/src/lockfree/atomic_ops.rs index 00873a235..fe06fbf8f 100644 --- a/trading_engine/src/lockfree/atomic_ops.rs +++ b/trading_engine/src/lockfree/atomic_ops.rs @@ -308,8 +308,7 @@ pub mod memory_fence { } } -// Re-export memory fence function at module level for convenience -pub use memory_fence::full as memory_fence; +// Removed pub use - use memory_fence::full directly where needed #[cfg(test)] mod tests { diff --git a/trading_engine/src/lockfree/mod.rs b/trading_engine/src/lockfree/mod.rs index e93f92705..76c221f30 100644 --- a/trading_engine/src/lockfree/mod.rs +++ b/trading_engine/src/lockfree/mod.rs @@ -52,11 +52,8 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; -// Re-export key types for easy access -pub use atomic_ops::{memory_fence, AtomicFlag, AtomicMetrics, MetricsSnapshot, SequenceGenerator}; -pub use mpsc_queue::{AtomicCounter, MPSCQueue}; -pub use ring_buffer::{LockFreeRingBuffer, SPSCQueue}; -pub use small_batch_ring::{BatchMode, SmallBatchOrdersSoA, SmallBatchRing}; +// Import key types for internal use +use crate::lockfree::ring_buffer::LockFreeRingBuffer; /// High-frequency trading message for inter-service communication #[repr(C)] diff --git a/trading_engine/src/lockfree/mod.rs.bak b/trading_engine/src/lockfree/mod.rs.bak new file mode 100644 index 000000000..e93f92705 --- /dev/null +++ b/trading_engine/src/lockfree/mod.rs.bak @@ -0,0 +1,286 @@ +#![allow(clippy::mod_module_files)] // Lock-free structures require modular organization +//! Memory-safe lock-free data structures for ultra-low latency HFT trading +//! +//! This module provides corrected lock-free implementations with proper memory ordering +//! to prevent data races and ensure correctness in high-frequency trading systems. +//! +//! ## Key Improvements +//! - Proper Acquire-Release memory ordering to prevent data races +//! - Hazard pointers to solve ABA problem in MPSC queue +//! - Memory-safe atomic operations with explicit ordering guarantees +//! - Comprehensive testing for concurrency correctness +//! +//! ## Available Structures +//! - `LockFreeRingBuffer`: SPSC queue optimized for single producer/consumer +//! - `MPSCQueue`: Multi-producer single-consumer queue with hazard pointers +//! - `AtomicCounter`: High-performance atomic counter with proper ordering +//! - `SequenceGenerator`: Monotonic sequence numbers for operation ordering + +#![deny( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::unimplemented, + clippy::todo, + clippy::unreachable, + clippy::indexing_slicing +)] +#![warn( + clippy::pedantic, + clippy::nursery, + clippy::perf, + clippy::complexity, + clippy::style, + clippy::correctness +)] +#![allow( + // Lock-free implementation allowances for HFT performance + clippy::module_name_repetitions, // Descriptive names for lock-free types + clippy::similar_names, // Memory ordering variables often have similar names + clippy::cast_possible_truncation, // Low-level atomic operations require type casts +)] + +// Re-export the corrected lock-free implementations +pub mod atomic_ops; +pub mod mpsc_queue; +pub mod ring_buffer; +pub mod small_batch_ring; + +// Legacy compatibility - keep original shared memory channel for existing code +// Note: alloc functions moved to individual modules where they're actually used +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +// Re-export key types for easy access +pub use atomic_ops::{memory_fence, AtomicFlag, AtomicMetrics, MetricsSnapshot, SequenceGenerator}; +pub use mpsc_queue::{AtomicCounter, MPSCQueue}; +pub use ring_buffer::{LockFreeRingBuffer, SPSCQueue}; +pub use small_batch_ring::{BatchMode, SmallBatchOrdersSoA, SmallBatchRing}; + +/// High-frequency trading message for inter-service communication +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub struct HftMessage { + pub msg_type: u32, + pub timestamp_ns: u64, + pub sequence: u64, + pub payload: [u64; 8], // 64 bytes of payload data +} + +impl HftMessage { + #[must_use] + pub fn new(msg_type: u32, payload: [u64; 8]) -> Self { + Self { + msg_type, + timestamp_ns: SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() as u64, + sequence: 0, + payload, + } + } +} + +/// Shared memory channel for bidirectional communication (UPDATED with corrected ring buffer) +pub struct SharedMemoryChannel { + pub producer_to_consumer: Arc>, + pub consumer_to_producer: Arc>, + pub stats: Arc, +} + +#[derive(Debug, Default)] +pub struct ChannelStats { + pub messages_sent: AtomicU64, + pub messages_received: AtomicU64, + pub send_failures: AtomicU64, + pub avg_latency_ns: AtomicU64, + pub max_latency_ns: AtomicU64, +} + +impl SharedMemoryChannel { + /// Create a new bidirectional shared memory channel + pub fn new(buffer_size: usize) -> Result { + Ok(Self { + producer_to_consumer: Arc::new(LockFreeRingBuffer::new(buffer_size)?), + consumer_to_producer: Arc::new(LockFreeRingBuffer::new(buffer_size)?), + stats: Arc::new(ChannelStats::default()), + }) + } + + /// Send message with latency tracking + #[inline(always)] + pub fn send(&self, message: HftMessage) -> Result<(), HftMessage> { + let start_ns = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() as u64; + + match self.producer_to_consumer.try_push(message) { + Ok(()) => { + let latency_ns = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() as u64 + - start_ns; + + self.stats.messages_sent.fetch_add(1, Ordering::Relaxed); + self.update_latency_stats(latency_ns); + Ok(()) + } + Err(msg) => { + self.stats.send_failures.fetch_add(1, Ordering::Relaxed); + Err(msg) + } + } + } + + /// Receive message (non-blocking) + #[inline(always)] + #[must_use] + pub fn try_receive(&self) -> Option { + if let Some(message) = self.producer_to_consumer.try_pop() { + self.stats.messages_received.fetch_add(1, Ordering::Relaxed); + Some(message) + } else { + None + } + } + + /// Update latency statistics + fn update_latency_stats(&self, latency_ns: u64) { + // Update average using exponential moving average + let current_avg = self.stats.avg_latency_ns.load(Ordering::Relaxed); + let new_avg = if current_avg == 0 { + latency_ns + } else { + // EMA with α = 0.1 + (current_avg * 9 + latency_ns) / 10 + }; + self.stats.avg_latency_ns.store(new_avg, Ordering::Relaxed); + + // Update maximum + loop { + let current_max = self.stats.max_latency_ns.load(Ordering::Relaxed); + if latency_ns <= current_max { + break; + } + if self + .stats + .max_latency_ns + .compare_exchange_weak( + current_max, + latency_ns, + Ordering::Relaxed, + Ordering::Relaxed, + ) + .is_ok() + { + break; + } + } + } + + /// Get channel performance statistics + #[must_use] + pub fn get_stats(&self) -> SharedMemoryStats { + SharedMemoryStats { + messages_sent: self.stats.messages_sent.load(Ordering::Relaxed), + messages_received: self.stats.messages_received.load(Ordering::Relaxed), + send_failures: self.stats.send_failures.load(Ordering::Relaxed), + avg_latency_ns: self.stats.avg_latency_ns.load(Ordering::Relaxed), + max_latency_ns: self.stats.max_latency_ns.load(Ordering::Relaxed), + buffer_utilization: self.producer_to_consumer.utilization(), + } + } +} + +#[derive(Debug, Clone)] +pub struct SharedMemoryStats { + pub messages_sent: u64, + pub messages_received: u64, + pub send_failures: u64, + pub avg_latency_ns: u64, + pub max_latency_ns: u64, + pub buffer_utilization: f64, +} + +/// Message types for HFT inter-service communication +pub mod message_types { + pub const ORDER_REQUEST: u32 = 1; + pub const ORDER_RESPONSE: u32 = 2; + pub const RISK_CHECK: u32 = 3; + pub const RISK_RESPONSE: u32 = 4; + pub const MARKET_DATA: u32 = 5; + pub const EXECUTION_REPORT: u32 = 6; + pub const HEARTBEAT: u32 = 7; +} + +#[cfg(test)] +mod tests { + use super::*; + use std::error::Error; + use std::thread; + use std::time::{Duration, Instant}; + // use crate::safe_operations; // DISABLED - module not found + + #[test] + fn test_corrected_lock_free_ring_buffer() -> Result<(), Box> { + let buffer = ring_buffer::LockFreeRingBuffer::::new(1024)?; + + // Test push/pop with corrected implementation + assert!(buffer.try_push(42).is_ok()); + assert_eq!(buffer.try_pop(), Some(42)); + assert_eq!(buffer.try_pop(), None); + + Ok(()) + } + + #[test] + fn test_shared_memory_channel() -> Result<(), Box> { + let channel = SharedMemoryChannel::new(1024)?; + let message = HftMessage::new(message_types::ORDER_REQUEST, [1, 2, 3, 4, 5, 6, 7, 8]); + + assert!(channel.send(message).is_ok()); + + if let Some(received) = channel.try_receive() { + assert_eq!(received.msg_type, message_types::ORDER_REQUEST); + assert_eq!(received.payload[0], 1); + } else { + return Err("Message not received".into()); + } + + Ok(()) + } + + #[test] + fn test_high_throughput() -> Result<(), Box> { + let channel = SharedMemoryChannel::new(8192)?; + let message = HftMessage::new(message_types::HEARTBEAT, [0; 8]); + + let start = Instant::now(); + for _ in 0..10000 { + if channel.send(message).is_err() { + thread::sleep(Duration::from_nanos(1)); + } + } + let duration = start.elapsed(); + + println!("Sent 10,000 messages in {:?}", duration); + println!("Average latency: {:?}", duration / 10000); + + // Verify performance meets HFT requirements (<1μs per operation) + let avg_latency_ns = duration.as_nanos() / 10000; + println!("Average latency: {}ns per operation", avg_latency_ns); + + // For HFT, we want sub-microsecond performance + assert!( + avg_latency_ns < 1000, + "Latency too high: {}ns > 1000ns", + avg_latency_ns + ); + + Ok(()) + } +} diff --git a/trading_engine/src/persistence/health.rs b/trading_engine/src/persistence/health.rs index e40063d83..196af702e 100644 --- a/trading_engine/src/persistence/health.rs +++ b/trading_engine/src/persistence/health.rs @@ -8,7 +8,12 @@ use std::time::{Duration, Instant}; use thiserror::Error; use tokio::time::timeout; -use super::{ClickHouseClient, InfluxClient, PostgresPool, RedisPool}; +use super::{ + clickhouse::ClickHouseClient, + influxdb::InfluxClient, + postgres::PostgresPool, + redis::RedisPool, +}; /// Health check errors #[derive(Debug, Error)] diff --git a/trading_engine/src/persistence/mod.rs b/trading_engine/src/persistence/mod.rs index e8c592606..6c084bc52 100644 --- a/trading_engine/src/persistence/mod.rs +++ b/trading_engine/src/persistence/mod.rs @@ -30,13 +30,7 @@ pub mod redis; #[cfg(test)] mod redis_integration_test; -pub use backup::create_full_backup; -pub use clickhouse::{ClickHouseClient, ClickHouseConfig, ClickHouseError}; -pub use health::{ComponentHealth, HealthStatus, PersistenceHealth, SystemStatus}; -pub use influxdb::{DataPoint, FieldValue, InfluxClient, InfluxConfig, InfluxError}; -pub use migrations::run_pending_migrations; -pub use postgres::{PostgresConfig, PostgresError, PostgresPool}; -pub use redis::{RedisConfig, RedisError, RedisPool}; +// DO NOT RE-EXPORT - Use explicit imports at usage sites use serde::{Deserialize, Serialize}; use std::time::Duration; diff --git a/trading_engine/src/persistence/mod.rs.bak b/trading_engine/src/persistence/mod.rs.bak new file mode 100644 index 000000000..e8c592606 --- /dev/null +++ b/trading_engine/src/persistence/mod.rs.bak @@ -0,0 +1,239 @@ +//! Core Persistence Layer for Foxhunt HFT Trading System +//! +//! This module provides the main database connectivity and data persistence +//! infrastructure for high-frequency trading operations. +//! +//! # Architecture +//! +//! ```text +//! ┌─────────────────────────────────────────────────────────────────┐ +//! │ Foxhunt Persistence Stack │ +//! ├─────────────────────────────────────────────────────────────────┤ +//! │ Trading Layer: Order Management, Position Tracking │ +//! ├─────────────────────────────────────────────────────────────────┤ +//! │ Persistence Layer: PostgreSQL, InfluxDB, Redis, ClickHouse │ +//! ├─────────────────────────────────────────────────────────────────┤ +//! │ Connection Management: Pools, Health Checks, Failover │ +//! ├─────────────────────────────────────────────────────────────────┤ +//! │ Performance Layer: Sub-1ms timeouts, Connection prewarming │ +//! └─────────────────────────────────────────────────────────────────┘ +//! ``` + +pub mod backup; +pub mod clickhouse; +pub mod health; +pub mod influxdb; +pub mod migrations; +pub mod postgres; +pub mod redis; + +#[cfg(test)] +mod redis_integration_test; + +pub use backup::create_full_backup; +pub use clickhouse::{ClickHouseClient, ClickHouseConfig, ClickHouseError}; +pub use health::{ComponentHealth, HealthStatus, PersistenceHealth, SystemStatus}; +pub use influxdb::{DataPoint, FieldValue, InfluxClient, InfluxConfig, InfluxError}; +pub use migrations::run_pending_migrations; +pub use postgres::{PostgresConfig, PostgresError, PostgresPool}; +pub use redis::{RedisConfig, RedisError, RedisPool}; + +use serde::{Deserialize, Serialize}; +use std::time::Duration; +use thiserror::Error; + +/// Core persistence configuration for all database systems +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct PersistenceConfig { + /// `PostgreSQL` configuration for main trading data + pub postgres: PostgresConfig, + /// `InfluxDB` configuration for time-series metrics + pub influx: InfluxConfig, + /// Redis configuration for caching and session data + pub redis: RedisConfig, + /// `ClickHouse` configuration for analytics (optional) + pub clickhouse: Option, + /// Global persistence settings + pub global: GlobalPersistenceConfig, +} + +/// Global persistence settings affecting all database connections +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct GlobalPersistenceConfig { + /// Environment (development, staging, production) + pub environment: String, + /// Enable detailed query logging for performance analysis + pub enable_query_logging: bool, + /// Enable connection pool monitoring + pub enable_pool_monitoring: bool, + /// Enable automatic health checks + pub enable_health_checks: bool, + /// Health check interval in seconds + pub health_check_interval_seconds: u64, + /// Maximum allowed query latency in microseconds for HFT operations + pub max_query_latency_micros: u64, +} + +impl Default for GlobalPersistenceConfig { + fn default() -> Self { + Self { + environment: "development".to_owned(), + enable_query_logging: true, + enable_pool_monitoring: true, + enable_health_checks: true, + health_check_interval_seconds: 30, + max_query_latency_micros: 800, // <1ms for HFT + } + } +} + +/// Unified error type for all persistence operations +#[derive(Debug, Error)] +pub enum PersistenceError { + #[error("PostgreSQL error: {0}")] + Postgres(#[from] PostgresError), + #[error("InfluxDB error: {0}")] + Influx(#[from] InfluxError), + #[error("Redis error: {0}")] + Redis(#[from] RedisError), + #[error("ClickHouse error: {0}")] + ClickHouse(#[from] ClickHouseError), + #[error("Configuration error: {0}")] + Configuration(String), + #[error("Health check failed: {0}")] + HealthCheck(String), + #[error( + "Performance violation: {operation} took {actual_micros}\u{3bc}s, max allowed {max_micros}\u{3bc}s" + )] + PerformanceViolation { + operation: String, + actual_micros: u64, + max_micros: u64, + }, +} + +/// Main persistence manager coordinating all database connections +pub struct PersistenceManager { + postgres: PostgresPool, + influx: InfluxClient, + redis: RedisPool, + clickhouse: Option, + config: PersistenceConfig, + health: PersistenceHealth, +} + +impl PersistenceManager { + /// Initialize the persistence manager with all database connections + pub async fn new(config: PersistenceConfig) -> Result { + // Initialize PostgreSQL connection pool for main trading data + let postgres = PostgresPool::new(config.postgres.clone()).await?; + + // Initialize InfluxDB client for time-series data + let influx = InfluxClient::new(config.influx.clone()).await?; + + // Initialize Redis connection pool for caching + let redis = RedisPool::new(config.redis.clone()).await?; + + // Initialize ClickHouse client if configured + let clickhouse = if let Some(ch_config) = &config.clickhouse { + Some(ClickHouseClient::new(ch_config.clone()).await?) + } else { + None + }; + + // Initialize health monitoring + let health = PersistenceHealth::new( + config.global.enable_health_checks, + Duration::from_secs(config.global.health_check_interval_seconds), + ); + + Ok(Self { + postgres, + influx, + redis, + clickhouse, + config, + health, + }) + } + + /// Get `PostgreSQL` connection pool + pub const fn postgres(&self) -> &PostgresPool { + &self.postgres + } + + /// Get `InfluxDB` client + pub const fn influx(&self) -> &InfluxClient { + &self.influx + } + + /// Get Redis connection pool + pub const fn redis(&self) -> &RedisPool { + &self.redis + } + + /// Get `ClickHouse` client (if configured) + pub const fn clickhouse(&self) -> Option<&ClickHouseClient> { + self.clickhouse.as_ref() + } + + /// Get persistence configuration + pub const fn config(&self) -> &PersistenceConfig { + &self.config + } + + /// Check health of all database connections + pub async fn health_check(&self) -> Result { + self.health + .check_all_systems( + &self.postgres, + &self.influx, + &self.redis, + self.clickhouse.as_ref(), + ) + .await + .map_err(|e| PersistenceError::Configuration(format!("Health check failed: {}", e))) + } + + /// Run database migrations on `PostgreSQL` + pub async fn run_migrations(&self) -> Result<(), PersistenceError> { + run_pending_migrations(self.postgres.pool()) + .await + .map(|_| ()) + .map_err(|e| PersistenceError::Configuration(format!("Migration failed: {}", e))) + } + + /// Perform backup operations + pub async fn backup(&self) -> Result<(), PersistenceError> { + create_full_backup(&self.config) + .await + .map(|_| ()) + .map_err(|e| PersistenceError::Configuration(format!("Backup failed: {}", e))) + } + + /// Get performance metrics from all systems + pub async fn get_performance_metrics(&self) -> Result { + Ok(PersistenceMetrics { + postgres: self.postgres.get_metrics().await?, + influx: self.influx.get_metrics().await?, + redis: self.redis.get_metrics().await?, + clickhouse: if let Some(ch) = &self.clickhouse { + Some(ch.get_metrics().await?) + } else { + None + }, + }) + } +} + +/// Performance metrics for all persistence systems +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PersistenceMetrics { + pub postgres: postgres::PostgresMetrics, + pub influx: influxdb::InfluxMetrics, + pub redis: redis::RedisMetrics, + pub clickhouse: Option, +} + +/// Result type for persistence operations +pub type PersistenceResult = Result; diff --git a/trading_engine/src/repositories/event_repository.rs b/trading_engine/src/repositories/event_repository.rs index cdd3f0c58..349b16d55 100644 --- a/trading_engine/src/repositories/event_repository.rs +++ b/trading_engine/src/repositories/event_repository.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; use std::sync::Arc; use thiserror::Error; -use crate::events::{EventMetricsSnapshot, TradingEvent}; +use crate::events::{EventMetricsSnapshot, event_types::TradingEvent}; // Removed unused prelude import - specific types imported as needed /// Errors that can occur in event repository operations diff --git a/trading_engine/src/repositories/mod.rs b/trading_engine/src/repositories/mod.rs index 4ca17eb98..aeb1a6eee 100644 --- a/trading_engine/src/repositories/mod.rs +++ b/trading_engine/src/repositories/mod.rs @@ -9,9 +9,7 @@ pub mod event_repository; pub mod migration_repository; // Re-export all repository traits -pub use compliance_repository::{ComplianceRepository, ComplianceRepositoryError}; -pub use event_repository::{EventRepository, EventRepositoryError}; -pub use migration_repository::{MigrationRepository, MigrationRepositoryError}; +// DO NOT RE-EXPORT - Use explicit imports at usage sites use async_trait::async_trait; use std::sync::Arc; diff --git a/trading_engine/src/repositories/mod.rs.bak b/trading_engine/src/repositories/mod.rs.bak new file mode 100644 index 000000000..4ca17eb98 --- /dev/null +++ b/trading_engine/src/repositories/mod.rs.bak @@ -0,0 +1,46 @@ +//! Repository Pattern Abstractions +//! +//! This module provides clean data access abstractions that eliminate direct database coupling +//! from business logic components. All database operations are encapsulated behind trait interfaces +//! that can be mocked, tested, and swapped out without affecting core business logic. + +pub mod compliance_repository; +pub mod event_repository; +pub mod migration_repository; + +// Re-export all repository traits +pub use compliance_repository::{ComplianceRepository, ComplianceRepositoryError}; +pub use event_repository::{EventRepository, EventRepositoryError}; +pub use migration_repository::{MigrationRepository, MigrationRepositoryError}; + +use async_trait::async_trait; +use std::sync::Arc; + +/// Repository factory for creating repository implementations +/// This allows dependency injection and easier testing +pub struct RepositoryFactory { + pub event_repository: Arc, + pub compliance_repository: Arc, + pub migration_repository: Arc, +} + +impl RepositoryFactory { + /// Create a new repository factory with provided implementations + pub fn new( + event_repository: Arc, + compliance_repository: Arc, + migration_repository: Arc, + ) -> Self { + Self { + event_repository, + compliance_repository, + migration_repository, + } + } +} + +/// Health check trait for repositories +#[async_trait] +pub trait HealthCheck: Send + Sync { + async fn health_check(&self) -> Result<(), String>; +} diff --git a/trading_engine/src/storage/mod.rs b/trading_engine/src/storage/mod.rs index 058abc6d6..d91bdd21a 100644 --- a/trading_engine/src/storage/mod.rs +++ b/trading_engine/src/storage/mod.rs @@ -10,7 +10,7 @@ //! - Archival and lifecycle management // Re-export from storage crate for backward compatibility -pub use storage::{ +// DO NOT RE-EXPORT - Use explicit imports at usage sites ArchivalDataType, ArchivalMetadata, ArchivalStats, S3Storage as S3ArchivalService, S3StorageConfig as S3ArchivalConfig, Storage as S3Archival, }; diff --git a/trading_engine/src/storage/mod.rs.bak b/trading_engine/src/storage/mod.rs.bak new file mode 100644 index 000000000..058abc6d6 --- /dev/null +++ b/trading_engine/src/storage/mod.rs.bak @@ -0,0 +1,16 @@ +//! Storage Management Module +//! +//! DEPRECATED: Storage functionality has been moved to the dedicated `storage` crate. +//! This module is kept for backward compatibility during migration. +//! +//! Please use `storage` crate for all storage operations: +//! - S3 operations with Vault integration +//! - Local filesystem storage +//! - Model checkpoint management +//! - Archival and lifecycle management + +// Re-export from storage crate for backward compatibility +pub use storage::{ + ArchivalDataType, ArchivalMetadata, ArchivalStats, S3Storage as S3ArchivalService, + S3StorageConfig as S3ArchivalConfig, Storage as S3Archival, +}; diff --git a/trading_engine/src/tests/trading_tests.rs b/trading_engine/src/tests/trading_tests.rs index 0c545c69b..7f3396743 100644 --- a/trading_engine/src/tests/trading_tests.rs +++ b/trading_engine/src/tests/trading_tests.rs @@ -7,6 +7,7 @@ mod comprehensive_trading_tests { use super::*; use crate::{CoreError, CoreResult}; + use crate::events::event_types::TradingEvent; // use futures; // TODO: Fix futures import or add futures to dependencies use std::mem::size_of; use uuid::Uuid; diff --git a/trading_engine/src/trading/broker_client.rs b/trading_engine/src/trading/broker_client.rs index d20e007d3..b774097b6 100644 --- a/trading_engine/src/trading/broker_client.rs +++ b/trading_engine/src/trading/broker_client.rs @@ -23,12 +23,12 @@ use common::types::{OrderId, OrderSide, OrderStatus, OrderType}; use tracing::{debug, error, info, warn}; use super::data_interface::{ - BrokerConnectionStatus, BrokerError, BrokerInterface, ExecutionReport, Position, + BrokerConnectionStatus, BrokerError, BrokerInterface, }; +use common::types::{Execution as ExecutionReport, Position}; use crate::trading_operations::TradingOrder; -// Re-export from data_interface (avoid duplicates) -pub use super::data_interface::ExecutionReport as RealExecutionReport; +// Removed pub use - import ExecutionReport directly where needed /// Interactive Brokers configuration #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/trading_engine/src/trading/data_interface.rs b/trading_engine/src/trading/data_interface.rs index 3816a9aba..bae8849d2 100644 --- a/trading_engine/src/trading/data_interface.rs +++ b/trading_engine/src/trading/data_interface.rs @@ -5,8 +5,7 @@ //! while still being able to work with different data sources. // OrderEvent replaced with MarketDataEvent for simplicity -// TODO: Define proper OrderEvent type if needed -pub use common::types::MarketDataEvent as OrderEvent; +// Removed pub use - import MarketDataEvent directly where needed use async_trait::async_trait; use std::fmt::Debug; use tokio::sync::broadcast; @@ -18,8 +17,7 @@ use common::types::OrderStatus; // TradeEvent functionality removed - no longer available without data crate dependency /// Market data event that can be sent through the system -// Use canonical MarketDataEvent from common crate -pub use common::types::MarketDataEvent; +// Removed pub use - import MarketDataEvent directly where needed // OrderBookEvent moved to common::types::OrderBookEvent @@ -164,8 +162,4 @@ pub enum BrokerError { // ExecutionReport renamed to Execution - using canonical type from common::types // Re-export Execution as ExecutionReport for broker interfaces -pub use common::types::Execution as ExecutionReport; - -// Position struct removed - using canonical type from common::types -// Re-export Position for broker interfaces -pub use common::types::Position; +// Removed pub use - import Execution and Position directly where needed diff --git a/trading_engine/src/trading/engine.rs b/trading_engine/src/trading/engine.rs index 5c3ee6b6b..1aa86686b 100644 --- a/trading_engine/src/trading/engine.rs +++ b/trading_engine/src/trading/engine.rs @@ -10,9 +10,13 @@ use tracing::info; use uuid::Uuid; use super::{ - data_interface::{DataProvider, DataType, MarketDataEvent, OrderEvent, Subscription}, - AccountManager, BrokerClient, OrderManager, PositionManager, + account_manager::AccountManager, + broker_client::BrokerClient, + data_interface::{DataProvider, DataType, Subscription}, + order_manager::OrderManager, + position_manager::PositionManager, }; +use common::types::MarketDataEvent; use crate::trading_operations::{ ArbitrageOpportunity, ExecutionResult, TradingOperations, TradingOrder, TradingStats, }; @@ -282,8 +286,7 @@ pub struct AccountInfo { /// Position structure // CANONICAL Position type imported from types::basic -// NO DUPLICATE TYPES - SINGLE TYPE SYSTEM ONLY -pub use common::types::Position; +// Removed pub use - import Position directly where needed #[cfg(test)] mod tests { diff --git a/trading_engine/src/trading/mod.rs b/trading_engine/src/trading/mod.rs index 504514cc3..252420c7e 100644 --- a/trading_engine/src/trading/mod.rs +++ b/trading_engine/src/trading/mod.rs @@ -10,11 +10,5 @@ pub mod engine; pub mod order_manager; pub mod position_manager; -pub use account_manager::AccountManager; -pub use broker_client::BrokerClient; -pub use data_interface::{ - BrokerInterface, DataProvider, MarketDataEvent, Subscription, -}; -pub use engine::TradingEngine; -pub use order_manager::OrderManager; -pub use position_manager::PositionManager; +// ELIMINATED: Re-exports removed to force explicit imports +// DO NOT RE-EXPORT - Use explicit imports at usage sites diff --git a/trading_engine/src/trading/mod.rs.bak b/trading_engine/src/trading/mod.rs.bak new file mode 100644 index 000000000..504514cc3 --- /dev/null +++ b/trading_engine/src/trading/mod.rs.bak @@ -0,0 +1,20 @@ +//! Core Trading Module +//! +//! This module contains the core trading business logic extracted from TLI. +//! TLI should only handle gRPC API endpoints and delegate to these core services. + +pub mod account_manager; +pub mod broker_client; +pub mod data_interface; +pub mod engine; +pub mod order_manager; +pub mod position_manager; + +pub use account_manager::AccountManager; +pub use broker_client::BrokerClient; +pub use data_interface::{ + BrokerInterface, DataProvider, MarketDataEvent, Subscription, +}; +pub use engine::TradingEngine; +pub use order_manager::OrderManager; +pub use position_manager::PositionManager; diff --git a/trading_engine/src/types/basic.rs b/trading_engine/src/types/basic.rs index f5f8789be..02468823a 100644 --- a/trading_engine/src/types/basic.rs +++ b/trading_engine/src/types/basic.rs @@ -1,7 +1,7 @@ -//! Basic types - Clean imports from canonical common::types +//! Trading Engine Basic Types - NO RE-EXPORTS //! -//! This module provides clean re-exports of types from the canonical common::types module. -//! All type definitions have been moved to common::types to establish a single source of truth. +//! This module contains only trading engine specific types. +//! All common types must be imported directly from common::types. #![deny( clippy::unwrap_used, @@ -12,51 +12,11 @@ )] #![warn(clippy::pedantic, clippy::nursery, clippy::perf)] -// ============================================================================ -// CANONICAL TYPE IMPORTS - Single Source of Truth from common::types -// ============================================================================ - -// Re-export all canonical types from common crate -// Core Trading Types -pub use common::types::{ - Order, OrderId, OrderSide, OrderStatus, OrderType, OrderRef, - Price, Quantity, Volume, Symbol, Currency, TimeInForce, BrokerType, - Decimal, // Add Decimal -}; - -// Market Data Types -pub use common::types::{ - MarketTick, TradingSignal, QuoteEvent, TradeEvent, BarEvent, Level2Update, - PriceLevel, MarketStatus, ConnectionEvent, ConnectionStatus, ErrorEvent, - OrderBookEvent, DataType, Subscription, MarketDataEvent, -}; -pub use common::trading::{TickType, MarketRegime}; - -// Identifiers -pub use common::types::{ - TradeId, EventId, FillId, AggregateId, AssetId, ClientId, AccountId, -}; - -// Infrastructure Types -pub use common::types::{ - ServiceId, ServiceStatus, ConfigVersion, RequestId, Timestamp, - ConnectionInfo, ResourceLimits, -}; - -// Time Types -pub use common::types::{HftTimestamp, GenericTimestamp}; - -// Financial Types -pub use common::types::{Position, Execution, Money}; - -// Aggregate Types -pub use common::types::Aggregate; - // Import error type from crate for backward compatibility use crate::types::errors::FoxhuntError; // ============================================================================ -// COMPATIBILITY BRIDGE FUNCTIONS +// TRADING ENGINE SPECIFIC TYPES ONLY // ============================================================================ /// Bridge functions for existing code that expects TradingError static methods @@ -95,57 +55,10 @@ impl TradingError { } } -// ============================================================================ -// TYPE ALIASES FOR BACKWARD COMPATIBILITY -// ============================================================================ - -// IMPORTANT: These are clean re-exports, not deprecated aliases -// They provide stable API for existing code while using canonical types - -/// DateTime alias for convenience -pub use chrono::{DateTime, Utc}; - -/// UUID for unique identifiers -pub use uuid::Uuid; - -// ============================================================================ -// PRELUDE FOR COMMON IMPORTS -// ============================================================================ - -/// Common types and traits that are frequently used together -pub mod prelude { - pub use super::{ - Order, OrderId, OrderSide, OrderStatus, OrderType, - Price, Quantity, Volume, Symbol, Currency, TimeInForce, - MarketTick, TickType, MarketRegime, TradingSignal, - TradeId, EventId, AccountId, HftTimestamp, Decimal, - }; - - pub use chrono::{DateTime, Utc}; - pub use rust_decimal::Decimal as RustDecimal; - pub use uuid::Uuid; -} - #[cfg(test)] mod tests { use super::*; - #[test] - fn test_canonical_imports() { - // Test that canonical types are properly imported - let price = Price::from_f64(100.0).expect("Valid price"); - assert!((price.to_f64() - 100.0).abs() < f64::EPSILON); - - let qty = Quantity::from_f64(50.0).expect("Valid quantity"); - assert!((qty.to_f64() - 50.0).abs() < f64::EPSILON); - - let symbol = Symbol::from("AAPL"); - assert_eq!(symbol.as_str(), "AAPL"); - - let order_id = OrderId::new(); - assert!(order_id.value() > 0); - } - #[test] fn test_compatibility_bridges() { // Test that compatibility bridges work diff --git a/trading_engine/src/types/error.rs b/trading_engine/src/types/error.rs index ec5675329..895e01bdb 100644 --- a/trading_engine/src/types/error.rs +++ b/trading_engine/src/types/error.rs @@ -1,8 +1,7 @@ //! Error module for Foxhunt HFT system. #![warn(clippy::all)] -// Re-export FoxhuntError from the local errors module -pub use super::errors::{ErrorSeverity, FoxhuntError, FoxhuntResult}; +// Removed pub use - import errors directly where needed use std::fmt; use std::time::Duration; diff --git a/trading_engine/src/types/financial.rs b/trading_engine/src/types/financial.rs index 8c350ddc8..97226a025 100644 --- a/trading_engine/src/types/financial.rs +++ b/trading_engine/src/types/financial.rs @@ -12,12 +12,7 @@ use serde::{Deserialize, Serialize}; // ============================================================================ // Re-export Decimal from rust_decimal as the canonical type for this module -// This ensures financial calculations use the same underlying type as the prelude -pub use rust_decimal::prelude::{FromPrimitive, ToPrimitive}; -pub use rust_decimal::Decimal; - -// Re-export the dec! macro for decimal literals -pub use rust_decimal_macros::dec; +// Removed pub use - import rust_decimal types directly where needed // Note: The types::prelude module re-exports these same types as the canonical // Decimal for the entire system. This ensures type compatibility. diff --git a/trading_engine/src/types/mod.rs b/trading_engine/src/types/mod.rs index b1d98bf66..189fcf767 100644 --- a/trading_engine/src/types/mod.rs +++ b/trading_engine/src/types/mod.rs @@ -59,8 +59,7 @@ pub mod financial; /// Trading engine validation utilities pub mod validation; -/// Prelude module for convenient imports -pub mod prelude; +// REMOVED: Prelude module - no more convenience re-exports // REMOVED: Service prelude - eliminated anti-pattern, use common::types::* instead @@ -91,27 +90,9 @@ pub enum TradingEngineError { // NOTE: basic::* not re-exported to prevent conflicts with lib.rs // All basic types are available from crate root via lib.rs -pub use metrics::*; -pub use type_registry::*; -pub use financial::*; -pub use validation::*; -// Re-export errors types explicitly to avoid conflicts -pub use errors::{ - ConversionError, ProtocolError, SymbolError, - FoxhuntError, ErrorContext, RecoveryStrategy, - ErrorSeverity as ErrorsErrorSeverity, // Alias to avoid conflict with events::ErrorSeverity -}; - -// Re-export events types explicitly to avoid conflicts -// Temporarily commented out due to import issues - these types may not be properly exported -// pub use events::{ -// EventLevel, EventMetadata, EventSequence, SystemEventType, TradingEvent, -// BufferManager, BufferStats, EventRingBuffer, -// }; - -// Re-export prelude for convenient access -pub use prelude::*; +// NOTE: NO MORE RE-EXPORTS - All types must be imported directly at usage sites +// This enforces proper dependency management and prevents circular imports #[cfg(test)] mod tests { diff --git a/trading_engine/src/types/mod.rs.bak b/trading_engine/src/types/mod.rs.bak new file mode 100644 index 000000000..b1d98bf66 --- /dev/null +++ b/trading_engine/src/types/mod.rs.bak @@ -0,0 +1,130 @@ +#![allow(clippy::mod_module_files)] // Complex module structure is more maintainable than single file +//! Trading Engine Internal Types +//! +//! This module provides internal types specific to the trading engine. +//! For common types, use the `common` crate instead. +//! +//! # Internal Trading Engine Types +//! - **Basic Types**: Trading engine specific primitives +//! - **Metrics**: Trading engine performance metrics +//! - **Type Registry**: Engine-specific type management + +// Note: #![warn(missing_docs)] temporarily disabled to focus on critical clippy fixes +// Will be re-enabled once comprehensive documentation pass is completed +#![deny( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::unimplemented, + clippy::unreachable, + clippy::indexing_slicing +)] +#![warn( + clippy::pedantic, + clippy::nursery, + clippy::perf, + clippy::complexity, + clippy::style, + clippy::correctness +)] +#![warn(missing_debug_implementations)] +#![warn(rust_2018_idioms)] + +// ============================================================================ +// TRADING ENGINE INTERNAL TYPE MODULES +// ============================================================================ + +// NOTE: basic module commented out to prevent conflicts with lib.rs exports +// All basic types are available from crate root via lib.rs +// pub mod basic; + +/// Trading engine specific timestamp utilities +pub mod timestamp_utils; + +/// Trading engine performance metrics +pub mod metrics; + +/// Trading engine type registry for dynamic type management +pub mod type_registry; + +/// Trading engine error types +pub mod errors; + +/// Trading engine events module +pub mod events; + +/// Trading engine financial types +pub mod financial; + +/// Trading engine validation utilities +pub mod validation; + +/// Prelude module for convenient imports +pub mod prelude; + +// REMOVED: Service prelude - eliminated anti-pattern, use common::types::* instead + +// ============================================================================ +// TRADING ENGINE INTERNAL ERROR TYPES +// ============================================================================ + +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +/// Trading engine specific errors +#[derive(Error, Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub enum TradingEngineError { + /// Engine initialization failed + #[error("Engine initialization failed: {0}")] + InitializationFailed(String), + /// Engine state invalid + #[error("Invalid engine state: {0}")] + InvalidState(String), + /// Engine execution error + #[error("Engine execution error: {0}")] + ExecutionError(String), +} + +// ============================================================================ +// TRADING ENGINE INTERNAL RE-EXPORTS +// ============================================================================ + +// NOTE: basic::* not re-exported to prevent conflicts with lib.rs +// All basic types are available from crate root via lib.rs +pub use metrics::*; +pub use type_registry::*; +pub use financial::*; +pub use validation::*; + +// Re-export errors types explicitly to avoid conflicts +pub use errors::{ + ConversionError, ProtocolError, SymbolError, + FoxhuntError, ErrorContext, RecoveryStrategy, + ErrorSeverity as ErrorsErrorSeverity, // Alias to avoid conflict with events::ErrorSeverity +}; + +// Re-export events types explicitly to avoid conflicts +// Temporarily commented out due to import issues - these types may not be properly exported +// pub use events::{ +// EventLevel, EventMetadata, EventSequence, SystemEventType, TradingEvent, +// BufferManager, BufferStats, EventRingBuffer, +// }; + +// Re-export prelude for convenient access +pub use prelude::*; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_trading_engine_error_variants() { + let init_error = TradingEngineError::InitializationFailed("Failed to init".to_string()); + let state_error = TradingEngineError::InvalidState("Bad state".to_string()); + let exec_error = TradingEngineError::ExecutionError("Execution failed".to_string()); + + assert_eq!(format!("{}", init_error), "Engine initialization failed: Failed to init"); + assert_eq!(format!("{}", state_error), "Invalid engine state: Bad state"); + assert_eq!(format!("{}", exec_error), "Engine execution error: Execution failed"); + } +} diff --git a/trading_engine/src/types/type_registry.rs b/trading_engine/src/types/type_registry.rs index 83b39158e..e22bc4624 100644 --- a/trading_engine/src/types/type_registry.rs +++ b/trading_engine/src/types/type_registry.rs @@ -9,18 +9,15 @@ /// /// This compile-time registry ensures that all types are imported from their /// canonical locations. Any attempt to duplicate types will be caught at compile time. -pub mod canonical_types { - // Import available canonical types from common crate (single source of truth) - // Public imports for type registry implementations - pub use common::types::{ - AccountId, ConfigVersion, ConnectionEvent, ConnectionInfo, ConnectionStatus, - Currency, DataType, ErrorEvent, Execution, GenericTimestamp, HftTimestamp, - Level2Update, MarketDataEvent, MarketStatus, Money, Order, OrderBookEvent, - OrderId, OrderSide, OrderStatus, OrderType, Position, Price, PriceLevel, - Quantity, QuoteEvent, RequestId, ResourceLimits, ServiceId, ServiceStatus, - Subscription, Symbol, TimeInForce, Timestamp, TradeEvent, TradeId, Volume, - }; -} +// Import canonical types for local use without re-exporting +use common::types::{ + AccountId, ConfigVersion, ConnectionEvent, ConnectionInfo, ConnectionStatus, + Currency, DataType, ErrorEvent, Execution, GenericTimestamp, HftTimestamp, + Level2Update, MarketDataEvent, MarketStatus, Money, Order, OrderBookEvent, + OrderId, OrderSide, OrderStatus, OrderType, Position, Price, PriceLevel, + Quantity, QuoteEvent, RequestId, ResourceLimits, ServiceId, ServiceStatus, + Subscription, Symbol, TimeInForce, Timestamp, TradeEvent, TradeId, Volume, +}; /// Compile-time type compliance checker ///