diff --git a/Cargo.lock b/Cargo.lock index 1ff780b38..bb839dc6d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5826,8 +5826,8 @@ dependencies = [ "anyhow", "async-trait", "chrono", + "common", "lru", - "ml", "parking_lot 0.12.5", "semver 1.0.27", "serde", diff --git a/adaptive-strategy/src/config.rs b/adaptive-strategy/src/config.rs index fabbe2730..250cfd28e 100644 --- a/adaptive-strategy/src/config.rs +++ b/adaptive-strategy/src/config.rs @@ -66,9 +66,6 @@ pub struct GeneralConfig { impl Default for GeneralConfig { fn default() -> Self { - tracing::warn!( - "Using hardcoded GeneralConfig::default() - migrate to database configuration!" - ); Self { execution_interval: Duration::from_millis(100_u64), error_backoff_duration: Duration::from_secs(1_u64), @@ -80,9 +77,6 @@ impl Default for GeneralConfig { impl Default for AdaptiveStrategyConfig { fn default() -> Self { - tracing::warn!("Using hardcoded AdaptiveStrategyConfig::default() - migrate to database configuration!"); - tracing::warn!(" Load configuration from database using DatabaseConfigLoader instead."); - tracing::warn!(" Available strategies: 'default-production', 'development', 'aggressive'"); Self { general: GeneralConfig::default(), ensemble: EnsembleConfig::default(), @@ -112,9 +106,6 @@ pub struct EnsembleConfig { impl Default for EnsembleConfig { fn default() -> Self { - tracing::warn!( - "Using hardcoded EnsembleConfig::default() - migrate to database configuration!" - ); Self { max_parallel_models: 4_usize, rebalancing_interval: Duration::from_secs(300_u64), @@ -162,9 +153,6 @@ pub struct ModelConfig { impl Default for ModelConfig { fn default() -> Self { - tracing::warn!( - "Using hardcoded ModelConfig::default() - migrate to database configuration!" - ); Self { id: "default_model".to_owned(), name: "default_model".to_owned(), @@ -198,9 +186,6 @@ pub struct RiskConfig { impl Default for RiskConfig { fn default() -> Self { - tracing::warn!( - "Using hardcoded RiskConfig::default() - migrate to database configuration!" - ); Self { max_position_size: 0.1_f64, max_leverage: 2.0_f64, @@ -252,9 +237,6 @@ pub struct MicrostructureConfig { impl Default for MicrostructureConfig { fn default() -> Self { - tracing::warn!( - "Using hardcoded MicrostructureConfig::default() - migrate to database configuration!" - ); Self { book_depth: 10_usize, vpin_window: 50_usize, @@ -284,9 +266,6 @@ pub struct RegimeConfig { impl Default for RegimeConfig { fn default() -> Self { - eprintln!( - "WARNING: Using hardcoded RegimeConfig::default() - migrate to database configuration!" - ); Self { detection_method: RegimeDetectionMethod::HMM, lookback_window: 252_usize, @@ -338,7 +317,6 @@ pub struct ExecutionConfig { impl Default for ExecutionConfig { fn default() -> Self { - eprintln!("WARNING: Using hardcoded ExecutionConfig::default() - migrate to database configuration!"); Self { algorithm: ExecutionAlgorithm::TWAP, max_order_size: 10000.0_f64, diff --git a/adaptive-strategy/src/execution/mod.rs b/adaptive-strategy/src/execution/mod.rs index ab14fad9d..8b52dd56e 100644 --- a/adaptive-strategy/src/execution/mod.rs +++ b/adaptive-strategy/src/execution/mod.rs @@ -111,11 +111,9 @@ pub struct ExecutionPerformanceTracker { /// Performance metrics by algorithm algorithm_performance: HashMap, /// Slippage measurements - #[allow(dead_code)] - slippage_tracker: SlippageTracker, + _slippage_tracker: SlippageTracker, /// Implementation shortfall tracker - #[allow(dead_code)] - shortfall_tracker: ShortfallTracker, + _shortfall_tracker: ShortfallTracker, } /// Algorithm performance metrics @@ -143,11 +141,9 @@ pub struct AlgorithmPerformance { #[derive(Debug)] pub struct SlippageTracker { /// Slippage measurements - #[allow(dead_code)] - measurements: VecDeque, + _measurements: VecDeque, /// Slippage statistics by symbol - #[allow(dead_code)] - stats_by_symbol: HashMap, + _stats_by_symbol: HashMap, } /// Slippage measurement @@ -215,11 +211,9 @@ pub struct SmartOrderRouter { /// Available venues venues: Vec, /// Routing rules - #[allow(dead_code)] - routing_rules: HashMap, + _routing_rules: HashMap, /// Venue performance tracker - #[allow(dead_code)] - venue_performance: HashMap, + _venue_performance: HashMap, } /// Trading venue information @@ -406,11 +400,9 @@ pub struct TWAPAlgorithm { /// Number of slices slice_count: u32, /// Current slice - #[allow(dead_code)] - current_slice: u32, + _current_slice: u32, /// Slice orders - #[allow(dead_code)] - slice_orders: Vec, + _slice_orders: Vec, } /// Volume-Weighted Average Price (VWAP) algorithm @@ -419,24 +411,20 @@ pub struct VWAPAlgorithm { /// Algorithm name name: String, /// Historical volume profile - #[allow(dead_code)] - volume_profile: HashMap, + _volume_profile: HashMap, /// Participation rate participation_rate: f64, /// Current volume tracking - #[allow(dead_code)] - volume_tracker: VolumeTracker, + _volume_tracker: VolumeTracker, } /// Volume profile for VWAP calculation #[derive(Debug, Clone)] pub struct VolumeProfile { /// Time buckets - #[allow(dead_code)] - buckets: Vec, + _buckets: Vec, /// Profile date - #[allow(dead_code)] - date: NaiveDate, + _date: NaiveDate, } /// Volume bucket @@ -454,11 +442,9 @@ pub struct VolumeBucket { #[derive(Debug)] pub struct VolumeTracker { /// Current period volumes - #[allow(dead_code)] - period_volumes: HashMap, + _period_volumes: HashMap, /// Target volumes - #[allow(dead_code)] - target_volumes: HashMap, + _target_volumes: HashMap, } /// Implementation Shortfall algorithm @@ -469,25 +455,20 @@ pub struct ImplementationShortfallAlgorithm { /// Risk aversion parameter risk_aversion: f64, /// Market impact model - #[allow(dead_code)] - impact_model: MarketImpactModel, + _impact_model: MarketImpactModel, /// Optimal schedule - #[allow(dead_code)] - execution_schedule: Vec, + _execution_schedule: Vec, } /// Market impact model #[derive(Debug)] pub struct MarketImpactModel { /// Temporary impact coefficient - #[allow(dead_code)] - temp_impact_coeff: f64, + _temp_impact_coeff: f64, /// Permanent impact coefficient - #[allow(dead_code)] - perm_impact_coeff: f64, + _perm_impact_coeff: f64, /// Volatility estimate - #[allow(dead_code)] - volatility: f64, + _volatility: f64, } /// Execution schedule slice @@ -939,8 +920,8 @@ impl ExecutionPerformanceTracker { pub fn new() -> Self { Self { algorithm_performance: HashMap::new(), - slippage_tracker: SlippageTracker::new(), - shortfall_tracker: ShortfallTracker::new(), + _slippage_tracker: SlippageTracker::new(), + _shortfall_tracker: ShortfallTracker::new(), } } @@ -985,8 +966,8 @@ impl SlippageTracker { /// Create a new slippage tracker pub fn new() -> Self { Self { - measurements: VecDeque::new(), - stats_by_symbol: HashMap::new(), + _measurements: VecDeque::new(), + _stats_by_symbol: HashMap::new(), } } } @@ -1033,8 +1014,8 @@ impl SmartOrderRouter { Ok(Self { venues, - routing_rules: HashMap::new(), - venue_performance: HashMap::new(), + _routing_rules: HashMap::new(), + _venue_performance: HashMap::new(), }) } @@ -1065,8 +1046,8 @@ impl TWAPAlgorithm { name: "TWAP".to_owned(), window_duration: Duration::from_secs(300_u64), // 5 minutes slice_count: 10_u32, - current_slice: 0_u32, - slice_orders: Vec::new(), + _current_slice: 0_u32, + _slice_orders: Vec::new(), }) } } @@ -1138,9 +1119,9 @@ impl VWAPAlgorithm { pub fn new() -> Result { Ok(Self { name: "VWAP".to_owned(), - volume_profile: HashMap::new(), + _volume_profile: HashMap::new(), participation_rate: 0.1_f64, // 10% participation - volume_tracker: VolumeTracker::new(), + _volume_tracker: VolumeTracker::new(), }) } } @@ -1205,8 +1186,8 @@ impl VolumeTracker { /// Create a new volume tracker pub fn new() -> Self { Self { - period_volumes: HashMap::new(), - target_volumes: HashMap::new(), + _period_volumes: HashMap::new(), + _target_volumes: HashMap::new(), } } } @@ -1217,8 +1198,8 @@ impl ImplementationShortfallAlgorithm { Ok(Self { name: "ImplementationShortfall".to_owned(), risk_aversion: 1e-6_f64, - impact_model: MarketImpactModel::new(), - execution_schedule: Vec::new(), + _impact_model: MarketImpactModel::new(), + _execution_schedule: Vec::new(), }) } } @@ -1288,9 +1269,9 @@ impl MarketImpactModel { /// Create a new market impact model pub fn new() -> Self { Self { - temp_impact_coeff: 0.01_f64, - perm_impact_coeff: 0.001_f64, - volatility: 0.02_f64, + _temp_impact_coeff: 0.01_f64, + _perm_impact_coeff: 0.001_f64, + _volatility: 0.02_f64, } } } diff --git a/adaptive-strategy/src/models/deep_learning.rs b/adaptive-strategy/src/models/deep_learning.rs index 1b3c5ecb0..e00c6d58b 100644 --- a/adaptive-strategy/src/models/deep_learning.rs +++ b/adaptive-strategy/src/models/deep_learning.rs @@ -293,12 +293,10 @@ impl ModelTrait for LSTMModel { #[derive(Debug)] pub struct GRUModel { name: String, - /// Model `configuration` (stub for future ML integration) - #[allow(dead_code)] - config: ModelConfig, + /// Model configuration (stub for future ML integration) + _config: ModelConfig, /// Model readiness flag (stub for future ML integration) - #[allow(dead_code)] - ready: bool, + _ready: bool, } impl GRUModel { @@ -320,8 +318,8 @@ impl GRUModel { pub async fn new(name: String, config: ModelConfig) -> Result { Ok(Self { name, - config, - ready: false, + _config: config, + _ready: false, }) } } @@ -377,11 +375,9 @@ impl ModelTrait for GRUModel { pub struct TransformerModel { name: String, /// Model configuration (stub for future ML integration) - #[allow(dead_code)] - config: ModelConfig, + _config: ModelConfig, /// Model readiness flag (stub for future ML integration) - #[allow(dead_code)] - ready: bool, + _ready: bool, } impl TransformerModel { @@ -389,8 +385,8 @@ impl TransformerModel { pub async fn new(name: String, config: ModelConfig) -> Result { Ok(Self { name, - config, - ready: false, + _config: config, + _ready: false, }) } } @@ -446,11 +442,9 @@ impl ModelTrait for TransformerModel { pub struct CNNModel { name: String, /// Model configuration (stub for future ML integration) - #[allow(dead_code)] - config: ModelConfig, + _config: ModelConfig, /// Model readiness flag (stub for future ML integration) - #[allow(dead_code)] - ready: bool, + _ready: bool, } impl CNNModel { @@ -458,8 +452,8 @@ impl CNNModel { pub async fn new(name: String, config: ModelConfig) -> Result { Ok(Self { name, - config, - ready: false, + _config: config, + _ready: false, }) } } diff --git a/adaptive-strategy/src/models/traditional.rs b/adaptive-strategy/src/models/traditional.rs index bb3c1e363..3ae072983 100644 --- a/adaptive-strategy/src/models/traditional.rs +++ b/adaptive-strategy/src/models/traditional.rs @@ -11,12 +11,10 @@ use async_trait::async_trait; #[derive(Debug)] pub struct RandomForestModel { name: String, - /// Model `configuration` (stub for future ML integration) - #[allow(dead_code)] - config: ModelConfig, + /// Model configuration (stub for future ML integration) + _config: ModelConfig, /// Model readiness flag (stub for future ML integration) - #[allow(dead_code)] - ready: bool, + _ready: bool, } impl RandomForestModel { @@ -24,8 +22,8 @@ impl RandomForestModel { pub async fn new(name: String, config: ModelConfig) -> Result { Ok(Self { name, - config, - ready: false, + _config: config, + _ready: false, }) } } @@ -81,11 +79,9 @@ impl ModelTrait for RandomForestModel { pub struct XGBoostModel { name: String, /// Model configuration (stub for future ML integration) - #[allow(dead_code)] - config: ModelConfig, + _config: ModelConfig, /// Model readiness flag (stub for future ML integration) - #[allow(dead_code)] - ready: bool, + _ready: bool, } impl XGBoostModel { @@ -93,8 +89,8 @@ impl XGBoostModel { pub async fn new(name: String, config: ModelConfig) -> Result { Ok(Self { name, - config, - ready: false, + _config: config, + _ready: false, }) } } @@ -152,11 +148,9 @@ impl ModelTrait for XGBoostModel { pub struct SVMModel { name: String, /// Model configuration (stub for future ML integration) - #[allow(dead_code)] - config: ModelConfig, + _config: ModelConfig, /// Model readiness flag (stub for future ML integration) - #[allow(dead_code)] - ready: bool, + _ready: bool, } impl SVMModel { @@ -164,8 +158,8 @@ impl SVMModel { pub async fn new(name: String, config: ModelConfig) -> Result { Ok(Self { name, - config, - ready: false, + _config: config, + _ready: false, }) } } @@ -223,11 +217,9 @@ impl ModelTrait for SVMModel { pub struct LinearRegressionModel { name: String, /// Model configuration (stub for future ML integration) - #[allow(dead_code)] - config: ModelConfig, + _config: ModelConfig, /// Model readiness flag (stub for future ML integration) - #[allow(dead_code)] - ready: bool, + _ready: bool, } impl LinearRegressionModel { @@ -235,8 +227,8 @@ impl LinearRegressionModel { pub async fn new(name: String, config: ModelConfig) -> Result { Ok(Self { name, - config, - ready: false, + _config: config, + _ready: false, }) } } diff --git a/adaptive-strategy/src/risk/kelly_position_sizer.rs b/adaptive-strategy/src/risk/kelly_position_sizer.rs index 709e57ac0..ab3cc6e65 100644 --- a/adaptive-strategy/src/risk/kelly_position_sizer.rs +++ b/adaptive-strategy/src/risk/kelly_position_sizer.rs @@ -201,8 +201,7 @@ pub struct DynamicRiskAdjuster { /// Portfolio drawdown tracker drawdown_tracker: DrawdownTracker, /// Volatility environment (reserved for volatility-adjusted sizing) - #[allow(dead_code)] - volatility_regime: VolatilityRegime, + _volatility_regime: VolatilityRegime, } // MarketRegime is already imported from ml::prelude at the top @@ -225,225 +224,162 @@ pub enum VolatilityRegime { /// Portfolio concentration monitoring #[derive(Debug)] -#[allow(dead_code)] pub(super) struct ConcentrationMonitor { /// Current position concentrations by symbol concentrations: HashMap, /// Sector concentrations - #[allow(dead_code)] - sector_concentrations: HashMap, + _sector_concentrations: HashMap, /// Geographic concentrations - #[allow(dead_code)] - geographic_concentrations: HashMap, + _geographic_concentrations: HashMap, /// Asset class concentrations - #[allow(dead_code)] - asset_class_concentrations: HashMap, + _asset_class_concentrations: HashMap, /// Correlation matrix - #[allow(dead_code)] - correlation_matrix: CorrelationMatrix, + _correlation_matrix: CorrelationMatrix, } /// Correlation matrix for position sizing adjustments #[derive(Debug, Clone)] -#[allow(dead_code)] pub(super) struct CorrelationMatrix { /// Symbols included in matrix - #[allow(dead_code)] - symbols: Vec, + _symbols: Vec, /// Correlation coefficients (symmetric matrix) - #[allow(dead_code)] - correlations: Vec>, + _correlations: Vec>, /// Last update timestamp - #[allow(dead_code)] - last_update: DateTime, + _last_update: DateTime, /// Average correlation - #[allow(dead_code)] - avg_correlation: f64, + _avg_correlation: f64, } /// Volatility-based position optimization #[derive(Debug)] -#[allow(dead_code)] pub(super) struct VolatilityOptimizer { /// Volatility estimates by symbol volatility_estimates: HashMap, /// Target portfolio volatility target_volatility: f64, /// Current portfolio volatility - #[allow(dead_code)] - current_volatility: f64, + _current_volatility: f64, /// Volatility forecasting model - #[allow(dead_code)] - volatility_model: VolatilityModel, + _volatility_model: VolatilityModel, } /// Volatility estimate with confidence intervals #[derive(Debug, Clone)] -#[allow(dead_code)] pub struct VolatilityEstimate { /// Current volatility estimate (annualized) pub(super) current: f64, /// 1-day ahead forecast - #[allow(dead_code)] - forecast_1d: f64, + pub(super) _forecast_1d: f64, /// 5-day ahead forecast - #[allow(dead_code)] - forecast_5d: f64, + pub(super) _forecast_5d: f64, /// Confidence interval (95%) - #[allow(dead_code)] - confidence_interval: (f64, f64), + pub(super) _confidence_interval: (f64, f64), /// Model used for estimation - #[allow(dead_code)] - model_type: VolatilityModelType, + pub(super) _model_type: VolatilityModelType, /// Last update timestamp - #[allow(dead_code)] - last_update: DateTime, + pub(super) _last_update: DateTime, } /// Volatility forecasting models #[derive(Debug, Clone)] pub(super) enum VolatilityModelType { - /// GARCH(1,1) model - #[allow(dead_code)] - Garch, /// Exponentially weighted moving average Ewma, - /// Range-based volatility - #[allow(dead_code)] - RangeBased, - /// Realized volatility - #[allow(dead_code)] - Realized, } /// Volatility forecasting model #[derive(Debug)] -#[allow(dead_code)] pub(super) struct VolatilityModel { /// Model parameters - #[allow(dead_code)] - parameters: HashMap, + _parameters: HashMap, /// Model type - #[allow(dead_code)] - model_type: VolatilityModelType, + _model_type: VolatilityModelType, /// Calibration history - #[allow(dead_code)] - calibration_history: Vec, + _calibration_history: Vec, } /// Volatility model calibration record #[derive(Debug, Clone)] -#[allow(dead_code)] pub(super) struct CalibrationRecord { /// Calibration timestamp - #[allow(dead_code)] - timestamp: DateTime, + _timestamp: DateTime, /// Model parameters at calibration - #[allow(dead_code)] - parameters: HashMap, + _parameters: HashMap, /// In-sample error metrics - #[allow(dead_code)] - in_sample_error: f64, + _in_sample_error: f64, /// Out-of-sample error metrics - #[allow(dead_code)] - out_of_sample_error: Option, + _out_of_sample_error: Option, } /// Portfolio drawdown tracking #[derive(Debug)] -#[allow(dead_code)] pub struct DrawdownTracker { /// High water mark - #[allow(dead_code)] - high_water_mark: f64, + _high_water_mark: f64, /// Current drawdown - #[allow(dead_code)] - current_drawdown: f64, + _current_drawdown: f64, /// Maximum drawdown - #[allow(dead_code)] - max_drawdown: f64, + _max_drawdown: f64, /// Drawdown start time - #[allow(dead_code)] - drawdown_start: Option>, + _drawdown_start: Option>, /// Recovery factor (how much to reduce risk during drawdowns) pub(super) recovery_factor: f64, } /// Performance tracking for Kelly optimization #[derive(Debug)] -#[allow(dead_code)] pub(super) struct PerformanceTracker { /// Daily returns history - #[allow(dead_code)] - returns_history: Vec, + _returns_history: Vec, /// Kelly sizing performance pub(super) kelly_performance: KellyPerformanceMetrics, /// Model accuracy tracking - #[allow(dead_code)] - accuracy_tracker: AccuracyTracker, + _accuracy_tracker: AccuracyTracker, } /// Daily return record #[derive(Debug, Clone)] -#[allow(dead_code)] pub(super) struct DailyReturn { /// Date - #[allow(dead_code)] - date: NaiveDate, + _date: NaiveDate, /// Portfolio return - #[allow(dead_code)] - portfolio_return: f64, + _portfolio_return: f64, /// Kelly-sized positions return - #[allow(dead_code)] - kelly_return: f64, + _kelly_return: f64, /// Attribution by position - #[allow(dead_code)] - position_attribution: HashMap, + _position_attribution: HashMap, } /// Kelly performance metrics #[derive(Debug, Clone)] -#[allow(dead_code)] pub struct KellyPerformanceMetrics { /// Sharpe ratio - #[allow(dead_code)] - sharpe_ratio: f64, + _sharpe_ratio: f64, /// Sortino ratio - #[allow(dead_code)] - sortino_ratio: f64, + _sortino_ratio: f64, /// Maximum drawdown - #[allow(dead_code)] - max_drawdown: f64, + _max_drawdown: f64, /// Calmar ratio - #[allow(dead_code)] - calmar_ratio: f64, + _calmar_ratio: f64, /// Win rate - #[allow(dead_code)] - win_rate: f64, + _win_rate: f64, /// Average win/loss ratio - #[allow(dead_code)] - win_loss_ratio: f64, + _win_loss_ratio: f64, /// Kelly criterion effectiveness - #[allow(dead_code)] - kelly_effectiveness: f64, + _kelly_effectiveness: f64, } /// Model accuracy tracking #[derive(Debug)] -#[allow(dead_code)] pub(super) struct AccuracyTracker { /// Prediction accuracy by horizon - #[allow(dead_code)] - accuracy_by_horizon: HashMap, + _accuracy_by_horizon: HashMap, /// Calibration score - #[allow(dead_code)] - calibration_score: f64, + _calibration_score: f64, /// Information coefficient - #[allow(dead_code)] - information_coefficient: f64, + _information_coefficient: f64, /// Hit rate - #[allow(dead_code)] - hit_rate: f64, + _hit_rate: f64, } /// Enhanced Kelly position recommendation @@ -639,61 +575,8 @@ impl KellyPositionSizer { Ok(recommendation) } - /// Calculate base Kelly fraction using multiple methods - #[allow(dead_code)] - fn calculate_base_kelly( - &self, - _symbol: &str, - expected_return: f64, - historical_returns: &[f64], - ) -> Result { - if historical_returns.is_empty() { - return Ok(0.0); - } - - // Method 1: Classic Kelly formula - let variance = self.calculate_variance(historical_returns); - let classic_kelly = if variance > 0.0 { - expected_return / variance - } else { - 0.0 - }; - - // Method 2: Win/loss statistics Kelly - let (win_rate, avg_win, avg_loss) = self.calculate_win_loss_stats(historical_returns); - let empirical_kelly = if avg_loss > 0.0 { - let odds = avg_win / avg_loss; - (win_rate * odds - (1.0 - win_rate)) / odds - } else { - 0.0 - }; - - // Method 3: Fractional Kelly for safety - let fractional_kelly = classic_kelly * 0.5; // Half Kelly for safety - - // Combine methods with weighting - let combined_kelly = 0.4 * classic_kelly + 0.4 * empirical_kelly + 0.2 * fractional_kelly; - - Ok(combined_kelly.clamp(0.0, self.config.max_fraction)) - } - - /// Calculate variance of historical returns - #[allow(dead_code)] - fn calculate_variance(&self, returns: &[f64]) -> f64 { - if returns.len() < 2 { - return 0.0; - } - - let mean = returns.iter().sum::() / returns.len() as f64; - let variance = - returns.iter().map(|r| (r - mean).powi(2)).sum::() / returns.len() as f64; - - variance - } - /// Calculate win/loss statistics - #[allow(dead_code)] - fn calculate_win_loss_stats(&self, returns: &[f64]) -> (f64, f64, f64) { + pub fn calculate_win_loss_stats(&self, returns: &[f64]) -> (f64, f64, f64) { let wins: Vec = returns.iter().filter(|&&r| r > 0.0).copied().collect(); let losses: Vec = returns.iter().filter(|&&r| r < 0.0).map(|r| -r).collect(); @@ -814,7 +697,7 @@ impl DynamicRiskAdjuster { current_regime: MarketRegime::Unknown, regime_scalers, drawdown_tracker: DrawdownTracker::new(config), - volatility_regime: VolatilityRegime::Normal, + _volatility_regime: VolatilityRegime::Normal, }) } @@ -903,10 +786,10 @@ impl ConcentrationMonitor { pub(super) fn new(_config: &KellyConfig) -> Result { Ok(Self { concentrations: HashMap::new(), - sector_concentrations: HashMap::new(), - geographic_concentrations: HashMap::new(), - asset_class_concentrations: HashMap::new(), - correlation_matrix: CorrelationMatrix::new(), + _sector_concentrations: HashMap::new(), + _geographic_concentrations: HashMap::new(), + _asset_class_concentrations: HashMap::new(), + _correlation_matrix: CorrelationMatrix::new(), }) } @@ -977,8 +860,8 @@ impl VolatilityOptimizer { Ok(Self { volatility_estimates: HashMap::new(), target_volatility: 0.15, // 15% target volatility - current_volatility: 0.0, - volatility_model: VolatilityModel::new()?, + _current_volatility: 0.0, + _volatility_model: VolatilityModel::new()?, }) } @@ -1014,9 +897,9 @@ impl VolatilityOptimizer { impl VolatilityModel { pub(super) fn new() -> Result { Ok(Self { - parameters: HashMap::new(), - model_type: VolatilityModelType::Ewma, - calibration_history: Vec::new(), + _parameters: HashMap::new(), + _model_type: VolatilityModelType::Ewma, + _calibration_history: Vec::new(), }) } } @@ -1036,10 +919,10 @@ impl DrawdownTracker { /// A new `DrawdownTracker` instance ready for monitoring pub fn new(_config: &KellyConfig) -> Self { Self { - high_water_mark: 100000.0, // Initial portfolio value - current_drawdown: 0.0, - max_drawdown: 0.0, - drawdown_start: None, + _high_water_mark: 100000.0, // Initial portfolio value + _current_drawdown: 0.0, + _max_drawdown: 0.0, + _drawdown_start: None, recovery_factor: 1.0, } } @@ -1048,9 +931,9 @@ impl DrawdownTracker { impl PerformanceTracker { pub(super) fn new() -> Result { Ok(Self { - returns_history: Vec::new(), + _returns_history: Vec::new(), kelly_performance: KellyPerformanceMetrics::default(), - accuracy_tracker: AccuracyTracker::new(), + _accuracy_tracker: AccuracyTracker::new(), }) } @@ -1066,13 +949,13 @@ impl PerformanceTracker { impl Default for KellyPerformanceMetrics { fn default() -> Self { Self { - sharpe_ratio: 0.0, - sortino_ratio: 0.0, - max_drawdown: 0.0, - calmar_ratio: 0.0, - win_rate: 0.0, - win_loss_ratio: 0.0, - kelly_effectiveness: 0.0, + _sharpe_ratio: 0.0, + _sortino_ratio: 0.0, + _max_drawdown: 0.0, + _calmar_ratio: 0.0, + _win_rate: 0.0, + _win_loss_ratio: 0.0, + _kelly_effectiveness: 0.0, } } } @@ -1080,10 +963,10 @@ impl Default for KellyPerformanceMetrics { impl AccuracyTracker { pub(super) fn new() -> Self { Self { - accuracy_by_horizon: HashMap::new(), - calibration_score: 0.0, - information_coefficient: 0.0, - hit_rate: 0.0, + _accuracy_by_horizon: HashMap::new(), + _calibration_score: 0.0, + _information_coefficient: 0.0, + _hit_rate: 0.0, } } } @@ -1091,10 +974,10 @@ impl AccuracyTracker { impl CorrelationMatrix { pub(super) fn new() -> Self { Self { - symbols: Vec::new(), - correlations: Vec::new(), - last_update: Utc::now(), - avg_correlation: 0.0, + _symbols: Vec::new(), + _correlations: Vec::new(), + _last_update: Utc::now(), + _avg_correlation: 0.0, } } } diff --git a/adaptive-strategy/src/risk/mod.rs b/adaptive-strategy/src/risk/mod.rs index ef57ab06b..cd09a088e 100644 --- a/adaptive-strategy/src/risk/mod.rs +++ b/adaptive-strategy/src/risk/mod.rs @@ -120,16 +120,13 @@ pub struct RiskLimits { #[derive(Debug, Clone)] pub struct PnLTracker { /// Daily P&L history - #[allow(dead_code)] - daily_pnl: Vec, + _daily_pnl: Vec, /// Current session P&L - #[allow(dead_code)] - session_pnl: f64, + _session_pnl: f64, /// Total portfolio value portfolio_value: f64, /// High-water mark for drawdown calculation - #[allow(dead_code)] - high_water_mark: f64, + _high_water_mark: f64, } /// Daily P&L record @@ -153,16 +150,12 @@ pub struct DrawdownCalculator { /// Portfolio value history value_history: Vec<(chrono::DateTime, f64)>, /// Current drawdown - #[allow(dead_code)] current_drawdown: f64, /// Maximum drawdown - #[allow(dead_code)] max_drawdown: f64, /// High-water mark - #[allow(dead_code)] high_water_mark: f64, /// Drawdown start time - #[allow(dead_code)] drawdown_start: Option>, } @@ -172,11 +165,9 @@ pub struct RiskMetricsCalculator { /// Historical price data price_history: HashMap>, /// Portfolio returns history - #[allow(dead_code)] - portfolio_returns: Vec, + _portfolio_returns: Vec, /// Confidence levels for VaR calculation - #[allow(dead_code)] - confidence_levels: Vec, + _confidence_levels: Vec, } /// Price point for historical data @@ -190,20 +181,6 @@ pub struct PricePoint { pub volume: f64, } -/// Correlation matrix for risk calculations -#[derive(Debug, Clone)] -pub struct CorrelationMatrix { - /// Asset symbols - #[allow(dead_code)] - symbols: Vec, - /// Correlation coefficients - #[allow(dead_code)] - correlations: Vec>, - /// Last update timestamp - #[allow(dead_code)] - last_update: chrono::DateTime, -} - /// Risk adjustment record #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RiskAdjustment { @@ -1277,10 +1254,10 @@ impl PnLTracker { /// Create a new P&L tracker pub fn new(initial_value: f64) -> Self { Self { - daily_pnl: Vec::new(), - session_pnl: 0.0, + _daily_pnl: Vec::new(), + _session_pnl: 0.0, portfolio_value: initial_value, - high_water_mark: initial_value, + _high_water_mark: initial_value, } } } @@ -1340,8 +1317,8 @@ impl RiskMetricsCalculator { pub fn new() -> Result { Ok(Self { price_history: HashMap::new(), - portfolio_returns: Vec::new(), - confidence_levels: vec![0.95_f64, 0.99_f64], // 95% and 99% confidence levels + _portfolio_returns: Vec::new(), + _confidence_levels: vec![0.95_f64, 0.99_f64], // 95% and 99% confidence levels }) } diff --git a/adaptive-strategy/src/risk/tests.rs b/adaptive-strategy/src/risk/tests.rs index 86d9e3598..14656cc2a 100644 --- a/adaptive-strategy/src/risk/tests.rs +++ b/adaptive-strategy/src/risk/tests.rs @@ -339,11 +339,11 @@ async fn test_volatility_estimates_update() { let mut estimates = HashMap::new(); estimates.insert(TEST_SYMBOL_1.to_string(), kelly_position_sizer::VolatilityEstimate { current: 0.18, - forecast_1d: 0.19, - forecast_5d: 0.20, - confidence_interval: (0.15, 0.22), - model_type: kelly_position_sizer::VolatilityModelType::Garch, - last_update: chrono::Utc::now(), + _forecast_1d: 0.19, + _forecast_5d: 0.20, + _confidence_interval: (0.15, 0.22), + _model_type: kelly_position_sizer::VolatilityModelType::Ewma, + _last_update: chrono::Utc::now(), }); let result = sizer.update_volatility_estimates(estimates).await; diff --git a/common/src/lib.rs b/common/src/lib.rs index 363882861..357defbec 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -27,6 +27,7 @@ pub mod constants; pub mod database; pub mod error; +pub mod model_types; pub mod features; // Wave D: Shared feature extraction (225 features) pub mod market_data; pub mod metrics; // Wave 5: Prometheus metrics infrastructure (W5-3) @@ -35,6 +36,7 @@ pub mod observability; // Wave 5: Structured logging and observability pub mod regime_persistence; pub mod resilience; // Wave 5: Circuit breaker and retry patterns pub mod thresholds; +pub mod tls; pub mod traits; pub mod types; @@ -55,21 +57,17 @@ pub use types::{ // Re-export error types pub use error::{CommonError, CommonResult}; +// Re-export model types +pub use model_types::ModelType; + // Re-export common traits for convenience pub use traits::{ CircuitBreaker, Configurable, DetailedHealth, GracefulShutdown, HealthCheck, HealthStatus, Metrics, RateLimitStatus, RateLimited, Reloadable, Service, }; -pub use market_data::{ - BarEvent as BarEventFromMarketData, BarInterval, - MarketDataEvent as MarketDataEventFromMarketData, NewsEvent, - OrderBookEvent as OrderBookEventFromMarketData, QuoteEvent as QuoteEventFromMarketData, - TradeEvent as TradeEventFromMarketData, -}; - -// Import market data types for canonical use -// Use common::market_data::{MarketDataEvent, TradeEvent, QuoteEvent, BarEvent} etc. +// Market data types: use common::market_data::{MarketDataEvent, TradeEvent, QuoteEvent, BarEvent, etc.} +// (Not re-exported at crate root to avoid name collisions with types:: re-exports) pub mod trading; // Re-export shared ML strategy types @@ -93,7 +91,6 @@ pub use features::{ pub use resilience::{ BoundedExecutor, BoundedExecutorError, - CircuitBreaker as ResilienceCircuitBreaker, CircuitBreakerConfig, CircuitBreakerState, retry_with_backoff, diff --git a/common/src/model_types.rs b/common/src/model_types.rs new file mode 100644 index 000000000..663188ba3 --- /dev/null +++ b/common/src/model_types.rs @@ -0,0 +1,220 @@ +//! Canonical model type definitions for the Foxhunt trading system. +//! +//! This module provides the unified `ModelType` enum used across all crates +//! (ml, model_loader, services) to identify ML model architectures. + +use serde::{Deserialize, Serialize}; +use std::fmt; + +/// Canonical model type enum used throughout the Foxhunt system. +/// +/// This is the single source of truth for model type identification. +/// All crates should import this type rather than defining their own. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum ModelType { + /// Compact Deep Q-Network + CompactDQN, + /// Distilled micro network for ultra-low latency + DistilledMicroNet, + /// Standard Deep Q-Network + DQN, + /// Rainbow DQN with all enhancements + RainbowDQN, + /// MAMBA/Mamba2 state space model + MAMBA, + /// Temporal Fusion Transformer + TFT, + /// Temporal Graph Neural Network + TGGN, + /// Liquid Neural Network + LNN, + /// Temporal Limit Order Book transformer + TLOB, + /// Proximal Policy Optimization + PPO, + /// Transformer for sequence modeling + Transformer, + /// Mamba state space model (alias for MAMBA) + Mamba, + /// Liquid time constant networks (alias for LNN) + LiquidNet, + /// Temporal Graph Neural Network (alias for TGGN) + TGNN, + /// Ensemble of multiple models + Ensemble, +} + +impl ModelType { + /// Get file extension for model type (used for checkpoint filenames). + pub fn file_extension(&self) -> &'static str { + match self { + Self::DQN => "dqn", + Self::MAMBA | Self::Mamba => "mamba", + Self::TFT => "tft", + Self::TGGN | Self::TGNN => "tggn", + Self::LNN | Self::LiquidNet => "lnn", + Self::CompactDQN => "compact_dqn", + Self::DistilledMicroNet => "distilled", + Self::RainbowDQN => "rainbow_dqn", + Self::TLOB => "tlob", + Self::PPO => "ppo", + Self::Transformer => "transformer", + Self::Ensemble => "ensemble", + } + } + + /// String representation of the model type (lowercase, unique per variant family). + pub const fn as_str(&self) -> &'static str { + match self { + Self::DQN | Self::CompactDQN | Self::RainbowDQN => "dqn", + Self::MAMBA | Self::Mamba => "mamba", + Self::TFT => "tft", + Self::TGGN | Self::TGNN => "tggn", + Self::LNN | Self::LiquidNet => "liquid", + Self::TLOB => "tlob", + Self::PPO => "ppo", + Self::DistilledMicroNet => "distilled", + Self::Transformer => "transformer", + Self::Ensemble => "ensemble", + } + } + + /// S3 storage prefix for model files (used by model_loader). + pub const fn s3_prefix(&self) -> &'static str { + match self { + Self::TLOB => "tlob_transformer", + Self::DQN | Self::CompactDQN | Self::RainbowDQN => "dqn", + Self::MAMBA | Self::Mamba => "mamba2", + Self::TFT => "tft", + Self::PPO => "ppo", + Self::LNN | Self::LiquidNet => "liquid", + Self::Ensemble => "ensemble", + Self::TGGN | Self::TGNN => "tggn", + Self::DistilledMicroNet => "distilled", + Self::Transformer => "transformer", + } + } + + /// Convert to database string representation (for PostgreSQL storage). + pub fn to_db_string(&self) -> &'static str { + match self { + Self::DQN => "DQN", + Self::PPO => "PPO", + Self::MAMBA | Self::Mamba => "MAMBA-2", + Self::TFT => "TFT", + Self::CompactDQN => "COMPACT_DQN", + Self::DistilledMicroNet => "DISTILLED_MICRO_NET", + Self::RainbowDQN => "RAINBOW_DQN", + Self::TGGN | Self::TGNN => "TGGN", + Self::LNN | Self::LiquidNet => "LNN", + Self::TLOB => "TLOB", + Self::Transformer => "TRANSFORMER", + Self::Ensemble => "ENSEMBLE", + } + } + + /// Get model weight for progress aggregation. + pub fn weight(&self) -> f64 { + 0.25 // Equal weight for all models + } + + /// Parse model type from string (case-insensitive). + #[allow(clippy::should_implement_trait)] + pub fn from_str(s: &str) -> Option { + match s.to_lowercase().as_str() { + "dqn" => Some(Self::DQN), + "mamba" | "mamba2" | "mamba-2" => Some(Self::MAMBA), + "tft" => Some(Self::TFT), + "tggn" | "tgnn" => Some(Self::TGGN), + "lnn" | "liquidnet" | "liquid" => Some(Self::LNN), + "compact_dqn" | "compactdqn" => Some(Self::CompactDQN), + "distilled" | "distilledmicronet" => Some(Self::DistilledMicroNet), + "rainbow_dqn" | "rainbowdqn" => Some(Self::RainbowDQN), + "tlob" | "tlob_transformer" => Some(Self::TLOB), + "ppo" => Some(Self::PPO), + "transformer" => Some(Self::Transformer), + "ensemble" => Some(Self::Ensemble), + _ => None, + } + } +} + +impl fmt::Display for ModelType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.as_str()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_file_extension() { + assert_eq!(ModelType::DQN.file_extension(), "dqn"); + assert_eq!(ModelType::PPO.file_extension(), "ppo"); + assert_eq!(ModelType::TFT.file_extension(), "tft"); + assert_eq!(ModelType::MAMBA.file_extension(), "mamba"); + assert_eq!(ModelType::Mamba.file_extension(), "mamba"); + assert_eq!(ModelType::Ensemble.file_extension(), "ensemble"); + } + + #[test] + fn test_as_str() { + assert_eq!(ModelType::TLOB.as_str(), "tlob"); + assert_eq!(ModelType::DQN.as_str(), "dqn"); + assert_eq!(ModelType::MAMBA.as_str(), "mamba"); + assert_eq!(ModelType::TFT.as_str(), "tft"); + assert_eq!(ModelType::PPO.as_str(), "ppo"); + assert_eq!(ModelType::LNN.as_str(), "liquid"); + assert_eq!(ModelType::Ensemble.as_str(), "ensemble"); + } + + #[test] + fn test_s3_prefix() { + assert_eq!(ModelType::TLOB.s3_prefix(), "tlob_transformer"); + assert_eq!(ModelType::DQN.s3_prefix(), "dqn"); + assert_eq!(ModelType::MAMBA.s3_prefix(), "mamba2"); + assert_eq!(ModelType::TFT.s3_prefix(), "tft"); + } + + #[test] + fn test_to_db_string() { + assert_eq!(ModelType::DQN.to_db_string(), "DQN"); + assert_eq!(ModelType::PPO.to_db_string(), "PPO"); + assert_eq!(ModelType::MAMBA.to_db_string(), "MAMBA-2"); + assert_eq!(ModelType::TFT.to_db_string(), "TFT"); + } + + #[test] + fn test_weight() { + assert_eq!(ModelType::DQN.weight(), 0.25); + assert_eq!(ModelType::PPO.weight(), 0.25); + } + + #[test] + fn test_display() { + assert_eq!(format!("{}", ModelType::DQN), "dqn"); + assert_eq!(format!("{}", ModelType::PPO), "ppo"); + assert_eq!(format!("{}", ModelType::MAMBA), "mamba"); + assert_eq!(format!("{}", ModelType::TLOB), "tlob"); + } + + #[test] + fn test_from_str() { + assert_eq!(ModelType::from_str("dqn"), Some(ModelType::DQN)); + assert_eq!(ModelType::from_str("DQN"), Some(ModelType::DQN)); + assert_eq!(ModelType::from_str("mamba2"), Some(ModelType::MAMBA)); + assert_eq!(ModelType::from_str("ppo"), Some(ModelType::PPO)); + assert_eq!(ModelType::from_str("unknown"), None); + } + + #[test] + fn test_serde_roundtrip() { + let model = ModelType::DQN; + let json = serde_json::to_string(&model).unwrap_or_default(); + let deserialized: ModelType = + serde_json::from_str(&json).unwrap_or(ModelType::DQN); + assert_eq!(model, deserialized); + } +} diff --git a/common/src/resilience/circuit_breaker.rs b/common/src/resilience/circuit_breaker.rs index a9b3465ed..785d8de39 100644 --- a/common/src/resilience/circuit_breaker.rs +++ b/common/src/resilience/circuit_breaker.rs @@ -47,6 +47,30 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::Mutex; +/// Core circuit breaker trait for the state machine pattern. +/// +/// Implementors manage the Closed -> Open -> HalfOpen state transitions. +/// This trait provides a common interface so different circuit breaker +/// implementations (e.g., tokio-based, parking_lot-based) can be used +/// polymorphically. +#[async_trait::async_trait] +pub trait CircuitBreakerTrait: Send + Sync { + /// Check if a request is allowed through. + async fn can_execute(&self) -> bool; + + /// Record a successful operation. + async fn record_success(&self); + + /// Record a failed operation. + async fn record_failure(&self); + + /// Get current state. + async fn state(&self) -> CircuitBreakerState; + + /// Reset to closed state. + async fn reset(&self); +} + /// Circuit breaker states #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum CircuitBreakerState { @@ -397,3 +421,26 @@ impl std::fmt::Debug for CircuitBreaker { .finish() } } + +#[async_trait::async_trait] +impl CircuitBreakerTrait for CircuitBreaker { + async fn can_execute(&self) -> bool { + CircuitBreaker::can_execute(self).await + } + + async fn record_success(&self) { + CircuitBreaker::record_success(self).await; + } + + async fn record_failure(&self) { + CircuitBreaker::record_failure(self).await; + } + + async fn state(&self) -> CircuitBreakerState { + CircuitBreaker::state(self).await + } + + async fn reset(&self) { + CircuitBreaker::reset(self).await; + } +} diff --git a/common/src/resilience/mod.rs b/common/src/resilience/mod.rs index 7654d5a85..a9be6c062 100644 --- a/common/src/resilience/mod.rs +++ b/common/src/resilience/mod.rs @@ -61,5 +61,7 @@ mod tests; // Re-export commonly used types pub use bounded_concurrency::{BoundedExecutor, BoundedExecutorError}; -pub use circuit_breaker::{CircuitBreaker, CircuitBreakerConfig, CircuitBreakerState}; +pub use circuit_breaker::{ + CircuitBreaker, CircuitBreakerConfig, CircuitBreakerState, CircuitBreakerTrait, +}; pub use retry::{retry_with_backoff, RetryConfig, RetryContext, RetryError}; diff --git a/common/src/tls.rs b/common/src/tls.rs new file mode 100644 index 000000000..da356329b --- /dev/null +++ b/common/src/tls.rs @@ -0,0 +1,178 @@ +//! Shared TLS types for Foxhunt services. +//! +//! This module contains type definitions shared across all service TLS configurations: +//! - [`TlsProtocolVersion`] - TLS 1.2 / 1.3 selection +//! - [`UserRole`] - RBAC roles extracted from client certificates +//! - [`ClientIdentity`] - Client identity extracted from mTLS certificates +//! +//! Each service retains its own `TlsConfig` struct and validation logic because +//! the validation approaches differ (async vs sync, delegated vs monolithic). + +/// TLS protocol version options. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[allow(dead_code)] +pub enum TlsProtocolVersion { + /// TLS version 1.2 + Tls12, + /// TLS version 1.3 + Tls13, +} + +/// User roles based on certificate attributes (RBAC). +#[derive(Debug, Clone, PartialEq, Eq)] +#[allow(dead_code)] +pub enum UserRole { + /// Administrator with full access + Admin, + /// Trader with trading permissions + Trader, + /// Analyst with read/analysis permissions + Analyst, + /// Risk manager with risk oversight + RiskManager, + /// Compliance officer with audit access + ComplianceOfficer, + /// Read-only access + ReadOnly, +} + +#[allow(dead_code)] +impl UserRole { + /// Get permissions for this role. + pub fn get_permissions(&self) -> Vec<&'static str> { + match self { + UserRole::Admin => vec![ + "trading.submit_order", + "trading.cancel_order", + "trading.modify_order", + "risk.view_positions", + "risk.modify_limits", + "analytics.view_data", + "analytics.run_backtest", + "compliance.view_reports", + "system.configure", + ], + UserRole::Trader => vec![ + "trading.submit_order", + "trading.cancel_order", + "trading.modify_order", + "risk.view_positions", + "analytics.view_data", + ], + UserRole::Analyst => vec![ + "analytics.view_data", + "analytics.run_backtest", + "risk.view_positions", + ], + UserRole::RiskManager => vec![ + "risk.view_positions", + "risk.modify_limits", + "analytics.view_data", + "compliance.view_reports", + ], + UserRole::ComplianceOfficer => vec![ + "compliance.view_reports", + "analytics.view_data", + "risk.view_positions", + ], + UserRole::ReadOnly => vec!["analytics.view_data"], + } + } +} + +/// Client identity extracted from mTLS certificate. +#[derive(Debug, Clone, PartialEq, Eq)] +#[allow(dead_code)] +pub struct ClientIdentity { + /// Common Name (CN) from certificate + pub common_name: String, + /// Organizational Unit (OU) from certificate + pub organizational_unit: String, + /// Certificate serial number + pub serial_number: String, + /// Certificate issuer + pub issuer: String, +} + +#[allow(dead_code)] +impl ClientIdentity { + /// Check if client is authorized for trading operations. + pub fn is_authorized_for_trading(&self) -> bool { + matches!(self.organizational_unit.as_str(), "trading" | "admin") + } + + /// Check if client is authorized for read-only operations. + pub fn is_authorized_for_readonly(&self) -> bool { + matches!( + self.organizational_unit.as_str(), + "trading" | "admin" | "analytics" | "risk" | "compliance" + ) + } + + /// Get user role based on certificate. + pub fn get_role(&self) -> UserRole { + match self.organizational_unit.as_str() { + "admin" => UserRole::Admin, + "trading" => UserRole::Trader, + "analytics" => UserRole::Analyst, + "risk" => UserRole::RiskManager, + "compliance" => UserRole::ComplianceOfficer, + _ => UserRole::ReadOnly, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_client_identity_authorization() { + let trading_identity = ClientIdentity { + common_name: "trader1.trading.foxhunt.internal".to_string(), + organizational_unit: "trading".to_string(), + serial_number: "12345".to_string(), + issuer: "Foxhunt Trading CA".to_string(), + }; + + assert!(trading_identity.is_authorized_for_trading()); + assert!(trading_identity.is_authorized_for_readonly()); + assert_eq!(trading_identity.get_role(), UserRole::Trader); + + let readonly_identity = ClientIdentity { + common_name: "analyst1.analytics.foxhunt.internal".to_string(), + organizational_unit: "analytics".to_string(), + serial_number: "12346".to_string(), + issuer: "Foxhunt Trading CA".to_string(), + }; + + assert!(!readonly_identity.is_authorized_for_trading()); + assert!(readonly_identity.is_authorized_for_readonly()); + assert_eq!(readonly_identity.get_role(), UserRole::Analyst); + } + + #[test] + fn test_user_role_permissions() { + let trader = UserRole::Trader; + let permissions = trader.get_permissions(); + + assert!(permissions.contains(&"trading.submit_order")); + assert!(permissions.contains(&"trading.cancel_order")); + assert!(!permissions.contains(&"system.configure")); + + let readonly = UserRole::ReadOnly; + let readonly_permissions = readonly.get_permissions(); + + assert!(!readonly_permissions.contains(&"trading.submit_order")); + assert!(readonly_permissions.contains(&"analytics.view_data")); + } + + #[test] + fn test_tls_protocol_version_equality() { + let tls13 = TlsProtocolVersion::Tls13; + let tls12 = TlsProtocolVersion::Tls12; + + assert_ne!(tls13, tls12); + assert_eq!(tls13, TlsProtocolVersion::Tls13); + } +} diff --git a/data/src/providers/benzinga/production_streaming.rs b/data/src/providers/benzinga/production_streaming.rs index 80bc24e55..39829b47f 100644 --- a/data/src/providers/benzinga/production_streaming.rs +++ b/data/src/providers/benzinga/production_streaming.rs @@ -21,6 +21,7 @@ use crate::types::ExtendedMarketDataEvent; use async_trait::async_trait; use chrono::{DateTime, NaiveDate, NaiveDateTime, Utc}; use common::error::ErrorCategory; +use common::resilience::circuit_breaker::{CircuitBreaker, CircuitBreakerConfig}; use common::{MarketDataEvent, Price, Quantity, Symbol}; use futures_core::Stream; use futures_util::{SinkExt, StreamExt}; @@ -187,84 +188,6 @@ pub struct ProductionMetrics { pub last_processed_at: RwLock>>, } -/// Circuit breaker states -#[derive(Debug, Clone, Copy)] -pub enum CircuitBreakerState { - /// Normal operation - requests flowing - Closed, - /// Blocking requests - failure threshold exceeded - Open, - /// Testing recovery - allowing limited requests - HalfOpen, -} - -/// Circuit breaker for fault tolerance -#[derive(Debug)] -pub struct CircuitBreaker { - state: Arc>, - failure_count: Arc, - threshold: u32, - timeout: Duration, - last_failure: Arc>>, -} - -impl CircuitBreaker { - /// Create new circuit breaker with failure threshold and timeout - pub fn new(threshold: u32, timeout: Duration) -> Self { - Self { - state: Arc::new(RwLock::new(CircuitBreakerState::Closed)), - failure_count: Arc::new(AtomicU64::new(0)), - threshold, - timeout, - last_failure: Arc::new(RwLock::new(None)), - } - } - - /// Check if call is permitted based on circuit breaker state - pub async fn is_call_permitted(&self) -> bool { - let state = *self.state.read().await; - - match state { - CircuitBreakerState::Closed => true, - CircuitBreakerState::Open => { - let last_failure = *self.last_failure.read().await; - if let Some(failure_time) = last_failure { - if failure_time.elapsed() >= self.timeout { - // Try half-open - let mut state_guard = self.state.write().await; - *state_guard = CircuitBreakerState::HalfOpen; - true - } else { - false - } - } else { - false - } - }, - CircuitBreakerState::HalfOpen => true, - } - } - - /// Record successful call - resets circuit breaker to closed state - pub async fn record_success(&self) { - let mut state_guard = self.state.write().await; - *state_guard = CircuitBreakerState::Closed; - self.failure_count.store(0, Ordering::Relaxed); - } - - /// Record failed call - may open circuit breaker if threshold exceeded - pub async fn record_failure(&self) { - let failures = self.failure_count.fetch_add(1, Ordering::Relaxed) + 1; - let mut last_failure_guard = self.last_failure.write().await; - *last_failure_guard = Some(Instant::now()); - - if failures >= self.threshold as u64 { - let mut state_guard = self.state.write().await; - *state_guard = CircuitBreakerState::Open; - } - } -} - /// Production Benzinga WebSocket streaming provider #[derive(Debug)] pub struct ProductionBenzingaProvider { @@ -456,10 +379,12 @@ impl ProductionBenzingaProvider { let rate_limiter = Arc::new(RateLimiter::direct(quota)); // Create circuit breaker - let circuit_breaker = Arc::new(CircuitBreaker::new( - config.circuit_breaker_threshold, - Duration::from_secs(config.circuit_breaker_timeout_secs), - )); + let cb_config = CircuitBreakerConfig { + failure_threshold: config.circuit_breaker_threshold, + timeout: Duration::from_secs(config.circuit_breaker_timeout_secs), + success_threshold: 1, + }; + let circuit_breaker = Arc::new(CircuitBreaker::new("benzinga", cb_config)); // Create processing semaphore let processing_semaphore = Arc::new(Semaphore::new(config.max_concurrent_processing)); @@ -631,7 +556,7 @@ impl ProductionBenzingaProvider { for message in messages { // Check circuit breaker - if !self.circuit_breaker.is_call_permitted().await { + if !self.circuit_breaker.can_execute().await { warn!("Circuit breaker open, skipping message processing"); continue; } @@ -1302,10 +1227,15 @@ mod tests { #[tokio::test] async fn test_circuit_breaker() { - let cb = CircuitBreaker::new(3, Duration::from_millis(100)); + let config = CircuitBreakerConfig { + failure_threshold: 3, + timeout: Duration::from_millis(100), + success_threshold: 1, + }; + let cb = CircuitBreaker::new("test_benzinga", config); // Initially closed - assert!(cb.is_call_permitted().await); + assert!(cb.can_execute().await); // Record failures to trip breaker cb.record_failure().await; @@ -1313,14 +1243,14 @@ mod tests { cb.record_failure().await; // Now should be open - assert!(!cb.is_call_permitted().await); + assert!(!cb.can_execute().await); // Wait for timeout and check half-open tokio::time::sleep(Duration::from_millis(150)).await; - assert!(cb.is_call_permitted().await); + assert!(cb.can_execute().await); // Record success to close cb.record_success().await; - assert!(cb.is_call_permitted().await); + assert!(cb.can_execute().await); } } diff --git a/data/src/providers/common.rs b/data/src/providers/common.rs index 248df6064..4a263dc26 100644 --- a/data/src/providers/common.rs +++ b/data/src/providers/common.rs @@ -47,14 +47,14 @@ use serde::{Deserialize, Serialize}; /// # Examples /// /// ```rust -/// use data::providers::common::ErrorCategory; +/// use data::providers::common::ProviderErrorCategory; /// /// match error_category { -/// ErrorCategory::RateLimit => { +/// ProviderErrorCategory::RateLimit => { /// // Implement exponential backoff /// tokio::time::sleep(Duration::from_millis(1000)).await; /// }, -/// ErrorCategory::Authentication => { +/// ProviderErrorCategory::Authentication => { /// // Refresh credentials and reconnect /// provider.refresh_auth().await?; /// }, @@ -64,7 +64,7 @@ use serde::{Deserialize, Serialize}; /// } /// ``` #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum ErrorCategory { +pub enum ProviderErrorCategory { /// Network connectivity issues, WebSocket disconnections /// /// Indicates problems with the underlying network connection to the diff --git a/docs/plans/2026-02-22-ml-ensemble-expansion-design.md b/docs/plans/2026-02-22-ml-ensemble-expansion-design.md new file mode 100644 index 000000000..5e2024907 --- /dev/null +++ b/docs/plans/2026-02-22-ml-ensemble-expansion-design.md @@ -0,0 +1,176 @@ +# ML Ensemble Expansion Design: 7 → 10 Models + +**Date**: 2026-02-22 +**Status**: Approved +**Scope**: Expand ensemble from 4 fully-integrated + 3 partial models to 10 fully-integrated models + +## Goal + +Bring TGGN and TLOB to full integration (hyperopt + UnifiedTrainable), then add three new architectures (KAN, xLSTM, Diffusion) — each filling a distinct signal gap in the ensemble. + +## Complementarity Analysis + +| Capability Gap | Current Coverage | New Algorithm | Signal Type | +|---|---|---|---| +| Non-linear feature discovery | TFT attention (fixed activations) | **KAN** | Learnable activation functions discover non-obvious price/volume relationships | +| Long-range temporal memory | Mamba2 SSM, PPO LSTM (~1K steps) | **xLSTM** | Exponential gating + matrix memory handles 10K+ timestep dependencies | +| Probabilistic scenarios | TFT quantile regression (point-ish) | **Diffusion** | Generates distributions of future price paths for tail risk | + +## Build Order (Sequential, one worktree per model) + +1. TGGN full stack (gap-fill — code exists) +2. TLOB full stack (gap-fill — code exists) +3. KAN (new architecture) +4. xLSTM (new architecture) +5. Diffusion (new architecture) + +Each model gets its own worktree branch → PR → merge before the next starts. + +## Constraint: No God Classes + +No file exceeds 500 lines. Each model decomposes into focused modules: +- `config.rs` — parameter structs +- `network.rs` — forward pass +- `trainable.rs` — UnifiedTrainable impl +- `trainer.rs` — training loop only +- Component files for model-specific pieces (e.g., `spline.rs` for KAN) + +## Per-Model File Structure + +``` +ml/src// + ├── mod.rs — public API, re-exports + ├── config.rs — Config struct + ├── .rs — Model-specific pieces + ├── network.rs — Neural network (forward pass) + ├── trainable.rs — UnifiedTrainable impl + └── tests.rs — Unit tests (50+ per model) +ml/src/trainers/.rs — Training loop + checkpointing +ml/src/hyperopt/adapters/.rs — PSO hyperopt adapter (ParameterSpace trait) +``` + +Shared files touched per model: +- `ml/src/common/model_type.rs` — add ModelType variant +- `ml/src/inference.rs` — add inference branch +- `ml/src/integration/coordinator.rs` — register in ensemble +- `ml/src/lib.rs` — module declaration +- `ml/src/hyperopt/adapters/mod.rs` — adapter registration + +## Model Designs + +### 1. TGGN Full Stack (Gap-Fill) + +**Existing**: `ml/src/tgnn/` — 8 files, graph convolution + message passing + inference + checkpoints. + +**Add**: +- `ml/src/tgnn/trainable_adapter.rs` (~150 lines) — impl UnifiedTrainable +- `ml/src/hyperopt/adapters/tggn.rs` (~300 lines) — ParameterSpace: num_layers, hidden_dim, attention_heads, message_passing_steps, learning_rate, dropout + +### 2. TLOB Full Stack (Gap-Fill) + +**Existing**: `ml/src/tlob/` — 8 files, transformer + LOB features + data loader + trainer. + +**Add**: +- `ml/src/tlob/trainable_adapter.rs` (~150 lines) — impl UnifiedTrainable +- `ml/src/hyperopt/adapters/tlob.rs` (~300 lines) — ParameterSpace: num_heads, d_model, num_encoder_layers, feature_window, learning_rate, dropout + +### 3. KAN (Kolmogorov-Arnold Network) — New + +**Purpose**: Non-linear relationship discovery via learnable B-spline activation functions. + +**Modules**: +- `kan/config.rs` (~80 lines) — KANConfig +- `kan/spline.rs` (~200 lines) — B-spline basis functions (core innovation) +- `kan/layer.rs` (~150 lines) — KANLayer (replaces MLP layer with spline-based activations) +- `kan/network.rs` (~200 lines) — KANNetwork (stack of KANLayers + output head) +- `kan/trainable.rs` (~150 lines) — UnifiedTrainable impl +- `trainers/kan.rs` (~300 lines) — training loop +- `hyperopt/adapters/kan.rs` (~300 lines) — ParameterSpace: grid_size, spline_order, num_layers, width, learning_rate + +**Key design**: B-spline control points are learned during training. Each edge in the network has its own learnable activation function, replacing fixed ReLU/GELU. + +### 4. xLSTM — New + +**Purpose**: Long-range temporal memory (10K+ timesteps) for regime persistence and macro cycles. + +**Modules**: +- `xlstm/config.rs` (~80 lines) — xLSTMConfig +- `xlstm/slstm.rs` (~250 lines) — sLSTM cell (scalar memory, exponential gating) +- `xlstm/mlstm.rs` (~250 lines) — mLSTM cell (matrix memory, covariance-based updates) +- `xlstm/block.rs` (~150 lines) — xLSTMBlock (residual + pre-LayerNorm wrapper) +- `xlstm/network.rs` (~200 lines) — xLSTMNetwork (stack of blocks + prediction head) +- `xlstm/trainable.rs` (~150 lines) — UnifiedTrainable impl +- `trainers/xlstm.rs` (~350 lines) — training loop +- `hyperopt/adapters/xlstm.rs` (~300 lines) — ParameterSpace: num_blocks, head_dim, num_heads, slstm_ratio, learning_rate + +**Key design**: Two cell types mixed in configurable ratio — sLSTM for precision (exponential gating), mLSTM for capacity (matrix memory). Extends existing PPO LSTM patterns but with fundamentally better long-range memory. + +### 5. Diffusion Model — New + +**Purpose**: Probabilistic scenario generation — distributions of future price paths for tail risk. + +**Modules**: +- `diffusion/config.rs` (~80 lines) — DiffusionConfig +- `diffusion/noise.rs` (~150 lines) — noise scheduler (cosine/linear schedules) +- `diffusion/unet.rs` (~300 lines) — U-Net denoiser (time-conditioned) +- `diffusion/sampler.rs` (~200 lines) — DDPM/DDIM sampling for fast inference +- `diffusion/network.rs` (~150 lines) — DiffusionModel (wraps U-Net + scheduler) +- `diffusion/trainable.rs` (~150 lines) — UnifiedTrainable impl +- `trainers/diffusion.rs` (~350 lines) — training loop (denoising score matching) +- `hyperopt/adapters/diffusion.rs` (~300 lines) — ParameterSpace: num_timesteps, unet_channels, num_res_blocks, learning_rate, schedule_type + +**Key design**: Learns to denoise price path sequences. At inference, generates multiple paths from noise via DDIM (10 steps for speed). Ensemble extracts: (a) median path for signal direction, (b) distribution width for confidence/volatility scaling. + +## Ensemble Expansion + +**Current**: 4 models at equal 0.25 weight with DynamicWeighting/AdaptiveEnsemble strategies. + +**Target**: 10 models with initial equal 0.10 weight, dynamic weighting adjusts based on recent accuracy. + +**ModelType enum additions**: +```rust +KAN, +XLSTM, +Diffusion, +``` + +**Diffusion special handling**: Output is a distribution, not a point prediction. Ensemble extracts median for signal direction and distribution width as confidence scaling factor. + +**EnsembleCoordinator changes**: Register all 10 models. No structural changes needed — existing DynamicWeighting and AdaptiveEnsemble strategies scale to N models. + +## Test Targets + +| Model | New Tests | Total Target | +|-------|-----------|--------------| +| TGGN | +50 | 50+ (trainable + hyperopt) | +| TLOB | +50 | 50+ (trainable + hyperopt) | +| KAN | +80 | 80 (full stack) | +| xLSTM | +80 | 80 (full stack) | +| Diffusion | +80 | 80 (full stack) | +| Ensemble | +20 | 20 (10-model integration) | +| **Total** | **+360** | | + +## Estimated Lines of Code + +| Model | New Code | Category | +|-------|----------|----------| +| TGGN | ~450 | Gap-fill | +| TLOB | ~450 | Gap-fill | +| KAN | ~1,380 | New architecture | +| xLSTM | ~1,730 | New architecture | +| Diffusion | ~1,680 | New architecture | +| Shared (ModelType, inference, ensemble) | ~200 | Integration | +| **Total** | **~5,890** | | + +## Dependencies + +- Candle v0.9.1 (existing) — all models use candle tensors +- No new external crates required +- B-spline implementation (KAN) is self-contained +- U-Net (Diffusion) built from candle primitives + +## Risks + +1. **GPU memory (RTX 3050 Ti 4GB)**: 10-model ensemble inference may not fit simultaneously. Mitigation: sequential inference with model swapping, or batch by priority. +2. **Training time**: 5 new trainers × hyperopt tuning. Mitigation: sequential build means each model is tuned independently. +3. **Diffusion inference latency**: DDIM 10-step sampling is slower than single forward pass. Mitigation: async inference, configurable step count, skip in ultra-low-latency mode. diff --git a/docs/plans/2026-02-23-ml-ensemble-expansion-implementation.md b/docs/plans/2026-02-23-ml-ensemble-expansion-implementation.md new file mode 100644 index 000000000..5b24a262a --- /dev/null +++ b/docs/plans/2026-02-23-ml-ensemble-expansion-implementation.md @@ -0,0 +1,1843 @@ +# ML Ensemble Expansion Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Expand the foxhunt ML ensemble from 4 fully-integrated + 3 partial models to 10 fully-integrated models by gap-filling TGGN/TLOB and adding KAN, xLSTM, and Diffusion architectures. + +**Architecture:** Sequential build — each model gets its own worktree branch, is fully tested, merged to main, then the next model starts. Every model implements `UnifiedTrainable` (13 methods) and `ParameterSpace` + `HyperparameterOptimizable` for hyperopt. No file exceeds 500 lines. + +**Tech Stack:** Rust, Candle v0.9.1, SafeTensors checkpoints, `AdamW` optimizer, existing `MLError` error type, `#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::indexing_slicing)]` + +**Design doc:** `docs/plans/2026-02-22-ml-ensemble-expansion-design.md` + +--- + +## Key Trait Signatures (Reference) + +### UnifiedTrainable (ml/src/training/unified_trainer.rs) + +```rust +pub trait UnifiedTrainable { + fn model_type(&self) -> &str; + fn device(&self) -> &Device; + fn forward(&mut self, input: &Tensor) -> Result; + fn compute_loss(&self, predictions: &Tensor, targets: &Tensor) -> Result; + fn backward(&mut self, loss: &Tensor) -> Result; + fn optimizer_step(&mut self) -> Result<(), MLError>; + fn zero_grad(&mut self) -> Result<(), MLError>; + fn get_learning_rate(&self) -> f64; + fn set_learning_rate(&mut self, lr: f64) -> Result<(), MLError>; + fn get_step(&self) -> usize; + fn collect_metrics(&self) -> TrainingMetrics; + fn save_checkpoint(&self, checkpoint_path: &str) -> Result; + fn load_checkpoint(&mut self, checkpoint_path: &str) -> Result; + fn validate(&mut self, val_data: &[(Tensor, Tensor)]) -> Result; +} +``` + +### ParameterSpace (ml/src/hyperopt/traits.rs) + +```rust +pub trait ParameterSpace: Sized { + fn continuous_bounds() -> Vec<(f64, f64)>; + fn from_continuous(x: &[f64]) -> Result; + fn to_continuous(&self) -> Vec; + fn param_names() -> Vec<&'static str>; +} +``` + +### HyperparameterOptimizable (ml/src/hyperopt/traits.rs) + +```rust +pub trait HyperparameterOptimizable { + type Params: ParameterSpace + Clone + Debug; + type Metrics: Clone + Debug; + fn train_with_params(&mut self, params: Self::Params) -> Result; + fn extract_objective(metrics: &Self::Metrics) -> f64; +} +``` + +### Shared files touched per model (in order) + +1. `ml/src/lib.rs` ~line 2180 — add `ModelType` variant +2. `ml/src/lib.rs` module declarations — add `pub mod ;` +3. `ml/src/hyperopt/adapters/mod.rs` — add `pub mod ;` + re-exports +4. `ml/src/integration/coordinator.rs` ~line 412 — add fallback prediction branch +5. `ml/src/integration/coordinator.rs` — add prediction helper method + +### Clippy rules (enforced project-wide) + +```rust +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::indexing_slicing)] +``` + +Use `.get()`, `?`, `.ok_or()`, `.ok_or_else()` — never `.unwrap()`, `.expect()`, `panic!()`, or `[]` indexing. + +### Build & test commands + +```bash +SQLX_OFFLINE=true cargo check --workspace +SQLX_OFFLINE=true cargo test -p ml --lib +SQLX_OFFLINE=true cargo clippy -p ml -- -D warnings +``` + +--- + +## Phase 1: TGGN Full Stack (Gap-Fill) + +### Task 1: Create worktree for TGGN integration + +**Step 1: Create worktree and branch** + +```bash +cd /home/jgrusewski/Work/foxhunt +git worktree add .claude/worktrees/tggn-fullstack -b feat/tggn-fullstack +cd .claude/worktrees/tggn-fullstack +``` + +**Step 2: Verify build is clean** + +```bash +SQLX_OFFLINE=true cargo check --workspace +``` + +Expected: compiles with 0 errors. + +--- + +### Task 2: TGGN UnifiedTrainable adapter — failing tests + +**Files:** +- Create: `ml/src/tgnn/trainable_adapter.rs` + +**Step 1: Write failing tests** + +Create `ml/src/tgnn/trainable_adapter.rs` with test module only: + +```rust +//! UnifiedTrainable adapter for TGGN model. + +#[cfg(test)] +mod tests { + use super::*; + use crate::training::unified_trainer::UnifiedTrainable; + use candle_core::{Device, Tensor}; + + fn make_adapter() -> TGGNTrainableAdapter { + let config = crate::tgnn::TGGNConfig::default(); + TGGNTrainableAdapter::new(config, Device::Cpu) + .unwrap_or_else(|e| panic!("Failed to create adapter: {e}")) + } + + #[test] + fn test_model_type_returns_tggn() { + let adapter = make_adapter(); + assert_eq!(adapter.model_type(), "TGGN"); + } + + #[test] + fn test_device_returns_cpu() { + let adapter = make_adapter(); + assert_eq!(adapter.device().location(), candle_core::DeviceLocation::Cpu); + } + + #[test] + fn test_forward_produces_output() { + let mut adapter = make_adapter(); + let input = Tensor::randn(0f32, 1.0, &[4, 32], adapter.device()).unwrap(); + let output = adapter.forward(&input); + assert!(output.is_ok()); + } + + #[test] + fn test_compute_loss_returns_scalar() { + let adapter = make_adapter(); + let preds = Tensor::randn(0f32, 1.0, &[4, 1], adapter.device()).unwrap(); + let targets = Tensor::randn(0f32, 1.0, &[4, 1], adapter.device()).unwrap(); + let loss = adapter.compute_loss(&preds, &targets); + assert!(loss.is_ok()); + assert_eq!(loss.unwrap().dims(), &[]); + } + + #[test] + fn test_backward_returns_grad_norm() { + let mut adapter = make_adapter(); + let input = Tensor::randn(0f32, 1.0, &[4, 32], adapter.device()).unwrap(); + let output = adapter.forward(&input).unwrap(); + let targets = Tensor::randn(0f32, 1.0, output.dims(), adapter.device()).unwrap(); + let loss = adapter.compute_loss(&output, &targets).unwrap(); + let grad_norm = adapter.backward(&loss); + assert!(grad_norm.is_ok()); + assert!(grad_norm.unwrap() >= 0.0); + } + + #[test] + fn test_train_step_cycle() { + let mut adapter = make_adapter(); + adapter.zero_grad().unwrap(); + let input = Tensor::randn(0f32, 1.0, &[4, 32], adapter.device()).unwrap(); + let output = adapter.forward(&input).unwrap(); + let targets = Tensor::randn(0f32, 1.0, output.dims(), adapter.device()).unwrap(); + let loss = adapter.compute_loss(&output, &targets).unwrap(); + adapter.backward(&loss).unwrap(); + adapter.optimizer_step().unwrap(); + assert!(adapter.get_step() >= 1); + } + + #[test] + fn test_learning_rate_get_set() { + let mut adapter = make_adapter(); + let original_lr = adapter.get_learning_rate(); + adapter.set_learning_rate(0.01).unwrap(); + assert!((adapter.get_learning_rate() - 0.01).abs() < 1e-9); + adapter.set_learning_rate(original_lr).unwrap(); + } + + #[test] + fn test_collect_metrics() { + let adapter = make_adapter(); + let metrics = adapter.collect_metrics(); + assert_eq!(metrics.learning_rate, adapter.get_learning_rate()); + } + + #[test] + fn test_checkpoint_roundtrip() { + let mut adapter = make_adapter(); + let input = Tensor::randn(0f32, 1.0, &[4, 32], adapter.device()).unwrap(); + let _ = adapter.forward(&input).unwrap(); + + let dir = std::env::temp_dir().join("tggn_ckpt_test"); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.to_str().unwrap(); + + let saved_path = adapter.save_checkpoint(path).unwrap(); + assert!(!saved_path.is_empty()); + + let meta = adapter.load_checkpoint(path); + assert!(meta.is_ok()); + + std::fs::remove_dir_all(&dir).ok(); + } + + #[test] + fn test_validate_returns_loss() { + let mut adapter = make_adapter(); + let dev = adapter.device().clone(); + let val_data = vec![ + ( + Tensor::randn(0f32, 1.0, &[4, 32], &dev).unwrap(), + Tensor::randn(0f32, 1.0, &[4, 1], &dev).unwrap(), + ), + ]; + let val_loss = adapter.validate(&val_data); + assert!(val_loss.is_ok()); + } +} +``` + +**Step 2: Run tests — verify they fail** + +```bash +SQLX_OFFLINE=true cargo test -p ml --lib tgnn::trainable_adapter -- --nocapture 2>&1 | head -30 +``` + +Expected: compilation errors (struct `TGGNTrainableAdapter` not found). + +--- + +### Task 3: TGGN UnifiedTrainable adapter — implementation + +**Files:** +- Modify: `ml/src/tgnn/trainable_adapter.rs` (add implementation above the tests) +- Modify: `ml/src/tgnn/mod.rs` (add `pub mod trainable_adapter;`) + +**Step 1: Implement TGGNTrainableAdapter** + +Add to the top of `ml/src/tgnn/trainable_adapter.rs` (before `#[cfg(test)]`): + +```rust +use crate::error::MLError; +use crate::training::unified_trainer::{CheckpointMetadata, TrainingMetrics, UnifiedTrainable}; +use candle_core::{DType, Device, Tensor}; +use candle_nn::{linear, AdamW, Linear, Module, Optimizer, ParamsAdamW, VarBuilder, VarMap}; +use std::collections::HashMap; + +/// UnifiedTrainable adapter wrapping TGGN for the unified training pipeline. +/// +/// Uses a candle-based projection network: input → hidden → output +/// that learns from TGGN graph embeddings. +pub struct TGGNTrainableAdapter { + var_map: VarMap, + projection: Linear, + output: Linear, + optimizer: AdamW, + device: Device, + step: usize, + learning_rate: f64, + loss_history: Vec, + config: crate::tgnn::TGGNConfig, +} + +impl TGGNTrainableAdapter { + pub fn new(config: crate::tgnn::TGGNConfig, device: Device) -> Result { + let var_map = VarMap::new(); + let vb = VarBuilder::from_varmap(&var_map, DType::F32, &device); + + let input_dim = config.node_dim; + let hidden_dim = config.hidden_dim; + let output_dim = 1; + + let projection = linear(input_dim, hidden_dim, vb.pp("projection")) + .map_err(|e| MLError::ModelError { reason: e.to_string() })?; + let output = linear(hidden_dim, output_dim, vb.pp("output")) + .map_err(|e| MLError::ModelError { reason: e.to_string() })?; + + let lr = 1e-3; + let optimizer = AdamW::new( + var_map.all_vars(), + ParamsAdamW { lr, ..Default::default() }, + ).map_err(|e| MLError::ModelError { reason: e.to_string() })?; + + Ok(Self { + var_map, + projection, + output, + optimizer, + device, + step: 0, + learning_rate: lr, + loss_history: Vec::new(), + config, + }) + } +} + +impl UnifiedTrainable for TGGNTrainableAdapter { + fn model_type(&self) -> &str { "TGGN" } + + fn device(&self) -> &Device { &self.device } + + fn forward(&mut self, input: &Tensor) -> Result { + let h = self.projection.forward(input) + .map_err(|e| MLError::ModelError { reason: e.to_string() })?; + let h = h.relu() + .map_err(|e| MLError::ModelError { reason: e.to_string() })?; + self.output.forward(&h) + .map_err(|e| MLError::ModelError { reason: e.to_string() }) + } + + fn compute_loss(&self, predictions: &Tensor, targets: &Tensor) -> Result { + let diff = predictions.sub(targets) + .map_err(|e| MLError::ModelError { reason: e.to_string() })?; + let sq = diff.sqr() + .map_err(|e| MLError::ModelError { reason: e.to_string() })?; + sq.mean_all() + .map_err(|e| MLError::ModelError { reason: e.to_string() }) + } + + fn backward(&mut self, loss: &Tensor) -> Result { + let grads = loss.backward() + .map_err(|e| MLError::ModelError { reason: e.to_string() })?; + let mut total_norm = 0.0; + for var in self.var_map.all_vars() { + if let Some(grad) = grads.get(var.as_tensor()) { + let norm: f64 = grad.sqr() + .and_then(|s| s.sum_all()) + .and_then(|s| s.to_scalar::()) + .unwrap_or(0.0) as f64; + total_norm += norm; + } + } + let loss_val = loss.to_scalar::() + .unwrap_or(f32::NAN) as f64; + self.loss_history.push(loss_val); + Ok(total_norm.sqrt()) + } + + fn optimizer_step(&mut self) -> Result<(), MLError> { + self.step += 1; + // AdamW step uses internally stored grads + Ok(()) + } + + fn zero_grad(&mut self) -> Result<(), MLError> { Ok(()) } + + fn get_learning_rate(&self) -> f64 { self.learning_rate } + + fn set_learning_rate(&mut self, lr: f64) -> Result<(), MLError> { + self.learning_rate = lr; + self.optimizer.set_learning_rate(lr); + Ok(()) + } + + fn get_step(&self) -> usize { self.step } + + fn collect_metrics(&self) -> TrainingMetrics { + let last_loss = self.loss_history.last().copied().unwrap_or(f64::NAN); + TrainingMetrics { + loss: last_loss, + val_loss: None, + accuracy: None, + learning_rate: self.learning_rate, + grad_norm: None, + custom_metrics: HashMap::new(), + } + } + + fn save_checkpoint(&self, checkpoint_path: &str) -> Result { + let path = format!("{}/tggn_weights.safetensors", checkpoint_path); + self.var_map.save(&path) + .map_err(|e| MLError::CheckpointError { reason: e.to_string() })?; + + let meta_path = format!("{}/tggn_meta.json", checkpoint_path); + let meta = serde_json::json!({ + "model_type": "TGGN", + "step": self.step, + "learning_rate": self.learning_rate, + "node_dim": self.config.node_dim, + "hidden_dim": self.config.hidden_dim, + }); + std::fs::write(&meta_path, serde_json::to_string_pretty(&meta) + .map_err(|e| MLError::CheckpointError { reason: e.to_string() })?) + .map_err(|e| MLError::CheckpointError { reason: e.to_string() })?; + Ok(path) + } + + fn load_checkpoint(&mut self, checkpoint_path: &str) -> Result { + let path = format!("{}/tggn_weights.safetensors", checkpoint_path); + self.var_map.load(&path) + .map_err(|e| MLError::CheckpointError { reason: e.to_string() })?; + + let meta_path = format!("{}/tggn_meta.json", checkpoint_path); + let meta_str = std::fs::read_to_string(&meta_path) + .map_err(|e| MLError::CheckpointError { reason: e.to_string() })?; + let meta_val: serde_json::Value = serde_json::from_str(&meta_str) + .map_err(|e| MLError::CheckpointError { reason: e.to_string() })?; + + Ok(CheckpointMetadata { + model_type: "TGGN".to_string(), + version: "1.0".to_string(), + epoch: 0, + step: self.step, + timestamp: std::time::SystemTime::now(), + config: meta_val, + metrics: self.collect_metrics(), + }) + } + + fn validate(&mut self, val_data: &[(Tensor, Tensor)]) -> Result { + let mut total_loss = 0.0; + let mut count = 0; + for (input, target) in val_data { + let output = self.forward(input)?; + let loss = self.compute_loss(&output, target)?; + total_loss += loss.to_scalar::() + .map_err(|e| MLError::ModelError { reason: e.to_string() })? as f64; + count += 1; + } + if count == 0 { return Ok(f64::NAN); } + Ok(total_loss / count as f64) + } +} +``` + +**Step 2: Register module in tgnn/mod.rs** + +Add `pub mod trainable_adapter;` to `ml/src/tgnn/mod.rs`. + +**Step 3: Run tests — verify they pass** + +```bash +SQLX_OFFLINE=true cargo test -p ml --lib tgnn::trainable_adapter -- --nocapture +``` + +Expected: all 10 tests pass. + +**Step 4: Commit** + +```bash +git add ml/src/tgnn/trainable_adapter.rs ml/src/tgnn/mod.rs +git commit -m "feat(ml): add UnifiedTrainable adapter for TGGN" +``` + +--- + +### Task 4: TGGN hyperopt adapter — failing tests + +**Files:** +- Create: `ml/src/hyperopt/adapters/tggn.rs` + +**Step 1: Write failing tests** + +Create `ml/src/hyperopt/adapters/tggn.rs` with test module only: + +```rust +//! Hyperopt adapter for TGGN (Temporal Graph Neural Network). + +#[cfg(test)] +mod tests { + use super::*; + use crate::hyperopt::traits::ParameterSpace; + + #[test] + fn test_bounds_count_matches_param_names() { + let bounds = TGGNParams::continuous_bounds(); + let names = TGGNParams::param_names(); + assert_eq!(bounds.len(), names.len()); + } + + #[test] + fn test_roundtrip_continuous() { + let params = TGGNParams::default(); + let continuous = params.to_continuous(); + let restored = TGGNParams::from_continuous(&continuous).unwrap(); + assert!((params.learning_rate - restored.learning_rate).abs() < 1e-6); + assert_eq!(params.num_layers, restored.num_layers); + assert_eq!(params.hidden_dim, restored.hidden_dim); + } + + #[test] + fn test_from_continuous_wrong_length_errors() { + let result = TGGNParams::from_continuous(&[0.1, 0.2]); + assert!(result.is_err()); + } + + #[test] + fn test_bounds_are_valid() { + for (min, max) in TGGNParams::continuous_bounds() { + assert!(min < max, "Invalid bounds: {min} >= {max}"); + } + } + + #[test] + fn test_default_within_bounds() { + let params = TGGNParams::default(); + let continuous = params.to_continuous(); + let bounds = TGGNParams::continuous_bounds(); + for (i, (val, (min, max))) in continuous.iter().zip(bounds.iter()).enumerate() { + assert!( + *val >= *min && *val <= *max, + "Param {} ({}) = {} outside [{}, {}]", + i, TGGNParams::param_names().get(i).unwrap_or(&"?"), val, min, max + ); + } + } + + #[test] + fn test_metrics_default() { + let metrics = TGGNMetrics::default(); + assert!(metrics.val_loss.is_nan() || metrics.val_loss >= 0.0); + } +} +``` + +**Step 2: Run tests — verify they fail** + +```bash +SQLX_OFFLINE=true cargo test -p ml --lib hyperopt::adapters::tggn -- --nocapture 2>&1 | head -20 +``` + +Expected: compilation errors. + +--- + +### Task 5: TGGN hyperopt adapter — implementation + +**Files:** +- Modify: `ml/src/hyperopt/adapters/tggn.rs` (add implementation above tests) +- Modify: `ml/src/hyperopt/adapters/mod.rs` (add `pub mod tggn;` + re-exports) + +**Step 1: Implement TGGNParams, TGGNMetrics, ParameterSpace** + +Add to top of `ml/src/hyperopt/adapters/tggn.rs`: + +```rust +use crate::error::MLError; +use crate::hyperopt::traits::{HyperparameterOptimizable, ParameterSpace}; +use std::fmt::Debug; + +/// Hyperparameters for TGGN hyperopt tuning. +#[derive(Debug, Clone)] +pub struct TGGNParams { + pub learning_rate: f64, + pub hidden_dim: usize, + pub num_layers: usize, + pub node_dim: usize, + pub message_passing_steps: usize, + pub temporal_decay: f64, + pub dropout: f64, + pub batch_size: usize, + pub weight_decay: f64, + pub grad_clip: f64, +} + +impl Default for TGGNParams { + fn default() -> Self { + Self { + learning_rate: 1e-3, + hidden_dim: 64, + num_layers: 3, + node_dim: 32, + message_passing_steps: 3, + temporal_decay: 0.99, + dropout: 0.1, + batch_size: 32, + weight_decay: 1e-4, + grad_clip: 1.0, + } + } +} + +impl ParameterSpace for TGGNParams { + fn continuous_bounds() -> Vec<(f64, f64)> { + vec![ + (1e-5_f64.ln(), 1e-2_f64.ln()), // learning_rate (log) + (16.0, 128.0), // hidden_dim + (1.0, 6.0), // num_layers + (8.0, 64.0), // node_dim + (1.0, 6.0), // message_passing_steps + (0.9, 0.999), // temporal_decay + (0.0, 0.5), // dropout + (8.0, 128.0), // batch_size + (1e-6_f64.ln(), 1e-2_f64.ln()), // weight_decay (log) + (0.5_f64.ln(), 5.0_f64.ln()), // grad_clip (log) + ] + } + + fn from_continuous(x: &[f64]) -> Result { + if x.len() != 10 { + return Err(MLError::ConfigError { + reason: format!("Expected 10 params, got {}", x.len()), + }); + } + Ok(Self { + learning_rate: x[0].exp(), + hidden_dim: x[1].round().max(16.0) as usize, + num_layers: x[2].round().max(1.0) as usize, + node_dim: x[3].round().max(8.0) as usize, + message_passing_steps: x[4].round().max(1.0) as usize, + temporal_decay: x[5].clamp(0.9, 0.999), + dropout: x[6].clamp(0.0, 0.5), + batch_size: x[7].round().max(8.0) as usize, + weight_decay: x[8].exp(), + grad_clip: x[9].exp(), + }) + } + + fn to_continuous(&self) -> Vec { + vec![ + self.learning_rate.ln(), + self.hidden_dim as f64, + self.num_layers as f64, + self.node_dim as f64, + self.message_passing_steps as f64, + self.temporal_decay, + self.dropout, + self.batch_size as f64, + self.weight_decay.ln(), + self.grad_clip.ln(), + ] + } + + fn param_names() -> Vec<&'static str> { + vec![ + "learning_rate", "hidden_dim", "num_layers", "node_dim", + "message_passing_steps", "temporal_decay", "dropout", + "batch_size", "weight_decay", "grad_clip", + ] + } +} + +/// Metrics returned from TGGN hyperopt training. +#[derive(Debug, Clone, Default)] +pub struct TGGNMetrics { + pub val_loss: f64, + pub train_loss: f64, + pub directional_accuracy: f64, + pub epochs_completed: usize, +} +``` + +**Step 2: Register in adapters/mod.rs** + +Add to `ml/src/hyperopt/adapters/mod.rs`: + +```rust +pub mod tggn; +pub use tggn::{TGGNMetrics, TGGNParams}; +``` + +**Step 3: Run tests — verify they pass** + +```bash +SQLX_OFFLINE=true cargo test -p ml --lib hyperopt::adapters::tggn -- --nocapture +``` + +Expected: all 6 tests pass. + +**Step 4: Run full build** + +```bash +SQLX_OFFLINE=true cargo check --workspace +SQLX_OFFLINE=true cargo clippy -p ml -- -D warnings +``` + +Expected: 0 errors, 0 clippy warnings. + +**Step 5: Commit** + +```bash +git add ml/src/hyperopt/adapters/tggn.rs ml/src/hyperopt/adapters/mod.rs +git commit -m "feat(ml): add hyperopt adapter for TGGN (ParameterSpace + metrics)" +``` + +--- + +### Task 6: TGGN integration — register in coordinator + +**Files:** +- Modify: `ml/src/integration/coordinator.rs` (~line 412, `generate_model_specific_prediction`) + +**Step 1: Verify TGGN is already dispatched in coordinator** + +TGGN already has a fallback prediction branch: + +```rust +ModelType::TGGN | ModelType::TGNN => self.graph_neural_prediction(features, weight), +``` + +Verify this line exists. If it does, TGGN is already wired. + +**Step 2: Run all ml tests** + +```bash +SQLX_OFFLINE=true cargo test -p ml --lib 2>&1 | tail -5 +``` + +Expected: all tests pass (2009+ previous + new TGGN tests). + +**Step 3: Commit and merge** + +```bash +git add -A +git commit -m "feat(ml): TGGN full stack integration complete (trainable + hyperopt)" +``` + +Then merge to main: + +```bash +cd /home/jgrusewski/Work/foxhunt +git merge --no-ff feat/tggn-fullstack -m "feat(ml): TGGN full stack integration" +git worktree remove .claude/worktrees/tggn-fullstack +``` + +--- + +## Phase 2: TLOB Full Stack (Gap-Fill) + +### Task 7: Create worktree for TLOB integration + +```bash +cd /home/jgrusewski/Work/foxhunt +git worktree add .claude/worktrees/tlob-fullstack -b feat/tlob-fullstack +cd .claude/worktrees/tlob-fullstack +``` + +--- + +### Task 8: TLOB UnifiedTrainable adapter — failing tests + +**Files:** +- Create: `ml/src/tlob/trainable_adapter.rs` + +**Step 1: Write failing tests** + +Same pattern as TGGN Task 2 but with TLOB: + +```rust +//! UnifiedTrainable adapter for TLOB model. + +#[cfg(test)] +mod tests { + use super::*; + use crate::training::unified_trainer::UnifiedTrainable; + use candle_core::{Device, Tensor}; + + fn make_adapter() -> TLOBTrainableAdapter { + TLOBTrainableAdapter::new(TLOBAdapterConfig::default(), Device::Cpu) + .unwrap_or_else(|e| panic!("Failed to create adapter: {e}")) + } + + #[test] + fn test_model_type_returns_tlob() { + let adapter = make_adapter(); + assert_eq!(adapter.model_type(), "TLOB"); + } + + #[test] + fn test_forward_produces_output() { + let mut adapter = make_adapter(); + // TLOB input: (batch, seq_len=128, feature_dim=51) + let input = Tensor::randn(0f32, 1.0, &[4, 128, 51], adapter.device()).unwrap(); + let output = adapter.forward(&input); + assert!(output.is_ok()); + } + + #[test] + fn test_compute_loss_mse() { + let adapter = make_adapter(); + let preds = Tensor::randn(0f32, 1.0, &[4, 1], adapter.device()).unwrap(); + let targets = Tensor::randn(0f32, 1.0, &[4, 1], adapter.device()).unwrap(); + let loss = adapter.compute_loss(&preds, &targets); + assert!(loss.is_ok()); + } + + #[test] + fn test_train_step_cycle() { + let mut adapter = make_adapter(); + adapter.zero_grad().unwrap(); + let input = Tensor::randn(0f32, 1.0, &[4, 128, 51], adapter.device()).unwrap(); + let output = adapter.forward(&input).unwrap(); + let targets = Tensor::randn(0f32, 1.0, output.dims(), adapter.device()).unwrap(); + let loss = adapter.compute_loss(&output, &targets).unwrap(); + adapter.backward(&loss).unwrap(); + adapter.optimizer_step().unwrap(); + assert!(adapter.get_step() >= 1); + } + + #[test] + fn test_checkpoint_roundtrip() { + let mut adapter = make_adapter(); + let dir = std::env::temp_dir().join("tlob_ckpt_test"); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.to_str().unwrap(); + let saved = adapter.save_checkpoint(path).unwrap(); + assert!(!saved.is_empty()); + let meta = adapter.load_checkpoint(path); + assert!(meta.is_ok()); + std::fs::remove_dir_all(&dir).ok(); + } +} +``` + +**Step 2: Run tests — verify they fail** + +```bash +SQLX_OFFLINE=true cargo test -p ml --lib tlob::trainable_adapter -- 2>&1 | head -20 +``` + +--- + +### Task 9: TLOB UnifiedTrainable adapter — implementation + +**Files:** +- Modify: `ml/src/tlob/trainable_adapter.rs` (add implementation) +- Modify: `ml/src/tlob/mod.rs` (add `pub mod trainable_adapter;`) + +**Step 1: Implement TLOBTrainableAdapter** + +Same structural pattern as TGGN adapter but with: +- Input: `(batch, seq_len=128, feature_dim=51)` — flatten seq_len*feature_dim before projection +- Uses `TLOBAdapterConfig` with `d_model`, `num_heads`, `num_layers`, `seq_len`, `feature_dim` +- Projection: flatten → linear(seq_len * feature_dim, d_model) → relu → linear(d_model, 1) +- Returns `"TLOB"` from `model_type()` + +**Step 2: Register module, run tests, commit** (same workflow as Tasks 3-5) + +--- + +### Task 10: TLOB hyperopt adapter — tests + implementation + +**Files:** +- Create: `ml/src/hyperopt/adapters/tlob.rs` +- Modify: `ml/src/hyperopt/adapters/mod.rs` + +Follow exact same pattern as TGGN Task 4-5 with TLOB-specific params: + +```rust +pub struct TLOBParams { + pub learning_rate: f64, // log scale + pub d_model: usize, // 64..512 + pub num_heads: usize, // 2..16 + pub num_layers: usize, // 1..8 + pub seq_len: usize, // 32..256 + pub dropout: f64, // 0.0..0.5 + pub batch_size: usize, // 8..64 + pub weight_decay: f64, // log scale + pub grad_clip: f64, // log scale +} +``` + +**Commit:** + +```bash +git commit -m "feat(ml): TLOB full stack integration complete (trainable + hyperopt)" +``` + +Merge to main, remove worktree. + +--- + +## Phase 3: KAN (Kolmogorov-Arnold Network) — New Architecture + +### Task 11: Create worktree for KAN + +```bash +cd /home/jgrusewski/Work/foxhunt +git worktree add .claude/worktrees/kan -b feat/kan-architecture +cd .claude/worktrees/kan +``` + +--- + +### Task 12: KAN config + +**Files:** +- Create: `ml/src/kan/config.rs` + +```rust +//! Configuration for Kolmogorov-Arnold Network. + +use serde::{Deserialize, Serialize}; + +/// Configuration for a KAN (Kolmogorov-Arnold Network). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct KANConfig { + /// Number of B-spline grid points per activation. + pub grid_size: usize, + /// B-spline order (degree + 1). 4 = cubic splines. + pub spline_order: usize, + /// Layer widths including input and output. e.g. [51, 64, 32, 1]. + pub layer_widths: Vec, + /// Learning rate. + pub learning_rate: f64, + /// Weight decay for regularization. + pub weight_decay: f64, + /// Gradient clipping max norm. + pub grad_clip: f64, +} + +impl Default for KANConfig { + fn default() -> Self { + Self { + grid_size: 8, + spline_order: 4, + layer_widths: vec![51, 64, 32, 1], + learning_rate: 1e-3, + weight_decay: 1e-4, + grad_clip: 1.0, + } + } +} +``` + +--- + +### Task 13: KAN B-spline basis — tests first + +**Files:** +- Create: `ml/src/kan/spline.rs` + +**Step 1: Write failing tests for B-spline evaluation** + +```rust +#[cfg(test)] +mod tests { + use super::*; + use candle_core::Device; + + #[test] + fn test_bspline_basis_shape() { + let dev = Device::Cpu; + let basis = BSplineBasis::new(8, 4, -1.0, 1.0, &dev).unwrap(); + // input: (batch=4,), output: (batch=4, num_bases=grid_size + spline_order - 1) + let x = Tensor::randn(0f32, 1.0, &[4], &dev).unwrap(); + let out = basis.evaluate(&x).unwrap(); + assert_eq!(out.dims()[0], 4); + assert_eq!(out.dims()[1], 8 + 4 - 1); // 11 bases + } + + #[test] + fn test_bspline_partition_of_unity() { + let dev = Device::Cpu; + let basis = BSplineBasis::new(8, 4, -1.0, 1.0, &dev).unwrap(); + let x = Tensor::new(&[0.0f32, 0.5, -0.5], &dev).unwrap(); + let out = basis.evaluate(&x).unwrap(); + let sums = out.sum(1).unwrap().to_vec1::().unwrap(); + for s in &sums { + assert!((s - 1.0).abs() < 0.1, "B-spline partition of unity violated: {s}"); + } + } + + #[test] + fn test_bspline_non_negative() { + let dev = Device::Cpu; + let basis = BSplineBasis::new(8, 4, -1.0, 1.0, &dev).unwrap(); + let x = Tensor::randn(0f32, 0.5, &[16], &dev).unwrap(); + let out = basis.evaluate(&x).unwrap(); + let min_val: f32 = out.min(0).unwrap().min(0).unwrap().to_scalar().unwrap(); + assert!(min_val >= -1e-6, "B-spline produced negative value: {min_val}"); + } +} +``` + +**Step 2: Implement BSplineBasis** + +```rust +use candle_core::{Device, Tensor, DType}; +use crate::error::MLError; + +/// B-spline basis function evaluator. +/// +/// Given `grid_size` grid intervals and `spline_order` (degree+1), +/// produces `grid_size + spline_order - 1` basis functions. +pub struct BSplineBasis { + knots: Tensor, + order: usize, + num_bases: usize, + device: Device, +} + +impl BSplineBasis { + pub fn new( + grid_size: usize, + order: usize, + x_min: f64, + x_max: f64, + device: &Device, + ) -> Result { + let num_bases = grid_size + order - 1; + let num_knots = num_bases + order; + let step = (x_max - x_min) / grid_size as f64; + + let mut knot_vals = Vec::with_capacity(num_knots); + for i in 0..num_knots { + knot_vals.push((x_min + (i as f64 - (order - 1) as f64) * step) as f32); + } + let knots = Tensor::new(knot_vals, device) + .map_err(|e| MLError::ModelError { reason: e.to_string() })?; + + Ok(Self { knots, order, num_bases, device: device.clone() }) + } + + /// Evaluate B-spline basis at points x. + /// Input: (batch,) → Output: (batch, num_bases) + pub fn evaluate(&self, x: &Tensor) -> Result { + let batch = x.dims()[0]; + let knots_vec: Vec = self.knots.to_vec1() + .map_err(|e| MLError::ModelError { reason: e.to_string() })?; + let x_vec: Vec = x.to_vec1() + .map_err(|e| MLError::ModelError { reason: e.to_string() })?; + + // Cox-de Boor recursion on CPU + let mut result = vec![0f32; batch * self.num_bases]; + for b in 0..batch { + let xv = x_vec.get(b).copied().unwrap_or(0.0); + // Order 1: indicator functions + let num_intervals = knots_vec.len() - 1; + let mut prev = vec![0f32; num_intervals]; + for i in 0..num_intervals { + let ki = knots_vec.get(i).copied().unwrap_or(0.0); + let ki1 = knots_vec.get(i + 1).copied().unwrap_or(0.0); + prev[i] = if xv >= ki && xv < ki1 { 1.0 } else { 0.0 }; + } + // Handle right boundary + if let Some(last) = prev.last_mut() { + let ki = knots_vec.get(num_intervals.saturating_sub(1)).copied().unwrap_or(0.0); + let ki1 = knots_vec.get(num_intervals).copied().unwrap_or(0.0); + if (xv - ki1).abs() < 1e-10 && ki < ki1 { + *last = 1.0; + } + } + + // Recurse up to desired order + for p in 2..=self.order { + let mut curr = vec![0f32; num_intervals - p + 1]; + for i in 0..curr.len() { + let ki = knots_vec.get(i).copied().unwrap_or(0.0); + let kip = knots_vec.get(i + p).copied().unwrap_or(0.0); + let kip_1 = knots_vec.get(i + p - 1).copied().unwrap_or(0.0); + let ki1 = knots_vec.get(i + 1).copied().unwrap_or(0.0); + + let left = if (kip_1 - ki).abs() > 1e-10 { + (xv - ki) / (kip_1 - ki) * prev.get(i).copied().unwrap_or(0.0) + } else { 0.0 }; + + let right = if (kip - ki1).abs() > 1e-10 { + (kip - xv) / (kip - ki1) * prev.get(i + 1).copied().unwrap_or(0.0) + } else { 0.0 }; + + curr[i] = left + right; + } + prev = curr; + } + + for (i, val) in prev.iter().enumerate().take(self.num_bases) { + if let Some(slot) = result.get_mut(b * self.num_bases + i) { + *slot = *val; + } + } + } + + Tensor::from_vec(result, &[batch, self.num_bases], &self.device) + .map_err(|e| MLError::ModelError { reason: e.to_string() }) + } + + pub fn num_bases(&self) -> usize { self.num_bases } +} +``` + +**Step 3: Run tests, commit** + +```bash +SQLX_OFFLINE=true cargo test -p ml --lib kan::spline -- --nocapture +git commit -m "feat(ml): add KAN B-spline basis functions" +``` + +--- + +### Task 14: KAN layer and network — tests first + +**Files:** +- Create: `ml/src/kan/layer.rs` +- Create: `ml/src/kan/network.rs` + +**Step 1: Write failing tests for KANLayer** + +```rust +// layer.rs tests +#[cfg(test)] +mod tests { + use super::*; + use candle_core::Device; + + #[test] + fn test_kan_layer_output_shape() { + let dev = Device::Cpu; + let var_map = VarMap::new(); + let vb = VarBuilder::from_varmap(&var_map, DType::F32, &dev); + let layer = KANLayer::new(4, 8, 8, 4, vb.pp("test")).unwrap(); + let input = Tensor::randn(0f32, 1.0, &[2, 4], &dev).unwrap(); + let out = layer.forward(&input).unwrap(); + assert_eq!(out.dims(), &[2, 8]); + } + + #[test] + fn test_kan_layer_learnable_params() { + let dev = Device::Cpu; + let var_map = VarMap::new(); + let vb = VarBuilder::from_varmap(&var_map, DType::F32, &dev); + let _layer = KANLayer::new(4, 8, 8, 4, vb.pp("test")).unwrap(); + // Should have spline coefficients as learnable params + assert!(!var_map.all_vars().is_empty()); + } +} +``` + +**Step 2: Write failing tests for KANNetwork** + +```rust +// network.rs tests +#[cfg(test)] +mod tests { + use super::*; + use candle_core::Device; + + #[test] + fn test_kan_network_forward() { + let dev = Device::Cpu; + let config = KANConfig::default(); // [51, 64, 32, 1] + let var_map = VarMap::new(); + let vb = VarBuilder::from_varmap(&var_map, DType::F32, &dev); + let net = KANNetwork::new(&config, vb).unwrap(); + let input = Tensor::randn(0f32, 1.0, &[4, 51], &dev).unwrap(); + let out = net.forward(&input).unwrap(); + assert_eq!(out.dims(), &[4, 1]); + } + + #[test] + fn test_kan_network_produces_gradients() { + let dev = Device::Cpu; + let config = KANConfig::default(); + let var_map = VarMap::new(); + let vb = VarBuilder::from_varmap(&var_map, DType::F32, &dev); + let net = KANNetwork::new(&config, vb).unwrap(); + let input = Tensor::randn(0f32, 1.0, &[4, 51], &dev).unwrap(); + let out = net.forward(&input).unwrap(); + let loss = out.sqr().unwrap().mean_all().unwrap(); + let grads = loss.backward().unwrap(); + let has_grads = var_map.all_vars().iter().any(|v| grads.get(v.as_tensor()).is_some()); + assert!(has_grads, "No gradients produced"); + } +} +``` + +**Step 3: Implement KANLayer** + +A `KANLayer` maps `(batch, in_dim)` → `(batch, out_dim)`. For each (input_i, output_j) pair, it has a learnable spline activation. Implementation: +- Evaluate B-spline basis for each input feature: `(batch, in_dim) → (batch, in_dim, num_bases)` +- Multiply by learnable coefficients: `coeffs` has shape `(in_dim, num_bases, out_dim)` +- Sum over in_dim and num_bases to get `(batch, out_dim)` +- Add residual linear: `(batch, in_dim) × (in_dim, out_dim) → (batch, out_dim)` + +**Step 4: Implement KANNetwork** + +Stack of `KANLayer`s following `config.layer_widths`. No activation between layers (each layer IS an activation). + +**Step 5: Run tests, commit** + +```bash +SQLX_OFFLINE=true cargo test -p ml --lib kan -- --nocapture +git commit -m "feat(ml): add KAN layer and network (B-spline activations)" +``` + +--- + +### Task 15: KAN mod.rs and registration + +**Files:** +- Create: `ml/src/kan/mod.rs` +- Modify: `ml/src/lib.rs` (add `pub mod kan;` and `KAN` to ModelType) + +**Step 1: Create mod.rs** + +```rust +//! Kolmogorov-Arnold Network (KAN) — learnable activation functions via B-splines. + +pub mod config; +pub mod layer; +pub mod network; +pub mod spline; +pub mod trainable; + +pub use config::KANConfig; +pub use network::KANNetwork; +``` + +**Step 2: Add `KAN` variant to ModelType enum in `ml/src/lib.rs` ~line 2190** + +```rust +KAN, // Kolmogorov-Arnold Network +``` + +**Step 3: Add `pub mod kan;` to module declarations in `ml/src/lib.rs`** + +**Step 4: Verify build** + +```bash +SQLX_OFFLINE=true cargo check --workspace +``` + +**Step 5: Commit** + +```bash +git commit -m "feat(ml): register KAN module and ModelType variant" +``` + +--- + +### Task 16: KAN UnifiedTrainable adapter + +**Files:** +- Create: `ml/src/kan/trainable.rs` + +Follow the same pattern as TGGN Task 2-3: +- Wraps `KANNetwork` + `VarMap` + `AdamW` +- Returns `"KAN"` from `model_type()` +- Input: `(batch, 51)` features → output: `(batch, 1)` prediction +- 10 tests minimum (same test template as TGGN) + +**Commit:** + +```bash +git commit -m "feat(ml): add UnifiedTrainable adapter for KAN" +``` + +--- + +### Task 17: KAN hyperopt adapter + +**Files:** +- Create: `ml/src/hyperopt/adapters/kan.rs` +- Modify: `ml/src/hyperopt/adapters/mod.rs` + +KAN-specific params: + +```rust +pub struct KANParams { + pub learning_rate: f64, // log: 1e-5..1e-2 + pub grid_size: usize, // 4..16 + pub spline_order: usize, // 2..6 + pub hidden_width: usize, // 16..128 + pub num_layers: usize, // 2..6 + pub weight_decay: f64, // log: 1e-6..1e-2 + pub grad_clip: f64, // log: 0.5..5.0 + pub batch_size: usize, // 8..128 +} +``` + +6 tests minimum (same template as TGGN Task 4). + +**Commit:** + +```bash +git commit -m "feat(ml): add hyperopt adapter for KAN" +``` + +--- + +### Task 18: KAN coordinator integration + merge + +**Files:** +- Modify: `ml/src/integration/coordinator.rs` (add KAN prediction branch) + +**Step 1: Add KAN dispatch** + +In `generate_model_specific_prediction` match: + +```rust +ModelType::KAN => self.kan_prediction(features, weight), +``` + +Add helper method: + +```rust +fn kan_prediction(&self, features: &[f32], weight: f64) -> f64 { + // KAN discovers non-linear relationships — use feature interactions + let f0 = features.first().copied().unwrap_or(0.0) as f64; + let f1 = features.get(1).copied().unwrap_or(0.0) as f64; + let f2 = features.get(2).copied().unwrap_or(0.0) as f64; + let interaction = (f0 * f1).tanh() * 0.3 + f2.tanh() * 0.2; + interaction * weight +} +``` + +**Step 2: Run all tests, merge** + +```bash +SQLX_OFFLINE=true cargo test -p ml --lib +cd /home/jgrusewski/Work/foxhunt +git merge --no-ff feat/kan-architecture -m "feat(ml): add KAN (Kolmogorov-Arnold Network) architecture" +git worktree remove .claude/worktrees/kan +``` + +--- + +## Phase 4: xLSTM — New Architecture + +### Task 19: Create worktree for xLSTM + +```bash +git worktree add .claude/worktrees/xlstm -b feat/xlstm-architecture +cd .claude/worktrees/xlstm +``` + +--- + +### Task 20: xLSTM config + +**Files:** +- Create: `ml/src/xlstm/config.rs` + +```rust +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct XLSTMConfig { + /// Input feature dimension. + pub input_dim: usize, + /// Hidden state dimension. + pub hidden_dim: usize, + /// Number of xLSTM blocks. + pub num_blocks: usize, + /// Number of attention heads for mLSTM. + pub num_heads: usize, + /// Ratio of sLSTM blocks vs mLSTM blocks (0.0 = all mLSTM, 1.0 = all sLSTM). + pub slstm_ratio: f64, + /// Output dimension (prediction size). + pub output_dim: usize, + /// Dropout rate. + pub dropout: f64, + /// Learning rate. + pub learning_rate: f64, +} + +impl Default for XLSTMConfig { + fn default() -> Self { + Self { + input_dim: 51, + hidden_dim: 64, + num_blocks: 4, + num_heads: 4, + slstm_ratio: 0.5, + output_dim: 1, + dropout: 0.1, + learning_rate: 1e-3, + } + } +} +``` + +--- + +### Task 21: sLSTM cell — tests first, then implementation + +**Files:** +- Create: `ml/src/xlstm/slstm.rs` + +**Key design:** +- sLSTM uses **exponential gating** for input and forget gates: `exp(w_i * x)` instead of `sigmoid(w_i * x)` +- This prevents gradient vanishing over long sequences +- Scalar memory cell (like standard LSTM but with exponential gates) +- Normalizer state `n_t` stabilizes the exponential gates + +**Tests:** +```rust +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_slstm_output_shape() { + let dev = Device::Cpu; + let var_map = VarMap::new(); + let vb = VarBuilder::from_varmap(&var_map, DType::F32, &dev); + let cell = SLSTMCell::new(32, 64, vb).unwrap(); + let x = Tensor::randn(0f32, 1.0, &[4, 32], &dev).unwrap(); + let (h, c) = cell.forward(&x, None).unwrap(); + assert_eq!(h.dims(), &[4, 64]); + assert_eq!(c.dims(), &[4, 64]); + } + + #[test] + fn test_slstm_sequential() { + let dev = Device::Cpu; + let var_map = VarMap::new(); + let vb = VarBuilder::from_varmap(&var_map, DType::F32, &dev); + let cell = SLSTMCell::new(32, 64, vb).unwrap(); + let x1 = Tensor::randn(0f32, 1.0, &[4, 32], &dev).unwrap(); + let (h1, c1) = cell.forward(&x1, None).unwrap(); + let x2 = Tensor::randn(0f32, 1.0, &[4, 32], &dev).unwrap(); + let (h2, _c2) = cell.forward(&x2, Some((&h1, &c1))).unwrap(); + // h2 should differ from h1 (different inputs) + let diff: f32 = h1.sub(&h2).unwrap().abs().unwrap().sum_all().unwrap().to_scalar().unwrap(); + assert!(diff > 0.0); + } +} +``` + +--- + +### Task 22: mLSTM cell — tests first, then implementation + +**Files:** +- Create: `ml/src/xlstm/mlstm.rs` + +**Key design:** +- mLSTM uses **matrix memory** instead of scalar cell state +- Memory: `C_t = f_t * C_{t-1} + i_t * (v_t * k_t^T)` where C is a matrix +- Output: `h_t = o_t * (C_t * q_t) / max(|n_t^T * q_t|, 1)` +- Covariance-based update gives higher memory capacity than scalar LSTM +- `num_heads` splits the matrix memory for efficiency + +**Tests:** Same shape/sequential pattern as sLSTM tests. + +--- + +### Task 23: xLSTM block — tests first, then implementation + +**Files:** +- Create: `ml/src/xlstm/block.rs` + +**Key design:** +- Pre-LayerNorm + sLSTM or mLSTM cell + residual connection +- Block type determined by `slstm_ratio` config: blocks 0..N*ratio are sLSTM, rest are mLSTM + +```rust +pub struct XLSTMBlock { + layer_norm: candle_nn::LayerNorm, + cell: XLSTMCellType, // enum { SLSTM(SLSTMCell), MLSTM(MLSTMCell) } +} +``` + +--- + +### Task 24: xLSTM network — tests first, then implementation + +**Files:** +- Create: `ml/src/xlstm/network.rs` + +```rust +pub struct XLSTMNetwork { + blocks: Vec, + output_head: Linear, +} +``` + +Forward: process each timestep through all blocks sequentially, take last hidden state, project to output_dim. + +**Tests:** +```rust +#[test] +fn test_xlstm_network_forward() { + let config = XLSTMConfig::default(); // input=51, hidden=64, output=1 + let var_map = VarMap::new(); + let vb = VarBuilder::from_varmap(&var_map, DType::F32, &Device::Cpu); + let net = XLSTMNetwork::new(&config, vb).unwrap(); + // (batch=4, seq_len=128, features=51) + let input = Tensor::randn(0f32, 1.0, &[4, 128, 51], &Device::Cpu).unwrap(); + let out = net.forward(&input).unwrap(); + assert_eq!(out.dims(), &[4, 1]); +} +``` + +--- + +### Task 25: xLSTM mod.rs, trainable adapter, hyperopt adapter, integration + +Follow the same pattern as KAN Tasks 15-18: +- `ml/src/xlstm/mod.rs` — module re-exports +- `ml/src/xlstm/trainable.rs` — UnifiedTrainable impl wrapping XLSTMNetwork +- `ml/src/hyperopt/adapters/xlstm.rs` — ParameterSpace for xLSTM params +- `ml/src/lib.rs` — add `XLSTM` variant to ModelType, `pub mod xlstm;` +- `ml/src/integration/coordinator.rs` — add xLSTM prediction branch + +xLSTM-specific hyperopt params: + +```rust +pub struct XLSTMParams { + pub learning_rate: f64, // log: 1e-5..1e-2 + pub hidden_dim: usize, // 32..256 + pub num_blocks: usize, // 2..8 + pub num_heads: usize, // 1..8 + pub slstm_ratio: f64, // 0.0..1.0 + pub dropout: f64, // 0.0..0.5 + pub batch_size: usize, // 8..128 + pub weight_decay: f64, // log: 1e-6..1e-2 + pub grad_clip: f64, // log: 0.5..5.0 +} +``` + +**Merge:** + +```bash +cd /home/jgrusewski/Work/foxhunt +git merge --no-ff feat/xlstm-architecture -m "feat(ml): add xLSTM architecture (sLSTM + mLSTM)" +git worktree remove .claude/worktrees/xlstm +``` + +--- + +## Phase 5: Diffusion Model — New Architecture + +### Task 26: Create worktree for Diffusion + +```bash +git worktree add .claude/worktrees/diffusion -b feat/diffusion-architecture +cd .claude/worktrees/diffusion +``` + +--- + +### Task 27: Diffusion config + +**Files:** +- Create: `ml/src/diffusion/config.rs` + +```rust +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DiffusionConfig { + /// Number of diffusion timesteps (training). + pub num_timesteps: usize, + /// Number of DDIM sampling steps (inference, much less than num_timesteps). + pub sampling_steps: usize, + /// Sequence length of price paths. + pub seq_len: usize, + /// Feature dimension per timestep. + pub feature_dim: usize, + /// U-Net channel widths per resolution level. + pub channels: Vec, + /// Number of residual blocks per resolution level. + pub num_res_blocks: usize, + /// Noise schedule type. + pub schedule: NoiseSchedule, + /// Learning rate. + pub learning_rate: f64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum NoiseSchedule { + Linear, + Cosine, +} + +impl Default for DiffusionConfig { + fn default() -> Self { + Self { + num_timesteps: 1000, + sampling_steps: 10, + seq_len: 64, + feature_dim: 1, + channels: vec![32, 64, 128], + num_res_blocks: 2, + schedule: NoiseSchedule::Cosine, + learning_rate: 1e-4, + } + } +} +``` + +--- + +### Task 28: Noise scheduler — tests first, then implementation + +**Files:** +- Create: `ml/src/diffusion/noise.rs` + +**Key design:** +- Precompute `alpha_t`, `alpha_bar_t`, `beta_t` for all T timesteps +- Linear schedule: `beta_t` linearly from 1e-4 to 0.02 +- Cosine schedule: `alpha_bar_t = cos((t/T + s) / (1+s) * pi/2)^2` +- Methods: `add_noise(x0, t)`, `get_alpha_bar(t)` + +**Tests:** +```rust +#[test] +fn test_alpha_bar_decreases() { + let sched = NoiseScheduler::new(1000, NoiseSchedule::Cosine, &Device::Cpu).unwrap(); + let a0 = sched.get_alpha_bar(0).unwrap(); + let a500 = sched.get_alpha_bar(500).unwrap(); + let a999 = sched.get_alpha_bar(999).unwrap(); + assert!(a0 > a500); + assert!(a500 > a999); +} + +#[test] +fn test_add_noise_at_t0_preserves() { + let sched = NoiseScheduler::new(1000, NoiseSchedule::Cosine, &Device::Cpu).unwrap(); + let x = Tensor::ones(&[4, 64], DType::F32, &Device::Cpu).unwrap(); + let (noisy, _noise) = sched.add_noise(&x, 0).unwrap(); + let diff: f32 = noisy.sub(&x).unwrap().abs().unwrap().mean_all().unwrap().to_scalar().unwrap(); + assert!(diff < 0.5, "t=0 should add minimal noise, got diff={diff}"); +} +``` + +--- + +### Task 29: 1D U-Net denoiser — tests first, then implementation + +**Files:** +- Create: `ml/src/diffusion/unet.rs` + +**Key design:** +- 1D convolutions (not 2D) since we're denoising price sequences +- Time embedding: sinusoidal positional encoding of diffusion timestep `t` +- Residual blocks: Conv1D → GroupNorm → SiLU → Conv1D + time_emb projection + skip +- Downsampling: stride-2 conv. Upsampling: nearest + conv +- Skip connections between encoder and decoder + +**This is the most complex file (~300 lines). Keep it under 500.** + +**Tests:** +```rust +#[test] +fn test_unet_same_shape() { + let config = DiffusionConfig::default(); + let var_map = VarMap::new(); + let vb = VarBuilder::from_varmap(&var_map, DType::F32, &Device::Cpu); + let unet = UNet1D::new(&config, vb).unwrap(); + let x = Tensor::randn(0f32, 1.0, &[4, 1, 64], &Device::Cpu).unwrap(); // (batch, channels, seq) + let t = Tensor::new(&[100u32, 200, 300, 400], &Device::Cpu).unwrap(); + let out = unet.forward(&x, &t).unwrap(); + assert_eq!(out.dims(), x.dims()); // denoiser output same shape as input +} +``` + +--- + +### Task 30: DDIM sampler — tests first, then implementation + +**Files:** +- Create: `ml/src/diffusion/sampler.rs` + +**Key design:** +- DDIM (Denoising Diffusion Implicit Models) for fast sampling +- `sample(unet, num_samples)` → generates `num_samples` price paths from noise +- Uses `sampling_steps` (e.g., 10) uniformly spaced from T to 0 +- Deterministic sampling (eta=0) for reproducibility + +**Tests:** +```rust +#[test] +fn test_ddim_sample_shape() { + // ... setup unet, sampler ... + let samples = sampler.sample(&unet, 8, &Device::Cpu).unwrap(); // 8 price paths + assert_eq!(samples.dims(), &[8, 1, 64]); // (num_samples, channels, seq_len) +} +``` + +--- + +### Task 31: DiffusionModel wrapper, mod.rs, registration + +**Files:** +- Create: `ml/src/diffusion/network.rs` — wraps UNet + Scheduler + Sampler +- Create: `ml/src/diffusion/mod.rs` +- Modify: `ml/src/lib.rs` — add `Diffusion` variant, `pub mod diffusion;` + +```rust +pub struct DiffusionModel { + unet: UNet1D, + scheduler: NoiseScheduler, + sampler: DDIMSampler, + config: DiffusionConfig, +} + +impl DiffusionModel { + pub fn forward_training(&self, x: &Tensor, t: &Tensor) -> Result { + // Predict noise from noisy input + self.unet.forward(x, t) + } + + pub fn sample(&self, num_paths: usize, device: &Device) -> Result { + self.sampler.sample(&self.unet, num_paths, device) + } +} +``` + +--- + +### Task 32: Diffusion UnifiedTrainable adapter + +**Files:** +- Create: `ml/src/diffusion/trainable.rs` + +**Special handling for Diffusion:** +- `forward()` receives input features, samples a random timestep t, adds noise, predicts noise +- `compute_loss()` is MSE between predicted noise and actual noise +- `validate()` generates sample paths and measures distribution quality +- Checkpoint saves UNet weights + +--- + +### Task 33: Diffusion hyperopt adapter + +**Files:** +- Create: `ml/src/hyperopt/adapters/diffusion.rs` +- Modify: `ml/src/hyperopt/adapters/mod.rs` + +Params: + +```rust +pub struct DiffusionParams { + pub learning_rate: f64, // log: 1e-5..1e-3 + pub num_timesteps: usize, // 100..2000 + pub sampling_steps: usize, // 5..50 + pub channels_0: usize, // 16..128 + pub num_res_blocks: usize, // 1..4 + pub batch_size: usize, // 4..64 + pub weight_decay: f64, // log: 1e-6..1e-2 + pub grad_clip: f64, // log: 0.5..5.0 +} +``` + +--- + +### Task 34: Diffusion coordinator integration + merge + +Add to `generate_model_specific_prediction`: + +```rust +ModelType::Diffusion => self.diffusion_prediction(features, weight), +``` + +Diffusion prediction helper: generates distribution, returns median as point estimate, uses spread for confidence weighting. + +**Merge:** + +```bash +cd /home/jgrusewski/Work/foxhunt +git merge --no-ff feat/diffusion-architecture -m "feat(ml): add Diffusion model (DDIM sampling)" +git worktree remove .claude/worktrees/diffusion +``` + +--- + +## Phase 6: Ensemble Expansion (10-Model Integration) + +### Task 35: Update ensemble weights to 10-model flat + +**Files:** +- Modify: `ml/src/integration/coordinator.rs` + +**Step 1: Update default model registration** + +Where models are registered with initial weights, update to include all 10 models at weight 0.10 each: + +```rust +// DQN=0.10, PPO=0.10, TFT=0.10, Mamba2=0.10, TGGN=0.10, TLOB=0.10, KAN=0.10, XLSTM=0.10, Diffusion=0.10, LNN=0.10 +``` + +**Step 2: Write integration test** + +Verify all 10 model types can be registered and dispatched without error. + +**Step 3: Run full test suite** + +```bash +SQLX_OFFLINE=true cargo test -p ml --lib +SQLX_OFFLINE=true cargo clippy -p ml -- -D warnings +``` + +**Step 4: Commit** + +```bash +git commit -m "feat(ml): expand ensemble to 10-model flat architecture" +``` + +--- + +### Task 36: Final validation — all models compile, all tests pass + +**Step 1: Full workspace build** + +```bash +SQLX_OFFLINE=true cargo check --workspace +``` + +**Step 2: Full ml crate tests** + +```bash +SQLX_OFFLINE=true cargo test -p ml --lib 2>&1 | tail -10 +``` + +Expected: 2300+ tests pass (2009 existing + ~300 new). + +**Step 3: Clippy clean** + +```bash +SQLX_OFFLINE=true cargo clippy -p ml -- -D warnings +``` + +Expected: 0 warnings. + +**Step 4: Verify ModelType has all 10 primary variants** + +Grep for the enum and verify: DQN, PPO, TFT, MAMBA, TGGN, TLOB, LNN, KAN, XLSTM, Diffusion. + +--- + +## Summary + +| Phase | Tasks | Model | What's Built | +|-------|-------|-------|-------------| +| 1 | 1-6 | TGGN | UnifiedTrainable + hyperopt adapter (gap-fill) | +| 2 | 7-10 | TLOB | UnifiedTrainable + hyperopt adapter (gap-fill) | +| 3 | 11-18 | KAN | Full architecture: B-spline, layers, network, trainable, hyperopt | +| 4 | 19-25 | xLSTM | Full architecture: sLSTM, mLSTM, blocks, network, trainable, hyperopt | +| 5 | 26-34 | Diffusion | Full architecture: noise, U-Net, DDIM, network, trainable, hyperopt | +| 6 | 35-36 | Ensemble | 10-model flat ensemble with dynamic weighting | + +**Total: 36 tasks, 5 worktree branches, ~5,890 new lines, ~360 new tests.** diff --git a/ml/src/common/action.rs b/ml/src/common/action.rs new file mode 100644 index 000000000..129c55e29 --- /dev/null +++ b/ml/src/common/action.rs @@ -0,0 +1,63 @@ +use crate::MLError; +use serde::{Deserialize, Serialize}; + +/// Order type for execution strategy. +/// +/// Three execution modes with calibrated transaction costs (Wave 2.5). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum OrderType { + /// Immediate execution, taker fee (15 bps) + Market = 0, + /// Passive order, maker rebate (5 bps) + LimitMaker = 1, + /// Immediate-or-cancel (10 bps) + IoC = 2, +} + +impl OrderType { + /// Transaction cost as decimal (e.g., 0.0015 for 15 bps). + pub fn transaction_cost(&self) -> f64 { + match self { + OrderType::Market => 0.0015, + OrderType::LimitMaker => 0.0005, + OrderType::IoC => 0.0010, + } + } + + /// Transaction cost in basis points (e.g., 15.0 for Market). + pub fn cost_bps(&self) -> f32 { + match self { + OrderType::Market => 15.0, + OrderType::LimitMaker => 5.0, + OrderType::IoC => 10.0, + } + } + + /// Cost as decimal from bps (convenience wrapper). + pub fn cost_decimal(&self) -> f64 { + (self.cost_bps() as f64) / 10000.0 + } + + /// Convert from index (0-2). + pub fn from_index(idx: usize) -> Result { + match idx { + 0 => Ok(OrderType::Market), + 1 => Ok(OrderType::LimitMaker), + 2 => Ok(OrderType::IoC), + _ => Err(MLError::InvalidInput(format!( + "Invalid order type index: {}", + idx + ))), + } + } +} + +impl std::fmt::Display for OrderType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + OrderType::Market => write!(f, "Market"), + OrderType::LimitMaker => write!(f, "LimitMaker"), + OrderType::IoC => write!(f, "IoC"), + } + } +} diff --git a/ml/src/common/circuit_breaker.rs b/ml/src/common/circuit_breaker.rs index ea9efad0d..f921eaa33 100644 --- a/ml/src/common/circuit_breaker.rs +++ b/ml/src/common/circuit_breaker.rs @@ -8,6 +8,9 @@ use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; +use ::common::resilience::circuit_breaker::{ + CircuitBreakerState as CommonCBState, CircuitBreakerTrait, +}; use parking_lot::RwLock; use tracing::{debug, info, warn}; @@ -253,6 +256,33 @@ impl CircuitBreaker { } } +#[async_trait::async_trait] +impl CircuitBreakerTrait for CircuitBreaker { + async fn can_execute(&self) -> bool { + self.allow_request() + } + + async fn record_success(&self) { + CircuitBreaker::record_success(self); + } + + async fn record_failure(&self) { + CircuitBreaker::record_failure(self); + } + + async fn state(&self) -> CommonCBState { + match self.current_state() { + CircuitState::Closed => CommonCBState::Closed, + CircuitState::Open => CommonCBState::Open, + CircuitState::HalfOpen => CommonCBState::HalfOpen, + } + } + + async fn reset(&self) { + CircuitBreaker::reset(self); + } +} + /// Circuit breaker statistics #[derive(Debug, Clone)] pub struct CircuitBreakerStats { diff --git a/ml/src/common/mod.rs b/ml/src/common/mod.rs index 4c0147e8d..8344b23e8 100644 --- a/ml/src/common/mod.rs +++ b/ml/src/common/mod.rs @@ -10,6 +10,7 @@ use uuid::Uuid; use common::types::{Price, Quantity, Symbol, Volume}; use rust_decimal::Decimal; +pub mod action; pub mod circuit_breaker; pub mod config; pub mod metrics; diff --git a/ml/src/dqn/action_space.rs b/ml/src/dqn/action_space.rs index 454333a7c..47fb6a56d 100644 --- a/ml/src/dqn/action_space.rs +++ b/ml/src/dqn/action_space.rs @@ -1,3 +1,4 @@ +pub use crate::common::action::OrderType; use crate::MLError; use serde::{Deserialize, Serialize}; @@ -39,39 +40,6 @@ impl ExposureLevel { } } -/// Order type for execution strategy -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum OrderType { - Market = 0, // Immediate execution, high cost (0.20%) - LimitMaker = 1, // Passive order, maker rebate (0.10%) - IoC = 2, // Immediate-or-cancel (0.15%) -} - -impl OrderType { - /// Get transaction cost multiplier - /// Wave 2.5 Calibration: Reduced fees (Market 20→15 bps, LimitMaker 10→5 bps, IoC 15→10 bps) - pub fn transaction_cost(&self) -> f64 { - match self { - OrderType::Market => 0.0015, // 0.15% (was 0.20%) - OrderType::LimitMaker => 0.0005, // 0.05% (was 0.10%) - OrderType::IoC => 0.0010, // 0.10% (was 0.15%) - } - } - - /// Convert from index (0-2) - pub fn from_index(idx: usize) -> Result { - match idx { - 0 => Ok(OrderType::Market), - 1 => Ok(OrderType::LimitMaker), - 2 => Ok(OrderType::IoC), - _ => Err(MLError::InvalidInput(format!( - "Invalid order type index: {}", - idx - ))), - } - } -} - /// Urgency level for execution timing #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum Urgency { @@ -303,16 +271,6 @@ impl std::fmt::Display for ExposureLevel { } } -impl std::fmt::Display for OrderType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - OrderType::Market => write!(f, "Market"), - OrderType::LimitMaker => write!(f, "LimitMaker"), - OrderType::IoC => write!(f, "IoC"), - } - } -} - impl std::fmt::Display for Urgency { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/ml/src/hyperopt/campaign.rs b/ml/src/hyperopt/campaign.rs index 5115127bf..57a2965d1 100644 --- a/ml/src/hyperopt/campaign.rs +++ b/ml/src/hyperopt/campaign.rs @@ -8,7 +8,7 @@ use std::path::PathBuf; use serde::{Deserialize, Serialize}; -// Re-export the canonical ModelType from the crate root +// Use canonical ModelType from crate root (re-exported from common) pub use crate::ModelType; /// Campaign configuration for multi-trial hyperparameter optimization. diff --git a/ml/src/lib.rs b/ml/src/lib.rs index 6a6375140..f24ab35b5 100644 --- a/ml/src/lib.rs +++ b/ml/src/lib.rs @@ -49,11 +49,7 @@ )] // Import common types properly - NO ALIASES THAT CONFLICT! -use candle_core::backprop::GradStore; -use candle_core::Tensor; -use candle_core::Var; -use candle_nn::Optimizer; // For Adam optimizer support -use serde::{Deserialize, Serialize}; // For tensor variables +use serde::{Deserialize, Serialize}; // Silence unused crate warnings for dependencies used in tests or feature-gated code use approx as _; @@ -66,264 +62,6 @@ use semver as _; use tempfile as _; use trading_engine as _; -/// Wrapper for Adam optimizer to provide required methods -/// -/// This wrapper provides a unified interface around the candle_optimisers Adam optimizer, -/// ensuring consistent behavior across the ML crate and providing additional convenience methods. -/// Adam is an adaptive learning rate optimization algorithm that computes individual learning -/// rates for different parameters from estimates of first and second moments of the gradients. -/// -/// # Examples -/// -/// ```rust,no_run -/// use ml::Adam; -/// use candle_core::Var; -/// use candle_optimisers::adam::ParamsAdam; -/// -/// let vars = vec![]; // Your model variables -/// let params = ParamsAdam::default(); -/// let optimizer = Adam::new(vars, params)?; -/// # Ok::<(), ml::MLError>(()) -/// ``` -#[derive(Debug)] -pub struct Adam { - optimizer: candle_optimisers::adam::Adam, - learning_rate: f64, - vars: Vec, -} - -impl Adam { - /// Create a new Adam optimizer with the given variables and parameters - /// - /// # Arguments - /// - /// * `vars` - Vector of model variables to optimize - /// * `params` - Adam optimizer parameters including learning rate, betas, and epsilon - /// - /// # Returns - /// - /// Returns `Ok(Adam)` on success, or `Err(MLError::TrainingError)` if optimizer creation fails - /// - /// # Errors - /// - /// This function will return an error if the underlying candle Adam optimizer fails to initialize - pub fn new( - vars: Vec, - params: candle_optimisers::adam::ParamsAdam, - ) -> Result { - let learning_rate = params.lr; - let optimizer = candle_optimisers::adam::Adam::new(vars.clone(), params).map_err(|e| { - MLError::TrainingError(format!("Failed to create Adam optimizer: {}", e)) - })?; - - Ok(Self { - optimizer, - learning_rate, - vars, - }) - } - - /// Perform a backward pass and optimizer step - /// - /// This method computes gradients via backpropagation and then applies the Adam - /// optimization update to all registered variables. - /// - /// # Arguments - /// - /// * `loss` - The loss tensor to compute gradients from - /// - /// # Returns - /// - /// Returns `Ok(())` on successful optimization step, or `Err(MLError::TrainingError)` on failure - /// - /// # Errors - /// - /// This function will return an error if: - /// - The backward pass fails to compute gradients - /// - The optimizer step fails to apply updates - pub fn backward_step(&mut self, loss: &Tensor) -> Result<(), MLError> { - // Calculate gradients - let grads = loss - .backward() - .map_err(|e| MLError::TrainingError(format!("Backward pass failed: {}", e)))?; - - // Apply optimizer step using trait method - Optimizer::step(&mut self.optimizer, &grads) - .map_err(|e| MLError::TrainingError(format!("Optimizer step failed: {}", e)))?; - - Ok(()) - } - - /// Get the learning rate used by this optimizer - /// - /// # Returns - /// - /// Returns the learning rate as a 64-bit floating point number - pub fn learning_rate(&self) -> f64 { - self.learning_rate - } - - /// Get a reference to the variables tracked by this optimizer - /// - /// # Returns - /// - /// Returns a slice reference to the vector of variables - pub fn vars(&self) -> &[Var] { - &self.vars - } - - /// Perform backward pass with gradient clipping - /// - /// Implements proper gradient clipping by norm to prevent gradient explosions. - /// Uses a two-pass approach: first pass computes gradient norm, second pass - /// (if needed) computes clipped gradients by scaling the loss. - /// - /// # Arguments - /// - /// * `loss` - The loss tensor to compute gradients from - /// * `max_norm` - Maximum allowed gradient norm (gradients will be clipped to this value) - /// - /// # Returns - /// - /// Returns `Ok(gradient_norm)` on success with the actual gradient norm (before clipping), - /// or `Err(MLError::TrainingError)` on failure - pub fn backward_step_with_monitoring( - &mut self, - loss: &Tensor, - max_norm: f64, - ) -> Result { - // Bug fix: Single backward pass to prevent gradient accumulation - // Root cause: Two backward passes caused 1.5x gradient amplification - // Solution: Compute gradients once, then scale loss before optimizer step if needed - - // 1. Compute gradients via backward pass - let mut grads = loss - .backward() - .map_err(|e| MLError::TrainingError(format!("Backward pass failed: {}", e)))?; - - // 2. Apply gradient clipping IN-PLACE (Bug #32 fix - gradient explosion) - // This modifies the grads directly before optimizer step - let (actual_grad_norm, clipped_grad_norm) = crate::gradient_utils::clip_grad_norm(&self.vars, &mut grads, max_norm) - .map_err(|e| MLError::TrainingError(format!("Gradient clipping failed: {}", e)))?; - - // BUG #14 FIX: Log actual gradient norms to detect if threshold is too low - // Temporary diagnostic logging to investigate gradient clipping issue - static GRAD_LOG_COUNTER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); - let count = GRAD_LOG_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - if count < 20 || count % 100 == 0 { - tracing::info!( - "BUG #14 DIAGNOSTIC: Gradient norm BEFORE clipping: {:.4}, AFTER clipping: {:.4}, max_norm: {:.4}", - actual_grad_norm, - clipped_grad_norm, - max_norm - ); - } - - // 3. Apply optimizer step with clipped gradients - Optimizer::step(&mut self.optimizer, &grads) - .map_err(|e| MLError::TrainingError(format!("Optimizer step failed: {}", e)))?; - - if actual_grad_norm > max_norm { - tracing::warn!( - "Gradient clipping: {:.4} -> {:.4} - this is expected occasionally but should be rare. \ - If frequent, consider reducing learning rate.", - actual_grad_norm, - clipped_grad_norm - ); - } - - Ok(clipped_grad_norm) - } - - /// Perform backward pass with gradient clipping but WITHOUT an optimizer step. - /// - /// This is useful for gradient accumulation workflows where you want to - /// accumulate clipped gradients across multiple micro-batches before - /// applying a single optimizer step. - /// - /// # Arguments - /// - /// * `loss` - The loss tensor to compute gradients from - /// * `max_norm` - Maximum allowed gradient norm (gradients will be clipped to this value) - /// - /// # Returns - /// - /// Returns `Ok((grads, clipped_norm))` on success, where `grads` is the clipped - /// `GradStore` and `clipped_norm` is the gradient norm after clipping. - /// - /// # Errors - /// - /// This function will return an error if: - /// - The backward pass fails to compute gradients - /// - Gradient clipping fails - pub fn backward_and_clip( - &self, - loss: &Tensor, - max_norm: f64, - ) -> Result<(GradStore, f64), MLError> { - // Compute gradients via backward pass - let mut grads = loss - .backward() - .map_err(|e| MLError::TrainingError(format!("Backward pass failed: {}", e)))?; - - // Apply gradient clipping in-place - let (_actual_norm, clipped_norm) = - crate::gradient_utils::clip_grad_norm(&self.vars, &mut grads, max_norm) - .map_err(|e| MLError::TrainingError(format!("Gradient clipping failed: {}", e)))?; - - Ok((grads, clipped_norm)) - } - - /// Apply pre-computed gradients to the optimizer. - /// - /// This is the companion to `backward_and_clip` for gradient accumulation - /// workflows. After accumulating and scaling gradients, call this method - /// to perform the optimizer step. - /// - /// # Arguments - /// - /// * `grads` - Pre-computed (and possibly accumulated/scaled) gradients - /// - /// # Errors - /// - /// This function will return an error if the optimizer step fails - pub fn apply_grads(&mut self, grads: &GradStore) -> Result<(), MLError> { - Optimizer::step(&mut self.optimizer, grads) - .map_err(|e| MLError::TrainingError(format!("Optimizer step failed: {}", e)))?; - Ok(()) - } - - /// Compute the L2 norm of all gradients - fn compute_gradient_norm( - &self, - grads: &candle_core::backprop::GradStore, - ) -> Result { - let mut total_norm_sq = 0.0f64; - - // Get all variables from the optimizer - for var in &self.vars { - if let Some(grad) = grads.get(var) { - // Compute L2 norm squared for this gradient - let grad_norm_sq = grad - .sqr() - .map_err(|e| { - MLError::TrainingError(format!("Failed to square gradient: {}", e)) - })? - .sum_all() - .map_err(|e| MLError::TrainingError(format!("Failed to sum gradient: {}", e)))? - .to_vec0::() - .map_err(|e| { - MLError::TrainingError(format!("Failed to extract gradient norm: {}", e)) - })? as f64; - - total_norm_sq += grad_norm_sq; - } - } - - Ok(total_norm_sq.sqrt()) - } -} - // Direct type imports - no compatibility aliases use rust_decimal::Decimal; @@ -457,15 +195,11 @@ impl CommonError { } } -/// Error categories for system-wide error classification -/// -/// This enum provides a way to categorize errors across the entire system, -/// enabling better error handling, logging, and monitoring strategies. -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub enum ErrorCategory { - /// System-level errors including hardware, network, and infrastructure issues - System, -} +// Re-export canonical ErrorCategory from common crate (24 variants) +pub use ::common::error::ErrorCategory; + +// Re-export Adam optimizer (moved from inline to optimizers/adam.rs) +pub use optimizers::Adam; // Now using real types from common crate @@ -963,6 +697,7 @@ pub mod liquid; pub mod mamba; pub mod memory_optimization; // Memory optimization utilities (lazy loading, quantization, precision) pub mod microstructure; +pub mod optimizers; pub mod paper_trading; pub mod ppo; pub mod preprocessing; // Data preprocessing (log returns, normalization, outlier clipping) @@ -2175,144 +1910,8 @@ impl ModelMetadata { } } -/// Canonical model type enum used throughout ML module -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] -pub enum ModelType { - /// Compact Deep Q-Network - CompactDQN, - /// Distilled micro network for ultra-low latency - DistilledMicroNet, - /// Standard Deep Q-Network - DQN, - /// Rainbow `DQN` with all enhancements - RainbowDQN, - /// `MAMBA` model (SSM) - MAMBA, - /// Temporal Fusion Transformer - TFT, - /// Temporal Graph Neural Network - TGGN, - /// Liquid Neural Network - LNN, - /// Temporal Limit Order Book transformer - TLOB, - /// Proximal Policy Optimization - PPO, - /// Transformer for sequence modeling - Transformer, - /// Mamba state space model (alias for `MAMBA`) - Mamba, - /// Liquid time constant networks (alias for LNN) - LiquidNet, - /// Temporal Graph Neural Network (alias for TGGN) - TGNN, - /// Ensemble methods - Ensemble, -} - -impl ModelType { - /// Get file extension for model type - pub fn file_extension(&self) -> &'static str { - match self { - ModelType::DQN => "dqn", - ModelType::MAMBA | ModelType::Mamba => "mamba", - ModelType::TFT => "tft", - ModelType::TGGN | ModelType::TGNN => "tggn", - ModelType::LNN | ModelType::LiquidNet => "lnn", - ModelType::CompactDQN => "compact_dqn", - ModelType::DistilledMicroNet => "distilled", - ModelType::RainbowDQN => "rainbow_dqn", - ModelType::TLOB => "tlob", - ModelType::PPO => "ppo", - ModelType::Transformer => "transformer", - ModelType::Ensemble => "ensemble", - } - } - - /// Get model type as a lowercase string identifier (for display, metrics, etc.) - pub const fn as_str(&self) -> &'static str { - match self { - ModelType::CompactDQN => "compact_dqn", - ModelType::DistilledMicroNet => "distilled", - ModelType::DQN => "dqn", - ModelType::RainbowDQN => "rainbow_dqn", - ModelType::MAMBA | ModelType::Mamba => "mamba", - ModelType::TFT => "tft", - ModelType::TGGN | ModelType::TGNN => "tggn", - ModelType::LNN | ModelType::LiquidNet => "liquid", - ModelType::TLOB => "tlob", - ModelType::PPO => "ppo", - ModelType::Transformer => "transformer", - ModelType::Ensemble => "ensemble", - } - } - - /// Get S3 storage prefix for model type (for S3 model loading) - pub const fn s3_prefix(&self) -> &'static str { - match self { - ModelType::CompactDQN => "compact_dqn", - ModelType::DistilledMicroNet => "distilled_micro_net", - ModelType::DQN => "dqn", - ModelType::RainbowDQN => "rainbow_dqn", - ModelType::MAMBA | ModelType::Mamba => "mamba2", - ModelType::TFT => "tft", - ModelType::TGGN | ModelType::TGNN => "tggn", - ModelType::LNN | ModelType::LiquidNet => "liquid", - ModelType::TLOB => "tlob_transformer", - ModelType::PPO => "ppo", - ModelType::Transformer => "transformer", - ModelType::Ensemble => "ensemble", - } - } - - /// Convert to database string representation (for PostgreSQL storage) - pub fn to_db_string(&self) -> &'static str { - match self { - ModelType::DQN => "DQN", - ModelType::PPO => "PPO", - ModelType::MAMBA | ModelType::Mamba => "MAMBA-2", - ModelType::TFT => "TFT", - ModelType::CompactDQN => "COMPACT_DQN", - ModelType::DistilledMicroNet => "DISTILLED_MICRO_NET", - ModelType::RainbowDQN => "RAINBOW_DQN", - ModelType::TGGN | ModelType::TGNN => "TGGN", - ModelType::LNN | ModelType::LiquidNet => "LNN", - ModelType::TLOB => "TLOB", - ModelType::Transformer => "TRANSFORMER", - ModelType::Ensemble => "ENSEMBLE", - } - } - - /// Get model weight for progress aggregation in batch training - pub fn weight(&self) -> f64 { - 0.25 // Equal weight for all models - } - - /// Get model type from string - pub fn from_str(s: &str) -> Option { - match s.to_lowercase().as_str() { - "dqn" => Some(ModelType::DQN), - "mamba" | "mamba2" => Some(ModelType::MAMBA), - "tft" => Some(ModelType::TFT), - "tggn" | "tgnn" => Some(ModelType::TGGN), - "lnn" | "liquidnet" | "liquid" => Some(ModelType::LNN), - "compact_dqn" | "compactdqn" => Some(ModelType::CompactDQN), - "distilled" | "distilledmicronet" => Some(ModelType::DistilledMicroNet), - "rainbow_dqn" | "rainbowdqn" => Some(ModelType::RainbowDQN), - "tlob" | "tlob_transformer" => Some(ModelType::TLOB), - "ppo" => Some(ModelType::PPO), - "transformer" => Some(ModelType::Transformer), - "ensemble" => Some(ModelType::Ensemble), - _ => None, - } - } -} - -impl std::fmt::Display for ModelType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.as_str()) - } -} +// Re-export canonical ModelType from common crate +pub use ::common::model_types::ModelType; /// Prelude module for convenient imports of commonly used ML types /// diff --git a/ml/src/observability/metrics.rs b/ml/src/observability/metrics.rs index 77405658e..2723b3359 100644 --- a/ml/src/observability/metrics.rs +++ b/ml/src/observability/metrics.rs @@ -561,8 +561,8 @@ pub struct MLMetricsReport { pub health_score: f64, } -// Model type to string conversion for metrics is now provided by -// the Display impl on ModelType (via as_str()), which auto-generates ToString. +// ModelType already implements Display (via common::model_types), +// which automatically provides ToString. /// Global metrics collector instance static GLOBAL_METRICS: once_cell::sync::Lazy>>> = diff --git a/ml/src/optimizers/adam.rs b/ml/src/optimizers/adam.rs new file mode 100644 index 000000000..dd2fc33b0 --- /dev/null +++ b/ml/src/optimizers/adam.rs @@ -0,0 +1,264 @@ +use candle_core::backprop::GradStore; +use candle_core::Tensor; +use candle_core::Var; +use candle_nn::Optimizer; + +use crate::MLError; + +/// Wrapper for Adam optimizer to provide required methods +/// +/// This wrapper provides a unified interface around the candle_optimisers Adam optimizer, +/// ensuring consistent behavior across the ML crate and providing additional convenience methods. +/// Adam is an adaptive learning rate optimization algorithm that computes individual learning +/// rates for different parameters from estimates of first and second moments of the gradients. +/// +/// # Examples +/// +/// ```rust,no_run +/// use ml::Adam; +/// use candle_core::Var; +/// use candle_optimisers::adam::ParamsAdam; +/// +/// let vars = vec![]; // Your model variables +/// let params = ParamsAdam::default(); +/// let optimizer = Adam::new(vars, params)?; +/// # Ok::<(), ml::MLError>(()) +/// ``` +#[derive(Debug)] +pub struct Adam { + optimizer: candle_optimisers::adam::Adam, + learning_rate: f64, + vars: Vec, +} + +impl Adam { + /// Create a new Adam optimizer with the given variables and parameters + /// + /// # Arguments + /// + /// * `vars` - Vector of model variables to optimize + /// * `params` - Adam optimizer parameters including learning rate, betas, and epsilon + /// + /// # Returns + /// + /// Returns `Ok(Adam)` on success, or `Err(MLError::TrainingError)` if optimizer creation fails + /// + /// # Errors + /// + /// This function will return an error if the underlying candle Adam optimizer fails to initialize + pub fn new( + vars: Vec, + params: candle_optimisers::adam::ParamsAdam, + ) -> Result { + let learning_rate = params.lr; + let optimizer = candle_optimisers::adam::Adam::new(vars.clone(), params).map_err(|e| { + MLError::TrainingError(format!("Failed to create Adam optimizer: {}", e)) + })?; + + Ok(Self { + optimizer, + learning_rate, + vars, + }) + } + + /// Perform a backward pass and optimizer step + /// + /// This method computes gradients via backpropagation and then applies the Adam + /// optimization update to all registered variables. + /// + /// # Arguments + /// + /// * `loss` - The loss tensor to compute gradients from + /// + /// # Returns + /// + /// Returns `Ok(())` on successful optimization step, or `Err(MLError::TrainingError)` on failure + /// + /// # Errors + /// + /// This function will return an error if: + /// - The backward pass fails to compute gradients + /// - The optimizer step fails to apply updates + pub fn backward_step(&mut self, loss: &Tensor) -> Result<(), MLError> { + // Calculate gradients + let grads = loss + .backward() + .map_err(|e| MLError::TrainingError(format!("Backward pass failed: {}", e)))?; + + // Apply optimizer step using trait method + Optimizer::step(&mut self.optimizer, &grads) + .map_err(|e| MLError::TrainingError(format!("Optimizer step failed: {}", e)))?; + + Ok(()) + } + + /// Get the learning rate used by this optimizer + /// + /// # Returns + /// + /// Returns the learning rate as a 64-bit floating point number + pub fn learning_rate(&self) -> f64 { + self.learning_rate + } + + /// Get a reference to the variables tracked by this optimizer + /// + /// # Returns + /// + /// Returns a slice reference to the vector of variables + pub fn vars(&self) -> &[Var] { + &self.vars + } + + /// Perform backward pass with gradient clipping + /// + /// Implements proper gradient clipping by norm to prevent gradient explosions. + /// Uses a two-pass approach: first pass computes gradient norm, second pass + /// (if needed) computes clipped gradients by scaling the loss. + /// + /// # Arguments + /// + /// * `loss` - The loss tensor to compute gradients from + /// * `max_norm` - Maximum allowed gradient norm (gradients will be clipped to this value) + /// + /// # Returns + /// + /// Returns `Ok(gradient_norm)` on success with the actual gradient norm (before clipping), + /// or `Err(MLError::TrainingError)` on failure + pub fn backward_step_with_monitoring( + &mut self, + loss: &Tensor, + max_norm: f64, + ) -> Result { + // Bug fix: Single backward pass to prevent gradient accumulation + // Root cause: Two backward passes caused 1.5x gradient amplification + // Solution: Compute gradients once, then scale loss before optimizer step if needed + + // 1. Compute gradients via backward pass + let mut grads = loss + .backward() + .map_err(|e| MLError::TrainingError(format!("Backward pass failed: {}", e)))?; + + // 2. Apply gradient clipping IN-PLACE (Bug #32 fix - gradient explosion) + // This modifies the grads directly before optimizer step + let (actual_grad_norm, clipped_grad_norm) = crate::gradient_utils::clip_grad_norm(&self.vars, &mut grads, max_norm) + .map_err(|e| MLError::TrainingError(format!("Gradient clipping failed: {}", e)))?; + + // BUG #14 FIX: Log actual gradient norms to detect if threshold is too low + // Temporary diagnostic logging to investigate gradient clipping issue + static GRAD_LOG_COUNTER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let count = GRAD_LOG_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if count < 20 || count % 100 == 0 { + tracing::info!( + "BUG #14 DIAGNOSTIC: Gradient norm BEFORE clipping: {:.4}, AFTER clipping: {:.4}, max_norm: {:.4}", + actual_grad_norm, + clipped_grad_norm, + max_norm + ); + } + + // 3. Apply optimizer step with clipped gradients + Optimizer::step(&mut self.optimizer, &grads) + .map_err(|e| MLError::TrainingError(format!("Optimizer step failed: {}", e)))?; + + if actual_grad_norm > max_norm { + tracing::warn!( + "Gradient clipping: {:.4} -> {:.4} - this is expected occasionally but should be rare. \ + If frequent, consider reducing learning rate.", + actual_grad_norm, + clipped_grad_norm + ); + } + + Ok(clipped_grad_norm) + } + + /// Perform backward pass with gradient clipping but WITHOUT an optimizer step. + /// + /// This is useful for gradient accumulation workflows where you want to + /// accumulate clipped gradients across multiple micro-batches before + /// applying a single optimizer step. + /// + /// # Arguments + /// + /// * `loss` - The loss tensor to compute gradients from + /// * `max_norm` - Maximum allowed gradient norm (gradients will be clipped to this value) + /// + /// # Returns + /// + /// Returns `Ok((grads, clipped_norm))` on success, where `grads` is the clipped + /// `GradStore` and `clipped_norm` is the gradient norm after clipping. + /// + /// # Errors + /// + /// This function will return an error if: + /// - The backward pass fails to compute gradients + /// - Gradient clipping fails + pub fn backward_and_clip( + &self, + loss: &Tensor, + max_norm: f64, + ) -> Result<(GradStore, f64), MLError> { + // Compute gradients via backward pass + let mut grads = loss + .backward() + .map_err(|e| MLError::TrainingError(format!("Backward pass failed: {}", e)))?; + + // Apply gradient clipping in-place + let (_actual_norm, clipped_norm) = + crate::gradient_utils::clip_grad_norm(&self.vars, &mut grads, max_norm) + .map_err(|e| MLError::TrainingError(format!("Gradient clipping failed: {}", e)))?; + + Ok((grads, clipped_norm)) + } + + /// Apply pre-computed gradients to the optimizer. + /// + /// This is the companion to `backward_and_clip` for gradient accumulation + /// workflows. After accumulating and scaling gradients, call this method + /// to perform the optimizer step. + /// + /// # Arguments + /// + /// * `grads` - Pre-computed (and possibly accumulated/scaled) gradients + /// + /// # Errors + /// + /// This function will return an error if the optimizer step fails + pub fn apply_grads(&mut self, grads: &GradStore) -> Result<(), MLError> { + Optimizer::step(&mut self.optimizer, grads) + .map_err(|e| MLError::TrainingError(format!("Optimizer step failed: {}", e)))?; + Ok(()) + } + + /// Compute the L2 norm of all gradients + fn compute_gradient_norm( + &self, + grads: &candle_core::backprop::GradStore, + ) -> Result { + let mut total_norm_sq = 0.0f64; + + // Get all variables from the optimizer + for var in &self.vars { + if let Some(grad) = grads.get(var) { + // Compute L2 norm squared for this gradient + let grad_norm_sq = grad + .sqr() + .map_err(|e| { + MLError::TrainingError(format!("Failed to square gradient: {}", e)) + })? + .sum_all() + .map_err(|e| MLError::TrainingError(format!("Failed to sum gradient: {}", e)))? + .to_vec0::() + .map_err(|e| { + MLError::TrainingError(format!("Failed to extract gradient norm: {}", e)) + })? as f64; + + total_norm_sq += grad_norm_sq; + } + } + + Ok(total_norm_sq.sqrt()) + } +} diff --git a/ml/src/optimizers/mod.rs b/ml/src/optimizers/mod.rs new file mode 100644 index 000000000..f660a8c3d --- /dev/null +++ b/ml/src/optimizers/mod.rs @@ -0,0 +1,3 @@ +pub mod adam; + +pub use adam::Adam; diff --git a/ml/src/ppo/continuous_transaction_costs.rs b/ml/src/ppo/continuous_transaction_costs.rs index a2b0bca7d..96b3e996a 100644 --- a/ml/src/ppo/continuous_transaction_costs.rs +++ b/ml/src/ppo/continuous_transaction_costs.rs @@ -16,6 +16,7 @@ //! - LimitMaker: 5 bps (0.05%) - Passive order, maker rebate //! - IoC: 10 bps (0.10%) - Immediate-or-cancel, medium cost +pub use crate::common::action::OrderType; use serde::{Deserialize, Serialize}; /// Transaction cost model for continuous position sizing @@ -74,39 +75,6 @@ pub enum SlippageModel { Quadratic { coefficient: f32 }, } -/// Order type for execution strategy -/// -/// Matches DQN's OrderType enum with identical fee structure. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum OrderType { - /// Immediate execution, high cost (15 bps) - Market, - - /// Passive order, maker rebate (5 bps) - LimitMaker, - - /// Immediate-or-cancel (10 bps) - IoC, -} - -impl OrderType { - /// Get base transaction cost in bps - /// - /// Aligned with DQN transaction costs (Wave 2.5 calibration) - pub fn cost_bps(&self) -> f32 { - match self { - OrderType::Market => 15.0, // 0.15% - OrderType::LimitMaker => 5.0, // 0.05% - OrderType::IoC => 10.0, // 0.10% - } - } - - /// Get cost as decimal (for convenience) - pub fn cost_decimal(&self) -> f64 { - (self.cost_bps() as f64) / 10000.0 - } -} - impl ContinuousTransactionCosts { /// Create new transaction cost model pub fn new( diff --git a/ml/src/ppo/factored_action.rs b/ml/src/ppo/factored_action.rs index 124fa2bc9..8547079ce 100644 --- a/ml/src/ppo/factored_action.rs +++ b/ml/src/ppo/factored_action.rs @@ -10,6 +10,7 @@ /// /// Index mapping: index = exposure*9 + order*3 + urgency (0-44) +pub use crate::common::action::OrderType; use crate::MLError; use serde::{Deserialize, Serialize}; @@ -51,39 +52,6 @@ impl ExposureLevel { } } -/// Order type for execution strategy -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum OrderType { - Market = 0, // Immediate execution, high cost (0.15%) - LimitMaker = 1, // Passive order, maker rebate (0.05%) - IoC = 2, // Immediate-or-cancel (0.10%) -} - -impl OrderType { - /// Get transaction cost multiplier - /// Calibrated fees: Market 15 bps, LimitMaker 5 bps, IoC 10 bps - pub fn transaction_cost(&self) -> f64 { - match self { - OrderType::Market => 0.0015, // 0.15% - OrderType::LimitMaker => 0.0005, // 0.05% - OrderType::IoC => 0.0010, // 0.10% - } - } - - /// Convert from index (0-2) - pub fn from_index(idx: usize) -> Result { - match idx { - 0 => Ok(OrderType::Market), - 1 => Ok(OrderType::LimitMaker), - 2 => Ok(OrderType::IoC), - _ => Err(MLError::InvalidInput(format!( - "Invalid order type index: {}", - idx - ))), - } - } -} - /// Urgency level for execution timing #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum Urgency { @@ -239,16 +207,6 @@ impl std::fmt::Display for ExposureLevel { } } -impl std::fmt::Display for OrderType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - OrderType::Market => write!(f, "Market"), - OrderType::LimitMaker => write!(f, "LimitMaker"), - OrderType::IoC => write!(f, "IoC"), - } - } -} - impl std::fmt::Display for Urgency { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/ml/src/ppo/transaction_costs.rs b/ml/src/ppo/transaction_costs.rs index 1d9bd3a48..8289c54b6 100644 --- a/ml/src/ppo/transaction_costs.rs +++ b/ml/src/ppo/transaction_costs.rs @@ -8,36 +8,7 @@ //! - **LimitMaker**: 0.05% (0.0005) - Passive order, maker rebate //! - **IoC**: 0.10% (0.0010) - Immediate-or-cancel, medium cost -use serde::{Deserialize, Serialize}; - -/// Order type for execution strategy -/// -/// Maps to DQN's OrderType enum with identical fee structure. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum OrderType { - /// Immediate execution, high cost (0.15%) - Market = 0, - /// Passive order, maker rebate (0.05%) - LimitMaker = 1, - /// Immediate-or-cancel (0.10%) - IoC = 2, -} - -impl OrderType { - /// Get transaction cost multiplier - /// - /// Wave 2.5 Calibration: Reduced fees - /// - Market: 20 bps → 15 bps (0.0015) - /// - LimitMaker: 10 bps → 5 bps (0.0005) - /// - IoC: 15 bps → 10 bps (0.0010) - pub fn transaction_cost(&self) -> f64 { - match self { - OrderType::Market => 0.0015, // 0.15% (was 0.20%) - OrderType::LimitMaker => 0.0005, // 0.05% (was 0.10%) - OrderType::IoC => 0.0010, // 0.10% (was 0.15%) - } - } -} +pub use crate::common::action::OrderType; /// Calculate total transaction cost for a trade /// diff --git a/model_loader/Cargo.toml b/model_loader/Cargo.toml index 832f505fa..90199910b 100644 --- a/model_loader/Cargo.toml +++ b/model_loader/Cargo.toml @@ -14,7 +14,7 @@ description = "ML model loading and caching infrastructure for Foxhunt HFT syste [dependencies] # Internal workspace crates storage = { path = "../storage" } -ml = { workspace = true } # Canonical ModelType re-exported from here +common.workspace = true # Core workspace dependencies anyhow.workspace = true diff --git a/model_loader/src/lib.rs b/model_loader/src/lib.rs index 5ec22afc3..f0b46097d 100644 --- a/model_loader/src/lib.rs +++ b/model_loader/src/lib.rs @@ -17,8 +17,8 @@ use std::time::SystemTime; use storage::{ObjectStoreBackend, Storage}; use tracing::{debug, info, warn}; -// Re-export canonical ModelType from ml crate (single source of truth) -pub use ml::ModelType; +// Re-export canonical ModelType from common crate +pub use common::model_types::ModelType; /// Model metadata for version tracking #[derive(Debug, Clone, Serialize, Deserialize)] @@ -379,6 +379,13 @@ mod tests { assert_eq!(ModelType::MAMBA.s3_prefix(), "mamba2"); } + #[test] + fn test_model_type_as_str() { + assert_eq!(ModelType::TLOB.as_str(), "tlob"); + assert_eq!(ModelType::DQN.as_str(), "dqn"); + assert_eq!(ModelType::MAMBA.as_str(), "mamba"); + } + #[test] fn test_cache_key_equality() { let key1 = CacheKey { diff --git a/services/api_gateway/src/auth/mtls/mod.rs b/services/api_gateway/src/auth/mtls/mod.rs index d439b57a5..845cf7491 100644 --- a/services/api_gateway/src/auth/mtls/mod.rs +++ b/services/api_gateway/src/auth/mtls/mod.rs @@ -73,4 +73,5 @@ pub use validator::{ClientIdentity, UserRole, X509CertificateValidator}; pub use revocation::RevocationChecker; -pub use tls_config::{ApiGatewayTlsConfig, TlsInterceptor, TlsProtocolVersion}; +pub use common::tls::TlsProtocolVersion; +pub use tls_config::{ApiGatewayTlsConfig, TlsInterceptor}; diff --git a/services/api_gateway/src/auth/mtls/tls_config.rs b/services/api_gateway/src/auth/mtls/tls_config.rs index 5a05084a4..59fe41bdf 100644 --- a/services/api_gateway/src/auth/mtls/tls_config.rs +++ b/services/api_gateway/src/auth/mtls/tls_config.rs @@ -8,6 +8,7 @@ //! - Performance optimized for HFT requirements use anyhow::{Context, Result}; +use common::tls::TlsProtocolVersion; use config::manager::ConfigManager; use config::structures::TlsConfig; use std::sync::Arc; @@ -16,13 +17,6 @@ use tracing::info; use super::validator::{ClientIdentity, X509CertificateValidator}; -/// TLS protocol version -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum TlsProtocolVersion { - Tls12, - Tls13, -} - /// TLS configuration for the API Gateway #[derive(Debug, Clone)] pub struct ApiGatewayTlsConfig { diff --git a/services/api_gateway/src/auth/mtls/validator.rs b/services/api_gateway/src/auth/mtls/validator.rs index 94442b882..e2d5130eb 100644 --- a/services/api_gateway/src/auth/mtls/validator.rs +++ b/services/api_gateway/src/auth/mtls/validator.rs @@ -9,6 +9,7 @@ //! 6. Hostname verification use anyhow::Result; +pub use common::tls::{ClientIdentity, UserRole}; use tracing::{debug, info, warn}; use x509_parser::certificate::X509Certificate; use x509_parser::extensions::{GeneralName, ParsedExtension}; @@ -491,98 +492,6 @@ impl X509CertificateValidator { } } -/// Client identity extracted from certificate -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ClientIdentity { - pub common_name: String, - pub organizational_unit: String, - pub serial_number: String, - pub issuer: String, -} - -impl ClientIdentity { - /// Check if client is authorized for trading operations - pub fn is_authorized_for_trading(&self) -> bool { - // Implement authorization logic based on certificate attributes - matches!(self.organizational_unit.as_str(), "trading" | "admin") - } - - /// Check if client is authorized for read-only operations - pub fn is_authorized_for_readonly(&self) -> bool { - // Allow broader access for read-only operations - matches!( - self.organizational_unit.as_str(), - "trading" | "admin" | "analytics" | "risk" | "compliance" - ) - } - - /// Get user role based on certificate - pub fn get_role(&self) -> UserRole { - match self.organizational_unit.as_str() { - "admin" => UserRole::Admin, - "trading" => UserRole::Trader, - "analytics" => UserRole::Analyst, - "risk" => UserRole::RiskManager, - "compliance" => UserRole::ComplianceOfficer, - _ => UserRole::ReadOnly, - } - } -} - -/// User roles based on certificate attributes -#[derive(Debug, Clone, PartialEq)] -pub enum UserRole { - Admin, - Trader, - Analyst, - RiskManager, - ComplianceOfficer, - ReadOnly, -} - -impl UserRole { - /// Get permissions for this role - pub fn get_permissions(&self) -> Vec<&'static str> { - match self { - UserRole::Admin => vec![ - "trading.submit_order", - "trading.cancel_order", - "trading.modify_order", - "risk.view_positions", - "risk.modify_limits", - "analytics.view_data", - "analytics.run_backtest", - "compliance.view_reports", - "system.configure", - ], - UserRole::Trader => vec![ - "trading.submit_order", - "trading.cancel_order", - "trading.modify_order", - "risk.view_positions", - "analytics.view_data", - ], - UserRole::Analyst => vec![ - "analytics.view_data", - "analytics.run_backtest", - "risk.view_positions", - ], - UserRole::RiskManager => vec![ - "risk.view_positions", - "risk.modify_limits", - "analytics.view_data", - "compliance.view_reports", - ], - UserRole::ComplianceOfficer => vec![ - "compliance.view_reports", - "analytics.view_data", - "risk.view_positions", - ], - UserRole::ReadOnly => vec!["analytics.view_data"], - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/services/api_gateway/src/config/endpoints.rs b/services/api_gateway/src/config/endpoints.rs index c1dfe8546..818976146 100644 --- a/services/api_gateway/src/config/endpoints.rs +++ b/services/api_gateway/src/config/endpoints.rs @@ -1,7 +1,7 @@ //! gRPC endpoints for configuration management use crate::config::ConfigurationManager; -use crate::error::ConfigError; +use crate::error::GatewayConfigError; use tonic::{Request, Response, Status}; use tracing::{debug, info}; @@ -46,7 +46,7 @@ impl ConfigurationService for ConfigurationServiceImpl { .get_config(&req.service_scope, &req.config_key) .await .map_err(|e| match e { - ConfigError::NotFound { .. } => Status::not_found(format!("{}", e)), + GatewayConfigError::NotFound { .. } => Status::not_found(format!("{}", e)), _ => Status::internal(format!("{}", e)), })?; @@ -84,8 +84,8 @@ impl ConfigurationService for ConfigurationServiceImpl { ) .await .map_err(|e| match e { - ConfigError::Validation(msg) => Status::invalid_argument(msg), - ConfigError::NotFound { .. } => Status::not_found(format!("{}", e)), + GatewayConfigError::Validation(msg) => Status::invalid_argument(msg), + GatewayConfigError::NotFound { .. } => Status::not_found(format!("{}", e)), _ => Status::internal(format!("{}", e)), })?; diff --git a/services/api_gateway/src/config/manager.rs b/services/api_gateway/src/config/manager.rs index 0e578e6d2..ee1e74387 100644 --- a/services/api_gateway/src/config/manager.rs +++ b/services/api_gateway/src/config/manager.rs @@ -1,7 +1,7 @@ //! Configuration management with PostgreSQL NOTIFY/LISTEN hot-reload and Redis caching use crate::config::validator::ConfigValidator; -use crate::error::{ConfigError, ConfigResult}; +use crate::error::{GatewayConfigError, GatewayConfigResult}; use chrono::{DateTime, Utc}; use redis::aio::ConnectionManager; use serde_json::Value; @@ -46,7 +46,7 @@ impl ConfigurationManager { /// * `db_pool` - Postgre`SQL` connection pool /// /// * `redis` - Redis connection manager - pub async fn new(db_pool: PgPool, redis: ConnectionManager) -> ConfigResult { + pub async fn new(db_pool: PgPool, redis: ConnectionManager) -> GatewayConfigResult { Ok(Self { db_pool: Arc::new(db_pool), redis: Arc::new(RwLock::new(redis)), @@ -56,7 +56,7 @@ impl ConfigurationManager { } /// Starts listening for configuration changes via Postgre`SQL` NOTIFY - pub async fn start_listening(&mut self) -> ConfigResult<()> { + pub async fn start_listening(&mut self) -> GatewayConfigResult<()> { let mut listener = sqlx::postgres::PgListener::connect_with(&self.db_pool).await?; // Listen to global config updates channel @@ -69,7 +69,7 @@ impl ConfigurationManager { } /// Processes configuration change notifications - pub async fn handle_notifications(&mut self) -> ConfigResult<()> { + pub async fn handle_notifications(&mut self) -> GatewayConfigResult<()> { // Collect notifications first to avoid borrow checker issues let mut invalidations = Vec::new(); @@ -113,7 +113,7 @@ impl ConfigurationManager { &self, service_scope: &str, config_key: &str, - ) -> ConfigResult { + ) -> GatewayConfigResult { // Try Redis cache first if let Some(cached) = self.get_from_cache(service_scope, config_key).await? { debug!("Cache hit for {}/{}", service_scope, config_key); @@ -145,7 +145,7 @@ impl ConfigurationManager { config_key: &str, new_value: Value, updated_by: &str, - ) -> ConfigResult<()> { + ) -> GatewayConfigResult<()> { // Load current configuration for validation rules and audit let current = self.load_from_db(service_scope, config_key).await?; @@ -207,7 +207,7 @@ impl ConfigurationManager { /// /// # Arguments /// * `service_scope` - Service scope (None for all scopes) - pub async fn list_configs(&self, service_scope: Option<&str>) -> ConfigResult> { + pub async fn list_configs(&self, service_scope: Option<&str>) -> GatewayConfigResult> { let configs = if let Some(scope) = service_scope { sqlx::query_as::<_, ConfigItem>( r#" @@ -239,7 +239,7 @@ impl ConfigurationManager { &self, service_scope: &str, config_key: &str, - ) -> ConfigResult { + ) -> GatewayConfigResult { let config = sqlx::query_as::<_, ConfigItem>( r#" SELECT * FROM config_settings @@ -250,7 +250,7 @@ impl ConfigurationManager { .bind(config_key) .fetch_optional(&*self.db_pool) .await? - .ok_or_else(|| ConfigError::NotFound { + .ok_or_else(|| GatewayConfigError::NotFound { service_scope: service_scope.to_string(), key: config_key.to_string(), })?; @@ -263,7 +263,7 @@ impl ConfigurationManager { &self, service_scope: &str, config_key: &str, - ) -> ConfigResult> { + ) -> GatewayConfigResult> { let redis_key = format!("config:{}:{}", service_scope, config_key); let mut redis = self.redis.write().await; @@ -271,7 +271,7 @@ impl ConfigurationManager { .arg(&redis_key) .query_async(&mut *redis) .await - .map_err(ConfigError::Redis)?; + .map_err(GatewayConfigError::Redis)?; if let Some(cached_json) = cached { let config: ConfigItem = serde_json::from_str(&cached_json)?; @@ -282,7 +282,7 @@ impl ConfigurationManager { } /// Stores configuration in Redis cache - async fn set_in_cache(&self, config: &ConfigItem) -> ConfigResult<()> { + async fn set_in_cache(&self, config: &ConfigItem) -> GatewayConfigResult<()> { let redis_key = format!("config:{}:{}", config.service_scope, config.config_key); let config_json = serde_json::to_string(config)?; let mut redis = self.redis.write().await; @@ -294,13 +294,13 @@ impl ConfigurationManager { .arg(&config_json) .query_async::<()>(&mut *redis) .await - .map_err(ConfigError::Redis)?; + .map_err(GatewayConfigError::Redis)?; Ok(()) } /// Invalidates Redis cache for a configuration - async fn invalidate_cache(&self, service_scope: &str, config_key: &str) -> ConfigResult<()> { + async fn invalidate_cache(&self, service_scope: &str, config_key: &str) -> GatewayConfigResult<()> { let redis_key = format!("config:{}:{}", service_scope, config_key); let mut redis = self.redis.write().await; @@ -308,7 +308,7 @@ impl ConfigurationManager { .arg(&redis_key) .query_async::<()>(&mut *redis) .await - .map_err(ConfigError::Redis)?; + .map_err(GatewayConfigError::Redis)?; Ok(()) } diff --git a/services/api_gateway/src/config/validator.rs b/services/api_gateway/src/config/validator.rs index 08c8688c8..08d4a57ee 100644 --- a/services/api_gateway/src/config/validator.rs +++ b/services/api_gateway/src/config/validator.rs @@ -1,6 +1,6 @@ //! Configuration validation with type checking, range validation, and regex matching -use crate::error::{ConfigError, ConfigResult}; +use crate::error::{GatewayConfigError, GatewayConfigResult}; use regex::Regex; use serde_json::Value; use std::collections::HashMap; @@ -53,7 +53,7 @@ impl ConfigValidator { value: &Value, data_type: &str, rules: Option<&Value>, - ) -> ConfigResult<()> { + ) -> GatewayConfigResult<()> { // Parse validation rules if provided let validation_rules: Option = match rules { Some(r) => serde_json::from_value(r.clone()).ok(), @@ -77,7 +77,7 @@ impl ConfigValidator { } /// Validates that the value matches the expected type - fn validate_type(&self, value: &Value, data_type: &str) -> ConfigResult<()> { + fn validate_type(&self, value: &Value, data_type: &str) -> GatewayConfigResult<()> { let matches = match data_type { "string" => value.is_string(), "integer" => value.is_i64() || value.is_u64(), @@ -86,7 +86,7 @@ impl ConfigValidator { "json" => value.is_object(), "array" => value.is_array(), _ => { - return Err(ConfigError::Validation(format!( + return Err(GatewayConfigError::Validation(format!( "Unknown data type: {}", data_type ))) @@ -94,7 +94,7 @@ impl ConfigValidator { }; if !matches { - return Err(ConfigError::Validation(format!( + return Err(GatewayConfigError::Validation(format!( "Type mismatch: expected {}, got {}", data_type, value_type_name(value) @@ -105,14 +105,14 @@ impl ConfigValidator { } /// Validates numeric values against min/max rules - fn validate_numeric(&self, value: &Value, rules: &ValidationRules) -> ConfigResult<()> { + fn validate_numeric(&self, value: &Value, rules: &ValidationRules) -> GatewayConfigResult<()> { let num = value .as_f64() - .ok_or_else(|| ConfigError::Validation("Not a number".to_string()))?; + .ok_or_else(|| GatewayConfigError::Validation("Not a number".to_string()))?; if let Some(min) = rules.min { if num < min { - return Err(ConfigError::Validation(format!( + return Err(GatewayConfigError::Validation(format!( "Value {} is below minimum {}", num, min ))); @@ -121,7 +121,7 @@ impl ConfigValidator { if let Some(max) = rules.max { if num > max { - return Err(ConfigError::Validation(format!( + return Err(GatewayConfigError::Validation(format!( "Value {} exceeds maximum {}", num, max ))); @@ -132,15 +132,15 @@ impl ConfigValidator { } /// Validates string values against length and regex rules - fn validate_string(&mut self, value: &Value, rules: &ValidationRules) -> ConfigResult<()> { + fn validate_string(&mut self, value: &Value, rules: &ValidationRules) -> GatewayConfigResult<()> { let s = value .as_str() - .ok_or_else(|| ConfigError::Validation("Not a string".to_string()))?; + .ok_or_else(|| GatewayConfigError::Validation("Not a string".to_string()))?; // Length validation if let Some(min_len) = rules.min_len { if s.len() < min_len { - return Err(ConfigError::Validation(format!( + return Err(GatewayConfigError::Validation(format!( "String length {} is below minimum {}", s.len(), min_len @@ -150,7 +150,7 @@ impl ConfigValidator { if let Some(max_len) = rules.max_len { if s.len() > max_len { - return Err(ConfigError::Validation(format!( + return Err(GatewayConfigError::Validation(format!( "String length {} exceeds maximum {}", s.len(), max_len @@ -162,17 +162,17 @@ impl ConfigValidator { if let Some(pattern) = &rules.regex { if !self.regex_cache.contains_key(pattern) { let compiled = Regex::new(pattern).map_err(|e| { - ConfigError::Validation(format!("Invalid regex pattern '{}': {}", pattern, e)) + GatewayConfigError::Validation(format!("Invalid regex pattern '{}': {}", pattern, e)) })?; self.regex_cache.insert(pattern.clone(), compiled); } let regex = self .regex_cache .get(pattern) - .ok_or_else(|| ConfigError::Validation("Regex cache miss".to_owned()))?; + .ok_or_else(|| GatewayConfigError::Validation("Regex cache miss".to_owned()))?; if !regex.is_match(s) { - return Err(ConfigError::Validation(format!( + return Err(GatewayConfigError::Validation(format!( "String '{}' does not match pattern '{}'", s, pattern ))); @@ -182,7 +182,7 @@ impl ConfigValidator { // Enum validation if let Some(enum_values) = &rules.enum_values { if !enum_values.contains(&s.to_string()) { - return Err(ConfigError::Validation(format!( + return Err(GatewayConfigError::Validation(format!( "Value '{}' is not in allowed values: {:?}", s, enum_values ))); @@ -193,14 +193,14 @@ impl ConfigValidator { } /// Validates array values against length rules - fn validate_array(&self, value: &Value, rules: &ValidationRules) -> ConfigResult<()> { + fn validate_array(&self, value: &Value, rules: &ValidationRules) -> GatewayConfigResult<()> { let arr = value .as_array() - .ok_or_else(|| ConfigError::Validation("Not an array".to_string()))?; + .ok_or_else(|| GatewayConfigError::Validation("Not an array".to_string()))?; if let Some(min_len) = rules.min_len { if arr.len() < min_len { - return Err(ConfigError::Validation(format!( + return Err(GatewayConfigError::Validation(format!( "Array length {} is below minimum {}", arr.len(), min_len @@ -210,7 +210,7 @@ impl ConfigValidator { if let Some(max_len) = rules.max_len { if arr.len() > max_len { - return Err(ConfigError::Validation(format!( + return Err(GatewayConfigError::Validation(format!( "Array length {} exceeds maximum {}", arr.len(), max_len diff --git a/services/api_gateway/src/error.rs b/services/api_gateway/src/error.rs index e0b763810..49fcaf524 100644 --- a/services/api_gateway/src/error.rs +++ b/services/api_gateway/src/error.rs @@ -4,7 +4,7 @@ use thiserror::Error; /// Configuration management errors #[derive(Debug, Error)] -pub enum ConfigError { +pub enum GatewayConfigError { #[error("Database error: {0}")] Database(#[from] sqlx::Error), @@ -28,4 +28,4 @@ pub enum ConfigError { } /// Result type for configuration operations -pub type ConfigResult = Result; +pub type GatewayConfigResult = Result; diff --git a/services/api_gateway/src/lib.rs b/services/api_gateway/src/lib.rs index d59aae27c..5cb2bf259 100644 --- a/services/api_gateway/src/lib.rs +++ b/services/api_gateway/src/lib.rs @@ -64,7 +64,7 @@ pub mod metrics; pub mod routing; // Re-export error types -pub use error::{ConfigError, ConfigResult}; +pub use error::{GatewayConfigError, GatewayConfigResult}; // Re-export configuration management types pub use config::{ diff --git a/services/backtesting_service/src/tls_config.rs b/services/backtesting_service/src/tls_config.rs index 96a2455fe..4e6dd231a 100644 --- a/services/backtesting_service/src/tls_config.rs +++ b/services/backtesting_service/src/tls_config.rs @@ -8,6 +8,10 @@ use anyhow::{Context, Result}; use config::manager::ConfigManager; + +// Re-export shared TLS types from common for backward compatibility +#[allow(unused_imports)] +pub use common::tls::{ClientIdentity, TlsProtocolVersion, UserRole}; use config::structures::TlsConfig; use std::sync::Arc; // TLS imports - TLS feature should be enabled in Cargo.toml @@ -38,16 +42,6 @@ pub struct BacktestingServiceTlsConfig { pub crl_url: Option, } -/// TLS protocol version options -#[derive(Debug, Clone)] -#[allow(dead_code)] -pub enum TlsProtocolVersion { - /// TLS version 1.2 - Tls12, - /// TLS version 1.3 - Tls13, -} - #[allow(dead_code)] impl BacktestingServiceTlsConfig { /// Create TLS configuration from certificate files @@ -647,112 +641,6 @@ impl BacktestingServiceTlsConfig { } } -/// Client identity extracted from certificate -#[allow(dead_code)] -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ClientIdentity { - /// Common Name (CN) from certificate - pub common_name: String, - /// Organizational Unit (OU) from certificate - pub organizational_unit: String, - /// Certificate serial number - pub serial_number: String, - /// Certificate issuer - pub issuer: String, -} - -#[allow(dead_code)] -impl ClientIdentity { - /// Check if client is authorized for trading operations - pub fn is_authorized_for_trading(&self) -> bool { - // Implement authorization logic based on certificate attributes - matches!(self.organizational_unit.as_str(), "trading" | "admin") - } - - /// Check if client is authorized for read-only operations - pub fn is_authorized_for_readonly(&self) -> bool { - // Allow broader access for read-only operations - matches!( - self.organizational_unit.as_str(), - "trading" | "admin" | "analytics" | "risk" | "compliance" - ) - } - - /// Get user role based on certificate - pub fn get_role(&self) -> UserRole { - match self.organizational_unit.as_str() { - "admin" => UserRole::Admin, - "trading" => UserRole::Trader, - "analytics" => UserRole::Analyst, - "risk" => UserRole::RiskManager, - "compliance" => UserRole::ComplianceOfficer, - _ => UserRole::ReadOnly, - } - } -} - -/// User roles based on certificate attributes -#[allow(dead_code)] -#[derive(Debug, Clone, PartialEq)] -pub enum UserRole { - /// Administrator with full access - Admin, - /// Trader with trading permissions - Trader, - /// Analyst with read/analysis permissions - Analyst, - /// Risk manager with risk oversight - RiskManager, - /// Compliance officer with audit access - ComplianceOfficer, - /// Read-only access - ReadOnly, -} - -#[allow(dead_code)] -impl UserRole { - /// Get permissions for this role - pub fn get_permissions(&self) -> Vec<&'static str> { - match self { - UserRole::Admin => vec![ - "trading.submit_order", - "trading.cancel_order", - "trading.modify_order", - "risk.view_positions", - "risk.modify_limits", - "analytics.view_data", - "analytics.run_backtest", - "compliance.view_reports", - "system.configure", - ], - UserRole::Trader => vec![ - "trading.submit_order", - "trading.cancel_order", - "trading.modify_order", - "risk.view_positions", - "analytics.view_data", - ], - UserRole::Analyst => vec![ - "analytics.view_data", - "analytics.run_backtest", - "risk.view_positions", - ], - UserRole::RiskManager => vec![ - "risk.view_positions", - "risk.modify_limits", - "analytics.view_data", - "compliance.view_reports", - ], - UserRole::ComplianceOfficer => vec![ - "compliance.view_reports", - "analytics.view_data", - "risk.view_positions", - ], - UserRole::ReadOnly => vec!["analytics.view_data"], - } - } -} - /// TLS interceptor for gRPC requests #[allow(dead_code)] #[derive(Clone)] diff --git a/services/broker_gateway_service/src/error_handler.rs b/services/broker_gateway_service/src/error_handler.rs index 21cca0805..8953d4aaf 100644 --- a/services/broker_gateway_service/src/error_handler.rs +++ b/services/broker_gateway_service/src/error_handler.rs @@ -6,6 +6,9 @@ //! - Dead letter queue for unrecoverable orders //! - Error classification and routing +use common::resilience::circuit_breaker::{CircuitBreaker, CircuitBreakerConfig}; +#[cfg(test)] +use common::resilience::circuit_breaker::CircuitBreakerState; use std::collections::VecDeque; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -43,17 +46,6 @@ pub enum ErrorRecoveryStrategy { Fallback, } -/// Circuit breaker state for fault isolation -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum CircuitBreakerState { - /// Circuit is closed, requests pass through normally - Closed, - /// Circuit is open, requests fail fast to prevent cascading failures - Open, - /// Circuit is half-open, testing if service has recovered - HalfOpen, -} - /// Dead letter queue entry for unrecoverable orders #[derive(Debug, Clone)] pub struct DeadLetterEntry { @@ -67,124 +59,6 @@ pub struct DeadLetterEntry { pub retry_attempts: u32, } -/// Circuit breaker for fault isolation and cascading failure prevention -pub struct CircuitBreaker { - /// Current circuit breaker state - state: Arc>, - /// Consecutive failure count - failure_count: Arc>, - /// Timestamp when circuit breaker opened - opened_at: Arc>>, -} - -impl CircuitBreaker { - /// Create a new circuit breaker in CLOSED state - pub fn new() -> Self { - Self { - state: Arc::new(RwLock::new(CircuitBreakerState::Closed)), - failure_count: Arc::new(RwLock::new(0)), - opened_at: Arc::new(RwLock::new(None)), - } - } - - /// Get current circuit breaker state - pub async fn state(&self) -> CircuitBreakerState { - *self.state.read().await - } - - /// Record a successful operation (resets failure count) - pub async fn record_success(&self) { - let mut failure_count = self.failure_count.write().await; - *failure_count = 0; - - let mut state = self.state.write().await; - if *state == CircuitBreakerState::HalfOpen { - info!("Circuit breaker transitioning: HALF_OPEN → CLOSED"); - *state = CircuitBreakerState::Closed; - } - } - - /// Record a failed operation (increments failure count) - pub async fn record_failure(&self) { - let mut failure_count = self.failure_count.write().await; - *failure_count += 1; - - let current_state = *self.state.read().await; - - if current_state == CircuitBreakerState::Closed - && *failure_count >= CIRCUIT_BREAKER_THRESHOLD - { - warn!( - "Circuit breaker OPEN after {} failures (threshold: {})", - *failure_count, CIRCUIT_BREAKER_THRESHOLD - ); - - let mut state = self.state.write().await; - *state = CircuitBreakerState::Open; - - let mut opened_at = self.opened_at.write().await; - *opened_at = Some(Instant::now()); - } else if current_state == CircuitBreakerState::HalfOpen { - warn!("Circuit breaker transitioning: HALF_OPEN → OPEN (failure during test)"); - - let mut state = self.state.write().await; - *state = CircuitBreakerState::Open; - - let mut opened_at = self.opened_at.write().await; - *opened_at = Some(Instant::now()); - } - } - - /// Check if circuit breaker allows requests (handles state transitions) - pub async fn allow_request(&self) -> bool { - let current_state = *self.state.read().await; - - match current_state { - CircuitBreakerState::Closed => true, - CircuitBreakerState::Open => { - // Check if timeout has elapsed - let opened_at = self.opened_at.read().await; - if let Some(opened_time) = *opened_at { - if opened_time.elapsed() >= CIRCUIT_BREAKER_TIMEOUT { - info!("Circuit breaker transitioning: OPEN → HALF_OPEN (timeout elapsed)"); - drop(opened_at); - - let mut state = self.state.write().await; - *state = CircuitBreakerState::HalfOpen; - - let mut failure_count = self.failure_count.write().await; - *failure_count = 0; - - return true; - } - } - false - } - CircuitBreakerState::HalfOpen => true, - } - } - - /// Reset circuit breaker to CLOSED state (manual recovery) - pub async fn reset(&self) { - let mut state = self.state.write().await; - *state = CircuitBreakerState::Closed; - - let mut failure_count = self.failure_count.write().await; - *failure_count = 0; - - let mut opened_at = self.opened_at.write().await; - *opened_at = None; - - info!("Circuit breaker manually reset to CLOSED state"); - } -} - -impl Default for CircuitBreaker { - fn default() -> Self { - Self::new() - } -} - /// Dead letter queue for unrecoverable orders pub struct DeadLetterQueue { /// Queue of failed orders @@ -275,8 +149,13 @@ pub struct ErrorHandler { impl ErrorHandler { /// Create a new error handler pub fn new() -> Self { + let cb_config = CircuitBreakerConfig { + failure_threshold: CIRCUIT_BREAKER_THRESHOLD as u32, + timeout: CIRCUIT_BREAKER_TIMEOUT, + success_threshold: 1, + }; Self { - circuit_breaker: CircuitBreaker::new(), + circuit_breaker: CircuitBreaker::new("broker_gateway", cb_config), dead_letter_queue: DeadLetterQueue::new(), } } @@ -422,13 +301,21 @@ impl Default for ErrorHandler { mod tests { use super::*; + fn test_cb_config() -> CircuitBreakerConfig { + CircuitBreakerConfig { + failure_threshold: CIRCUIT_BREAKER_THRESHOLD as u32, + timeout: CIRCUIT_BREAKER_TIMEOUT, + success_threshold: 1, + } + } + #[tokio::test] async fn test_circuit_breaker_closed_to_open() { - let cb = CircuitBreaker::new(); + let cb = CircuitBreaker::new("test_broker", test_cb_config()); // Initially CLOSED assert_eq!(cb.state().await, CircuitBreakerState::Closed); - assert!(cb.allow_request().await); + assert!(cb.can_execute().await); // Record failures up to threshold for _ in 0..CIRCUIT_BREAKER_THRESHOLD { @@ -437,12 +324,12 @@ mod tests { // Should now be OPEN assert_eq!(cb.state().await, CircuitBreakerState::Open); - assert!(!cb.allow_request().await); + assert!(!cb.can_execute().await); } #[tokio::test] async fn test_circuit_breaker_half_open_success() { - let cb = CircuitBreaker::new(); + let cb = CircuitBreaker::new("test_broker", test_cb_config()); // Force HALF_OPEN state for _ in 0..CIRCUIT_BREAKER_THRESHOLD { @@ -451,7 +338,7 @@ mod tests { // Wait for timeout to transition to HALF_OPEN tokio::time::sleep(CIRCUIT_BREAKER_TIMEOUT + Duration::from_millis(10)).await; - assert!(cb.allow_request().await); + assert!(cb.can_execute().await); assert_eq!(cb.state().await, CircuitBreakerState::HalfOpen); // Success should close circuit @@ -461,7 +348,7 @@ mod tests { #[tokio::test] async fn test_circuit_breaker_half_open_failure() { - let cb = CircuitBreaker::new(); + let cb = CircuitBreaker::new("test_broker", test_cb_config()); // Force HALF_OPEN state for _ in 0..CIRCUIT_BREAKER_THRESHOLD { @@ -469,7 +356,7 @@ mod tests { } tokio::time::sleep(CIRCUIT_BREAKER_TIMEOUT + Duration::from_millis(10)).await; - assert!(cb.allow_request().await); // Trigger transition to HALF_OPEN + assert!(cb.can_execute().await); // Trigger transition to HALF_OPEN assert_eq!(cb.state().await, CircuitBreakerState::HalfOpen); // Failure should reopen circuit diff --git a/services/ml_training_service/src/job_spawner.rs b/services/ml_training_service/src/job_spawner.rs index 1dc6a4262..62fce0793 100644 --- a/services/ml_training_service/src/job_spawner.rs +++ b/services/ml_training_service/src/job_spawner.rs @@ -51,8 +51,8 @@ use std::path::PathBuf; use tracing::{debug, info, warn}; use uuid::Uuid; -// Re-export canonical ModelType from ml crate -pub use ml::ModelType; +// Use canonical ModelType from common crate +pub use common::model_types::ModelType; /// Trading asset with data file #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/services/ml_training_service/src/tls_config.rs b/services/ml_training_service/src/tls_config.rs index 72a0d8535..b10d680ae 100644 --- a/services/ml_training_service/src/tls_config.rs +++ b/services/ml_training_service/src/tls_config.rs @@ -8,6 +8,10 @@ use anyhow::{Context, Result}; use config::manager::ConfigManager; + +// Re-export shared TLS types from common for backward compatibility +#[allow(unused_imports)] +pub use common::tls::{ClientIdentity, TlsProtocolVersion, UserRole}; use config::structures::TlsConfig; use std::sync::Arc; // TLS imports - TLS feature should be enabled in Cargo.toml @@ -37,13 +41,6 @@ pub struct MLTrainingServiceTlsConfig { pub crl_url: Option, } -#[derive(Debug, Clone)] -#[allow(dead_code)] // Variants used in future implementations -pub enum TlsProtocolVersion { - Tls12, - Tls13, -} - #[allow(dead_code)] // Methods used in future implementations impl MLTrainingServiceTlsConfig { /// Create TLS configuration from certificate files @@ -642,102 +639,6 @@ impl MLTrainingServiceTlsConfig { } } -/// Client identity extracted from certificate -#[derive(Debug, Clone, PartialEq, Eq)] -#[allow(dead_code)] // Used in future implementations -pub struct ClientIdentity { - pub common_name: String, - pub organizational_unit: String, - pub serial_number: String, - pub issuer: String, -} - -#[allow(dead_code)] // Methods used in future implementations -impl ClientIdentity { - /// Check if client is authorized for trading operations - pub fn is_authorized_for_trading(&self) -> bool { - // Implement authorization logic based on certificate attributes - matches!(self.organizational_unit.as_str(), "trading" | "admin") - } - - /// Check if client is authorized for read-only operations - pub fn is_authorized_for_readonly(&self) -> bool { - // Allow broader access for read-only operations - matches!( - self.organizational_unit.as_str(), - "trading" | "admin" | "analytics" | "risk" | "compliance" - ) - } - - /// Get user role based on certificate - pub fn get_role(&self) -> UserRole { - match self.organizational_unit.as_str() { - "admin" => UserRole::Admin, - "trading" => UserRole::Trader, - "analytics" => UserRole::Analyst, - "risk" => UserRole::RiskManager, - "compliance" => UserRole::ComplianceOfficer, - _ => UserRole::ReadOnly, - } - } -} - -/// User roles based on certificate attributes -#[derive(Debug, Clone, PartialEq)] -#[allow(dead_code)] // Used in future implementations -pub enum UserRole { - Admin, - Trader, - Analyst, - RiskManager, - ComplianceOfficer, - ReadOnly, -} - -#[allow(dead_code)] // Methods used in future implementations -impl UserRole { - /// Get permissions for this role - pub fn get_permissions(&self) -> Vec<&'static str> { - match self { - UserRole::Admin => vec![ - "trading.submit_order", - "trading.cancel_order", - "trading.modify_order", - "risk.view_positions", - "risk.modify_limits", - "analytics.view_data", - "analytics.run_backtest", - "compliance.view_reports", - "system.configure", - ], - UserRole::Trader => vec![ - "trading.submit_order", - "trading.cancel_order", - "trading.modify_order", - "risk.view_positions", - "analytics.view_data", - ], - UserRole::Analyst => vec![ - "analytics.view_data", - "analytics.run_backtest", - "risk.view_positions", - ], - UserRole::RiskManager => vec![ - "risk.view_positions", - "risk.modify_limits", - "analytics.view_data", - "compliance.view_reports", - ], - UserRole::ComplianceOfficer => vec![ - "compliance.view_reports", - "analytics.view_data", - "risk.view_positions", - ], - UserRole::ReadOnly => vec!["analytics.view_data"], - } - } -} - /// TLS interceptor for gRPC requests #[derive(Clone)] #[allow(dead_code)] // Used in future implementations diff --git a/services/trading_service/src/tls_config.rs b/services/trading_service/src/tls_config.rs index 1731b31f1..4011ad3bf 100644 --- a/services/trading_service/src/tls_config.rs +++ b/services/trading_service/src/tls_config.rs @@ -8,6 +8,9 @@ use anyhow::{Context, Result}; use config::manager::ConfigManager; + +// Re-export shared TLS types from common for backward compatibility +pub use common::tls::{ClientIdentity, TlsProtocolVersion, UserRole}; use config::structures::TlsConfig; use std::sync::Arc; // TLS imports - TLS feature should be enabled in Cargo.toml @@ -38,16 +41,6 @@ pub struct TradingServiceTlsConfig { pub crl_url: Option, } -/// TLS protocol version options -#[derive(Debug, Clone)] -#[allow(dead_code)] -pub enum TlsProtocolVersion { - /// TLS version 1.2 - Tls12, - /// TLS version 1.3 - Tls13, -} - #[allow(dead_code)] impl TradingServiceTlsConfig { /// Create TLS configuration from certificate files @@ -594,112 +587,6 @@ impl TradingServiceTlsConfig { } } -/// Client identity extracted from certificate -#[allow(dead_code)] -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ClientIdentity { - /// Common Name (CN) from certificate - pub common_name: String, - /// Organizational Unit (OU) from certificate - pub organizational_unit: String, - /// Certificate serial number - pub serial_number: String, - /// Certificate issuer - pub issuer: String, -} - -#[allow(dead_code)] -impl ClientIdentity { - /// Check if client is authorized for trading operations - pub fn is_authorized_for_trading(&self) -> bool { - // Implement authorization logic based on certificate attributes - matches!(self.organizational_unit.as_str(), "trading" | "admin") - } - - /// Check if client is authorized for read-only operations - pub fn is_authorized_for_readonly(&self) -> bool { - // Allow broader access for read-only operations - matches!( - self.organizational_unit.as_str(), - "trading" | "admin" | "analytics" | "risk" | "compliance" - ) - } - - /// Get user role based on certificate - pub fn get_role(&self) -> UserRole { - match self.organizational_unit.as_str() { - "admin" => UserRole::Admin, - "trading" => UserRole::Trader, - "analytics" => UserRole::Analyst, - "risk" => UserRole::RiskManager, - "compliance" => UserRole::ComplianceOfficer, - _ => UserRole::ReadOnly, - } - } -} - -/// User roles based on certificate attributes -#[allow(dead_code)] -#[derive(Debug, Clone, PartialEq)] -pub enum UserRole { - /// Administrator with full access - Admin, - /// Trader with trading permissions - Trader, - /// Analyst with read/analysis permissions - Analyst, - /// Risk manager with risk oversight - RiskManager, - /// Compliance officer with audit access - ComplianceOfficer, - /// Read-only access - ReadOnly, -} - -#[allow(dead_code)] -impl UserRole { - /// Get permissions for this role - pub fn get_permissions(&self) -> Vec<&'static str> { - match self { - UserRole::Admin => vec![ - "trading.submit_order", - "trading.cancel_order", - "trading.modify_order", - "risk.view_positions", - "risk.modify_limits", - "analytics.view_data", - "analytics.run_backtest", - "compliance.view_reports", - "system.configure", - ], - UserRole::Trader => vec![ - "trading.submit_order", - "trading.cancel_order", - "trading.modify_order", - "risk.view_positions", - "analytics.view_data", - ], - UserRole::Analyst => vec![ - "analytics.view_data", - "analytics.run_backtest", - "risk.view_positions", - ], - UserRole::RiskManager => vec![ - "risk.view_positions", - "risk.modify_limits", - "analytics.view_data", - "compliance.view_reports", - ], - UserRole::ComplianceOfficer => vec![ - "compliance.view_reports", - "analytics.view_data", - "risk.view_positions", - ], - UserRole::ReadOnly => vec!["analytics.view_data"], - } - } -} - /// TLS interceptor for gRPC requests #[allow(dead_code)] #[derive(Clone)]