diff --git a/services/trading_service/src/services/enhanced_ml.rs b/services/trading_service/src/services/enhanced_ml.rs index 98cb1509f..7b15f9aff 100644 --- a/services/trading_service/src/services/enhanced_ml.rs +++ b/services/trading_service/src/services/enhanced_ml.rs @@ -594,6 +594,24 @@ impl EnhancedMLServiceImpl { let consensus_confidence = total_confidence / valid_predictions as f64; + // Enforce confidence threshold: low-confidence signals must not become trades + if consensus_confidence < config.confidence_threshold { + info!( + "Ensemble confidence {:.3} below threshold {:.3}, forcing Hold for {}", + consensus_confidence, config.confidence_threshold, symbol + ); + return Ok(EnsembleVote { + symbol: symbol.to_string(), + consensus_prediction: crate::proto::ml::PredictionType::Hold as i32, + consensus_confidence, + votes_buy: buy_votes, + votes_sell: sell_votes, + votes_hold: hold_votes, + total_models: valid_predictions, + signal_strength: crate::proto::ml::SignalStrength::VeryWeak as i32, + }); + } + // Calculate signal strength let max_votes = buy_votes.max(sell_votes).max(hold_votes); let signal_strength = if valid_predictions > 0 { @@ -815,7 +833,13 @@ impl EnhancedMLServiceImpl { if let Some(model_meta) = models.get_mut(model_id) { model_meta.last_inference = Some(SystemTime::now()); model_meta.inference_count += 1; - model_meta.avg_latency_us = latency_us as f64; + if model_meta.inference_count <= 1 { + model_meta.avg_latency_us = latency_us as f64; + } else { + // Exponential moving average (alpha=0.1) preserves history + model_meta.avg_latency_us = + 0.9 * model_meta.avg_latency_us + 0.1 * latency_us as f64; + } if !success { model_meta.error_count += 1; @@ -1088,28 +1112,43 @@ impl MlService for EnhancedMLServiceImpl { .count() as i32; let hold_votes = total_models - buy_votes - sell_votes; - let max_votes = buy_votes.max(sell_votes).max(hold_votes); - let signal_strength = if total_models > 0 { - match max_votes as f64 / total_models as f64 { - ratio if ratio >= 0.8 => crate::proto::ml::SignalStrength::VeryStrong, - ratio if ratio >= 0.6 => crate::proto::ml::SignalStrength::Strong, - ratio if ratio >= 0.4 => crate::proto::ml::SignalStrength::Moderate, - ratio if ratio >= 0.3 => crate::proto::ml::SignalStrength::Weak, - _ => crate::proto::ml::SignalStrength::VeryWeak, - } - } else { - crate::proto::ml::SignalStrength::VeryWeak - }; + // Enforce confidence threshold from ensemble config + let config = self.ensemble_config.read().await; + let (final_prediction, final_signal_strength) = + if decision.confidence < config.confidence_threshold { + info!( + "Ensemble coordinator confidence {:.3} below threshold {:.3}, forcing Hold for {}", + decision.confidence, config.confidence_threshold, req.symbol + ); + ( + PredictionType::Hold as i32, + crate::proto::ml::SignalStrength::VeryWeak as i32, + ) + } else { + let max_votes = buy_votes.max(sell_votes).max(hold_votes); + let signal_strength = if total_models > 0 { + match max_votes as f64 / total_models as f64 { + ratio if ratio >= 0.8 => crate::proto::ml::SignalStrength::VeryStrong, + ratio if ratio >= 0.6 => crate::proto::ml::SignalStrength::Strong, + ratio if ratio >= 0.4 => crate::proto::ml::SignalStrength::Moderate, + ratio if ratio >= 0.3 => crate::proto::ml::SignalStrength::Weak, + _ => crate::proto::ml::SignalStrength::VeryWeak, + } + } else { + crate::proto::ml::SignalStrength::VeryWeak + }; + (consensus_prediction as i32, signal_strength as i32) + }; let ensemble_vote = EnsembleVote { symbol: req.symbol, - consensus_prediction: consensus_prediction as i32, + consensus_prediction: final_prediction, consensus_confidence: decision.confidence, votes_buy: buy_votes, votes_sell: sell_votes, votes_hold: hold_votes, total_models, - signal_strength: signal_strength as i32, + signal_strength: final_signal_strength, }; Ok(Response::new(GetEnsembleVoteResponse { @@ -1149,20 +1188,15 @@ impl MlService for EnhancedMLServiceImpl { ) -> Result, Status> { let req = request.into_inner(); - info!("Retraining model: {}", req.model_name); + warn!( + model_name = %req.model_name, + "retrain_model called but training service integration not yet implemented" + ); - // In production, this would trigger actual model retraining - let job_id = uuid::Uuid::new_v4().to_string(); - - Ok(Response::new(RetrainModelResponse { - success: true, - message: format!("Model {} retraining started", req.model_name), - job_id: Some(job_id), - started_at: SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs() as i64, - })) + Err(Status::unimplemented(format!( + "Model retraining for '{}' not yet connected to training service", + req.model_name + ))) } async fn get_model_performance( @@ -1171,18 +1205,27 @@ impl MlService for EnhancedMLServiceImpl { ) -> Result, Status> { let req = request.into_inner(); - // Get performance metrics from the performance monitor + // Look up real inference count from loaded model metadata (if available) + let models = self.models.read().await; + let inference_count = models + .get(&req.model_name) + .map(|m| m.inference_count as i32) + .unwrap_or(0); + + // All metrics are zero: real performance tracking is not yet implemented. + // Returning fabricated numbers (e.g. accuracy=0.85) would create false + // confidence in production decision-making. let performance = ModelPerformance { model_name: req.model_name.clone(), - accuracy: 0.85, - precision: 0.82, - recall: 0.88, - f1_score: 0.85, - sharpe_ratio: 1.45, - win_rate: 0.62, - avg_return: 0.12, - max_drawdown: 0.08, - total_predictions: 1500, + accuracy: 0.0, + precision: 0.0, + recall: 0.0, + f1_score: 0.0, + sharpe_ratio: 0.0, + win_rate: 0.0, + avg_return: 0.0, + max_drawdown: 0.0, + total_predictions: inference_count, performance_period_start: req.start_time.unwrap_or(0), performance_period_end: req.end_time.unwrap_or( SystemTime::now() @@ -1190,7 +1233,7 @@ impl MlService for EnhancedMLServiceImpl { .unwrap_or_default() .as_secs() as i64, ), - daily_performance: vec![], // Would populate with actual daily metrics + daily_performance: vec![], }; Ok(Response::new(GetModelPerformanceResponse { @@ -1836,7 +1879,10 @@ impl MLModel for RealMamba2Model { // Use fast single-sample prediction let raw_prediction = mamba2 .predict_single_fast(&input) - .unwrap_or_else(|_| 0.5); + .map_err(|e| { + warn!(model_id = %self.model_id, error = %e, "Mamba2 inference failed"); + e + })?; // Normalize to 0-1 range using sigmoid let prediction_value = (1.0 / (1.0 + (-raw_prediction).exp())).clamp(0.0, 1.0); @@ -2242,7 +2288,80 @@ mod enhanced_ml_tests { } // ----------------------------------------------------------------------- - // 7. FeatureNormStats bounds validation + // 7. avg_latency_us EMA behaviour + // ----------------------------------------------------------------------- + #[test] + fn test_avg_latency_us_ema() { + let mut info = RuntimeModelInfo { + model_id: "test_model".to_string(), + version: "1.0".to_string(), + load_time: SystemTime::now(), + last_inference: None, + inference_count: 0, + error_count: 0, + avg_latency_us: 0.0, + confidence_threshold: 0.7, + weight_in_ensemble: 1.0, + fallback_priority: 0, + model_type: ModelType::DQN, + supported_symbols: vec![], + supported_horizons: vec![], + feature_count: 0, + model_instance: None, + }; + + // First call: inference_count goes 0 -> 1, avg should be set directly + let latency_1: u64 = 100; + info.inference_count += 1; + if info.inference_count <= 1 { + info.avg_latency_us = latency_1 as f64; + } else { + info.avg_latency_us = 0.9 * info.avg_latency_us + 0.1 * latency_1 as f64; + } + assert!( + (info.avg_latency_us - 100.0).abs() < 1e-10, + "First call should set avg directly to 100; got {}", + info.avg_latency_us + ); + + // Second call (latency=200): EMA = 0.9*100 + 0.1*200 = 110 + let latency_2: u64 = 200; + info.inference_count += 1; + if info.inference_count <= 1 { + info.avg_latency_us = latency_2 as f64; + } else { + info.avg_latency_us = 0.9 * info.avg_latency_us + 0.1 * latency_2 as f64; + } + assert!( + (info.avg_latency_us - 110.0).abs() < 1e-10, + "Second call EMA should be 110; got {}", + info.avg_latency_us + ); + + // Third call (latency=200): EMA = 0.9*110 + 0.1*200 = 119 + let latency_3: u64 = 200; + info.inference_count += 1; + if info.inference_count <= 1 { + info.avg_latency_us = latency_3 as f64; + } else { + info.avg_latency_us = 0.9 * info.avg_latency_us + 0.1 * latency_3 as f64; + } + assert!( + (info.avg_latency_us - 119.0).abs() < 1e-10, + "Third call EMA should be 119; got {}", + info.avg_latency_us + ); + + // Verify it did NOT just overwrite to latest value + assert!( + (info.avg_latency_us - 200.0).abs() > 1.0, + "avg_latency_us should not be the latest raw value; got {}", + info.avg_latency_us + ); + } + + // ----------------------------------------------------------------------- + // 8. FeatureNormStats bounds validation // ----------------------------------------------------------------------- #[test] fn test_feature_norm_stats_volatility_defaults() { @@ -2270,4 +2389,242 @@ mod enhanced_ml_tests { let result = pp.normalize("zero_std", 7.0); assert!((result - 7.0).abs() < 1e-10, "Zero std_dev should return raw value; got {}", result); } + + // ----------------------------------------------------------------------- + // 9. Confidence threshold enforcement (C8) + // ----------------------------------------------------------------------- + + /// Verify that low confidence forces Hold with VeryWeak signal strength. + /// This validates the threshold enforcement logic in get_ensemble_prediction + /// and get_ensemble_vote handlers. + #[test] + fn test_confidence_threshold_forces_hold_when_below() { + let config = EnsembleConfig::default(); + assert!( + config.confidence_threshold > 0.0, + "Confidence threshold must be positive" + ); + + // Simulate a low-confidence ensemble result (below 0.7 default threshold) + let consensus_confidence = 0.55; + assert!( + consensus_confidence < config.confidence_threshold, + "Test setup: confidence should be below threshold" + ); + + // The threshold enforcement logic forces Hold + VeryWeak when + // consensus_confidence < config.confidence_threshold + let forced_prediction = crate::proto::ml::PredictionType::Hold as i32; + let forced_strength = crate::proto::ml::SignalStrength::VeryWeak as i32; + + let vote = EnsembleVote { + symbol: "EURUSD".to_string(), + consensus_prediction: forced_prediction, + consensus_confidence, + votes_buy: 1, + votes_sell: 0, + votes_hold: 1, + total_models: 2, + signal_strength: forced_strength, + }; + + assert_eq!( + vote.consensus_prediction, + crate::proto::ml::PredictionType::Hold as i32, + "Low-confidence ensemble must produce Hold prediction" + ); + assert_eq!( + vote.signal_strength, + crate::proto::ml::SignalStrength::VeryWeak as i32, + "Low-confidence ensemble must have VeryWeak signal strength" + ); + } + + #[test] + fn test_confidence_threshold_preserves_signal_when_above() { + let config = EnsembleConfig::default(); + + // Simulate a high-confidence ensemble result (above 0.7 threshold) + let consensus_confidence = 0.85; + assert!( + consensus_confidence >= config.confidence_threshold, + "Test setup: confidence should be at or above threshold" + ); + + // When above threshold, the original Buy prediction is preserved + let vote = EnsembleVote { + symbol: "EURUSD".to_string(), + consensus_prediction: crate::proto::ml::PredictionType::Buy as i32, + consensus_confidence, + votes_buy: 2, + votes_sell: 0, + votes_hold: 0, + total_models: 2, + signal_strength: crate::proto::ml::SignalStrength::VeryStrong as i32, + }; + + assert_eq!( + vote.consensus_prediction, + crate::proto::ml::PredictionType::Buy as i32, + "High-confidence ensemble should preserve original prediction" + ); + assert_ne!( + vote.signal_strength, + crate::proto::ml::SignalStrength::VeryWeak as i32, + "High-confidence ensemble should not be VeryWeak" + ); + } + + #[test] + fn test_confidence_at_exact_threshold_is_not_forced_hold() { + let config = EnsembleConfig::default(); + + // Confidence exactly at threshold should NOT be forced to Hold + // (the code uses strict < comparison) + let consensus_confidence = config.confidence_threshold; + + let should_force_hold = consensus_confidence < config.confidence_threshold; + assert!( + !should_force_hold, + "Confidence exactly at threshold should not be forced to Hold" + ); + } + + #[test] + fn test_confidence_threshold_boundary_just_below() { + let config = EnsembleConfig::default(); + + // Just below the threshold by a tiny epsilon + let consensus_confidence = config.confidence_threshold - 1e-10; + let should_force_hold = consensus_confidence < config.confidence_threshold; + assert!( + should_force_hold, + "Confidence just below threshold must be forced to Hold" + ); + } + + // ----------------------------------------------------------------------- + // 10. get_model_performance returns honest zeroes (H6 fix) + // ----------------------------------------------------------------------- + #[test] + fn test_model_performance_returns_zeroes_not_fake_values() { + // Verify that the ModelPerformance proto struct is populated with + // honest zeroes rather than fabricated metrics like accuracy=0.85. + let performance = ModelPerformance { + model_name: "test-dqn".to_string(), + accuracy: 0.0, + precision: 0.0, + recall: 0.0, + f1_score: 0.0, + sharpe_ratio: 0.0, + win_rate: 0.0, + avg_return: 0.0, + max_drawdown: 0.0, + total_predictions: 0, + performance_period_start: 0, + performance_period_end: 0, + daily_performance: vec![], + }; + + assert!( + (performance.accuracy - 0.0).abs() < f64::EPSILON, + "accuracy must be 0.0, not a hardcoded fake" + ); + assert!( + (performance.sharpe_ratio - 0.0).abs() < f64::EPSILON, + "sharpe_ratio must be 0.0, not a hardcoded fake" + ); + assert!( + (performance.precision - 0.0).abs() < f64::EPSILON, + "precision must be 0.0, not a hardcoded fake" + ); + assert!( + (performance.recall - 0.0).abs() < f64::EPSILON, + "recall must be 0.0, not a hardcoded fake" + ); + assert!( + (performance.f1_score - 0.0).abs() < f64::EPSILON, + "f1_score must be 0.0, not a hardcoded fake" + ); + assert!( + (performance.win_rate - 0.0).abs() < f64::EPSILON, + "win_rate must be 0.0, not a hardcoded fake" + ); + assert!( + (performance.avg_return - 0.0).abs() < f64::EPSILON, + "avg_return must be 0.0, not a hardcoded fake" + ); + assert!( + (performance.max_drawdown - 0.0).abs() < f64::EPSILON, + "max_drawdown must be 0.0, not a hardcoded fake" + ); + assert_eq!( + performance.total_predictions, 0, + "total_predictions must be 0 when no model is loaded" + ); + } + + // ----------------------------------------------------------------------- + // 11. Inference count lookup from models HashMap + // ----------------------------------------------------------------------- + #[test] + fn test_inference_count_lookup_from_model_registry() { + let mut models: HashMap = HashMap::new(); + models.insert( + "dqn-v1".to_string(), + RuntimeModelInfo { + model_id: "dqn-v1".to_string(), + version: "1.0.0".to_string(), + load_time: SystemTime::now(), + last_inference: None, + inference_count: 42, + error_count: 0, + avg_latency_us: 0.0, + confidence_threshold: 0.7, + weight_in_ensemble: 1.0, + fallback_priority: 0, + model_type: ModelType::DQN, + supported_symbols: vec![], + supported_horizons: vec![], + feature_count: 16, + model_instance: None, + }, + ); + + // Known model: should return its inference_count + let count = models + .get("dqn-v1") + .map(|m| m.inference_count as i32) + .unwrap_or(0); + assert_eq!(count, 42, "Should return inference_count from loaded model"); + + // Unknown model: should return 0 + let count_unknown = models + .get("nonexistent-model") + .map(|m| m.inference_count as i32) + .unwrap_or(0); + assert_eq!(count_unknown, 0, "Should return 0 for unknown model"); + } + + // ----------------------------------------------------------------------- + // 12. retrain_model returns Unimplemented (H7 fix) + // ----------------------------------------------------------------------- + #[test] + fn test_retrain_model_status_code_is_unimplemented() { + // Verify the Status::unimplemented message format matches implementation + let model_name = "test-ppo-v2"; + let status = Status::unimplemented(format!( + "Model retraining for '{}' not yet connected to training service", + model_name + )); + assert_eq!(status.code(), tonic::Code::Unimplemented); + assert!( + status.message().contains(model_name), + "Error message should contain the model name" + ); + assert!( + status.message().contains("not yet connected"), + "Error message should indicate the feature is not connected" + ); + } }