//! Confidence-weighted voting and uncertainty quantification for ensemble predictions //! //! This module provides confidence aggregation mechanisms that combine predictions //! from multiple models while accounting for prediction uncertainty, model reliability, //! and ensemble confidence intervals. //! //! ## Key types //! - [`ConfidenceAggregator`] — main entry point for uncertainty-aware ensemble voting //! - [`UncertaintyQuantifier`] — decomposes uncertainty into epistemic / aleatoric / disagreement //! - [`ReliabilityScorer`] — exponential-decay model reliability tracking //! - [`IntervalCombiner`] — prediction intervals at 68% / 95% / 99% confidence levels use anyhow::Result; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use tracing::{debug, warn}; use crate::ModelPrediction; // ── Backwards-compatible stub kept for aggregator.rs ────────────────────────── /// Simple confidence calculator (used by [`super::aggregator::SignalAggregator`]). #[derive(Debug)] pub struct ConfidenceCalculator; impl ConfidenceCalculator { pub fn new() -> Self { Self } } // ── Core aggregator ─────────────────────────────────────────────────────────── /// Confidence aggregator for uncertainty-aware ensemble voting. /// /// Combines predictions from multiple models with proper uncertainty /// decomposition, reliability tracking, and prediction interval estimation. #[derive(Debug)] pub struct ConfidenceAggregator { /// Uncertainty quantification engine uncertainty_quantifier: UncertaintyQuantifier, /// Model reliability scoring system reliability_scorer: ReliabilityScorer, /// Prediction interval combination engine interval_combiner: IntervalCombiner, /// Aggregation configuration config: AggregationConfig, } // ── Sub-components ──────────────────────────────────────────────────────────── /// Uncertainty quantification engine. /// /// Decomposes total prediction uncertainty into: /// - **Epistemic** — model disagreement (reducible with more data / models) /// - **Aleatoric** — inherent data noise (irreducible) /// - **Disagreement** — recent historical model spread #[derive(Debug)] pub struct UncertaintyQuantifier { epistemic_config: EpistemicConfig, aleatoric_config: AleatoricConfig, disagreement_tracker: DisagreementTracker, } /// Model reliability scoring system with exponential decay. #[derive(Debug)] pub struct ReliabilityScorer { reliability_history: HashMap>, decay_factor: f64, min_observations: usize, } /// Prediction interval combination engine. #[derive(Debug)] pub struct IntervalCombiner { confidence_levels: Vec, } /// Model disagreement tracker with weighted recency. #[derive(Debug)] pub struct DisagreementTracker { disagreement_history: Vec, max_history_length: usize, warning_threshold: f64, } // ── Output types ────────────────────────────────────────────────────────────── /// Enhanced ensemble prediction with full uncertainty quantification. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EnsemblePredictionWithUncertainty { /// Central prediction value pub prediction: f64, /// Overall ensemble confidence (0.0–1.0) pub confidence: f64, /// Uncertainty bounds (lower, upper) at 2-sigma pub uncertainty_bounds: (f64, f64), /// Prediction intervals at multiple confidence levels pub prediction_intervals: Vec, /// Per-model contributions with reliability scores pub model_contributions: HashMap, /// Uncertainty decomposition into epistemic / aleatoric / disagreement pub uncertainty_decomposition: UncertaintyDecomposition, /// Weight-averaged ensemble reliability (0.0–1.0) pub ensemble_reliability: f64, /// Prediction timestamp pub timestamp: chrono::DateTime, } /// Individual model contribution to ensemble prediction. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ModelContribution { /// Model's raw prediction pub prediction: f64, /// Model's confidence pub confidence: f64, /// Model's reliability score pub reliability: f64, /// Model's weight in ensemble pub weight: f64, /// Weighted contribution to final prediction pub weighted_contribution: f64, /// Model's prediction interval pub prediction_interval: Option, } /// Prediction interval at a specific confidence level. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PredictionInterval { /// Confidence level (e.g., 0.95 for 95%) pub confidence_level: f64, /// Lower bound pub lower_bound: f64, /// Upper bound pub upper_bound: f64, /// Interval width pub width: f64, } /// Uncertainty decomposition into components. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct UncertaintyDecomposition { /// Epistemic uncertainty (model uncertainty — reducible) pub epistemic: f64, /// Aleatoric uncertainty (data uncertainty — irreducible) pub aleatoric: f64, /// Model disagreement component pub disagreement: f64, /// Total uncertainty = sqrt(epistemic² + aleatoric² + disagreement²) pub total: f64, } /// Reliability record for a model. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ReliabilityRecord { pub timestamp: chrono::DateTime, pub accuracy: f64, pub calibration: f64, pub confidence_reliability: f64, pub actual_outcome: Option, } /// Disagreement record between models. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DisagreementRecord { pub timestamp: chrono::DateTime, pub magnitude: f64, pub models: Vec, pub prediction_spread: f64, } // ── Configuration types ─────────────────────────────────────────────────────── /// Aggregation configuration. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AggregationConfig { /// Weight confidence by reliability pub weight_by_reliability: bool, /// Minimum confidence threshold pub min_confidence_threshold: f64, /// Maximum uncertainty allowed pub max_uncertainty_threshold: f64, /// Outlier detection enabled pub outlier_detection_enabled: bool, /// Outlier threshold (in standard deviations) pub outlier_threshold: f64, } /// Epistemic uncertainty configuration. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EpistemicConfig { /// Use model disagreement as proxy for epistemic uncertainty pub use_model_disagreement: bool, /// Bayesian uncertainty estimation pub bayesian_estimation: bool, /// Monte Carlo dropout samples pub mc_dropout_samples: usize, } /// Aleatoric uncertainty configuration. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AleatoricConfig { /// Heteroscedastic noise modeling pub heteroscedastic_noise: bool, /// Noise variance estimation method pub variance_estimation_method: VarianceEstimationMethod, /// Historical volatility window pub volatility_window: usize, } /// Calibration parameters for prediction intervals. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CalibrationParams { /// Temperature scaling parameter pub temperature: f64, /// Platt scaling enabled pub platt_scaling: bool, /// Isotonic regression enabled pub isotonic_regression: bool, } // ── Enums ───────────────────────────────────────────────────────────────────── /// Methods for combining prediction intervals. #[derive(Debug, Clone, Serialize, Deserialize)] pub enum CombinationMethod { /// Weighted average of intervals WeightedAverage, /// Conservative union of intervals ConservativeUnion, /// Bayesian model averaging BayesianAveraging, /// Mixture of Gaussians MixtureOfGaussians, } /// Methods for variance estimation. #[derive(Debug, Clone, Serialize, Deserialize)] pub enum VarianceEstimationMethod { /// Rolling window variance RollingWindow, /// EWMA variance EWMA, /// GARCH modeling GARCH, /// Realized volatility RealizedVolatility, } // ── ConfidenceAggregator implementation ─────────────────────────────────────── impl ConfidenceAggregator { /// Create a new confidence aggregator with the given config. pub fn new(config: AggregationConfig) -> Self { let uncertainty_quantifier = UncertaintyQuantifier::new(EpistemicConfig::default(), AleatoricConfig::default()); let reliability_scorer = ReliabilityScorer::new(0.95_f64, 10_usize); let interval_combiner = IntervalCombiner::new( CombinationMethod::WeightedAverage, vec![0.68_f64, 0.95_f64, 0.99_f64], CalibrationParams::default(), ); Self { uncertainty_quantifier, reliability_scorer, interval_combiner, config, } } /// Aggregate predictions with full uncertainty quantification. /// /// Returns an [`EnsemblePredictionWithUncertainty`] containing the central /// prediction, confidence, uncertainty bounds, per-model contributions, and /// a full epistemic / aleatoric / disagreement decomposition. pub fn aggregate_with_uncertainty( &mut self, predictions: HashMap, weights: &HashMap, ) -> Result { debug!( "Aggregating {} predictions with uncertainty quantification", predictions.len() ); if predictions.is_empty() { anyhow::bail!("Cannot aggregate empty predictions"); } // Filter outliers if enabled let filtered_predictions = if self.config.outlier_detection_enabled { self.filter_outliers(predictions)? } else { predictions }; // Calculate model reliability scores let reliability_scores = self.calculate_reliability_scores(&filtered_predictions); // Compute ensemble prediction let (ensemble_prediction, model_contributions) = self.compute_weighted_prediction(&filtered_predictions, weights, &reliability_scores)?; // Quantify uncertainty components let uncertainty_decomposition = self .uncertainty_quantifier .quantify_uncertainty(&filtered_predictions, weights)?; // Calculate prediction intervals let prediction_intervals = self .interval_combiner .combine_intervals(&filtered_predictions, weights)?; // Calculate uncertainty bounds (2-sigma) let uncertainty_bounds = self.calculate_uncertainty_bounds(ensemble_prediction, &uncertainty_decomposition); // Calculate ensemble reliability let ensemble_reliability = self.calculate_ensemble_reliability(&reliability_scores, weights); // Compute overall confidence let confidence = self.compute_ensemble_confidence(&uncertainty_decomposition, ensemble_reliability); // Update disagreement tracking self.update_disagreement_tracking(&filtered_predictions); Ok(EnsemblePredictionWithUncertainty { prediction: ensemble_prediction, confidence, uncertainty_bounds, prediction_intervals, model_contributions, uncertainty_decomposition, ensemble_reliability, timestamp: chrono::Utc::now(), }) } /// Update reliability history for a model. pub fn update_reliability(&mut self, model_name: String, record: ReliabilityRecord) { self.reliability_scorer .update_reliability(model_name, record); } /// Get current model reliability scores. pub fn get_reliability_scores(&self) -> HashMap { self.reliability_scorer.get_current_scores() } // ── private helpers ─────────────────────────────────────────────────── fn calculate_reliability_scores( &self, predictions: &HashMap, ) -> HashMap { predictions .keys() .map(|name| { let reliability = self.reliability_scorer.get_model_reliability(name); (name.clone(), reliability) }) .collect() } fn compute_weighted_prediction( &self, predictions: &HashMap, weights: &HashMap, reliability_scores: &HashMap, ) -> Result<(f64, HashMap)> { let mut weighted_sum = 0.0_f64; let mut total_weight = 0.0_f64; let mut model_contributions = HashMap::new(); for (model_name, prediction) in predictions { let base_weight = weights.get(model_name).copied().unwrap_or(0.0_f64); let reliability = reliability_scores .get(model_name) .copied() .unwrap_or(0.5_f64); let final_weight = if self.config.weight_by_reliability { base_weight * reliability } else { base_weight }; let weighted_contribution = prediction.value * final_weight; weighted_sum += weighted_contribution; total_weight += final_weight; model_contributions.insert( model_name.clone(), ModelContribution { prediction: prediction.value, confidence: prediction.confidence, reliability, weight: final_weight, weighted_contribution, prediction_interval: None, }, ); } if total_weight == 0.0_f64 { anyhow::bail!("Total weight is zero — cannot compute ensemble prediction"); } let ensemble_prediction = weighted_sum / total_weight; Ok((ensemble_prediction, model_contributions)) } fn filter_outliers( &self, predictions: HashMap, ) -> Result> { if predictions.len() < 3 { return Ok(predictions); } let values: Vec = predictions.values().map(|p| p.value).collect(); let mean = values.iter().sum::() / values.len() as f64; let variance = values.iter().map(|v| (v - mean).powi(2)).sum::() / values.len() as f64; let std_dev = variance.sqrt(); let threshold = self.config.outlier_threshold * std_dev; let mut filtered = HashMap::new(); for (model_name, prediction) in &predictions { if (prediction.value - mean).abs() <= threshold { filtered.insert(model_name.clone(), prediction.clone()); } else { warn!( "Filtered outlier prediction from {}: {}", model_name, prediction.value ); } } if filtered.is_empty() { warn!("All predictions filtered as outliers, using original set"); Ok(predictions) } else { Ok(filtered) } } fn calculate_uncertainty_bounds( &self, prediction: f64, uncertainty: &UncertaintyDecomposition, ) -> (f64, f64) { let uncertainty_width = 2.0_f64 * uncertainty.total; (prediction - uncertainty_width, prediction + uncertainty_width) } fn calculate_ensemble_reliability( &self, reliability_scores: &HashMap, weights: &HashMap, ) -> f64 { let mut weighted_reliability = 0.0_f64; let mut total_weight = 0.0_f64; for (model_name, &reliability) in reliability_scores { if let Some(&weight) = weights.get(model_name) { weighted_reliability += reliability * weight; total_weight += weight; } } if total_weight > 0.0_f64 { weighted_reliability / total_weight } else { 0.5 } } fn compute_ensemble_confidence( &self, uncertainty: &UncertaintyDecomposition, reliability: f64, ) -> f64 { let uncertainty_factor = 1.0_f64 - (uncertainty.total / (1.0_f64 + uncertainty.total)); (uncertainty_factor * reliability).clamp(0.0_f64, 1.0_f64) } fn update_disagreement_tracking(&mut self, predictions: &HashMap) { if predictions.len() < 2 { return; } let values: Vec = predictions.values().map(|p| p.value).collect(); let mean = values.iter().sum::() / values.len() as f64; let spread = values .iter() .map(|v| (v - mean).abs()) .fold(0.0_f64, f64::max); let disagreement_magnitude = spread / mean.abs().max(1e-6); let record = DisagreementRecord { timestamp: chrono::Utc::now(), magnitude: disagreement_magnitude, models: predictions.keys().cloned().collect(), prediction_spread: spread, }; self.uncertainty_quantifier .disagreement_tracker .add_record(record); if disagreement_magnitude > self .uncertainty_quantifier .disagreement_tracker .warning_threshold { warn!( "High model disagreement detected: {:.3}", disagreement_magnitude ); } } } // ── UncertaintyQuantifier implementation ────────────────────────────────────── impl UncertaintyQuantifier { pub fn new(epistemic_config: EpistemicConfig, aleatoric_config: AleatoricConfig) -> Self { let disagreement_tracker = DisagreementTracker::new(1000_usize, 0.2_f64); Self { epistemic_config, aleatoric_config, disagreement_tracker, } } /// Quantify total uncertainty by decomposing into three components. pub fn quantify_uncertainty( &self, predictions: &HashMap, weights: &HashMap, ) -> Result { let epistemic = self.calculate_epistemic_uncertainty(predictions, weights); let aleatoric = self.calculate_aleatoric_uncertainty(predictions); let disagreement = self.calculate_disagreement_uncertainty(); let total = (epistemic.powi(2) + aleatoric.powi(2) + disagreement.powi(2)).sqrt(); Ok(UncertaintyDecomposition { epistemic, aleatoric, disagreement, total, }) } /// Epistemic uncertainty via model disagreement (standard deviation of predictions). fn calculate_epistemic_uncertainty( &self, predictions: &HashMap, _weights: &HashMap, ) -> f64 { if predictions.len() < 2 || !self.epistemic_config.use_model_disagreement { return 0.0_f64; } let values: Vec = predictions.values().map(|p| p.value).collect(); let mean = values.iter().sum::() / values.len() as f64; let variance = values.iter().map(|v| (v - mean).powi(2)).sum::() / values.len() as f64; variance.sqrt() } /// Aleatoric uncertainty from average model self-reported uncertainty (1 − confidence). fn calculate_aleatoric_uncertainty( &self, predictions: &HashMap, ) -> f64 { let uncertainties: Vec = predictions .values() .map(|p| 1.0_f64 - p.confidence) .collect(); if uncertainties.is_empty() { return 0.0_f64; } uncertainties.iter().sum::() / uncertainties.len() as f64 } /// Historical disagreement component from recent tracking window. fn calculate_disagreement_uncertainty(&self) -> f64 { self.disagreement_tracker.get_recent_disagreement() } } // ── ReliabilityScorer implementation ────────────────────────────────────────── impl ReliabilityScorer { pub fn new(decay_factor: f64, min_observations: usize) -> Self { Self { reliability_history: HashMap::new(), decay_factor, min_observations, } } /// Record a reliability observation for a model. pub fn update_reliability(&mut self, model_name: String, record: ReliabilityRecord) { let history = self .reliability_history .entry(model_name.clone()) .or_default(); history.push(record); // Cap history at 1000 entries if history.len() > 1000 { history.remove(0); } debug!( "Updated reliability history for {}: {} records", model_name, history.len() ); } /// Get current reliability scores for all tracked models. pub fn get_current_scores(&self) -> HashMap { self.reliability_history .iter() .map(|(name, history)| (name.clone(), self.calculate_model_reliability(history))) .collect() } /// Get reliability for a specific model (0.5 default for unknown models). pub fn get_model_reliability(&self, model_name: &str) -> f64 { self.reliability_history .get(model_name) .map(|history| self.calculate_model_reliability(history)) .unwrap_or(0.5_f64) } fn calculate_model_reliability(&self, history: &[ReliabilityRecord]) -> f64 { if history.len() < self.min_observations { return 0.5_f64; } let mut weighted_sum = 0.0_f64; let mut weight_sum = 0.0_f64; let current_time = chrono::Utc::now(); for (i, record) in history.iter().rev().enumerate() { let age = (current_time - record.timestamp).num_hours() as f64; let weight = self.decay_factor.powf(age / 24.0_f64); let reliability_score = (record.accuracy + record.calibration + record.confidence_reliability) / 3.0_f64; weighted_sum += reliability_score * weight; weight_sum += weight; if i >= 100_usize { break; } } if weight_sum > 0.0_f64 { (weighted_sum / weight_sum).clamp(0.0_f64, 1.0_f64) } else { 0.5_f64 } } } // ── IntervalCombiner implementation ─────────────────────────────────────────── impl IntervalCombiner { pub fn new( _combination_method: CombinationMethod, confidence_levels: Vec, _calibration_params: CalibrationParams, ) -> Self { Self { confidence_levels } } /// Compute prediction intervals at each configured confidence level. pub fn combine_intervals( &self, predictions: &HashMap, weights: &HashMap, ) -> Result> { let mut intervals = Vec::new(); for &confidence_level in &self.confidence_levels { let interval = self.compute_combined_interval(predictions, weights, confidence_level)?; intervals.push(interval); } Ok(intervals) } fn compute_combined_interval( &self, predictions: &HashMap, _weights: &HashMap, confidence_level: f64, ) -> Result { let values: Vec = predictions.values().map(|p| p.value).collect(); let mean = values.iter().sum::() / values.len() as f64; let variance = values.iter().map(|v| (v - mean).powi(2)).sum::() / values.len() as f64; let std_dev = variance.sqrt(); // Normal approximation z-scores let z_score = match confidence_level { x if x >= 0.99_f64 => 2.576_f64, x if x >= 0.95_f64 => 1.96_f64, x if x >= 0.90_f64 => 1.645_f64, x if x >= 0.68_f64 => 1.0_f64, _ => 1.96_f64, }; let margin = z_score * std_dev; let lower_bound = mean - margin; let upper_bound = mean + margin; Ok(PredictionInterval { confidence_level, lower_bound, upper_bound, width: upper_bound - lower_bound, }) } } // ── DisagreementTracker implementation ──────────────────────────────────────── impl DisagreementTracker { pub fn new(max_history_length: usize, warning_threshold: f64) -> Self { Self { disagreement_history: Vec::new(), max_history_length, warning_threshold, } } pub fn add_record(&mut self, record: DisagreementRecord) { self.disagreement_history.push(record); if self.disagreement_history.len() > self.max_history_length { self.disagreement_history.remove(0); } } /// Exponentially-weighted average of last 20 disagreement records. pub fn get_recent_disagreement(&self) -> f64 { if self.disagreement_history.is_empty() { return 0.0_f64; } let mut weighted_sum = 0.0_f64; let mut weight_sum = 0.0_f64; for (i, record) in self .disagreement_history .iter() .rev() .take(20_usize) .enumerate() { let weight = 0.9_f64.powi(i as i32); weighted_sum += record.magnitude * weight; weight_sum += weight; } if weight_sum > 0.0_f64 { weighted_sum / weight_sum } else { 0.0_f64 } } } // ── Default implementations ─────────────────────────────────────────────────── impl Default for AggregationConfig { fn default() -> Self { Self { weight_by_reliability: true, min_confidence_threshold: 0.1_f64, max_uncertainty_threshold: 1.0_f64, outlier_detection_enabled: true, outlier_threshold: 2.5_f64, } } } impl Default for EpistemicConfig { fn default() -> Self { Self { use_model_disagreement: true, bayesian_estimation: true, mc_dropout_samples: 100_usize, } } } impl Default for AleatoricConfig { fn default() -> Self { Self { heteroscedastic_noise: true, variance_estimation_method: VarianceEstimationMethod::EWMA, volatility_window: 50_usize, } } } impl Default for CalibrationParams { fn default() -> Self { Self { temperature: 1.0_f64, platt_scaling: false, isotonic_regression: false, } } } // ── Tests ───────────────────────────────────────────────────────────────────── #[cfg(test)] #[allow(clippy::assertions_on_result_states)] mod tests { use super::*; fn make_prediction(model_id: &str, value: f64, confidence: f64) -> ModelPrediction { ModelPrediction::new(model_id.to_owned(), value, confidence) } #[test] fn test_confidence_aggregator_creation() { let config = AggregationConfig::default(); let aggregator = ConfidenceAggregator::new(config); assert!(aggregator.config.weight_by_reliability); } #[test] fn test_uncertainty_quantification() { let mut aggregator = ConfidenceAggregator::new(AggregationConfig::default()); let mut predictions = HashMap::new(); predictions.insert("model1".to_owned(), make_prediction("model1", 0.5, 0.8)); predictions.insert("model2".to_owned(), make_prediction("model2", 0.6, 0.7)); let mut weights = HashMap::new(); weights.insert("model1".to_owned(), 0.6_f64); weights.insert("model2".to_owned(), 0.4_f64); let result = aggregator.aggregate_with_uncertainty(predictions, &weights); assert!(result.is_ok()); let ensemble_pred = result.expect("checked is_ok above"); assert!(ensemble_pred.confidence >= 0.0_f64 && ensemble_pred.confidence <= 1.0_f64); assert!(ensemble_pred.uncertainty_decomposition.total >= 0.0_f64); assert_eq!(ensemble_pred.prediction_intervals.len(), 3); } #[test] fn test_disagreement_tracker() { let mut tracker = DisagreementTracker::new(100_usize, 0.2_f64); let record = DisagreementRecord { timestamp: chrono::Utc::now(), magnitude: 0.15_f64, models: vec!["model1".to_owned(), "model2".to_owned()], prediction_spread: 0.1_f64, }; tracker.add_record(record); let disagreement = tracker.get_recent_disagreement(); assert_eq!(disagreement, 0.15_f64); } #[test] fn test_reliability_scorer() { let mut scorer = ReliabilityScorer::new(0.95_f64, 5_usize); let record = ReliabilityRecord { timestamp: chrono::Utc::now(), accuracy: 0.8_f64, calibration: 0.75_f64, confidence_reliability: 0.85_f64, actual_outcome: None, }; scorer.update_reliability("test_model".to_owned(), record); let reliability = scorer.get_model_reliability("test_model"); assert!((0.0_f64..=1.0_f64).contains(&reliability)); } #[test] fn test_single_model_aggregation() { let mut aggregator = ConfidenceAggregator::new(AggregationConfig::default()); let mut predictions = HashMap::new(); predictions.insert("solo".to_owned(), make_prediction("solo", 1.23, 0.9)); let mut weights = HashMap::new(); weights.insert("solo".to_owned(), 1.0_f64); let result = aggregator .aggregate_with_uncertainty(predictions, &weights) .expect("single model should succeed"); // Single model → prediction equals that model's value (no outlier filtering below 3) assert!((result.prediction - 1.23).abs() < 1e-9); assert!(result.confidence > 0.0); } #[test] fn test_empty_weights_returns_error() { let mut aggregator = ConfidenceAggregator::new(AggregationConfig::default()); let mut predictions = HashMap::new(); predictions.insert("m1".to_owned(), make_prediction("m1", 0.5, 0.8)); // No weights → effective weight is 0.0 after reliability multiplication let weights: HashMap = HashMap::new(); let result = aggregator.aggregate_with_uncertainty(predictions, &weights); assert!(result.is_err()); } }