diff --git a/ml/src/asset_selection/scorer.rs b/ml/src/asset_selection/scorer.rs index efe2cd35b..9a6d8d6f6 100644 --- a/ml/src/asset_selection/scorer.rs +++ b/ml/src/asset_selection/scorer.rs @@ -1 +1,365 @@ -//! Scorer module - TBD +//! Predictability scorer — tracks rolling ML prediction accuracy per asset. +//! +//! The key insight: assets where our ensemble models predict direction +//! accurately should receive more capital allocation. This module records +//! prediction outcomes (predicted vs actual direction) and computes a +//! rolling accuracy window that feeds into composite asset scores. + +use chrono::{DateTime, Utc}; +use std::collections::{HashMap, VecDeque}; + +// --------------------------------------------------------------------------- +// Data types +// --------------------------------------------------------------------------- + +/// A single prediction outcome for one asset at one point in time. +#[derive(Debug, Clone)] +pub struct PredictionResult { + /// Ticker symbol (e.g. "AAPL"). + pub symbol: String, + /// Model's predicted price direction (positive = up, negative = down). + pub predicted_direction: f64, + /// Actual price direction observed after the prediction horizon. + pub actual_direction: f64, + /// Timestamp of the prediction. + pub timestamp: DateTime, +} + +impl PredictionResult { + /// A prediction is correct when predicted and actual directions agree + /// (both positive **or** both negative). Zero is treated as negative + /// for tie-breaking purposes. + #[must_use] + pub fn is_correct(&self) -> bool { + (self.predicted_direction > 0.0 && self.actual_direction > 0.0) + || (self.predicted_direction <= 0.0 && self.actual_direction <= 0.0) + } +} + +/// Composite score for a single asset combining predictability, liquidity, +/// and regime fit. +#[derive(Debug, Clone)] +pub struct AssetScore { + /// Ticker symbol. + pub symbol: String, + /// Rolling prediction accuracy in `[0.0, 1.0]`. + pub predictability: f64, + /// Normalised liquidity metric in `[0.0, 1.0]`. + pub liquidity: f64, + /// How well the asset fits the current market regime in `[0.0, 1.0]`. + pub regime_fit: f64, + /// Weighted composite of the three components. + pub composite: f64, +} + +/// Weights used to combine the three scoring dimensions. +#[derive(Debug, Clone)] +pub struct ScoringWeights { + /// Weight for predictability component. + pub predictability: f64, + /// Weight for liquidity component. + pub liquidity: f64, + /// Weight for regime-fit component. + pub regime_fit: f64, +} + +impl Default for ScoringWeights { + fn default() -> Self { + Self { + predictability: 0.5, + liquidity: 0.3, + regime_fit: 0.2, + } + } +} + +// --------------------------------------------------------------------------- +// Scorer +// --------------------------------------------------------------------------- + +/// Tracks per-asset ML prediction accuracy over a rolling window and +/// produces composite scores for capital allocation decisions. +#[derive(Debug)] +pub struct PredictabilityScorer { + /// Per-symbol ring buffer of recent prediction outcomes. + history: HashMap>, + /// Maximum number of results retained per symbol. + window_size: usize, + /// Weighting scheme for composite score calculation. + weights: ScoringWeights, +} + +impl Default for PredictabilityScorer { + fn default() -> Self { + Self::new() + } +} + +impl PredictabilityScorer { + /// Create a scorer with default window (100) and default weights. + #[must_use] + pub fn new() -> Self { + Self { + history: HashMap::new(), + window_size: 100, + weights: ScoringWeights::default(), + } + } + + /// Create a scorer with explicit window size and weights. + #[must_use] + pub fn with_config(window_size: usize, weights: ScoringWeights) -> Self { + Self { + history: HashMap::new(), + window_size: if window_size == 0 { 1 } else { window_size }, + weights, + } + } + + /// Record a prediction outcome. The ring buffer is trimmed to + /// `window_size` entries per symbol. + pub fn record(&mut self, result: PredictionResult) { + let deque = self + .history + .entry(result.symbol.clone()) + .or_insert_with(VecDeque::new); + deque.push_back(result); + while deque.len() > self.window_size { + deque.pop_front(); + } + } + + /// Rolling accuracy for `symbol`, or `None` if no history exists. + #[must_use] + pub fn accuracy(&self, symbol: &str) -> Option { + let deque = self.history.get(symbol)?; + if deque.is_empty() { + return None; + } + let correct = deque.iter().filter(|r| r.is_correct()).count(); + #[allow(clippy::cast_precision_loss)] + Some(correct as f64 / deque.len() as f64) + } + + /// Number of recorded predictions for `symbol`. + #[must_use] + pub fn prediction_count(&self, symbol: &str) -> usize { + self.history + .get(symbol) + .map_or(0, VecDeque::len) + } + + /// Compute a composite [`AssetScore`] for a single symbol. + /// + /// When no prediction history exists the predictability defaults to 0.5 + /// (no demonstrated edge). + #[must_use] + pub fn score(&self, symbol: &str, liquidity: f64, regime_fit: f64) -> AssetScore { + let predictability = self.accuracy(symbol).unwrap_or(0.5); + let composite = self.weights.predictability * predictability + + self.weights.liquidity * liquidity + + self.weights.regime_fit * regime_fit; + AssetScore { + symbol: symbol.to_string(), + predictability, + liquidity, + regime_fit, + composite, + } + } + + /// Score every symbol present in the liquidity map and sort descending + /// by composite score. Symbols present in `liquidity_scores` but absent + /// from `regime_scores` default to `0.0` regime fit (and vice-versa). + #[must_use] + pub fn score_all( + &self, + liquidity_scores: &HashMap, + regime_scores: &HashMap, + ) -> Vec { + // Collect the union of symbols from both maps and history. + let mut symbols: Vec<&String> = liquidity_scores.keys().collect(); + for k in regime_scores.keys() { + if !liquidity_scores.contains_key(k) { + symbols.push(k); + } + } + + let mut scores: Vec = symbols + .iter() + .map(|sym| { + let liq = liquidity_scores.get(*sym).copied().unwrap_or(0.0); + let regime = regime_scores.get(*sym).copied().unwrap_or(0.0); + self.score(sym, liq, regime) + }) + .collect(); + + scores.sort_by(|a, b| { + b.composite + .partial_cmp(&a.composite) + .unwrap_or(std::cmp::Ordering::Equal) + }); + scores + } + + /// Return the set of symbols that have at least one recorded prediction. + #[must_use] + pub fn tracked_symbols(&self) -> Vec { + self.history + .keys() + .cloned() + .collect() + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + + fn make_result(symbol: &str, predicted: f64, actual: f64) -> PredictionResult { + PredictionResult { + symbol: symbol.to_string(), + predicted_direction: predicted, + actual_direction: actual, + timestamp: Utc::now(), + } + } + + #[test] + fn test_prediction_correct() { + // Both positive => correct + assert!(make_result("X", 1.0, 0.5).is_correct()); + // Both negative => correct + assert!(make_result("X", -0.3, -1.0).is_correct()); + // Opposite => incorrect + assert!(!make_result("X", 1.0, -0.5).is_correct()); + assert!(!make_result("X", -1.0, 0.5).is_correct()); + } + + #[test] + fn test_accuracy_no_data() { + let scorer = PredictabilityScorer::new(); + assert!(scorer.accuracy("AAPL").is_none()); + } + + #[test] + fn test_accuracy_all_correct() { + let mut scorer = PredictabilityScorer::new(); + for _ in 0..10 { + scorer.record(make_result("AAPL", 1.0, 1.0)); + } + let acc = scorer.accuracy("AAPL").unwrap_or(0.0); + assert!((acc - 1.0).abs() < f64::EPSILON); + } + + #[test] + fn test_accuracy_half_correct() { + let mut scorer = PredictabilityScorer::new(); + for _ in 0..5 { + scorer.record(make_result("AAPL", 1.0, 1.0)); // correct + } + for _ in 0..5 { + scorer.record(make_result("AAPL", 1.0, -1.0)); // incorrect + } + let acc = scorer.accuracy("AAPL").unwrap_or(0.0); + assert!((acc - 0.5).abs() < f64::EPSILON); + } + + #[test] + fn test_window_trims() { + let weights = ScoringWeights::default(); + let mut scorer = PredictabilityScorer::with_config(5, weights); + for _ in 0..10 { + scorer.record(make_result("AAPL", 1.0, 1.0)); + } + assert_eq!(scorer.prediction_count("AAPL"), 5); + } + + #[test] + fn test_composite_score() { + // 80% accuracy, liquidity=0.7, regime=0.6 + // composite = 0.5*0.8 + 0.3*0.7 + 0.2*0.6 = 0.40 + 0.21 + 0.12 = 0.73 + let mut scorer = PredictabilityScorer::new(); + for _ in 0..8 { + scorer.record(make_result("AAPL", 1.0, 1.0)); // correct + } + for _ in 0..2 { + scorer.record(make_result("AAPL", 1.0, -1.0)); // incorrect + } + let s = scorer.score("AAPL", 0.7, 0.6); + assert!((s.predictability - 0.8).abs() < f64::EPSILON); + assert!((s.composite - 0.73).abs() < 1e-9); + } + + #[test] + fn test_score_no_history_defaults_50pct() { + let scorer = PredictabilityScorer::new(); + // No history => predictability = 0.5 + // composite = 0.5*0.5 + 0.3*0.7 + 0.2*0.6 = 0.25 + 0.21 + 0.12 = 0.58 + let s = scorer.score("AAPL", 0.7, 0.6); + assert!((s.predictability - 0.5).abs() < f64::EPSILON); + assert!((s.composite - 0.58).abs() < 1e-9); + } + + #[test] + fn test_score_all_sorted_by_composite() { + let mut scorer = PredictabilityScorer::new(); + // AAPL: 80% accuracy + for _ in 0..8 { + scorer.record(make_result("AAPL", 1.0, 1.0)); + } + for _ in 0..2 { + scorer.record(make_result("AAPL", 1.0, -1.0)); + } + // MSFT: 40% accuracy + for _ in 0..4 { + scorer.record(make_result("MSFT", 1.0, 1.0)); + } + for _ in 0..6 { + scorer.record(make_result("MSFT", 1.0, -1.0)); + } + + let mut liq = HashMap::new(); + liq.insert("AAPL".to_string(), 0.7); + liq.insert("MSFT".to_string(), 0.7); + + let mut regime = HashMap::new(); + regime.insert("AAPL".to_string(), 0.6); + regime.insert("MSFT".to_string(), 0.6); + + let scores = scorer.score_all(&liq, ®ime); + assert_eq!(scores.len(), 2); + // AAPL (higher accuracy) should come first. + assert_eq!( + scores.first().map(|s| s.symbol.as_str()).unwrap_or(""), + "AAPL" + ); + assert_eq!( + scores.get(1).map(|s| s.symbol.as_str()).unwrap_or(""), + "MSFT" + ); + } + + #[test] + fn test_multiple_symbols_independent() { + let mut scorer = PredictabilityScorer::new(); + // AAPL: all correct + scorer.record(make_result("AAPL", 1.0, 1.0)); + // MSFT: all wrong + scorer.record(make_result("MSFT", 1.0, -1.0)); + + let aapl_acc = scorer.accuracy("AAPL").unwrap_or(0.0); + let msft_acc = scorer.accuracy("MSFT").unwrap_or(0.0); + assert!((aapl_acc - 1.0).abs() < f64::EPSILON); + assert!(msft_acc.abs() < f64::EPSILON); + + let mut tracked = scorer.tracked_symbols(); + tracked.sort(); + assert_eq!(tracked, vec!["AAPL", "MSFT"]); + } +}