feat(ml): add PredictabilityScorer with rolling ML accuracy tracking
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
@@ -1 +1,365 @@
|
||||
//! Scorer module - TBD
|
||||
//! Predictability scorer — tracks rolling ML prediction accuracy per asset.
|
||||
//!
|
||||
//! The key insight: assets where our ensemble models predict direction
|
||||
//! accurately should receive more capital allocation. This module records
|
||||
//! prediction outcomes (predicted vs actual direction) and computes a
|
||||
//! rolling accuracy window that feeds into composite asset scores.
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Data types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// A single prediction outcome for one asset at one point in time.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PredictionResult {
|
||||
/// Ticker symbol (e.g. "AAPL").
|
||||
pub symbol: String,
|
||||
/// Model's predicted price direction (positive = up, negative = down).
|
||||
pub predicted_direction: f64,
|
||||
/// Actual price direction observed after the prediction horizon.
|
||||
pub actual_direction: f64,
|
||||
/// Timestamp of the prediction.
|
||||
pub timestamp: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl PredictionResult {
|
||||
/// A prediction is correct when predicted and actual directions agree
|
||||
/// (both positive **or** both negative). Zero is treated as negative
|
||||
/// for tie-breaking purposes.
|
||||
#[must_use]
|
||||
pub fn is_correct(&self) -> bool {
|
||||
(self.predicted_direction > 0.0 && self.actual_direction > 0.0)
|
||||
|| (self.predicted_direction <= 0.0 && self.actual_direction <= 0.0)
|
||||
}
|
||||
}
|
||||
|
||||
/// Composite score for a single asset combining predictability, liquidity,
|
||||
/// and regime fit.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AssetScore {
|
||||
/// Ticker symbol.
|
||||
pub symbol: String,
|
||||
/// Rolling prediction accuracy in `[0.0, 1.0]`.
|
||||
pub predictability: f64,
|
||||
/// Normalised liquidity metric in `[0.0, 1.0]`.
|
||||
pub liquidity: f64,
|
||||
/// How well the asset fits the current market regime in `[0.0, 1.0]`.
|
||||
pub regime_fit: f64,
|
||||
/// Weighted composite of the three components.
|
||||
pub composite: f64,
|
||||
}
|
||||
|
||||
/// Weights used to combine the three scoring dimensions.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ScoringWeights {
|
||||
/// Weight for predictability component.
|
||||
pub predictability: f64,
|
||||
/// Weight for liquidity component.
|
||||
pub liquidity: f64,
|
||||
/// Weight for regime-fit component.
|
||||
pub regime_fit: f64,
|
||||
}
|
||||
|
||||
impl Default for ScoringWeights {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
predictability: 0.5,
|
||||
liquidity: 0.3,
|
||||
regime_fit: 0.2,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scorer
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Tracks per-asset ML prediction accuracy over a rolling window and
|
||||
/// produces composite scores for capital allocation decisions.
|
||||
#[derive(Debug)]
|
||||
pub struct PredictabilityScorer {
|
||||
/// Per-symbol ring buffer of recent prediction outcomes.
|
||||
history: HashMap<String, VecDeque<PredictionResult>>,
|
||||
/// Maximum number of results retained per symbol.
|
||||
window_size: usize,
|
||||
/// Weighting scheme for composite score calculation.
|
||||
weights: ScoringWeights,
|
||||
}
|
||||
|
||||
impl Default for PredictabilityScorer {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl PredictabilityScorer {
|
||||
/// Create a scorer with default window (100) and default weights.
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
history: HashMap::new(),
|
||||
window_size: 100,
|
||||
weights: ScoringWeights::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a scorer with explicit window size and weights.
|
||||
#[must_use]
|
||||
pub fn with_config(window_size: usize, weights: ScoringWeights) -> Self {
|
||||
Self {
|
||||
history: HashMap::new(),
|
||||
window_size: if window_size == 0 { 1 } else { window_size },
|
||||
weights,
|
||||
}
|
||||
}
|
||||
|
||||
/// Record a prediction outcome. The ring buffer is trimmed to
|
||||
/// `window_size` entries per symbol.
|
||||
pub fn record(&mut self, result: PredictionResult) {
|
||||
let deque = self
|
||||
.history
|
||||
.entry(result.symbol.clone())
|
||||
.or_insert_with(VecDeque::new);
|
||||
deque.push_back(result);
|
||||
while deque.len() > self.window_size {
|
||||
deque.pop_front();
|
||||
}
|
||||
}
|
||||
|
||||
/// Rolling accuracy for `symbol`, or `None` if no history exists.
|
||||
#[must_use]
|
||||
pub fn accuracy(&self, symbol: &str) -> Option<f64> {
|
||||
let deque = self.history.get(symbol)?;
|
||||
if deque.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let correct = deque.iter().filter(|r| r.is_correct()).count();
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
Some(correct as f64 / deque.len() as f64)
|
||||
}
|
||||
|
||||
/// Number of recorded predictions for `symbol`.
|
||||
#[must_use]
|
||||
pub fn prediction_count(&self, symbol: &str) -> usize {
|
||||
self.history
|
||||
.get(symbol)
|
||||
.map_or(0, VecDeque::len)
|
||||
}
|
||||
|
||||
/// Compute a composite [`AssetScore`] for a single symbol.
|
||||
///
|
||||
/// When no prediction history exists the predictability defaults to 0.5
|
||||
/// (no demonstrated edge).
|
||||
#[must_use]
|
||||
pub fn score(&self, symbol: &str, liquidity: f64, regime_fit: f64) -> AssetScore {
|
||||
let predictability = self.accuracy(symbol).unwrap_or(0.5);
|
||||
let composite = self.weights.predictability * predictability
|
||||
+ self.weights.liquidity * liquidity
|
||||
+ self.weights.regime_fit * regime_fit;
|
||||
AssetScore {
|
||||
symbol: symbol.to_string(),
|
||||
predictability,
|
||||
liquidity,
|
||||
regime_fit,
|
||||
composite,
|
||||
}
|
||||
}
|
||||
|
||||
/// Score every symbol present in the liquidity map and sort descending
|
||||
/// by composite score. Symbols present in `liquidity_scores` but absent
|
||||
/// from `regime_scores` default to `0.0` regime fit (and vice-versa).
|
||||
#[must_use]
|
||||
pub fn score_all(
|
||||
&self,
|
||||
liquidity_scores: &HashMap<String, f64>,
|
||||
regime_scores: &HashMap<String, f64>,
|
||||
) -> Vec<AssetScore> {
|
||||
// Collect the union of symbols from both maps and history.
|
||||
let mut symbols: Vec<&String> = liquidity_scores.keys().collect();
|
||||
for k in regime_scores.keys() {
|
||||
if !liquidity_scores.contains_key(k) {
|
||||
symbols.push(k);
|
||||
}
|
||||
}
|
||||
|
||||
let mut scores: Vec<AssetScore> = symbols
|
||||
.iter()
|
||||
.map(|sym| {
|
||||
let liq = liquidity_scores.get(*sym).copied().unwrap_or(0.0);
|
||||
let regime = regime_scores.get(*sym).copied().unwrap_or(0.0);
|
||||
self.score(sym, liq, regime)
|
||||
})
|
||||
.collect();
|
||||
|
||||
scores.sort_by(|a, b| {
|
||||
b.composite
|
||||
.partial_cmp(&a.composite)
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
});
|
||||
scores
|
||||
}
|
||||
|
||||
/// Return the set of symbols that have at least one recorded prediction.
|
||||
#[must_use]
|
||||
pub fn tracked_symbols(&self) -> Vec<String> {
|
||||
self.history
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use chrono::Utc;
|
||||
|
||||
fn make_result(symbol: &str, predicted: f64, actual: f64) -> PredictionResult {
|
||||
PredictionResult {
|
||||
symbol: symbol.to_string(),
|
||||
predicted_direction: predicted,
|
||||
actual_direction: actual,
|
||||
timestamp: Utc::now(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_prediction_correct() {
|
||||
// Both positive => correct
|
||||
assert!(make_result("X", 1.0, 0.5).is_correct());
|
||||
// Both negative => correct
|
||||
assert!(make_result("X", -0.3, -1.0).is_correct());
|
||||
// Opposite => incorrect
|
||||
assert!(!make_result("X", 1.0, -0.5).is_correct());
|
||||
assert!(!make_result("X", -1.0, 0.5).is_correct());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_accuracy_no_data() {
|
||||
let scorer = PredictabilityScorer::new();
|
||||
assert!(scorer.accuracy("AAPL").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_accuracy_all_correct() {
|
||||
let mut scorer = PredictabilityScorer::new();
|
||||
for _ in 0..10 {
|
||||
scorer.record(make_result("AAPL", 1.0, 1.0));
|
||||
}
|
||||
let acc = scorer.accuracy("AAPL").unwrap_or(0.0);
|
||||
assert!((acc - 1.0).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_accuracy_half_correct() {
|
||||
let mut scorer = PredictabilityScorer::new();
|
||||
for _ in 0..5 {
|
||||
scorer.record(make_result("AAPL", 1.0, 1.0)); // correct
|
||||
}
|
||||
for _ in 0..5 {
|
||||
scorer.record(make_result("AAPL", 1.0, -1.0)); // incorrect
|
||||
}
|
||||
let acc = scorer.accuracy("AAPL").unwrap_or(0.0);
|
||||
assert!((acc - 0.5).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_window_trims() {
|
||||
let weights = ScoringWeights::default();
|
||||
let mut scorer = PredictabilityScorer::with_config(5, weights);
|
||||
for _ in 0..10 {
|
||||
scorer.record(make_result("AAPL", 1.0, 1.0));
|
||||
}
|
||||
assert_eq!(scorer.prediction_count("AAPL"), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_composite_score() {
|
||||
// 80% accuracy, liquidity=0.7, regime=0.6
|
||||
// composite = 0.5*0.8 + 0.3*0.7 + 0.2*0.6 = 0.40 + 0.21 + 0.12 = 0.73
|
||||
let mut scorer = PredictabilityScorer::new();
|
||||
for _ in 0..8 {
|
||||
scorer.record(make_result("AAPL", 1.0, 1.0)); // correct
|
||||
}
|
||||
for _ in 0..2 {
|
||||
scorer.record(make_result("AAPL", 1.0, -1.0)); // incorrect
|
||||
}
|
||||
let s = scorer.score("AAPL", 0.7, 0.6);
|
||||
assert!((s.predictability - 0.8).abs() < f64::EPSILON);
|
||||
assert!((s.composite - 0.73).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_score_no_history_defaults_50pct() {
|
||||
let scorer = PredictabilityScorer::new();
|
||||
// No history => predictability = 0.5
|
||||
// composite = 0.5*0.5 + 0.3*0.7 + 0.2*0.6 = 0.25 + 0.21 + 0.12 = 0.58
|
||||
let s = scorer.score("AAPL", 0.7, 0.6);
|
||||
assert!((s.predictability - 0.5).abs() < f64::EPSILON);
|
||||
assert!((s.composite - 0.58).abs() < 1e-9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_score_all_sorted_by_composite() {
|
||||
let mut scorer = PredictabilityScorer::new();
|
||||
// AAPL: 80% accuracy
|
||||
for _ in 0..8 {
|
||||
scorer.record(make_result("AAPL", 1.0, 1.0));
|
||||
}
|
||||
for _ in 0..2 {
|
||||
scorer.record(make_result("AAPL", 1.0, -1.0));
|
||||
}
|
||||
// MSFT: 40% accuracy
|
||||
for _ in 0..4 {
|
||||
scorer.record(make_result("MSFT", 1.0, 1.0));
|
||||
}
|
||||
for _ in 0..6 {
|
||||
scorer.record(make_result("MSFT", 1.0, -1.0));
|
||||
}
|
||||
|
||||
let mut liq = HashMap::new();
|
||||
liq.insert("AAPL".to_string(), 0.7);
|
||||
liq.insert("MSFT".to_string(), 0.7);
|
||||
|
||||
let mut regime = HashMap::new();
|
||||
regime.insert("AAPL".to_string(), 0.6);
|
||||
regime.insert("MSFT".to_string(), 0.6);
|
||||
|
||||
let scores = scorer.score_all(&liq, ®ime);
|
||||
assert_eq!(scores.len(), 2);
|
||||
// AAPL (higher accuracy) should come first.
|
||||
assert_eq!(
|
||||
scores.first().map(|s| s.symbol.as_str()).unwrap_or(""),
|
||||
"AAPL"
|
||||
);
|
||||
assert_eq!(
|
||||
scores.get(1).map(|s| s.symbol.as_str()).unwrap_or(""),
|
||||
"MSFT"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multiple_symbols_independent() {
|
||||
let mut scorer = PredictabilityScorer::new();
|
||||
// AAPL: all correct
|
||||
scorer.record(make_result("AAPL", 1.0, 1.0));
|
||||
// MSFT: all wrong
|
||||
scorer.record(make_result("MSFT", 1.0, -1.0));
|
||||
|
||||
let aapl_acc = scorer.accuracy("AAPL").unwrap_or(0.0);
|
||||
let msft_acc = scorer.accuracy("MSFT").unwrap_or(0.0);
|
||||
assert!((aapl_acc - 1.0).abs() < f64::EPSILON);
|
||||
assert!(msft_acc.abs() < f64::EPSILON);
|
||||
|
||||
let mut tracked = scorer.tracked_symbols();
|
||||
tracked.sort();
|
||||
assert_eq!(tracked, vec!["AAPL", "MSFT"]);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user