From b7597a754335946a9fa76ca234d67cb47bb1e0af Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 8 Mar 2026 13:54:22 +0100 Subject: [PATCH] refactor(ml): extract universe, backtesting, asset-selection into sub-crates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ml-universe (1.7K lines): correlation, liquidity, momentum, volatility modules. Only depends on ml-core (MLError, PRECISION_FACTOR). 6 tests passing. - ml-backtesting (1.1K lines): action_loader, barrier_backtest, report modules. Only depends on ml-core (MLError). Discovered and included previously undeclared report.rs module. 10 tests passing. - ml-asset-selection (1.2K lines): scorer, selector modules with AssetClass, AssetUniverse, PredictabilityScorer, ActiveSetSelector. Only depends on ml-core (MLError). 33 tests passing. All three replaced with thin facade re-exports in ml — existing `use ml::universe::*` / `use ml::backtesting::*` / `use ml::asset_selection::*` paths continue to work. Total: 15 sub-crates extracted from ml monolith. Workspace: 0 errors, ml tests 902 passed + 49 in sub-crates. Co-Authored-By: Claude Opus 4.6 --- Cargo.lock | 40 + Cargo.toml | 6 + crates/ml-asset-selection/Cargo.toml | 27 + crates/ml-asset-selection/src/lib.rs | 514 +++++++++++ .../src}/scorer.rs | 0 .../src}/selector.rs | 0 crates/ml-backtesting/Cargo.toml | 27 + crates/ml-backtesting/src/action_loader.rs | 197 +++++ crates/ml-backtesting/src/barrier_backtest.rs | 451 ++++++++++ crates/ml-backtesting/src/lib.rs | 17 + crates/ml-backtesting/src/report.rs | 464 ++++++++++ crates/ml-universe/Cargo.toml | 27 + .../src}/correlation.rs | 158 ---- crates/ml-universe/src/lib.rs | 818 ++++++++++++++++++ crates/ml-universe/src/liquidity.rs | 4 + crates/ml-universe/src/momentum.rs | 4 + .../src}/volatility.rs | 15 +- crates/ml/Cargo.toml | 3 + crates/ml/src/asset_selection/mod.rs | 510 +---------- crates/ml/src/backtesting/mod.rs | 10 +- crates/ml/src/universe/liquidity.rs | 80 -- crates/ml/src/universe/mod.rs | 813 +---------------- crates/ml/src/universe/momentum.rs | 61 -- 23 files changed, 2612 insertions(+), 1634 deletions(-) create mode 100644 crates/ml-asset-selection/Cargo.toml create mode 100644 crates/ml-asset-selection/src/lib.rs rename crates/{ml/src/asset_selection => ml-asset-selection/src}/scorer.rs (100%) rename crates/{ml/src/asset_selection => ml-asset-selection/src}/selector.rs (100%) create mode 100644 crates/ml-backtesting/Cargo.toml create mode 100644 crates/ml-backtesting/src/action_loader.rs create mode 100644 crates/ml-backtesting/src/barrier_backtest.rs create mode 100644 crates/ml-backtesting/src/lib.rs create mode 100644 crates/ml-backtesting/src/report.rs create mode 100644 crates/ml-universe/Cargo.toml rename crates/{ml/src/universe => ml-universe/src}/correlation.rs (61%) create mode 100644 crates/ml-universe/src/lib.rs create mode 100644 crates/ml-universe/src/liquidity.rs create mode 100644 crates/ml-universe/src/momentum.rs rename crates/{ml/src/universe => ml-universe/src}/volatility.rs (94%) delete mode 100644 crates/ml/src/universe/liquidity.rs delete mode 100644 crates/ml/src/universe/momentum.rs diff --git a/Cargo.lock b/Cargo.lock index 73c51ac73..f38a61bc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6262,6 +6262,8 @@ dependencies = [ "lru", "memmap2", "mimalloc", + "ml-asset-selection", + "ml-backtesting", "ml-checkpoint", "ml-core", "ml-data-validation", @@ -6274,6 +6276,7 @@ dependencies = [ "ml-regime", "ml-risk", "ml-supervised", + "ml-universe", "ml-validation", "nalgebra 0.33.2", "ndarray", @@ -6318,6 +6321,30 @@ dependencies = [ "zstd", ] +[[package]] +name = "ml-asset-selection" +version = "1.0.0" +dependencies = [ + "chrono", + "ml-core", + "serde", + "serde_json", + "tempfile", + "toml", +] + +[[package]] +name = "ml-backtesting" +version = "1.0.0" +dependencies = [ + "anyhow", + "chrono", + "csv", + "ml-core", + "serde", + "tempfile", +] + [[package]] name = "ml-checkpoint" version = "1.0.0" @@ -6700,6 +6727,19 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ml-universe" +version = "1.0.0" +dependencies = [ + "chrono", + "common", + "ml-core", + "ndarray", + "serde", + "tokio", + "tracing", +] + [[package]] name = "ml-validation" version = "1.0.0" diff --git a/Cargo.toml b/Cargo.toml index 7e4e6d16e..7e4807cac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -123,6 +123,9 @@ members = [ "crates/ml-data-validation", "crates/ml-validation", "crates/ml-risk", + "crates/ml-backtesting", + "crates/ml-asset-selection", + "crates/ml-universe", "crates/data", "crates/backtesting", "crates/common", @@ -410,6 +413,9 @@ ml-checkpoint = { path = "crates/ml-checkpoint" } ml-data-validation = { path = "crates/ml-data-validation" } ml-validation = { path = "crates/ml-validation" } ml-risk = { path = "crates/ml-risk" } +ml-backtesting = { path = "crates/ml-backtesting" } +ml-asset-selection = { path = "crates/ml-asset-selection" } +ml-universe = { path = "crates/ml-universe" } common = { path = "crates/common" } storage = { path = "crates/storage" } market-data = { path = "crates/market-data" } diff --git a/crates/ml-asset-selection/Cargo.toml b/crates/ml-asset-selection/Cargo.toml new file mode 100644 index 000000000..b4995478a --- /dev/null +++ b/crates/ml-asset-selection/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "ml-asset-selection" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +publish.workspace = true +keywords.workspace = true +categories.workspace = true +description = "Asset universe configuration and selection for the Foxhunt trading pipeline" + +[dependencies] +ml-core.workspace = true +serde = { workspace = true, features = ["derive"] } +toml.workspace = true +chrono.workspace = true + +[dev-dependencies] +serde_json.workspace = true +tempfile.workspace = true + +[lints] +workspace = true diff --git a/crates/ml-asset-selection/src/lib.rs b/crates/ml-asset-selection/src/lib.rs new file mode 100644 index 000000000..1f44d5063 --- /dev/null +++ b/crates/ml-asset-selection/src/lib.rs @@ -0,0 +1,514 @@ +#![deny(clippy::unwrap_used, clippy::expect_used)] +#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used))] +#![allow(clippy::module_name_repetitions)] +//! Asset universe configuration and selection for the trading pipeline. +//! +//! This crate defines which assets are eligible for trading, including +//! their classification, exchange, sector, and minimum volume requirements. +//! It also provides predictability scoring and active-set selection. + +pub use ml_core::MLError; + +pub mod scorer; +pub mod selector; + +use std::path::Path; + +use serde::{Deserialize, Serialize}; + +pub use scorer::{AssetScore, PredictabilityScorer, PredictionResult, ScoringWeights}; +pub use selector::{ActiveSetConfig, ActiveSetSelector, TradingCandidate}; + +/// Classification of a tradable asset. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum AssetClass { + /// Individual equity (stock). + Equity, + /// Exchange-traded fund. + ETF, + /// Futures contract. + Future, +} + +/// A single asset in the trading universe. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UniverseAsset { + /// Ticker symbol (e.g. "ES.FUT") -- used for data/training. + pub symbol: String, + /// Execution symbol (e.g. "MES.FUT") -- used for order routing. + /// When `None`, orders are routed to `symbol` directly. + pub trading_symbol: Option, + /// MIC exchange code (e.g. "GLBX.MDP3"). + pub exchange: String, + /// Asset classification. + pub asset_class: AssetClass, + /// Optional GICS sector. + pub sector: Option, + /// Minimum average daily volume required for eligibility. + pub min_daily_volume: f64, + /// Whether this asset is enabled for trading. + pub enabled: bool, +} + +impl UniverseAsset { + /// The symbol to use for order execution. + /// Returns `trading_symbol` if set, otherwise `symbol`. + #[must_use] + pub fn execution_symbol(&self) -> &str { + self.trading_symbol.as_deref().unwrap_or(&self.symbol) + } +} + +/// The full set of assets available for trading. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AssetUniverse { + /// All configured assets. + pub assets: Vec, +} + +// --------------------------------------------------------------------------- +// TOML config deserialization +// --------------------------------------------------------------------------- + +#[derive(Deserialize)] +struct UniverseConfigFile { + #[allow(dead_code)] // Deserialized for validation; will feed DatasetSpec once download automation lands + universe: UniverseConfigMeta, + symbols: Vec, +} + +/// Parsed metadata from the `[universe]` section of the TOML config. +/// Fields are currently used only for validation/documentation during parsing; +/// they will feed into `DatasetSpec` construction once download automation lands. +#[derive(Deserialize)] +#[allow(dead_code)] +struct UniverseConfigMeta { + name: String, + description: Option, + date_range_start: Option, + date_range_end: Option, + bar_size: Option, + databento_dataset: Option, + databento_schema: Option, +} + +#[derive(Deserialize)] +struct SymbolConfigEntry { + symbol: String, + trading_symbol: Option, + exchange: String, + asset_class: String, + min_daily_volume: Option, +} + +impl AssetUniverse { + /// Create an empty universe. + #[must_use] + pub fn new() -> Self { + Self { assets: Vec::new() } + } + + /// Load a universe from a TOML config file. + /// + /// # Errors + /// Returns `MLError::ConfigError` on I/O or parse failures. + pub fn from_config(path: &Path) -> Result { + let contents = std::fs::read_to_string(path).map_err(|e| MLError::ConfigError(format!("Failed to read universe config {}: {e}", path.display())))?; + + let config: UniverseConfigFile = + toml::from_str(&contents).map_err(|e| MLError::ConfigError(format!("Failed to parse universe config {}: {e}", path.display())))?; + + let assets = config + .symbols + .into_iter() + .map(|s| { + let asset_class = match s.asset_class.as_str() { + "Future" | "future" => AssetClass::Future, + "Equity" | "equity" => AssetClass::Equity, + "ETF" | "etf" => AssetClass::ETF, + other => { + return Err(MLError::ConfigError(format!("Unknown asset class: {other}"))) + } + }; + Ok(UniverseAsset { + symbol: s.symbol, + trading_symbol: s.trading_symbol, + exchange: s.exchange, + asset_class, + sector: None, + min_daily_volume: s.min_daily_volume.unwrap_or(0.0), + enabled: true, + }) + }) + .collect::, _>>()?; + + Ok(Self { assets }) + } + + /// Return only enabled assets. + #[must_use] + pub fn eligible(&self) -> Vec<&UniverseAsset> { + self.assets.iter().filter(|a| a.enabled).collect() + } + + /// Count of enabled assets. + #[must_use] + pub fn eligible_count(&self) -> usize { + self.assets.iter().filter(|a| a.enabled).count() + } + + /// Find an asset by symbol (case-sensitive). + #[must_use] + pub fn find(&self, symbol: &str) -> Option<&UniverseAsset> { + self.assets.iter().find(|a| a.symbol == symbol) + } + + /// List all asset symbols. + #[must_use] + pub fn symbols(&self) -> Vec<&str> { + self.assets.iter().map(|a| a.symbol.as_str()).collect() + } + + /// Preset: US large-cap starter universe (7 highly liquid equities/ETFs). + #[must_use] + pub fn us_starter() -> Self { + Self { + assets: vec![ + UniverseAsset { + symbol: "SPY".to_owned(), + trading_symbol: None, + exchange: "XNYS".to_owned(), + asset_class: AssetClass::ETF, + sector: None, + min_daily_volume: 50_000_000.0, + enabled: true, + }, + UniverseAsset { + symbol: "QQQ".to_owned(), + trading_symbol: None, + exchange: "XNAS".to_owned(), + asset_class: AssetClass::ETF, + sector: None, + min_daily_volume: 30_000_000.0, + enabled: true, + }, + UniverseAsset { + symbol: "AAPL".to_owned(), + trading_symbol: None, + exchange: "XNAS".to_owned(), + asset_class: AssetClass::Equity, + sector: Some("Technology".to_owned()), + min_daily_volume: 50_000_000.0, + enabled: true, + }, + UniverseAsset { + symbol: "MSFT".to_owned(), + trading_symbol: None, + exchange: "XNAS".to_owned(), + asset_class: AssetClass::Equity, + sector: Some("Technology".to_owned()), + min_daily_volume: 20_000_000.0, + enabled: true, + }, + UniverseAsset { + symbol: "NVDA".to_owned(), + trading_symbol: None, + exchange: "XNAS".to_owned(), + asset_class: AssetClass::Equity, + sector: Some("Technology".to_owned()), + min_daily_volume: 30_000_000.0, + enabled: true, + }, + UniverseAsset { + symbol: "AMZN".to_owned(), + trading_symbol: None, + exchange: "XNAS".to_owned(), + asset_class: AssetClass::Equity, + sector: Some("Consumer".to_owned()), + min_daily_volume: 20_000_000.0, + enabled: true, + }, + UniverseAsset { + symbol: "IWM".to_owned(), + trading_symbol: None, + exchange: "XNYS".to_owned(), + asset_class: AssetClass::ETF, + sector: None, + min_daily_volume: 20_000_000.0, + enabled: true, + }, + ], + } + } + + /// Preset: CME futures baseline universe (4 assets for limited capital). + /// + /// Training is done on full-size contracts (ES, NQ, ZN, 6E). + /// Execution routes to micro contracts where available (MES, MNQ, M6E). + #[must_use] + pub fn futures_baseline() -> Self { + Self { + assets: vec![ + UniverseAsset { + symbol: "ES.FUT".to_owned(), + trading_symbol: Some("MES.FUT".to_owned()), + exchange: "GLBX.MDP3".to_owned(), + asset_class: AssetClass::Future, + sector: None, + min_daily_volume: 1_000_000.0, + enabled: true, + }, + UniverseAsset { + symbol: "NQ.FUT".to_owned(), + trading_symbol: Some("MNQ.FUT".to_owned()), + exchange: "GLBX.MDP3".to_owned(), + asset_class: AssetClass::Future, + sector: None, + min_daily_volume: 500_000.0, + enabled: true, + }, + UniverseAsset { + symbol: "ZN.FUT".to_owned(), + trading_symbol: None, + exchange: "GLBX.MDP3".to_owned(), + asset_class: AssetClass::Future, + sector: None, + min_daily_volume: 500_000.0, + enabled: true, + }, + UniverseAsset { + symbol: "6E.FUT".to_owned(), + trading_symbol: Some("M6E.FUT".to_owned()), + exchange: "GLBX.MDP3".to_owned(), + asset_class: AssetClass::Future, + sector: None, + min_daily_volume: 200_000.0, + enabled: true, + }, + ], + } + } +} + +impl Default for AssetUniverse { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_us_starter_universe() { + let universe = AssetUniverse::us_starter(); + assert_eq!(universe.assets.len(), 7); + assert_eq!(universe.eligible_count(), 7); + } + + #[test] + fn test_eligible_filters_disabled() { + let mut universe = AssetUniverse::us_starter(); + if let Some(first) = universe.assets.get_mut(0) { + first.enabled = false; + } + assert_eq!(universe.eligible_count(), 6); + } + + #[test] + fn test_find_symbol() { + let universe = AssetUniverse::us_starter(); + let aapl = universe.find("AAPL"); + assert!(aapl.is_some()); + assert_eq!( + aapl.map(|a| a.exchange.as_str()).unwrap_or_default(), + "XNAS" + ); + } + + #[test] + fn test_find_missing_symbol() { + let universe = AssetUniverse::us_starter(); + assert!(universe.find("DOESNOTEXIST").is_none()); + } + + #[test] + fn test_symbols_list() { + let universe = AssetUniverse::us_starter(); + let symbols = universe.symbols(); + assert_eq!(symbols.len(), 7); + assert!(symbols.contains(&"SPY")); + assert!(symbols.contains(&"AAPL")); + } + + #[test] + fn test_empty_universe() { + let universe = AssetUniverse::new(); + assert!(universe.eligible().is_empty()); + } + + #[test] + fn test_serde_roundtrip() { + let universe = AssetUniverse::us_starter(); + let json = serde_json::to_string(&universe).unwrap_or_default(); + let restored: AssetUniverse = + serde_json::from_str(&json).unwrap_or_else(|_| AssetUniverse::new()); + assert_eq!(restored.assets.len(), 7); + } + + #[test] + fn test_trading_symbol_mapping() { + let asset = UniverseAsset { + symbol: "ES.FUT".to_owned(), + trading_symbol: Some("MES.FUT".to_owned()), + exchange: "GLBX.MDP3".to_owned(), + asset_class: AssetClass::Future, + sector: None, + min_daily_volume: 1_000_000.0, + enabled: true, + }; + assert_eq!(asset.execution_symbol(), "MES.FUT"); + } + + #[test] + fn test_trading_symbol_none_falls_back() { + let asset = UniverseAsset { + symbol: "ZN.FUT".to_owned(), + trading_symbol: None, + exchange: "GLBX.MDP3".to_owned(), + asset_class: AssetClass::Future, + sector: None, + min_daily_volume: 500_000.0, + enabled: true, + }; + assert_eq!(asset.execution_symbol(), "ZN.FUT"); + } + + #[test] + fn test_futures_baseline_universe() { + let universe = AssetUniverse::futures_baseline(); + assert_eq!(universe.assets.len(), 4); + assert_eq!(universe.eligible_count(), 4); + + // All are futures + for asset in &universe.assets { + assert_eq!(asset.asset_class, AssetClass::Future); + } + } + + #[test] + fn test_futures_baseline_symbols() { + let universe = AssetUniverse::futures_baseline(); + let symbols = universe.symbols(); + assert!(symbols.contains(&"ES.FUT")); + assert!(symbols.contains(&"NQ.FUT")); + assert!(symbols.contains(&"ZN.FUT")); + assert!(symbols.contains(&"6E.FUT")); + } + + #[test] + fn test_futures_baseline_micro_mapping() { + let universe = AssetUniverse::futures_baseline(); + + assert_eq!( + universe.find("ES.FUT").map(|a| a.execution_symbol()), + Some("MES.FUT") + ); + assert_eq!( + universe.find("NQ.FUT").map(|a| a.execution_symbol()), + Some("MNQ.FUT") + ); + // ZN has no micro -- executes as ZN.FUT + assert_eq!( + universe.find("ZN.FUT").map(|a| a.execution_symbol()), + Some("ZN.FUT") + ); + assert_eq!( + universe.find("6E.FUT").map(|a| a.execution_symbol()), + Some("M6E.FUT") + ); + } + + #[test] + fn test_futures_baseline_serde_roundtrip() { + let universe = AssetUniverse::futures_baseline(); + let json = serde_json::to_string(&universe).unwrap_or_default(); + let restored: AssetUniverse = + serde_json::from_str(&json).unwrap_or_else(|_| AssetUniverse::new()); + assert_eq!(restored.assets.len(), 4); + // Verify trading_symbol survives roundtrip + let es = restored.find("ES.FUT"); + assert_eq!(es.map(|a| a.execution_symbol()), Some("MES.FUT")); + } + + #[test] + fn test_from_config_toml() { + let toml_content = r#" +[universe] +name = "test-universe" +description = "Test" + +[[symbols]] +symbol = "ES.FUT" +trading_symbol = "MES.FUT" +exchange = "GLBX.MDP3" +asset_class = "Future" +min_daily_volume = 1000000.0 + +[[symbols]] +symbol = "ZN.FUT" +exchange = "GLBX.MDP3" +asset_class = "Future" +min_daily_volume = 500000.0 +"#; + let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("tmpdir: {e}")); + let path = dir.path().join("universe.toml"); + std::fs::write(&path, toml_content).unwrap_or_else(|e| panic!("write: {e}")); + + let universe = AssetUniverse::from_config(&path); + assert!(universe.is_ok(), "from_config failed: {:?}", universe.err()); + let universe = universe.unwrap_or_else(|_| AssetUniverse::new()); + assert_eq!(universe.assets.len(), 2); + assert_eq!( + universe.find("ES.FUT").map(|a| a.execution_symbol()), + Some("MES.FUT") + ); + assert_eq!( + universe.find("ZN.FUT").map(|a| a.trading_symbol.as_deref()), + Some(None) + ); + } + + #[test] + fn test_load_futures_baseline_config() { + let config_path = std::path::Path::new("../config/universe-futures-baseline.toml"); + if !config_path.exists() { + // Skip in CI where config may not be at expected relative path + return; + } + let universe = AssetUniverse::from_config(config_path); + assert!(universe.is_ok(), "Failed to load config: {:?}", universe.err()); + let universe = universe.unwrap_or_else(|_| AssetUniverse::new()); + assert_eq!(universe.assets.len(), 4); + assert_eq!( + universe.find("ES.FUT").map(|a| a.execution_symbol()), + Some("MES.FUT") + ); + } + + #[test] + fn test_from_config_missing_file() { + let result = AssetUniverse::from_config(std::path::Path::new("/nonexistent/path.toml")); + assert!(result.is_err()); + } + + #[test] + fn test_from_config_invalid_toml() { + let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("tmpdir: {e}")); + let path = dir.path().join("bad.toml"); + std::fs::write(&path, "not valid toml {{{{").unwrap_or_else(|e| panic!("write: {e}")); + let result = AssetUniverse::from_config(&path); + assert!(result.is_err()); + } +} diff --git a/crates/ml/src/asset_selection/scorer.rs b/crates/ml-asset-selection/src/scorer.rs similarity index 100% rename from crates/ml/src/asset_selection/scorer.rs rename to crates/ml-asset-selection/src/scorer.rs diff --git a/crates/ml/src/asset_selection/selector.rs b/crates/ml-asset-selection/src/selector.rs similarity index 100% rename from crates/ml/src/asset_selection/selector.rs rename to crates/ml-asset-selection/src/selector.rs diff --git a/crates/ml-backtesting/Cargo.toml b/crates/ml-backtesting/Cargo.toml new file mode 100644 index 000000000..ca908f077 --- /dev/null +++ b/crates/ml-backtesting/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "ml-backtesting" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +publish.workspace = true +keywords.workspace = true +categories.workspace = true +description = "ML backtesting framework for barrier optimization and report generation" + +[dependencies] +ml-core.workspace = true +serde = { workspace = true, features = ["derive"] } +chrono.workspace = true +anyhow.workspace = true +csv.workspace = true + +[dev-dependencies] +tempfile = "3.12" + +[lints] +workspace = true diff --git a/crates/ml-backtesting/src/action_loader.rs b/crates/ml-backtesting/src/action_loader.rs new file mode 100644 index 000000000..41aca0425 --- /dev/null +++ b/crates/ml-backtesting/src/action_loader.rs @@ -0,0 +1,197 @@ +// ml-backtesting/src/action_loader.rs +// DQN action loader for CSV-based backtesting + +use chrono::{DateTime, Utc}; +use serde::Deserialize; +use std::fs::File; +use std::io::BufReader; +use std::path::Path; + +/// DQN action record from CSV (13,552 actions, 10 columns) +/// Format: timestamp,action,q_buy,q_sell,q_hold,open,high,low,close,volume +#[derive(Debug, Clone, Deserialize)] +pub struct DQNActionRecord { + /// ISO8601 timestamp (e.g., "2024-10-20T23:31:00.000000000Z") + pub timestamp: DateTime, + + /// Action: 0=Buy, 1=Sell, 2=Hold + pub action: u8, + + /// Q-value for buy action + pub q_buy: f64, + + /// Q-value for sell action + pub q_sell: f64, + + /// Q-value for hold action + pub q_hold: f64, + + /// OHLCV data (for validation/debugging) + pub open: f64, + pub high: f64, + pub low: f64, + pub close: f64, + pub volume: u64, +} + +/// Load DQN actions from CSV file +/// +/// # Arguments +/// * `csv_path` - Path to CSV file (e.g., "/tmp/dqn_actions_wave3.csv") +/// +/// # Returns +/// * `Ok(Vec)` - Parsed and validated actions +/// * `Err(String)` - Validation errors (action bounds, Q-values, timestamps) +/// +/// # Validations +/// 1. Action bounds: 0 <= action <= 2 +/// 2. Finite Q-values: q_buy, q_sell, q_hold must be finite (no NaN/Inf) +/// 3. Timestamp ordering: timestamps must be monotonically increasing +/// +/// # Example +/// ```ignore +/// let actions = load_actions_from_csv("/tmp/dqn_actions_wave3.csv")?; +/// assert_eq!(actions.len(), 13_552); +/// ``` +pub fn load_actions_from_csv>(csv_path: P) -> Result, String> { + let path = csv_path.as_ref(); + + // Open CSV file + let file = File::open(path) + .map_err(|e| format!("Failed to open CSV file '{}': {}", path.display(), e))?; + + let reader = BufReader::new(file); + let mut csv_reader = csv::Reader::from_reader(reader); + + let mut actions = Vec::new(); + let mut prev_timestamp: Option> = None; + let mut row_num = 1; // Start at 1 (header is row 0) + + for result in csv_reader.deserialize() { + row_num += 1; + + let record: DQNActionRecord = + result.map_err(|e| format!("CSV parsing error at row {}: {}", row_num, e))?; + + // Validation 1: Action bounds (0-2) + if record.action > 2 { + return Err(format!( + "Invalid action {} at row {} (timestamp {}): action must be 0 (Buy), 1 (Sell), or 2 (Hold)", + record.action, + row_num, + record.timestamp + )); + } + + // Validation 2: Finite Q-values + if !record.q_buy.is_finite() { + return Err(format!( + "Invalid q_buy {} at row {} (timestamp {}): Q-value must be finite", + record.q_buy, row_num, record.timestamp + )); + } + if !record.q_sell.is_finite() { + return Err(format!( + "Invalid q_sell {} at row {} (timestamp {}): Q-value must be finite", + record.q_sell, row_num, record.timestamp + )); + } + if !record.q_hold.is_finite() { + return Err(format!( + "Invalid q_hold {} at row {} (timestamp {}): Q-value must be finite", + record.q_hold, row_num, record.timestamp + )); + } + + // Validation 3: Timestamp ordering + if let Some(prev_ts) = prev_timestamp { + if record.timestamp < prev_ts { + return Err(format!( + "Timestamp ordering violation at row {}: {} is before previous timestamp {}", + row_num, record.timestamp, prev_ts + )); + } + } + prev_timestamp = Some(record.timestamp); + + actions.push(record); + } + + if actions.is_empty() { + return Err(format!( + "CSV file '{}' contains no data rows", + path.display() + )); + } + + Ok(actions) +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeZone; + + #[test] + fn test_valid_action_record() { + // Test DQNActionRecord struct initialization + let record = DQNActionRecord { + timestamp: Utc.with_ymd_and_hms(2024, 10, 20, 23, 31, 0).unwrap(), + action: 2, // Hold + q_buy: -658.8440, + q_sell: 355.0268, + q_hold: 538.5875, + open: 5914.50, + high: 5914.75, + low: 5914.25, + close: 5914.25, + volume: 27, + }; + + assert_eq!(record.action, 2); + assert!(record.q_buy.is_finite()); + assert!(record.q_sell.is_finite()); + assert!(record.q_hold.is_finite()); + } + + #[test] + fn test_action_bounds_validation() { + // Test action bounds: 0 <= action <= 2 + // Create a temporary CSV with invalid action + use std::io::Write; + let mut tmpfile = tempfile::NamedTempFile::new().unwrap(); + writeln!( + tmpfile, + "timestamp,action,q_buy,q_sell,q_hold,open,high,low,close,volume" + ) + .unwrap(); + writeln!(tmpfile, "2024-10-20T23:31:00.000000000Z,3,-658.8440,355.0268,538.5875,5914.50,5914.75,5914.25,5914.25,27").unwrap(); + tmpfile.flush().unwrap(); + + let result = load_actions_from_csv(tmpfile.path()); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(err.contains("Invalid action 3")); + assert!(err.contains("action must be 0 (Buy), 1 (Sell), or 2 (Hold)")); + } + + #[test] + fn test_finite_qvalue_validation() { + // Test finite Q-value validation (NaN/Inf detection) + use std::io::Write; + let mut tmpfile = tempfile::NamedTempFile::new().unwrap(); + writeln!( + tmpfile, + "timestamp,action,q_buy,q_sell,q_hold,open,high,low,close,volume" + ) + .unwrap(); + writeln!(tmpfile, "2024-10-20T23:31:00.000000000Z,2,NaN,355.0268,538.5875,5914.50,5914.75,5914.25,5914.25,27").unwrap(); + tmpfile.flush().unwrap(); + + let result = load_actions_from_csv(tmpfile.path()); + assert!(result.is_err()); + let err = result.unwrap_err(); + assert!(err.contains("Invalid q_buy")); + assert!(err.contains("Q-value must be finite")); + } +} diff --git a/crates/ml-backtesting/src/barrier_backtest.rs b/crates/ml-backtesting/src/barrier_backtest.rs new file mode 100644 index 000000000..3736ae7ba --- /dev/null +++ b/crates/ml-backtesting/src/barrier_backtest.rs @@ -0,0 +1,451 @@ +// ml-backtesting/src/barrier_backtest.rs +// Barrier parameter optimization backtesting framework + +use crate::MLError; +use anyhow::Result; + +/// Barrier parameters for triple barrier labeling +#[derive(Debug, Clone, Copy)] +pub struct BarrierParams { + pub profit_target: f64, + pub stop_loss: f64, + pub max_holding_periods: usize, +} + +impl BarrierParams { + /// Validate barrier parameters + pub fn validate(&self) -> Result<(), MLError> { + if self.profit_target <= 0.0 { + return Err(MLError::ValidationError { + message: "Profit target must be positive".to_owned(), + }); + } + + if self.stop_loss <= 0.0 { + return Err(MLError::ValidationError { + message: "Stop loss must be positive".to_owned(), + }); + } + + if self.max_holding_periods == 0 { + return Err(MLError::ValidationError { + message: "Max holding periods must be greater than zero".to_owned(), + }); + } + + Ok(()) + } +} + +/// Results from barrier backtesting +#[derive(Debug, Clone)] +pub struct BacktestResults { + pub sharpe_ratio: f64, + pub win_rate: f64, + pub max_drawdown: f64, + pub label_distribution: (usize, usize, usize), // (buy, sell, hold) + pub stability_score: f64, +} + +/// Barrier backtester with walk-forward validation +#[derive(Debug)] +pub struct BarrierBacktester { + walk_forward_windows: usize, + train_test_split: f64, +} + +impl BarrierBacktester { + /// Create new barrier backtester + pub fn new(walk_forward_windows: usize, train_test_split: f64) -> Self { + Self { + walk_forward_windows, + train_test_split, + } + } + + /// Get walk-forward windows configuration + pub fn walk_forward_windows(&self) -> usize { + self.walk_forward_windows + } + + /// Get train/test split ratio + pub fn train_test_split(&self) -> f64 { + self.train_test_split + } + + /// Run backtesting with walk-forward validation + pub fn run(&self, prices: &[f64], params: BarrierParams) -> Result { + // Validate inputs + if prices.is_empty() { + return Err(MLError::ValidationError { + message: "Empty price series".to_owned(), + } + .into()); + } + + params.validate()?; + + // Check if we have enough data for walk-forward windows + let min_samples_per_window = 20; // Minimum samples needed per window + let min_total_samples = min_samples_per_window * self.walk_forward_windows; + + if prices.len() < min_total_samples { + return Err(MLError::InsufficientData(format!( + "Need at least {} samples for {} windows, got {}", + min_total_samples, + self.walk_forward_windows, + prices.len() + )) + .into()); + } + + // Run walk-forward validation + let window_results = self.walk_forward_backtest(prices, params)?; + + // Aggregate results + self.aggregate_results(&window_results, prices) + } + + /// Walk-forward backtesting across multiple windows + fn walk_forward_backtest( + &self, + prices: &[f64], + params: BarrierParams, + ) -> Result> { + let window_size = prices.len() / self.walk_forward_windows; + let mut window_results = Vec::new(); + + for window_idx in 0..self.walk_forward_windows { + let start_idx = window_idx * window_size; + let end_idx = if window_idx == self.walk_forward_windows - 1 { + prices.len() + } else { + (window_idx + 1) * window_size + }; + + let window_prices = &prices[start_idx..end_idx]; + + // Split into train/test + let train_size = (window_prices.len() as f64 * self.train_test_split) as usize; + let test_prices = &window_prices[train_size..]; + + if test_prices.is_empty() { + continue; + } + + // Run labeling on test set + let labels = self.label_bars(test_prices, params)?; + + // Calculate window metrics + let window_result = self.calculate_window_metrics(test_prices, &labels)?; + window_results.push(window_result); + } + + Ok(window_results) + } + + /// Label bars using triple barrier method + fn label_bars(&self, prices: &[f64], params: BarrierParams) -> Result> { + let mut labels = Vec::with_capacity(prices.len()); + + for (i, ¤t_price) in prices.iter().enumerate() { + if i + params.max_holding_periods >= prices.len() { + // Not enough future data for labeling + labels.push(0); // Hold + continue; + } + + let future_prices = &prices[i + 1..=i + params.max_holding_periods]; + let label = self.apply_triple_barrier(current_price, future_prices, params); + labels.push(label); + } + + Ok(labels) + } + + /// Apply triple barrier method to determine label + fn apply_triple_barrier( + &self, + entry_price: f64, + future_prices: &[f64], + params: BarrierParams, + ) -> i8 { + let upper_barrier = entry_price * (1.0 + params.profit_target); + let lower_barrier = entry_price * (1.0 - params.stop_loss); + + for &price in future_prices { + if price >= upper_barrier { + return 1; // Profit target hit (Buy signal) + } + if price <= lower_barrier { + return -1; // Stop loss hit (Sell signal) + } + } + + // Timeout - determine label based on final price + let final_price = future_prices.last().copied().unwrap_or(entry_price); + if final_price > entry_price { + 1 // Positive return + } else if final_price < entry_price { + -1 // Negative return + } else { + 0 // No change + } + } + + /// Calculate metrics for a single window + fn calculate_window_metrics(&self, prices: &[f64], labels: &[i8]) -> Result { + let mut returns = Vec::new(); + let mut equity_curve = Vec::new(); + let mut current_equity = 1.0; + + let mut wins = 0; + let mut total_trades = 0; + + for (i, &label) in labels.iter().enumerate() { + if i + 1 >= prices.len() { + break; + } + + let price_return = (prices[i + 1] / prices[i]) - 1.0; + + // Simulate strategy return based on label + let strategy_return = match label { + 1 => price_return, // Buy signal + -1 => -price_return, // Sell signal + _ => 0.0, // Hold + }; + + if label != 0 { + total_trades += 1; + if strategy_return > 0.0 { + wins += 1; + } + } + + returns.push(strategy_return); + current_equity *= 1.0 + strategy_return; + equity_curve.push(current_equity); + } + + // Calculate Sharpe ratio + let sharpe = if !returns.is_empty() { + calculate_sharpe_ratio(&returns) + } else { + 0.0 + }; + + // Calculate max drawdown + let max_dd = calculate_max_drawdown(&equity_curve); + + // Calculate win rate + let win_rate = if total_trades > 0 { + wins as f64 / total_trades as f64 + } else { + 0.0 + }; + + // Count label distribution + let buys = labels.iter().filter(|&&l| l == 1).count(); + let sells = labels.iter().filter(|&&l| l == -1).count(); + let holds = labels.iter().filter(|&&l| l == 0).count(); + + Ok(WindowResult { + sharpe_ratio: sharpe, + win_rate, + max_drawdown: max_dd, + label_distribution: (buys, sells, holds), + }) + } + + /// Aggregate results across all windows + fn aggregate_results( + &self, + window_results: &[WindowResult], + prices: &[f64], + ) -> Result { + if window_results.is_empty() { + return Err( + MLError::InsufficientData("No window results available".to_owned()).into(), + ); + } + + // Average Sharpe ratio + let avg_sharpe = window_results.iter().map(|w| w.sharpe_ratio).sum::() + / window_results.len() as f64; + + // Average win rate + let avg_win_rate = + window_results.iter().map(|w| w.win_rate).sum::() / window_results.len() as f64; + + // Worst max drawdown + let worst_dd = window_results + .iter() + .map(|w| w.max_drawdown) + .min_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)) + .unwrap_or(0.0); + + // Aggregate label distribution + let total_buys: usize = window_results.iter().map(|w| w.label_distribution.0).sum(); + let total_sells: usize = window_results.iter().map(|w| w.label_distribution.1).sum(); + let total_holds: usize = window_results.iter().map(|w| w.label_distribution.2).sum(); + + // Calculate stability score (variance of Sharpe ratios across windows) + let stability_score = if window_results.len() > 1 { + let sharpe_variance = calculate_variance( + &window_results + .iter() + .map(|w| w.sharpe_ratio) + .collect::>(), + ); + sharpe_variance + } else { + 0.0 + }; + + // Ensure total labels match price series length + let total_labels = total_buys + total_sells + total_holds; + if total_labels != prices.len() { + // Adjust for any discrepancies + let holds_adjustment = prices.len() - total_labels; + return Ok(BacktestResults { + sharpe_ratio: avg_sharpe, + win_rate: avg_win_rate, + max_drawdown: worst_dd, + label_distribution: (total_buys, total_sells, total_holds + holds_adjustment), + stability_score, + }); + } + + Ok(BacktestResults { + sharpe_ratio: avg_sharpe, + win_rate: avg_win_rate, + max_drawdown: worst_dd, + label_distribution: (total_buys, total_sells, total_holds), + stability_score, + }) + } +} + +/// Results from a single walk-forward window +#[derive(Debug, Clone)] +struct WindowResult { + sharpe_ratio: f64, + win_rate: f64, + max_drawdown: f64, + label_distribution: (usize, usize, usize), +} + +/// Calculate Sharpe ratio from returns +fn calculate_sharpe_ratio(returns: &[f64]) -> f64 { + if returns.is_empty() { + return 0.0; + } + + let mean_return = returns.iter().sum::() / returns.len() as f64; + let std_dev = calculate_std_dev(returns, mean_return); + + if std_dev == 0.0 { + return 0.0; + } + + // Annualized Sharpe ratio (assuming daily returns) + let sharpe = mean_return / std_dev; + sharpe * (252.0_f64).sqrt() // 252 trading days +} + +/// Calculate standard deviation +fn calculate_std_dev(values: &[f64], mean: f64) -> f64 { + if values.is_empty() { + return 0.0; + } + + let variance = values + .iter() + .map(|&v| { + let diff = v - mean; + diff * diff + }) + .sum::() + / values.len() as f64; + + variance.sqrt() +} + +/// Calculate variance +fn calculate_variance(values: &[f64]) -> f64 { + if values.is_empty() { + return 0.0; + } + + let mean = values.iter().sum::() / values.len() as f64; + calculate_std_dev(values, mean).powi(2) +} + +/// Calculate maximum drawdown +fn calculate_max_drawdown(equity_curve: &[f64]) -> f64 { + if equity_curve.is_empty() { + return 0.0; + } + + let mut max_equity = equity_curve[0]; + let mut max_dd = 0.0; + + for &equity in equity_curve { + if equity > max_equity { + max_equity = equity; + } + + let drawdown = (equity - max_equity) / max_equity; + if drawdown < max_dd { + max_dd = drawdown; + } + } + + max_dd +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_sharpe_ratio_calculation() { + let returns = vec![0.01, -0.005, 0.015, 0.02, -0.01]; + let sharpe = calculate_sharpe_ratio(&returns); + assert!(sharpe.is_finite()); + } + + #[test] + fn test_max_drawdown_calculation() { + let equity = vec![1.0, 1.1, 1.05, 0.95, 1.15]; + let max_dd = calculate_max_drawdown(&equity); + assert!(max_dd <= 0.0); + assert!(max_dd.is_finite()); + } + + #[test] + fn test_variance_calculation() { + let values = vec![1.0, 2.0, 3.0, 4.0, 5.0]; + let variance = calculate_variance(&values); + assert!(variance > 0.0); + assert!(variance.is_finite()); + } + + #[test] + fn test_barrier_params_validation() { + let valid_params = BarrierParams { + profit_target: 0.02, + stop_loss: 0.01, + max_holding_periods: 10, + }; + assert!(valid_params.validate().is_ok()); + + let invalid_params = BarrierParams { + profit_target: -0.02, + stop_loss: 0.01, + max_holding_periods: 10, + }; + assert!(invalid_params.validate().is_err()); + } +} diff --git a/crates/ml-backtesting/src/lib.rs b/crates/ml-backtesting/src/lib.rs new file mode 100644 index 000000000..25974a992 --- /dev/null +++ b/crates/ml-backtesting/src/lib.rs @@ -0,0 +1,17 @@ +//! ML Backtesting Framework +//! +//! Provides barrier parameter optimization, DQN action loading from CSV, +//! and comprehensive report generation for model evaluation. + +// Workspace lints deny unwrap/expect/indexing at warn level; override to allow in tests only. +#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing))] + +pub use ml_core::MLError; + +pub mod action_loader; +pub mod barrier_backtest; +pub mod report; + +pub use action_loader::{load_actions_from_csv, DQNActionRecord}; +pub use barrier_backtest::{BacktestResults, BarrierBacktester, BarrierParams}; +pub use report::{BacktestReport, PerformanceMetrics, Recommendation}; diff --git a/crates/ml-backtesting/src/report.rs b/crates/ml-backtesting/src/report.rs new file mode 100644 index 000000000..1b2d8a6b1 --- /dev/null +++ b/crates/ml-backtesting/src/report.rs @@ -0,0 +1,464 @@ +//! Backtesting Report Generation Module +//! +//! Provides comprehensive markdown report generation for comparing DQN model performance +//! against baseline models. Includes deployment recommendations based on production criteria. +//! +//! # Features +//! +//! - **Markdown Report Generation**: Professional formatted reports with tables +//! - **Production Criteria Validation**: Automated APPROVE/REJECT/REVIEW recommendations +//! - **Baseline Comparison**: Side-by-side comparison with reference model (Trial #35) +//! - **Comprehensive Metrics**: Returns, Sharpe, drawdown, win rate, alpha, trade stats +//! +//! # Usage +//! +//! ```rust +//! use ml_backtesting::report::{BacktestReport, PerformanceMetrics}; +//! +//! let new_results = PerformanceMetrics { +//! total_return_pct: 15.2, +//! sharpe_ratio: 2.3, +//! max_drawdown_pct: 12.5, +//! win_rate: 0.58, +//! alpha: 3.2, +//! total_trades: 145, +//! avg_trade_return_pct: 0.105, +//! }; +//! +//! let baseline = Some(PerformanceMetrics { +//! total_return_pct: 12.1, +//! sharpe_ratio: 1.8, +//! max_drawdown_pct: 18.3, +//! win_rate: 0.52, +//! alpha: 1.5, +//! total_trades: 138, +//! avg_trade_return_pct: 0.088, +//! }); +//! +//! let report = BacktestReport { +//! model_name: "DQN-Wave3-Entropy".to_owned(), +//! baseline_name: "DQN-Trial35-Baseline".to_owned(), +//! new_results, +//! baseline_results: baseline, +//! }; +//! +//! let markdown = report.generate_markdown(); +//! std::fs::write("backtest_report.md", markdown).unwrap(); +//! ``` + +use chrono::Utc; +use serde::{Deserialize, Serialize}; + +/// Performance metrics for backtesting evaluation +/// +/// Simplified metrics struct focused on production criteria validation. +/// Matches the key metrics from `backtesting/src/metrics.rs` but optimized +/// for report generation and comparison purposes. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PerformanceMetrics { + /// Total return as percentage (e.g., 15.2 = 15.2%) + pub total_return_pct: f64, + /// Sharpe ratio (risk-adjusted return) + pub sharpe_ratio: f64, + /// Maximum drawdown as percentage (e.g., 12.5 = 12.5%) + pub max_drawdown_pct: f64, + /// Win rate (0.0-1.0, e.g., 0.58 = 58%) + pub win_rate: f64, + /// Alpha (excess return vs benchmark, percentage) + pub alpha: f64, + /// Total number of trades executed + pub total_trades: usize, + /// Average trade return as percentage + pub avg_trade_return_pct: f64, +} + +/// Deployment recommendation structure +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Recommendation { + /// Status string (✅ APPROVE, ⚠️ REVIEW, ❌ REJECT) + pub status: String, + /// Human-readable reasoning for the recommendation + pub reasoning: String, +} + +/// Complete backtesting report structure +/// +/// Compares a new model against an optional baseline and generates +/// deployment recommendations based on production criteria. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BacktestReport { + /// Name of the new model being evaluated + pub model_name: String, + /// Name of the baseline model for comparison + pub baseline_name: String, + /// Performance metrics for the new model + pub new_results: PerformanceMetrics, + /// Optional baseline performance metrics (e.g., Trial #35) + pub baseline_results: Option, +} + +impl BacktestReport { + /// Generate comprehensive markdown report + /// + /// Creates a professional formatted report with: + /// - Header with model names and timestamp + /// - Performance summary table with production criteria + /// - Baseline comparison table (if baseline provided) + /// - Deployment recommendation with reasoning + /// - Trade statistics summary + /// + /// # Returns + /// + /// A formatted markdown string ready to write to a file + pub fn generate_markdown(&self) -> String { + let mut report = String::new(); + + // Header + report.push_str("# DQN Backtesting Report\n\n"); + report.push_str(&format!("**Model**: {}\n", self.model_name)); + if self.baseline_results.is_some() { + report.push_str(&format!("**Baseline**: {}\n", self.baseline_name)); + } + report.push_str(&format!("**Generated**: {}\n\n", Utc::now().format("%Y-%m-%d %H:%M:%S UTC"))); + + report.push_str("---\n\n"); + + // Performance Summary + report.push_str("## Performance Summary\n\n"); + report.push_str("| Metric | Value | Target | Status |\n"); + report.push_str("|--------|-------|--------|--------|\n"); + + let m = &self.new_results; + + // Total Return + report.push_str(&format!( + "| Total Return | {:.2}% | >0% | {} |\n", + m.total_return_pct, + if m.total_return_pct > 0.0 { "✅" } else { "❌" } + )); + + // Sharpe Ratio + report.push_str(&format!( + "| Sharpe Ratio | {:.2} | >1.5 | {} |\n", + m.sharpe_ratio, + if m.sharpe_ratio > 1.5 { "✅" } else { "❌" } + )); + + // Max Drawdown + report.push_str(&format!( + "| Max Drawdown | {:.2}% | <20% | {} |\n", + m.max_drawdown_pct, + if m.max_drawdown_pct < 20.0 { "✅" } else { "❌" } + )); + + // Win Rate + report.push_str(&format!( + "| Win Rate | {:.1}% | >50% | {} |\n", + m.win_rate * 100.0, + if m.win_rate > 0.50 { "✅" } else { "❌" } + )); + + // Alpha + report.push_str(&format!( + "| Alpha vs B&H | {:.2}% | >0% | {} |\n", + m.alpha, + if m.alpha > 0.0 { "✅" } else { "❌" } + )); + + report.push_str("\n"); + + // Comparison to baseline (if provided) + if let Some(baseline) = &self.baseline_results { + report.push_str("## Comparison to Baseline\n\n"); + report.push_str("| Metric | Baseline | New Model | Change | Direction |\n"); + report.push_str("|--------|----------|-----------|--------|----------|\n"); + + // Returns comparison + let return_change = m.total_return_pct - baseline.total_return_pct; + let return_arrow = if return_change > 0.0 { "↗️" } else if return_change < 0.0 { "↘️" } else { "→" }; + report.push_str(&format!( + "| Returns | {:.2}% | {:.2}% | {:+.2}% | {} |\n", + baseline.total_return_pct, m.total_return_pct, return_change, return_arrow + )); + + // Sharpe comparison + let sharpe_change = m.sharpe_ratio - baseline.sharpe_ratio; + let sharpe_arrow = if sharpe_change > 0.0 { "↗️" } else if sharpe_change < 0.0 { "↘️" } else { "→" }; + report.push_str(&format!( + "| Sharpe | {:.2} | {:.2} | {:+.2} | {} |\n", + baseline.sharpe_ratio, m.sharpe_ratio, sharpe_change, sharpe_arrow + )); + + // Drawdown comparison (lower is better) + let dd_change = m.max_drawdown_pct - baseline.max_drawdown_pct; + let dd_arrow = if dd_change < 0.0 { "↗️" } else if dd_change > 0.0 { "↘️" } else { "→" }; + report.push_str(&format!( + "| Drawdown | {:.2}% | {:.2}% | {:+.2}% | {} |\n", + baseline.max_drawdown_pct, m.max_drawdown_pct, dd_change, dd_arrow + )); + + // Win Rate comparison + let wr_change = (m.win_rate - baseline.win_rate) * 100.0; + let wr_arrow = if wr_change > 0.0 { "↗️" } else if wr_change < 0.0 { "↘️" } else { "→" }; + report.push_str(&format!( + "| Win Rate | {:.1}% | {:.1}% | {:+.1}% | {} |\n", + baseline.win_rate * 100.0, m.win_rate * 100.0, wr_change, wr_arrow + )); + + // Alpha comparison + let alpha_change = m.alpha - baseline.alpha; + let alpha_arrow = if alpha_change > 0.0 { "↗️" } else if alpha_change < 0.0 { "↘️" } else { "→" }; + report.push_str(&format!( + "| Alpha | {:.2}% | {:.2}% | {:+.2}% | {} |\n", + baseline.alpha, m.alpha, alpha_change, alpha_arrow + )); + + report.push_str("\n"); + } + + // Trade Statistics + report.push_str("## Trade Statistics\n\n"); + report.push_str("| Metric | Value |\n"); + report.push_str("|--------|-------|\n"); + report.push_str(&format!("| Total Trades | {} |\n", m.total_trades)); + report.push_str(&format!("| Avg Trade Return | {:.3}% |\n", m.avg_trade_return_pct)); + report.push_str(&format!("| Win Rate | {:.2}% |\n", m.win_rate * 100.0)); + + if let Some(baseline) = &self.baseline_results { + let trade_diff = m.total_trades as i64 - baseline.total_trades as i64; + report.push_str(&format!("| Trades vs Baseline | {:+} |\n", trade_diff)); + } + + report.push_str("\n"); + + // Deployment Recommendation + report.push_str("## Deployment Recommendation\n\n"); + let recommendation = self.get_recommendation(); + report.push_str(&format!("**Status**: {}\n\n", recommendation.status)); + report.push_str(&format!("{}\n\n", recommendation.reasoning)); + + // Production Criteria Summary + report.push_str("### Production Criteria Checklist\n\n"); + let criteria_passed = self.count_criteria_passed(); + report.push_str(&format!("- **Criteria Passed**: {}/5\n", criteria_passed)); + report.push_str(&format!("- **Total Return**: {} ({})\n", + if m.total_return_pct > 0.0 { "✅ PASS" } else { "❌ FAIL" }, + format!("{:.2}% > 0%", m.total_return_pct) + )); + report.push_str(&format!("- **Sharpe Ratio**: {} ({})\n", + if m.sharpe_ratio > 1.5 { "✅ PASS" } else { "❌ FAIL" }, + format!("{:.2} > 1.5", m.sharpe_ratio) + )); + report.push_str(&format!("- **Max Drawdown**: {} ({})\n", + if m.max_drawdown_pct < 20.0 { "✅ PASS" } else { "❌ FAIL" }, + format!("{:.2}% < 20%", m.max_drawdown_pct) + )); + report.push_str(&format!("- **Win Rate**: {} ({})\n", + if m.win_rate > 0.50 { "✅ PASS" } else { "❌ FAIL" }, + format!("{:.1}% > 50%", m.win_rate * 100.0) + )); + report.push_str(&format!("- **Alpha vs B&H**: {} ({})\n", + if m.alpha > 0.0 { "✅ PASS" } else { "❌ FAIL" }, + format!("{:.2}% > 0%", m.alpha) + )); + + report.push_str("\n"); + + // Footer + report.push_str("---\n\n"); + report.push_str("*Report generated automatically by Foxhunt ML Evaluation Framework*\n"); + + report + } + + /// Count how many production criteria are passed + /// + /// # Returns + /// + /// Number of criteria passed (0-5) + fn count_criteria_passed(&self) -> usize { + let m = &self.new_results; + [ + m.total_return_pct > 0.0, + m.sharpe_ratio > 1.5, + m.max_drawdown_pct < 20.0, + m.win_rate > 0.50, + m.alpha > 0.0, + ] + .iter() + .filter(|&&x| x) + .count() + } + + /// Generate deployment recommendation based on production criteria + /// + /// # Recommendation Logic + /// + /// - **APPROVE (4-5 criteria)**: Model is production-ready + /// - **REVIEW (2-3 criteria)**: Marginal performance, needs review + /// - **REJECT (0-1 criteria)**: Not production-ready + /// + /// # Returns + /// + /// A `Recommendation` with status and reasoning + pub fn get_recommendation(&self) -> Recommendation { + let m = &self.new_results; + let passes = self.count_criteria_passed(); + + if passes >= 4 { + Recommendation { + status: "✅ APPROVE - Ready for Production".to_owned(), + reasoning: format!( + "Model passes {}/5 production criteria. Strong performance with {} return, {} Sharpe ratio, and {} win rate. \ + Risk is acceptable with {} max drawdown. Model demonstrates profitability with {} alpha vs buy-and-hold. \ + \n\n**Action**: Proceed with production deployment after final validation.", + passes, + format!("{:.2}%", m.total_return_pct), + format!("{:.2}", m.sharpe_ratio), + format!("{:.1}%", m.win_rate * 100.0), + format!("{:.2}%", m.max_drawdown_pct), + format!("{:.2}%", m.alpha) + ), + } + } else if passes >= 2 { + Recommendation { + status: "⚠️ REVIEW - Marginal Performance".to_owned(), + reasoning: format!( + "Model passes {}/5 production criteria. Performance is marginal and requires careful review. \ + \n\n**Concerns**:\n{} + \n**Action**: Conduct detailed risk assessment and consider additional testing before deployment.", + passes, + self.generate_concerns_list() + ), + } + } else { + Recommendation { + status: "❌ REJECT - Not Production Ready".to_owned(), + reasoning: format!( + "Model only passes {}/5 production criteria. Performance is insufficient for production deployment. \ + \n\n**Critical Issues**:\n{} + \n**Action**: Do not deploy. Retrain model with improved hyperparameters or different architecture.", + passes, + self.generate_concerns_list() + ), + } + } + } + + /// Generate list of concerns for models not passing all criteria + /// + /// # Returns + /// + /// Markdown-formatted list of failed criteria + fn generate_concerns_list(&self) -> String { + let m = &self.new_results; + let mut concerns = Vec::new(); + + if m.total_return_pct <= 0.0 { + concerns.push(format!("- ❌ Negative total return ({:.2}%)", m.total_return_pct)); + } + if m.sharpe_ratio <= 1.5 { + concerns.push(format!("- ❌ Low Sharpe ratio ({:.2} < 1.5)", m.sharpe_ratio)); + } + if m.max_drawdown_pct >= 20.0 { + concerns.push(format!("- ❌ Excessive drawdown ({:.2}% > 20%)", m.max_drawdown_pct)); + } + if m.win_rate <= 0.50 { + concerns.push(format!("- ❌ Poor win rate ({:.1}% < 50%)", m.win_rate * 100.0)); + } + if m.alpha <= 0.0 { + concerns.push(format!("- ❌ Negative alpha ({:.2}%)", m.alpha)); + } + + if concerns.is_empty() { + "- No critical issues identified".to_owned() + } else { + concerns.join("\n") + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_report_generation_approve() { + let report = BacktestReport { + model_name: "DQN-Test-Model".to_owned(), + baseline_name: "DQN-Baseline".to_owned(), + new_results: PerformanceMetrics { + total_return_pct: 15.2, + sharpe_ratio: 2.3, + max_drawdown_pct: 12.5, + win_rate: 0.58, + alpha: 3.2, + total_trades: 145, + avg_trade_return_pct: 0.105, + }, + baseline_results: None, + }; + + let markdown = report.generate_markdown(); + + assert!(markdown.contains("# DQN Backtesting Report")); + assert!(markdown.contains("DQN-Test-Model")); + assert!(markdown.contains("✅ APPROVE")); + assert!(markdown.contains("5/5")); + } + + #[test] + fn test_report_generation_reject() { + let report = BacktestReport { + model_name: "DQN-Poor-Model".to_owned(), + baseline_name: "DQN-Baseline".to_owned(), + new_results: PerformanceMetrics { + total_return_pct: -5.2, + sharpe_ratio: 0.8, + max_drawdown_pct: 35.0, + win_rate: 0.42, + alpha: -2.1, + total_trades: 120, + avg_trade_return_pct: -0.043, + }, + baseline_results: None, + }; + + let markdown = report.generate_markdown(); + + assert!(markdown.contains("❌ REJECT")); + assert!(markdown.contains("0/5")); + } + + #[test] + fn test_baseline_comparison() { + let report = BacktestReport { + model_name: "DQN-New".to_owned(), + baseline_name: "DQN-Trial35".to_owned(), + new_results: PerformanceMetrics { + total_return_pct: 18.5, + sharpe_ratio: 2.5, + max_drawdown_pct: 10.2, + win_rate: 0.62, + alpha: 4.5, + total_trades: 150, + avg_trade_return_pct: 0.123, + }, + baseline_results: Some(PerformanceMetrics { + total_return_pct: 12.1, + sharpe_ratio: 1.8, + max_drawdown_pct: 18.3, + win_rate: 0.52, + alpha: 1.5, + total_trades: 138, + avg_trade_return_pct: 0.088, + }), + }; + + let markdown = report.generate_markdown(); + + assert!(markdown.contains("Comparison to Baseline")); + assert!(markdown.contains("DQN-Trial35")); + assert!(markdown.contains("+6.40%")); // Return improvement (18.5 - 12.1 = 6.4) + } +} diff --git a/crates/ml-universe/Cargo.toml b/crates/ml-universe/Cargo.toml new file mode 100644 index 000000000..9adf72bb2 --- /dev/null +++ b/crates/ml-universe/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "ml-universe" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +publish.workspace = true +keywords.workspace = true +categories.workspace = true + +[dependencies] +ml-core.workspace = true +common.workspace = true +serde = { workspace = true, features = ["derive"] } +tracing.workspace = true +chrono.workspace = true +ndarray.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util", "macros"] } + +[lints] +workspace = true diff --git a/crates/ml/src/universe/correlation.rs b/crates/ml-universe/src/correlation.rs similarity index 61% rename from crates/ml/src/universe/correlation.rs rename to crates/ml-universe/src/correlation.rs index dd34148cc..17d97f339 100644 --- a/crates/ml/src/universe/correlation.rs +++ b/crates/ml-universe/src/correlation.rs @@ -289,161 +289,3 @@ impl BreakdownDetector { Ok(breakdowns) } } - -// DISABLED: Tests require types not available -// // #[cfg(test)] -// mod tests { -// use super::*; -// -// #[test] -// fn test_correlation_analysis_engine_creation() { -// let config = CorrelationAnalysisConfig::default(); -// let engine = CorrelationAnalysisEngine::new(config); -// -// assert!(engine.is_ok()); -// let engine = engine?; -// assert_eq!(engine.total_updates, 0); -// assert!(engine.return_data.is_empty()); -// } -// -// #[test] -// fn test_return_data_update() { -// let mut engine = CorrelationAnalysisEngine::new(CorrelationAnalysisConfig::default())?; -// let test_symbol = "TEST_SYM_1"; -// -// let result = engine.update_return_data(test_symbol.to_string(), PRECISION_FACTOR / 100); // 1% return -// assert!(result.is_ok()); -// assert_eq!(engine.total_updates, 1); -// assert_eq!(engine.return_data.len(), 1); -// } -// -// #[test] -// fn test_pearson_correlation() { -// let engine = CorrelationAnalysisEngine::new(CorrelationAnalysisConfig::default())?; -// -// // Test perfect positive correlation -// let returns1: VecDeque = vec![ -// PRECISION_FACTOR / 10, // 0.1 -// PRECISION_FACTOR / 5, // 0.2 -// PRECISION_FACTOR / 3, // 0.33 -// ] -// .into_iter() -// .collect(); -// -// let returns2: VecDeque = vec![ -// PRECISION_FACTOR / 5, // 0.2 (2x returns1) -// PRECISION_FACTOR * 2 / 5, // 0.4 -// PRECISION_FACTOR * 2 / 3, // 0.66 -// ] -// .into_iter() -// .collect(); -// -// let correlation = engine.pearson_correlation(&returns1, &returns2)?; -// -// // Should be close to 1.0 (perfect positive correlation) -// assert!(correlation > PRECISION_FACTOR * 9 / 10); // > 0.9 -// } -// -// #[test] -// fn test_correlation_matrix_calculation() { -// let mut engine = CorrelationAnalysisEngine::new(CorrelationAnalysisConfig::default())?; -// -// // Add return data for multiple assets -// let test_symbol_1 = "TEST_SYM_1"; -// let test_symbol_2 = "TEST_SYM_2"; -// -// for i in 0..50 { -// let return_sym1 = if i % 2 == 0 { -// PRECISION_FACTOR / 100 -// } else { -// -PRECISION_FACTOR / 100 -// }; -// let return_sym2 = if i % 3 == 0 { -// PRECISION_FACTOR / 50 -// } else { -// -PRECISION_FACTOR / 50 -// }; -// -// engine.update_return_data(test_symbol_1.to_string(), return_sym1)?; -// engine.update_return_data(test_symbol_2.to_string(), return_sym2)?; -// } -// -// let matrix = engine.calculate_correlation_matrix()?; -// -// assert_eq!(matrix.assets.len(), 2); -// assert_eq!(matrix.matrix.nrows(), 2); -// assert_eq!(matrix.matrix.ncols(), 2); -// -// // Diagonal should be 1.0 -// assert_eq!(matrix.matrix[[0, 0]], PRECISION_FACTOR); -// assert_eq!(matrix.matrix[[1, 1]], PRECISION_FACTOR); -// -// // Off-diagonal should be symmetric -// assert_eq!(matrix.matrix[[0, 1]], matrix.matrix[[1, 0]]); -// } -// -// #[test] -// fn test_rank_calculation() { -// let engine = CorrelationAnalysisEngine::new(CorrelationAnalysisConfig::default())?; -// -// let values = vec![10, 20, 30, 20, 40]; // Note: 20 appears twice -// let ranks = engine.calculate_ranks(&values)?; -// -// assert_eq!(ranks.len(), 5); -// // Check that tied values get the same average rank -// assert_eq!(ranks[1], ranks[3]); // Both 20s should have same rank -// } -// -// #[test] -// fn test_breakdown_detection() { -// let config = BreakdownDetectionConfig::default(); -// let mut detector = BreakdownDetector::new(config)?; -// -// // Create two correlation matrices with different correlations -// let test_symbol_1 = "TEST_SYM_1"; -// let test_symbol_2 = "TEST_SYM_2"; -// let assets = vec![test_symbol_1.to_string(), test_symbol_2.to_string()]; -// -// let matrix1 = EnhancedCorrelationMatrix { -// assets: assets.clone(), -// matrix: ndarray::arr2(&[ -// [PRECISION_FACTOR, PRECISION_FACTOR / 2], -// [PRECISION_FACTOR / 2, PRECISION_FACTOR], -// ]), -// confidence_intervals: None, -// significance_flags: Array2::from_elem((2, 2), true), -// timestamp: 1000, -// n_observations: 100, -// condition_number: PRECISION_FACTOR as f64, -// }; -// -// let matrix2 = EnhancedCorrelationMatrix { -// assets: assets.clone(), -// matrix: ndarray::arr2(&[ -// [PRECISION_FACTOR, PRECISION_FACTOR * 8 / 10], -// [PRECISION_FACTOR * 8 / 10, PRECISION_FACTOR], -// ]), -// confidence_intervals: None, -// significance_flags: Array2::from_elem((2, 2), true), -// timestamp: 1001, -// n_observations: 100, -// condition_number: PRECISION_FACTOR as f64, -// }; -// -// // First call should return no breakdowns -// let breakdowns1 = detector.detect_breakdowns(&matrix1)?; -// assert!(breakdowns1.is_empty()); -// -// // Second call should detect breakdown -// let breakdowns2 = detector.detect_breakdowns(&matrix2)?; -// assert_eq!(breakdowns2.len(), 1); -// -// let breakdown = &breakdowns2[0]; -// assert_eq!(breakdown.pre_correlation, PRECISION_FACTOR / 2); -// assert_eq!(breakdown.post_correlation, PRECISION_FACTOR * 8 / 10); -// assert!(matches!( -// breakdown.breakdown_type, -// BreakdownType::CorrelationIncrease -// )); -// } -// } diff --git a/crates/ml-universe/src/lib.rs b/crates/ml-universe/src/lib.rs new file mode 100644 index 000000000..53eb7836d --- /dev/null +++ b/crates/ml-universe/src/lib.rs @@ -0,0 +1,818 @@ +//! # Universe Selection ML Module +//! +//! Dynamic universe selection using machine learning for optimal asset filtering. +//! Implements liquidity scoring, momentum ranking, volatility clustering, and correlation analysis. + +#![deny(clippy::unwrap_used, clippy::expect_used)] +#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used))] +// Workspace denies that need local overrides for universe code patterns +#![allow(clippy::indexing_slicing)] // Matrix indexing in correlation/volatility code +#![allow(clippy::shadow_reuse)] // Common in scoring/ranking code + +pub use ml_core::MLError; +pub use ml_core::PRECISION_FACTOR; + +pub mod correlation; +pub mod liquidity; +pub mod momentum; +pub mod volatility; + +use chrono::{DateTime, Utc}; +use std::collections::HashMap; +use std::time::SystemTime; + +use serde::{Deserialize, Serialize}; + +use common::{Price, Symbol, Volume}; + +// Missing types that need to be defined +#[derive(Debug)] +pub struct LiquidityScorer { + config: LiquidityScorerConfig, +} + +impl Default for LiquidityScorer { + fn default() -> Self { + Self { + config: LiquidityScorerConfig::default(), + } + } +} + +impl LiquidityScorer { + pub fn new(config: LiquidityScorerConfig) -> Result { + Ok(Self { config }) + } + + pub fn calculate_liquidity_score( + &self, + asset: &AssetMetadata, + ) -> Result { + let base_score = asset.market_cap_usd * self.config.volume_weight; + let spread_penalty = asset.spread * self.config.spread_weight; + let depth_bonus = asset.depth * self.config.depth_weight; + + let score = base_score - spread_penalty + depth_bonus; + + Ok(LiquidityScore { + overall_score: score, + score, + volume: asset.volume, + spread: asset.spread, + depth: asset.depth, + timestamp: SystemTime::now(), + }) + } +} + +#[derive(Debug, Clone, Default)] +pub struct LiquidityScorerConfig { + pub min_volume_threshold: f64, + pub spread_weight: f64, + pub depth_weight: f64, + pub volume_weight: f64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LiquidityScore { + pub overall_score: f64, + pub score: f64, + pub volume: f64, + pub spread: f64, + pub depth: f64, + pub timestamp: SystemTime, +} + +impl Default for LiquidityScore { + fn default() -> Self { + Self { + overall_score: 0.0, + score: 0.0, + volume: 0.0, + spread: 0.0, + depth: 0.0, + timestamp: SystemTime::now(), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MomentumScore { + pub overall_score: f64, + pub score: f64, + pub momentum_1d: f64, + pub momentum_7d: f64, + pub momentum_30d: f64, + pub timestamp: SystemTime, +} + +impl Default for MomentumScore { + fn default() -> Self { + Self { + overall_score: 0.0, + score: 0.0, + momentum_1d: 0.0, + momentum_7d: 0.0, + momentum_30d: 0.0, + timestamp: SystemTime::now(), + } + } +} + +#[derive(Debug)] +pub struct MomentumRanker { + config: MomentumRankerConfig, +} + +impl Default for MomentumRanker { + fn default() -> Self { + Self { + config: MomentumRankerConfig::default(), + } + } +} + +impl MomentumRanker { + pub fn new(config: MomentumRankerConfig) -> Result { + Ok(Self { config }) + } + + pub fn calculate_momentum_score( + &self, + asset: &AssetMetadata, + ) -> Result { + let momentum = asset.momentum * self.config.momentum_weight; + let reversal_penalty = asset.reversal_risk * self.config.reversal_weight; + + let score = momentum - reversal_penalty; + + Ok(MomentumScore { + overall_score: score, + score, + momentum_1d: asset.momentum, + momentum_7d: asset.momentum * 0.8, // Simplified + momentum_30d: asset.momentum * 0.6, // Simplified + timestamp: SystemTime::now(), + }) + } +} + +#[derive(Debug, Clone, Default)] +pub struct MomentumRankerConfig { + pub lookback_days: u32, + pub momentum_weight: f64, + pub reversal_weight: f64, +} + +#[derive(Debug)] +pub struct CorrelationAnalyzer { + config: CorrelationAnalyzerConfig, +} + +impl CorrelationAnalyzer { + pub fn new(config: CorrelationAnalyzerConfig) -> Result { + Ok(Self { config }) + } +} + +impl Default for CorrelationAnalyzer { + fn default() -> Self { + Self { + config: CorrelationAnalyzerConfig::default(), + } + } +} + +#[derive(Debug, Clone, Default)] +pub struct CorrelationAnalyzerConfig { + pub correlation_threshold: f64, + pub window_size: usize, + pub update_frequency_ms: u64, +} + +#[derive(Debug)] +pub struct VolatilityClusterEngine { + config: VolatilityClusterEngineConfig, +} + +impl VolatilityClusterEngine { + pub fn new(config: VolatilityClusterEngineConfig) -> Result { + Ok(Self { config }) + } +} + +impl Default for VolatilityClusterEngine { + fn default() -> Self { + Self { + config: VolatilityClusterEngineConfig::default(), + } + } +} + +#[derive(Debug, Clone, Default)] +pub struct VolatilityClusterEngineConfig { + pub cluster_count: usize, + pub volatility_window: usize, + pub min_volume_threshold: f64, +} + +// Additional required types +#[derive(Debug, Clone, Default)] +pub struct AssetMetadata { + pub symbol: String, + pub market_cap_usd: f64, + pub price: f64, + pub volume_24h: f64, + pub volume: f64, + pub spread: f64, + pub depth: f64, + pub momentum: f64, + pub reversal_risk: f64, + pub volatility: f64, + pub sector: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CorrelationMatrix { + pub correlations: HashMap<(Symbol, Symbol), f64>, + pub eigenvalues: Vec, + pub condition_number: f64, + pub timestamp: SystemTime, +} + +impl Default for CorrelationMatrix { + fn default() -> Self { + Self { + correlations: HashMap::new(), + eigenvalues: Vec::new(), + condition_number: 1.0, + timestamp: SystemTime::now(), + } + } +} + +// Additional types are defined above - main configuration types follow + +/// Configuration for universe selection engine +#[derive(Debug, Clone, Serialize, Deserialize)] +/// UniverseSelectionConfig component. +pub struct UniverseSelectionConfig { + /// Maximum number of assets to include in the universe + pub max_universe_size: usize, + + /// Minimum market cap threshold + pub min_market_cap: f64, + + /// Minimum trading `volume` + pub min_volume: f64, + + /// Minimum liquidity score + pub min_liquidity_score: f64, + + /// Minimum momentum score + pub min_momentum_score: f64, + + /// Enable correlation filtering + pub enable_correlation_filtering: bool, + + /// Maximum correlation threshold + pub max_correlation: f64, + + /// Rebalancing frequency in days + pub rebalancing_frequency_days: usize, + + /// Cache expiry time in seconds + pub cache_expiry_seconds: u64, +} + +impl Default for UniverseSelectionConfig { + fn default() -> Self { + Self { + max_universe_size: 500, + min_market_cap: 1_000_000_000.0, // $1B minimum market cap + min_volume: 1_000_000.0, // $1M daily volume + min_liquidity_score: 0.3, + min_momentum_score: 0.3, + enable_correlation_filtering: true, + max_correlation: 0.8, + rebalancing_frequency_days: 30, + cache_expiry_seconds: 300, // 5 minutes + } + } +} + +/// Asset data for universe selection +#[derive(Debug, Clone, Serialize, Deserialize)] +/// AssetData component. +pub struct AssetData { + pub asset_id: Symbol, + pub symbol: String, + pub price: Price, + pub volume: Volume, + pub market_cap: u64, + pub sector: String, + pub exchange: String, + pub last_trade_time: DateTime, +} + +/// Universe selection criteria +#[derive(Debug, Clone, Serialize, Deserialize)] +/// SelectionCriteria component. +pub struct SelectionCriteria { + pub min_liquidity_score: f64, + pub min_momentum_score: f64, + pub max_correlation: f64, + pub volatility_regime: Option, // Simplified for now + pub sector_limits: HashMap, + pub max_assets: usize, + pub min_market_cap: u64, + pub exclude_penny_stocks: bool, +} + +impl Default for SelectionCriteria { + fn default() -> Self { + Self { + min_liquidity_score: 0.6, + min_momentum_score: 0.5, + max_correlation: 0.8, + volatility_regime: None, + sector_limits: HashMap::new(), + max_assets: 100, + min_market_cap: 1_000_000_000, // $1B minimum market cap + exclude_penny_stocks: true, + } + } +} + +/// Selected universe with scores +#[derive(Debug, Clone, Serialize, Deserialize)] +/// SelectedUniverse component. +pub struct SelectedUniverse { + pub assets: Vec, + pub liquidity_scores: HashMap, + pub momentum_scores: HashMap, + pub correlation_matrix: CorrelationMatrix, + pub diversification_score: f64, // Replace with DiversificationScore when diversification module is implemented + pub selection_timestamp: DateTime, + pub rebalance_needed: bool, +} + +/// Universe selection performance metrics +#[derive(Debug, Clone, Serialize, Deserialize)] +/// SelectionMetrics component. +pub struct SelectionMetrics { + pub total_candidates: usize, + pub selected_assets: usize, + pub avg_liquidity_score: f64, + pub avg_momentum_score: f64, + pub correlation_efficiency: f64, + pub sector_distribution: HashMap, + pub selection_latency_micros: u64, + pub cache_hit_rate: f64, +} + +/// Configuration for universe selection models +#[derive(Debug, Clone, Serialize, Deserialize)] +/// UniverseConfig component. +pub struct UniverseConfig { + pub update_frequency_minutes: u32, + pub lookback_days: u32, + pub feature_dimensions: usize, + pub batch_size: usize, + pub cache_size: usize, + pub enable_real_time: bool, + pub enable_regime_detection: bool, + pub parallel_processing: bool, +} + +impl Default for UniverseConfig { + fn default() -> Self { + Self { + update_frequency_minutes: 60, + lookback_days: 252, // 1 year of trading days + feature_dimensions: 64, + batch_size: 64, // Default batch size + cache_size: 10000, // Default cache size + enable_real_time: true, + enable_regime_detection: true, + parallel_processing: true, + } + } +} + +/// Asset ranking for universe selection +#[derive(Debug, Clone, Serialize, Deserialize)] +/// AssetRanking component. +pub struct AssetRanking { + pub asset_id: Symbol, + pub symbol: Symbol, + pub liquidity_rank: usize, + pub momentum_rank: usize, + pub volatility_rank: usize, + pub correlation_score: f64, + pub composite_score: f64, + pub percentile_rank: f64, + pub sector: String, + pub market_cap_rank: usize, + pub is_selected: bool, + pub selection_confidence: f64, +} + +/// Universe update event +#[derive(Debug, Clone, Serialize, Deserialize)] +/// UniverseUpdate component. +pub struct UniverseUpdate { + pub timestamp: SystemTime, + pub added_assets: Vec, + pub removed_assets: Vec, + pub ranking_changes: HashMap, + pub selection_criteria_used: SelectionCriteria, + pub update_reason: UniverseUpdateReason, + pub performance_metrics: UniversePerformanceMetrics, +} + +/// Reason for universe update +#[derive(Debug, Clone, Serialize, Deserialize)] +/// UniverseUpdateReason component. +pub enum UniverseUpdateReason { + Scheduled, + ThresholdBreached, + MarketRegimeChange, + LiquidityChange, + VolatilitySpike, + Manual, +} + +/// Universe performance metrics +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct UniversePerformanceMetrics { + pub total_universe_size: usize, + pub avg_liquidity_score: f64, + pub avg_volatility: f64, + pub sector_diversification: f64, + pub correlation_efficiency: f64, + pub turnover_rate: f64, + pub selection_latency_ms: u64, +} + +/// Universe selection engine +#[derive(Debug)] +pub struct UniverseSelectionEngine { + config: UniverseConfig, + criteria: SelectionCriteria, + + // Scoring engines + liquidity_scorer: LiquidityScorer, + momentum_ranker: MomentumRanker, + correlation_analyzer: CorrelationAnalyzer, + volatility_engine: VolatilityClusterEngine, + + // Current state + current_universe: HashMap, + candidate_assets: Vec, + performance_history: Vec, + + // Cache and optimization + scoring_cache: HashMap)>, + last_update: SystemTime, + update_count: u64, +} + +impl UniverseSelectionEngine { + /// Create new universe selection engine + pub fn new(config: UniverseConfig, criteria: SelectionCriteria) -> Result { + let liquidity_scorer = LiquidityScorer::default(); + let momentum_ranker = MomentumRanker::default(); + let correlation_analyzer = CorrelationAnalyzer::default(); + let volatility_engine = VolatilityClusterEngine::default(); + + Ok(Self { + config, + criteria, + liquidity_scorer, + momentum_ranker, + correlation_analyzer, + volatility_engine, + current_universe: HashMap::new(), + candidate_assets: Vec::new(), + performance_history: Vec::new(), + scoring_cache: HashMap::new(), + last_update: SystemTime::now(), + update_count: 0, + }) + } + + /// Update candidate assets + pub fn update_candidates(&mut self, assets: Vec) -> Result<(), MLError> { + self.candidate_assets = assets; + Ok(()) + } + + /// Select universe based on current criteria + pub fn select_universe(&mut self) -> Result { + let start_time = SystemTime::now(); + + // Score all candidate assets + let mut asset_rankings = Vec::new(); + let candidate_assets = self.candidate_assets.clone(); // Clone to avoid borrowing issues + + for asset in &candidate_assets { + if let Ok(ranking) = self.score_asset(asset) { + asset_rankings.push(ranking); + } + } + + // Sort by composite score + asset_rankings.sort_by(|a, b| { + b.composite_score + .partial_cmp(&a.composite_score) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + // Apply selection criteria + let selected_assets = self.apply_selection_criteria(&asset_rankings)?; + + // Calculate changes from current universe + let mut added_assets = Vec::new(); + let mut removed_assets = Vec::new(); + let mut ranking_changes = HashMap::new(); + + for ranking in &selected_assets { + if !self.current_universe.contains_key(&ranking.asset_id) { + added_assets.push(ranking.asset_id.clone()); + } + ranking_changes.insert(ranking.asset_id.clone(), ranking.clone()); + } + + for asset_id in self.current_universe.keys() { + if !selected_assets.iter().any(|r| r.asset_id == *asset_id) { + removed_assets.push(asset_id.clone()); + } + } + + // Update current universe + self.current_universe.clear(); + for ranking in selected_assets { + let asset_id = ranking.asset_id.clone(); + self.current_universe.insert(asset_id, ranking); + } + + // Calculate performance metrics + let performance_metrics = self.calculate_performance_metrics()?; + self.performance_history.push(performance_metrics.clone()); + + // Create update event + let update = UniverseUpdate { + timestamp: start_time, + added_assets, + removed_assets, + ranking_changes, + selection_criteria_used: self.criteria.clone(), + update_reason: UniverseUpdateReason::Scheduled, + performance_metrics, + }; + + self.last_update = start_time; + self.update_count += 1; + + Ok(update) + } + + /// Score a single asset + fn score_asset(&mut self, asset: &AssetData) -> Result { + // Check cache first + if let Some((timestamp, cached_scores)) = self.scoring_cache.get(&asset.asset_id) { + if timestamp.elapsed().unwrap_or_default().as_secs() < 300 { + // 5 minute cache + return self.create_ranking_from_cache(asset, cached_scores); + } + } + + // Create asset metadata from asset data + let asset_metadata = AssetMetadata { + symbol: asset.symbol.clone(), + market_cap_usd: 1_000_000_000.0, // Default market cap + price: asset.price.to_f64(), + volume_24h: asset.volume.to_f64(), + volume: asset.volume.to_f64(), + spread: 0.001, // Default spread + depth: 100_000.0, // Default depth + momentum: 0.05, // Default momentum + reversal_risk: 0.1, // Default reversal risk + volatility: 0.15, // Default volatility + sector: asset.sector.clone(), + }; + + // Calculate fresh scores + let liquidity_score = self + .liquidity_scorer + .calculate_liquidity_score(&asset_metadata)?; + + let momentum_score = self + .momentum_ranker + .calculate_momentum_score(&asset_metadata)?; + + // Simplified scoring - in production would be more sophisticated + let composite_score = + liquidity_score.overall_score * 0.4 + momentum_score.overall_score * 0.6; + + let ranking = AssetRanking { + asset_id: asset.asset_id.clone(), + symbol: asset.symbol.clone().into(), + liquidity_rank: 0, // Would be calculated after sorting + momentum_rank: 0, // Would be calculated after sorting + volatility_rank: 0, + correlation_score: 0.5, // Default neutral correlation + composite_score, + percentile_rank: 0.0, // Would be calculated after sorting + sector: asset.sector.clone(), + market_cap_rank: 0, + is_selected: false, + selection_confidence: composite_score.min(1.0), + }; + + // Cache the scores + let mut scores = HashMap::new(); + scores.insert("liquidity".to_owned(), liquidity_score.overall_score); + scores.insert("momentum".to_owned(), momentum_score.overall_score); + scores.insert("composite".to_owned(), composite_score); + + self.scoring_cache + .insert(asset.asset_id.clone(), (SystemTime::now(), scores)); + + Ok(ranking) + } + + /// Create ranking from cached scores + fn create_ranking_from_cache( + &self, + asset: &AssetData, + cached_scores: &HashMap, + ) -> Result { + let composite_score = cached_scores.get("composite").cloned().unwrap_or(0.0); + + Ok(AssetRanking { + asset_id: asset.asset_id.clone(), + symbol: asset.symbol.clone().into(), + liquidity_rank: 0, + momentum_rank: 0, + volatility_rank: 0, + correlation_score: 0.5, + composite_score, + percentile_rank: 0.0, + sector: asset.sector.clone(), + market_cap_rank: 0, + is_selected: false, + selection_confidence: composite_score.min(1.0), + }) + } + + /// Apply selection criteria to ranked assets + fn apply_selection_criteria( + &self, + rankings: &[AssetRanking], + ) -> Result, MLError> { + let mut selected = Vec::new(); + let mut sector_counts: HashMap = HashMap::new(); + + for ranking in rankings { + // Check liquidity threshold + if ranking.selection_confidence < self.criteria.min_liquidity_score { + continue; + } + + // Check sector limits + let sector_count = sector_counts.get(&ranking.sector).cloned().unwrap_or(0); + if let Some(&limit) = self.criteria.sector_limits.get(&ranking.sector) { + if sector_count >= limit { + continue; + } + } + + // Check max assets limit + if selected.len() >= self.criteria.max_assets { + break; + } + + let mut selected_ranking = ranking.clone(); + selected_ranking.is_selected = true; + selected.push(selected_ranking); + + *sector_counts.entry(ranking.sector.clone()).or_insert(0) += 1; + } + + Ok(selected) + } + + /// Calculate performance metrics + fn calculate_performance_metrics(&self) -> Result { + let universe_assets: Vec<&AssetRanking> = self.current_universe.values().collect(); + + let total_universe_size = universe_assets.len(); + + let avg_liquidity_score = if !universe_assets.is_empty() { + universe_assets + .iter() + .map(|a| a.selection_confidence) + .sum::() + / total_universe_size as f64 + } else { + 0.0 + }; + + // Simplified metrics - in production would be more comprehensive + Ok(UniversePerformanceMetrics { + total_universe_size, + avg_liquidity_score, + avg_volatility: 0.15, // Production + sector_diversification: self.calculate_sector_diversification(&universe_assets), + correlation_efficiency: 0.7, // Production + turnover_rate: 0.1, // Production + selection_latency_ms: 50, // Production + }) + } + + /// Calculate sector diversification score + fn calculate_sector_diversification(&self, assets: &[&AssetRanking]) -> f64 { + if assets.is_empty() { + return 0.0; + } + + let mut sector_counts: HashMap = HashMap::new(); + for asset in assets { + *sector_counts.entry(asset.sector.clone()).or_insert(0) += 1; + } + + let _num_sectors = sector_counts.len() as f64; + let total_assets = assets.len() as f64; + + // Calculate Herfindahl-Hirschman Index for diversification + let hhi: f64 = sector_counts + .values() + .map(|&count| { + let proportion = count as f64 / total_assets; + proportion * proportion + }) + .sum(); + + // Convert to diversification score (lower HHI = higher diversification) + (1.0 - hhi).max(0.0) + } + + /// Get current universe + pub fn get_current_universe(&self) -> &HashMap { + &self.current_universe + } + + /// Get performance metrics + pub fn get_performance_metrics(&self) -> Option<&UniversePerformanceMetrics> { + self.performance_history.last() + } + + /// Update liquidity scores for all tracked assets + pub async fn update_liquidity_scores(&mut self) -> Result<(), MLError> { + // Production implementation - updates liquidity scores in background + tracing::debug!( + "Updating liquidity scores for {} assets", + self.current_universe.len() + ); + Ok(()) + } + + /// Update momentum rankings for all tracked assets + pub async fn update_momentum_rankings(&mut self) -> Result<(), MLError> { + // Production implementation - updates momentum rankings in background + tracing::debug!( + "Updating momentum rankings for {} assets", + self.current_universe.len() + ); + Ok(()) + } + + /// Update volatility clustering for all tracked assets + pub async fn update_volatility_clustering(&mut self) -> Result<(), MLError> { + // Production implementation - updates volatility clusters in background + tracing::debug!( + "Updating volatility clustering for {} assets", + self.current_universe.len() + ); + Ok(()) + } + + /// Generate a universe update event + pub fn generate_universe_update(&self) -> Result { + let added_assets: Vec = self.current_universe.keys().cloned().collect(); + let removed_assets = Vec::new(); // Would be populated with rejected assets + let ranking_changes = self.current_universe.clone(); + + Ok(UniverseUpdate { + added_assets, + removed_assets, + timestamp: SystemTime::now(), + ranking_changes, + selection_criteria_used: self.criteria.clone(), + update_reason: UniverseUpdateReason::Scheduled, + performance_metrics: self.performance_history.last().cloned().unwrap_or_default(), + }) + } +} diff --git a/crates/ml-universe/src/liquidity.rs b/crates/ml-universe/src/liquidity.rs new file mode 100644 index 000000000..35de6201a --- /dev/null +++ b/crates/ml-universe/src/liquidity.rs @@ -0,0 +1,4 @@ +//! # Liquidity Scoring Module +//! +//! ML-based liquidity assessment for universe selection. +//! Uses multiple metrics including bid-ask spreads, market impact, and volume patterns. diff --git a/crates/ml-universe/src/momentum.rs b/crates/ml-universe/src/momentum.rs new file mode 100644 index 000000000..9ae9eca45 --- /dev/null +++ b/crates/ml-universe/src/momentum.rs @@ -0,0 +1,4 @@ +//! # Momentum Ranking Module +//! +//! ML-based momentum analysis for universe selection. +//! Implements multi-timeframe momentum scoring, cross-sectional ranking, and momentum regime detection. diff --git a/crates/ml/src/universe/volatility.rs b/crates/ml-universe/src/volatility.rs similarity index 94% rename from crates/ml/src/universe/volatility.rs rename to crates/ml-universe/src/volatility.rs index 138b23752..82fab6c3e 100644 --- a/crates/ml/src/universe/volatility.rs +++ b/crates/ml-universe/src/volatility.rs @@ -8,8 +8,7 @@ use std::collections::{HashMap, VecDeque}; use serde::{Deserialize, Serialize}; -use super::*; -use crate::{MLError, PRECISION_FACTOR}; +use crate::{MLError, PRECISION_FACTOR, VolatilityClusterEngineConfig}; /// Volatility regime classification #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -131,14 +130,14 @@ impl GarchModel { /// Enhanced volatility cluster engine with missing fields #[derive(Debug)] -pub struct VolatilityClusterEngine { +pub struct EnhancedVolatilityClusterEngine { config: VolatilityClusterEngineConfig, pub total_updates: u64, pub volatility_features: HashMap>, price_history: HashMap>, } -impl VolatilityClusterEngine { +impl EnhancedVolatilityClusterEngine { pub fn new(config: VolatilityClusterEngineConfig) -> Result { Ok(Self { config, @@ -222,7 +221,7 @@ mod tests { #[test] fn test_volatility_cluster_engine_creation() -> Result<(), MLError> { let config = VolatilityClusterEngineConfig::default(); - let engine = VolatilityClusterEngine::new(config); + let engine = EnhancedVolatilityClusterEngine::new(config); assert!(engine.is_ok()); let engine = engine?; @@ -233,7 +232,7 @@ mod tests { #[test] fn test_price_data_update() -> Result<(), MLError> { - let mut engine = VolatilityClusterEngine::new(VolatilityClusterEngineConfig::default())?; + let mut engine = EnhancedVolatilityClusterEngine::new(VolatilityClusterEngineConfig::default())?; let price_point = PricePoint { timestamp: 1640995200, // 2022-01-01 @@ -253,7 +252,7 @@ mod tests { #[test] fn test_volatility_calculations() -> Result<(), MLError> { - let engine = VolatilityClusterEngine::new(VolatilityClusterEngineConfig::default())?; + let engine = EnhancedVolatilityClusterEngine::new(VolatilityClusterEngineConfig::default())?; // Test returns calculation let mut prices = VecDeque::new(); @@ -318,7 +317,7 @@ mod tests { #[test] fn test_integer_sqrt() -> Result<(), MLError> { - let engine = VolatilityClusterEngine::new(VolatilityClusterEngineConfig::default())?; + let engine = EnhancedVolatilityClusterEngine::new(VolatilityClusterEngineConfig::default())?; let sqrt_result = engine.integer_sqrt(PRECISION_FACTOR * 4)?; // sqrt(4) assert_eq!(sqrt_result, 2 * PRECISION_FACTOR); // Should be 2.0 in fixed point diff --git a/crates/ml/Cargo.toml b/crates/ml/Cargo.toml index 888c4c6b1..8880391cf 100644 --- a/crates/ml/Cargo.toml +++ b/crates/ml/Cargo.toml @@ -81,6 +81,9 @@ ml-checkpoint.workspace = true ml-data-validation.workspace = true ml-validation.workspace = true ml-risk.workspace = true +ml-backtesting.workspace = true +ml-asset-selection.workspace = true +ml-universe.workspace = true config.workspace = true common = { workspace = true, features = ["questdb"] } risk = { path = "../risk" } diff --git a/crates/ml/src/asset_selection/mod.rs b/crates/ml/src/asset_selection/mod.rs index e4e77a2e5..da399fd4d 100644 --- a/crates/ml/src/asset_selection/mod.rs +++ b/crates/ml/src/asset_selection/mod.rs @@ -1,508 +1,2 @@ -//! Asset universe configuration and selection for the trading pipeline. -//! -//! This module defines which assets are eligible for trading, including -//! their classification, exchange, sector, and minimum volume requirements. - -use std::path::Path; - -use serde::{Deserialize, Serialize}; - -use crate::MLError; - -pub mod scorer; -pub mod selector; - -pub use scorer::{AssetScore, PredictabilityScorer, PredictionResult, ScoringWeights}; -pub use selector::{ActiveSetConfig, ActiveSetSelector, TradingCandidate}; - -/// Classification of a tradable asset. -#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -pub enum AssetClass { - /// Individual equity (stock). - Equity, - /// Exchange-traded fund. - ETF, - /// Futures contract. - Future, -} - -/// A single asset in the trading universe. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct UniverseAsset { - /// Ticker symbol (e.g. "ES.FUT") -- used for data/training. - pub symbol: String, - /// Execution symbol (e.g. "MES.FUT") -- used for order routing. - /// When `None`, orders are routed to `symbol` directly. - pub trading_symbol: Option, - /// MIC exchange code (e.g. "GLBX.MDP3"). - pub exchange: String, - /// Asset classification. - pub asset_class: AssetClass, - /// Optional GICS sector. - pub sector: Option, - /// Minimum average daily volume required for eligibility. - pub min_daily_volume: f64, - /// Whether this asset is enabled for trading. - pub enabled: bool, -} - -impl UniverseAsset { - /// The symbol to use for order execution. - /// Returns `trading_symbol` if set, otherwise `symbol`. - #[must_use] - pub fn execution_symbol(&self) -> &str { - self.trading_symbol.as_deref().unwrap_or(&self.symbol) - } -} - -/// The full set of assets available for trading. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AssetUniverse { - /// All configured assets. - pub assets: Vec, -} - -// --------------------------------------------------------------------------- -// TOML config deserialization -// --------------------------------------------------------------------------- - -#[derive(Deserialize)] -struct UniverseConfigFile { - universe: UniverseConfigMeta, - symbols: Vec, -} - -/// Parsed metadata from the `[universe]` section of the TOML config. -/// Fields are currently used only for validation/documentation during parsing; -/// they will feed into `DatasetSpec` construction once download automation lands. -#[derive(Deserialize)] -struct UniverseConfigMeta { - name: String, - description: Option, - date_range_start: Option, - date_range_end: Option, - bar_size: Option, - databento_dataset: Option, - databento_schema: Option, -} - -#[derive(Deserialize)] -struct SymbolConfigEntry { - symbol: String, - trading_symbol: Option, - exchange: String, - asset_class: String, - min_daily_volume: Option, -} - -impl AssetUniverse { - /// Create an empty universe. - #[must_use] - pub fn new() -> Self { - Self { assets: Vec::new() } - } - - /// Load a universe from a TOML config file. - /// - /// # Errors - /// Returns `MLError::ConfigError` on I/O or parse failures. - pub fn from_config(path: &Path) -> Result { - let contents = std::fs::read_to_string(path).map_err(|e| MLError::ConfigError(format!("Failed to read universe config {}: {e}", path.display())))?; - - let config: UniverseConfigFile = - toml::from_str(&contents).map_err(|e| MLError::ConfigError(format!("Failed to parse universe config {}: {e}", path.display())))?; - - let assets = config - .symbols - .into_iter() - .map(|s| { - let asset_class = match s.asset_class.as_str() { - "Future" | "future" => AssetClass::Future, - "Equity" | "equity" => AssetClass::Equity, - "ETF" | "etf" => AssetClass::ETF, - other => { - return Err(MLError::ConfigError(format!("Unknown asset class: {other}"))) - } - }; - Ok(UniverseAsset { - symbol: s.symbol, - trading_symbol: s.trading_symbol, - exchange: s.exchange, - asset_class, - sector: None, - min_daily_volume: s.min_daily_volume.unwrap_or(0.0), - enabled: true, - }) - }) - .collect::, _>>()?; - - Ok(Self { assets }) - } - - /// Return only enabled assets. - #[must_use] - pub fn eligible(&self) -> Vec<&UniverseAsset> { - self.assets.iter().filter(|a| a.enabled).collect() - } - - /// Count of enabled assets. - #[must_use] - pub fn eligible_count(&self) -> usize { - self.assets.iter().filter(|a| a.enabled).count() - } - - /// Find an asset by symbol (case-sensitive). - #[must_use] - pub fn find(&self, symbol: &str) -> Option<&UniverseAsset> { - self.assets.iter().find(|a| a.symbol == symbol) - } - - /// List all asset symbols. - #[must_use] - pub fn symbols(&self) -> Vec<&str> { - self.assets.iter().map(|a| a.symbol.as_str()).collect() - } - - /// Preset: US large-cap starter universe (7 highly liquid equities/ETFs). - #[must_use] - pub fn us_starter() -> Self { - Self { - assets: vec![ - UniverseAsset { - symbol: "SPY".to_owned(), - trading_symbol: None, - exchange: "XNYS".to_owned(), - asset_class: AssetClass::ETF, - sector: None, - min_daily_volume: 50_000_000.0, - enabled: true, - }, - UniverseAsset { - symbol: "QQQ".to_owned(), - trading_symbol: None, - exchange: "XNAS".to_owned(), - asset_class: AssetClass::ETF, - sector: None, - min_daily_volume: 30_000_000.0, - enabled: true, - }, - UniverseAsset { - symbol: "AAPL".to_owned(), - trading_symbol: None, - exchange: "XNAS".to_owned(), - asset_class: AssetClass::Equity, - sector: Some("Technology".to_owned()), - min_daily_volume: 50_000_000.0, - enabled: true, - }, - UniverseAsset { - symbol: "MSFT".to_owned(), - trading_symbol: None, - exchange: "XNAS".to_owned(), - asset_class: AssetClass::Equity, - sector: Some("Technology".to_owned()), - min_daily_volume: 20_000_000.0, - enabled: true, - }, - UniverseAsset { - symbol: "NVDA".to_owned(), - trading_symbol: None, - exchange: "XNAS".to_owned(), - asset_class: AssetClass::Equity, - sector: Some("Technology".to_owned()), - min_daily_volume: 30_000_000.0, - enabled: true, - }, - UniverseAsset { - symbol: "AMZN".to_owned(), - trading_symbol: None, - exchange: "XNAS".to_owned(), - asset_class: AssetClass::Equity, - sector: Some("Consumer".to_owned()), - min_daily_volume: 20_000_000.0, - enabled: true, - }, - UniverseAsset { - symbol: "IWM".to_owned(), - trading_symbol: None, - exchange: "XNYS".to_owned(), - asset_class: AssetClass::ETF, - sector: None, - min_daily_volume: 20_000_000.0, - enabled: true, - }, - ], - } - } - - /// Preset: CME futures baseline universe (4 assets for limited capital). - /// - /// Training is done on full-size contracts (ES, NQ, ZN, 6E). - /// Execution routes to micro contracts where available (MES, MNQ, M6E). - #[must_use] - pub fn futures_baseline() -> Self { - Self { - assets: vec![ - UniverseAsset { - symbol: "ES.FUT".to_owned(), - trading_symbol: Some("MES.FUT".to_owned()), - exchange: "GLBX.MDP3".to_owned(), - asset_class: AssetClass::Future, - sector: None, - min_daily_volume: 1_000_000.0, - enabled: true, - }, - UniverseAsset { - symbol: "NQ.FUT".to_owned(), - trading_symbol: Some("MNQ.FUT".to_owned()), - exchange: "GLBX.MDP3".to_owned(), - asset_class: AssetClass::Future, - sector: None, - min_daily_volume: 500_000.0, - enabled: true, - }, - UniverseAsset { - symbol: "ZN.FUT".to_owned(), - trading_symbol: None, - exchange: "GLBX.MDP3".to_owned(), - asset_class: AssetClass::Future, - sector: None, - min_daily_volume: 500_000.0, - enabled: true, - }, - UniverseAsset { - symbol: "6E.FUT".to_owned(), - trading_symbol: Some("M6E.FUT".to_owned()), - exchange: "GLBX.MDP3".to_owned(), - asset_class: AssetClass::Future, - sector: None, - min_daily_volume: 200_000.0, - enabled: true, - }, - ], - } - } -} - -impl Default for AssetUniverse { - fn default() -> Self { - Self::new() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_us_starter_universe() { - let universe = AssetUniverse::us_starter(); - assert_eq!(universe.assets.len(), 7); - assert_eq!(universe.eligible_count(), 7); - } - - #[test] - fn test_eligible_filters_disabled() { - let mut universe = AssetUniverse::us_starter(); - if let Some(first) = universe.assets.get_mut(0) { - first.enabled = false; - } - assert_eq!(universe.eligible_count(), 6); - } - - #[test] - fn test_find_symbol() { - let universe = AssetUniverse::us_starter(); - let aapl = universe.find("AAPL"); - assert!(aapl.is_some()); - assert_eq!( - aapl.map(|a| a.exchange.as_str()).unwrap_or_default(), - "XNAS" - ); - } - - #[test] - fn test_find_missing_symbol() { - let universe = AssetUniverse::us_starter(); - assert!(universe.find("DOESNOTEXIST").is_none()); - } - - #[test] - fn test_symbols_list() { - let universe = AssetUniverse::us_starter(); - let symbols = universe.symbols(); - assert_eq!(symbols.len(), 7); - assert!(symbols.contains(&"SPY")); - assert!(symbols.contains(&"AAPL")); - } - - #[test] - fn test_empty_universe() { - let universe = AssetUniverse::new(); - assert!(universe.eligible().is_empty()); - } - - #[test] - fn test_serde_roundtrip() { - let universe = AssetUniverse::us_starter(); - let json = serde_json::to_string(&universe).unwrap_or_default(); - let restored: AssetUniverse = - serde_json::from_str(&json).unwrap_or_else(|_| AssetUniverse::new()); - assert_eq!(restored.assets.len(), 7); - } - - #[test] - fn test_trading_symbol_mapping() { - let asset = UniverseAsset { - symbol: "ES.FUT".to_owned(), - trading_symbol: Some("MES.FUT".to_owned()), - exchange: "GLBX.MDP3".to_owned(), - asset_class: AssetClass::Future, - sector: None, - min_daily_volume: 1_000_000.0, - enabled: true, - }; - assert_eq!(asset.execution_symbol(), "MES.FUT"); - } - - #[test] - fn test_trading_symbol_none_falls_back() { - let asset = UniverseAsset { - symbol: "ZN.FUT".to_owned(), - trading_symbol: None, - exchange: "GLBX.MDP3".to_owned(), - asset_class: AssetClass::Future, - sector: None, - min_daily_volume: 500_000.0, - enabled: true, - }; - assert_eq!(asset.execution_symbol(), "ZN.FUT"); - } - - #[test] - fn test_futures_baseline_universe() { - let universe = AssetUniverse::futures_baseline(); - assert_eq!(universe.assets.len(), 4); - assert_eq!(universe.eligible_count(), 4); - - // All are futures - for asset in &universe.assets { - assert_eq!(asset.asset_class, AssetClass::Future); - } - } - - #[test] - fn test_futures_baseline_symbols() { - let universe = AssetUniverse::futures_baseline(); - let symbols = universe.symbols(); - assert!(symbols.contains(&"ES.FUT")); - assert!(symbols.contains(&"NQ.FUT")); - assert!(symbols.contains(&"ZN.FUT")); - assert!(symbols.contains(&"6E.FUT")); - } - - #[test] - fn test_futures_baseline_micro_mapping() { - let universe = AssetUniverse::futures_baseline(); - - assert_eq!( - universe.find("ES.FUT").map(|a| a.execution_symbol()), - Some("MES.FUT") - ); - assert_eq!( - universe.find("NQ.FUT").map(|a| a.execution_symbol()), - Some("MNQ.FUT") - ); - // ZN has no micro -- executes as ZN.FUT - assert_eq!( - universe.find("ZN.FUT").map(|a| a.execution_symbol()), - Some("ZN.FUT") - ); - assert_eq!( - universe.find("6E.FUT").map(|a| a.execution_symbol()), - Some("M6E.FUT") - ); - } - - #[test] - fn test_futures_baseline_serde_roundtrip() { - let universe = AssetUniverse::futures_baseline(); - let json = serde_json::to_string(&universe).unwrap_or_default(); - let restored: AssetUniverse = - serde_json::from_str(&json).unwrap_or_else(|_| AssetUniverse::new()); - assert_eq!(restored.assets.len(), 4); - // Verify trading_symbol survives roundtrip - let es = restored.find("ES.FUT"); - assert_eq!(es.map(|a| a.execution_symbol()), Some("MES.FUT")); - } - - #[test] - fn test_from_config_toml() { - let toml_content = r#" -[universe] -name = "test-universe" -description = "Test" - -[[symbols]] -symbol = "ES.FUT" -trading_symbol = "MES.FUT" -exchange = "GLBX.MDP3" -asset_class = "Future" -min_daily_volume = 1000000.0 - -[[symbols]] -symbol = "ZN.FUT" -exchange = "GLBX.MDP3" -asset_class = "Future" -min_daily_volume = 500000.0 -"#; - let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("tmpdir: {e}")); - let path = dir.path().join("universe.toml"); - std::fs::write(&path, toml_content).unwrap_or_else(|e| panic!("write: {e}")); - - let universe = AssetUniverse::from_config(&path); - assert!(universe.is_ok(), "from_config failed: {:?}", universe.err()); - let universe = universe.unwrap_or_else(|_| AssetUniverse::new()); - assert_eq!(universe.assets.len(), 2); - assert_eq!( - universe.find("ES.FUT").map(|a| a.execution_symbol()), - Some("MES.FUT") - ); - assert_eq!( - universe.find("ZN.FUT").map(|a| a.trading_symbol.as_deref()), - Some(None) - ); - } - - #[test] - fn test_load_futures_baseline_config() { - let config_path = std::path::Path::new("../config/universe-futures-baseline.toml"); - if !config_path.exists() { - // Skip in CI where config may not be at expected relative path - return; - } - let universe = AssetUniverse::from_config(config_path); - assert!(universe.is_ok(), "Failed to load config: {:?}", universe.err()); - let universe = universe.unwrap_or_else(|_| AssetUniverse::new()); - assert_eq!(universe.assets.len(), 4); - assert_eq!( - universe.find("ES.FUT").map(|a| a.execution_symbol()), - Some("MES.FUT") - ); - } - - #[test] - fn test_from_config_missing_file() { - let result = AssetUniverse::from_config(std::path::Path::new("/nonexistent/path.toml")); - assert!(result.is_err()); - } - - #[test] - fn test_from_config_invalid_toml() { - let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("tmpdir: {e}")); - let path = dir.path().join("bad.toml"); - std::fs::write(&path, "not valid toml {{{{").unwrap_or_else(|e| panic!("write: {e}")); - let result = AssetUniverse::from_config(&path); - assert!(result.is_err()); - } -} +//! Asset Selection -- thin facade re-exporting from ml-asset-selection crate. +pub use ml_asset_selection::*; diff --git a/crates/ml/src/backtesting/mod.rs b/crates/ml/src/backtesting/mod.rs index 8b554b742..c23132251 100644 --- a/crates/ml/src/backtesting/mod.rs +++ b/crates/ml/src/backtesting/mod.rs @@ -1,8 +1,2 @@ -// ml/src/backtesting/mod.rs -// Backtesting modules for barrier optimization and DQN action loading - -pub mod action_loader; -pub mod barrier_backtest; - -pub use action_loader::{load_actions_from_csv, DQNActionRecord}; -pub use barrier_backtest::{BacktestResults, BarrierBacktester, BarrierParams}; +//! Backtesting -- thin facade re-exporting from ml-backtesting crate. +pub use ml_backtesting::*; diff --git a/crates/ml/src/universe/liquidity.rs b/crates/ml/src/universe/liquidity.rs deleted file mode 100644 index 8c2eb8d58..000000000 --- a/crates/ml/src/universe/liquidity.rs +++ /dev/null @@ -1,80 +0,0 @@ -//! # Liquidity Scoring Module -//! -//! ML-based liquidity assessment for universe selection. -//! Uses multiple metrics including bid-ask spreads, market impact, and volume patterns. - -// Price imported from crate root (lib.rs) - -// DISABLED: Tests require LiquidityScorer, Price, Volume types not exported from ml crate -// #[cfg(test)] -// mod tests { -// use super::*; -// use serde::{Serialize, Deserialize}; -// -// #[test] -// fn test_liquidity_scoring() { -// let config = LiquidityConfig::default(); -// let mut scorer = LiquidityScorer::new(config)?; -// let asset_id = "TEST_ASSET".to_owned(); -// let data = create_test_microstructure_data(); -// let score = scorer.score_liquidity(asset_id, &data)?; -// assert!(score.overall_score >= 0.0 && score.overall_score <= 1.0); -// assert!(score.spread_score >= 0.0 && score.spread_score <= 1.0); -// assert!(score.volume_score >= 0.0 && score.volume_score <= 1.0); -// } -// -// /// Microstructure data for liquidity analysis -// #[derive(Debug, Clone, Serialize, Deserialize)] -// struct MicrostructureData { -// /// Timestamp in nanoseconds -// pub timestamp: u64, -// /// Best bid price -// pub bid_price: Price, -// /// Best ask price -// pub ask_price: Price, -// /// Bid volume -// pub bid_volume: Volume, -// /// Ask volume -// pub ask_volume: Volume, -// /// Last trade price -// pub trade_price: Option, -// /// Last trade volume -// pub trade_volume: Option, -// /// Mid-price (bid + ask) / 2 -// pub mid_price: Price, -// /// Spread (ask - bid) -// pub spread: Price, -// } -// -// fn create_test_microstructure_data() -> Vec { -// use chrono::Utc; -// -// (0..200) -// .map(|i| { -// let base_price = 100.0 + (i as f64 * 0.01); -// let spread = 0.02 + (i as f64 * 0.0001); // Growing spread -// let bid_price = Price::from_f64(base_price - spread / 2.0).unwrap_or_default(); -// let ask_price = Price::from_f64(base_price + spread / 2.0).unwrap_or_default(); -// let mid_price = Price::from_f64(base_price).unwrap_or_default(); -// let spread_price = Price::from_f64(spread).unwrap_or_default(); -// -// MicrostructureData { -// timestamp: (Utc::now().timestamp_nanos_opt().unwrap_or(0) as u64) -// + (i as u64 * 1_000_000), // 1ms intervals -// bid_price, -// ask_price, -// bid_volume: Volume::new(1000.0 + (i as f64 * 10.0)).unwrap_or_default(), -// ask_volume: Volume::new(1000.0 + (i as f64 * 10.0)).unwrap_or_default(), -// trade_price: if i % 3 == 0 { Some(mid_price) } else { None }, // Sparse trades -// trade_volume: if i % 3 == 0 { -// Some(Volume::new(500.0).unwrap_or_default()) -// } else { -// None -// }, -// mid_price, -// spread: spread_price, -// } -// }) -// .collect() -// } -// } diff --git a/crates/ml/src/universe/mod.rs b/crates/ml/src/universe/mod.rs index 85a389c37..91c410a2c 100644 --- a/crates/ml/src/universe/mod.rs +++ b/crates/ml/src/universe/mod.rs @@ -1,811 +1,2 @@ -//! # Universe Selection ML Module -//! -//! Dynamic universe selection using machine learning for optimal asset filtering. -//! Implements liquidity scoring, momentum ranking, volatility clustering, and correlation analysis. - -pub mod correlation; -pub mod liquidity; -pub mod momentum; -pub mod volatility; - -use chrono::{DateTime, Utc}; -use std::collections::HashMap; -use std::time::SystemTime; - -use serde::{Deserialize, Serialize}; - -// Regime detection integration planned for future release -use crate::MLError; -use common::{Price, Symbol, Volume}; - -// Missing types that need to be defined -#[derive(Debug)] -pub struct LiquidityScorer { - config: LiquidityScorerConfig, -} - -impl Default for LiquidityScorer { - fn default() -> Self { - Self { - config: LiquidityScorerConfig::default(), - } - } -} - -impl LiquidityScorer { - pub fn new(config: LiquidityScorerConfig) -> Result { - Ok(Self { config }) - } - - pub fn calculate_liquidity_score( - &self, - asset: &AssetMetadata, - ) -> Result { - let base_score = asset.market_cap_usd * self.config.volume_weight; - let spread_penalty = asset.spread * self.config.spread_weight; - let depth_bonus = asset.depth * self.config.depth_weight; - - let score = base_score - spread_penalty + depth_bonus; - - Ok(LiquidityScore { - overall_score: score, - score, - volume: asset.volume, - spread: asset.spread, - depth: asset.depth, - timestamp: SystemTime::now(), - }) - } -} - -#[derive(Debug, Clone, Default)] -pub struct LiquidityScorerConfig { - pub min_volume_threshold: f64, - pub spread_weight: f64, - pub depth_weight: f64, - pub volume_weight: f64, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct LiquidityScore { - pub overall_score: f64, - pub score: f64, - pub volume: f64, - pub spread: f64, - pub depth: f64, - pub timestamp: SystemTime, -} - -impl Default for LiquidityScore { - fn default() -> Self { - Self { - overall_score: 0.0, - score: 0.0, - volume: 0.0, - spread: 0.0, - depth: 0.0, - timestamp: SystemTime::now(), - } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MomentumScore { - pub overall_score: f64, - pub score: f64, - pub momentum_1d: f64, - pub momentum_7d: f64, - pub momentum_30d: f64, - pub timestamp: SystemTime, -} - -impl Default for MomentumScore { - fn default() -> Self { - Self { - overall_score: 0.0, - score: 0.0, - momentum_1d: 0.0, - momentum_7d: 0.0, - momentum_30d: 0.0, - timestamp: SystemTime::now(), - } - } -} - -#[derive(Debug)] -pub struct MomentumRanker { - config: MomentumRankerConfig, -} - -impl Default for MomentumRanker { - fn default() -> Self { - Self { - config: MomentumRankerConfig::default(), - } - } -} - -impl MomentumRanker { - pub fn new(config: MomentumRankerConfig) -> Result { - Ok(Self { config }) - } - - pub fn calculate_momentum_score( - &self, - asset: &AssetMetadata, - ) -> Result { - let momentum = asset.momentum * self.config.momentum_weight; - let reversal_penalty = asset.reversal_risk * self.config.reversal_weight; - - let score = momentum - reversal_penalty; - - Ok(MomentumScore { - overall_score: score, - score, - momentum_1d: asset.momentum, - momentum_7d: asset.momentum * 0.8, // Simplified - momentum_30d: asset.momentum * 0.6, // Simplified - timestamp: SystemTime::now(), - }) - } -} - -#[derive(Debug, Clone, Default)] -pub struct MomentumRankerConfig { - pub lookback_days: u32, - pub momentum_weight: f64, - pub reversal_weight: f64, -} - -#[derive(Debug)] -pub struct CorrelationAnalyzer { - config: CorrelationAnalyzerConfig, -} - -impl CorrelationAnalyzer { - pub fn new(config: CorrelationAnalyzerConfig) -> Result { - Ok(Self { config }) - } -} - -impl Default for CorrelationAnalyzer { - fn default() -> Self { - Self { - config: CorrelationAnalyzerConfig::default(), - } - } -} - -#[derive(Debug, Clone, Default)] -pub struct CorrelationAnalyzerConfig { - pub correlation_threshold: f64, - pub window_size: usize, - pub update_frequency_ms: u64, -} - -#[derive(Debug)] -pub struct VolatilityClusterEngine { - config: VolatilityClusterEngineConfig, -} - -impl VolatilityClusterEngine { - pub fn new(config: VolatilityClusterEngineConfig) -> Result { - Ok(Self { config }) - } -} - -impl Default for VolatilityClusterEngine { - fn default() -> Self { - Self { - config: VolatilityClusterEngineConfig::default(), - } - } -} - -#[derive(Debug, Clone, Default)] -pub struct VolatilityClusterEngineConfig { - pub cluster_count: usize, - pub volatility_window: usize, - pub min_volume_threshold: f64, -} - -// Additional required types -#[derive(Debug, Clone, Default)] -pub struct AssetMetadata { - pub symbol: String, - pub market_cap_usd: f64, - pub price: f64, - pub volume_24h: f64, - pub volume: f64, - pub spread: f64, - pub depth: f64, - pub momentum: f64, - pub reversal_risk: f64, - pub volatility: f64, - pub sector: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CorrelationMatrix { - pub correlations: HashMap<(Symbol, Symbol), f64>, - pub eigenvalues: Vec, - pub condition_number: f64, - pub timestamp: SystemTime, -} - -impl Default for CorrelationMatrix { - fn default() -> Self { - Self { - correlations: HashMap::new(), - eigenvalues: Vec::new(), - condition_number: 1.0, - timestamp: SystemTime::now(), - } - } -} - -// Additional types are defined above - main configuration types follow - -/// Configuration for universe selection engine -#[derive(Debug, Clone, Serialize, Deserialize)] -/// UniverseSelectionConfig component. -pub struct UniverseSelectionConfig { - /// Maximum number of assets to include in the universe - pub max_universe_size: usize, - - /// Minimum market cap threshold - pub min_market_cap: f64, - - /// Minimum trading `volume` - pub min_volume: f64, - - /// Minimum liquidity score - pub min_liquidity_score: f64, - - /// Minimum momentum score - pub min_momentum_score: f64, - - /// Enable correlation filtering - pub enable_correlation_filtering: bool, - - /// Maximum correlation threshold - pub max_correlation: f64, - - /// Rebalancing frequency in days - pub rebalancing_frequency_days: usize, - - /// Cache expiry time in seconds - pub cache_expiry_seconds: u64, -} - -impl Default for UniverseSelectionConfig { - fn default() -> Self { - Self { - max_universe_size: 500, - min_market_cap: 1_000_000_000.0, // $1B minimum market cap - min_volume: 1_000_000.0, // $1M daily volume - min_liquidity_score: 0.3, - min_momentum_score: 0.3, - enable_correlation_filtering: true, - max_correlation: 0.8, - rebalancing_frequency_days: 30, - cache_expiry_seconds: 300, // 5 minutes - } - } -} - -/// Asset data for universe selection -#[derive(Debug, Clone, Serialize, Deserialize)] -/// AssetData component. -pub struct AssetData { - pub asset_id: Symbol, - pub symbol: String, - pub price: Price, - pub volume: Volume, - pub market_cap: u64, - pub sector: String, - pub exchange: String, - pub last_trade_time: DateTime, -} - -/// Universe selection criteria -#[derive(Debug, Clone, Serialize, Deserialize)] -/// SelectionCriteria component. -pub struct SelectionCriteria { - pub min_liquidity_score: f64, - pub min_momentum_score: f64, - pub max_correlation: f64, - pub volatility_regime: Option, // Simplified for now - pub sector_limits: HashMap, - pub max_assets: usize, - pub min_market_cap: u64, - pub exclude_penny_stocks: bool, -} - -impl Default for SelectionCriteria { - fn default() -> Self { - Self { - min_liquidity_score: 0.6, - min_momentum_score: 0.5, - max_correlation: 0.8, - volatility_regime: None, - sector_limits: HashMap::new(), - max_assets: 100, - min_market_cap: 1_000_000_000, // $1B minimum market cap - exclude_penny_stocks: true, - } - } -} - -/// Selected universe with scores -#[derive(Debug, Clone, Serialize, Deserialize)] -/// SelectedUniverse component. -pub struct SelectedUniverse { - pub assets: Vec, - pub liquidity_scores: HashMap, - pub momentum_scores: HashMap, - pub correlation_matrix: CorrelationMatrix, - pub diversification_score: f64, // Replace with DiversificationScore when diversification module is implemented - pub selection_timestamp: DateTime, - pub rebalance_needed: bool, -} - -/// Universe selection performance metrics -#[derive(Debug, Clone, Serialize, Deserialize)] -/// SelectionMetrics component. -pub struct SelectionMetrics { - pub total_candidates: usize, - pub selected_assets: usize, - pub avg_liquidity_score: f64, - pub avg_momentum_score: f64, - pub correlation_efficiency: f64, - pub sector_distribution: HashMap, - pub selection_latency_micros: u64, - pub cache_hit_rate: f64, -} - -/// Configuration for universe selection models -#[derive(Debug, Clone, Serialize, Deserialize)] -/// UniverseConfig component. -pub struct UniverseConfig { - pub update_frequency_minutes: u32, - pub lookback_days: u32, - pub feature_dimensions: usize, - pub batch_size: usize, - pub cache_size: usize, - pub enable_real_time: bool, - pub enable_regime_detection: bool, - pub parallel_processing: bool, -} - -impl Default for UniverseConfig { - fn default() -> Self { - Self { - update_frequency_minutes: 60, - lookback_days: 252, // 1 year of trading days - feature_dimensions: 64, - batch_size: 64, // Default batch size - cache_size: 10000, // Default cache size - enable_real_time: true, - enable_regime_detection: true, - parallel_processing: true, - } - } -} - -/// Asset ranking for universe selection -#[derive(Debug, Clone, Serialize, Deserialize)] -/// AssetRanking component. -pub struct AssetRanking { - pub asset_id: Symbol, - pub symbol: Symbol, - pub liquidity_rank: usize, - pub momentum_rank: usize, - pub volatility_rank: usize, - pub correlation_score: f64, - pub composite_score: f64, - pub percentile_rank: f64, - pub sector: String, - pub market_cap_rank: usize, - pub is_selected: bool, - pub selection_confidence: f64, -} - -/// Universe update event -#[derive(Debug, Clone, Serialize, Deserialize)] -/// UniverseUpdate component. -pub struct UniverseUpdate { - pub timestamp: SystemTime, - pub added_assets: Vec, - pub removed_assets: Vec, - pub ranking_changes: HashMap, - pub selection_criteria_used: SelectionCriteria, - pub update_reason: UniverseUpdateReason, - pub performance_metrics: UniversePerformanceMetrics, -} - -/// Reason for universe update -#[derive(Debug, Clone, Serialize, Deserialize)] -/// UniverseUpdateReason component. -pub enum UniverseUpdateReason { - Scheduled, - ThresholdBreached, - MarketRegimeChange, - LiquidityChange, - VolatilitySpike, - Manual, -} - -/// Universe performance metrics -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -pub struct UniversePerformanceMetrics { - pub total_universe_size: usize, - pub avg_liquidity_score: f64, - pub avg_volatility: f64, - pub sector_diversification: f64, - pub correlation_efficiency: f64, - pub turnover_rate: f64, - pub selection_latency_ms: u64, -} - -/// Universe selection engine -#[derive(Debug)] -pub struct UniverseSelectionEngine { - config: UniverseConfig, - criteria: SelectionCriteria, - - // Scoring engines - liquidity_scorer: LiquidityScorer, - momentum_ranker: MomentumRanker, - correlation_analyzer: CorrelationAnalyzer, - volatility_engine: VolatilityClusterEngine, - - // Current state - current_universe: HashMap, - candidate_assets: Vec, - performance_history: Vec, - - // Cache and optimization - scoring_cache: HashMap)>, - last_update: SystemTime, - update_count: u64, -} - -impl UniverseSelectionEngine { - /// Create new universe selection engine - pub fn new(config: UniverseConfig, criteria: SelectionCriteria) -> Result { - let liquidity_scorer = LiquidityScorer::default(); - let momentum_ranker = MomentumRanker::default(); - let correlation_analyzer = CorrelationAnalyzer::default(); - let volatility_engine = VolatilityClusterEngine::default(); - - Ok(Self { - config, - criteria, - liquidity_scorer, - momentum_ranker, - correlation_analyzer, - volatility_engine, - current_universe: HashMap::new(), - candidate_assets: Vec::new(), - performance_history: Vec::new(), - scoring_cache: HashMap::new(), - last_update: SystemTime::now(), - update_count: 0, - }) - } - - /// Update candidate assets - pub fn update_candidates(&mut self, assets: Vec) -> Result<(), MLError> { - self.candidate_assets = assets; - Ok(()) - } - - /// Select universe based on current criteria - pub fn select_universe(&mut self) -> Result { - let start_time = SystemTime::now(); - - // Score all candidate assets - let mut asset_rankings = Vec::new(); - let candidate_assets = self.candidate_assets.clone(); // Clone to avoid borrowing issues - - for asset in &candidate_assets { - if let Ok(ranking) = self.score_asset(asset) { - asset_rankings.push(ranking); - } - } - - // Sort by composite score - asset_rankings.sort_by(|a, b| { - b.composite_score - .partial_cmp(&a.composite_score) - .unwrap_or(std::cmp::Ordering::Equal) - }); - - // Apply selection criteria - let selected_assets = self.apply_selection_criteria(&asset_rankings)?; - - // Calculate changes from current universe - let mut added_assets = Vec::new(); - let mut removed_assets = Vec::new(); - let mut ranking_changes = HashMap::new(); - - for ranking in &selected_assets { - if !self.current_universe.contains_key(&ranking.asset_id) { - added_assets.push(ranking.asset_id.clone()); - } - ranking_changes.insert(ranking.asset_id.clone(), ranking.clone()); - } - - for asset_id in self.current_universe.keys() { - if !selected_assets.iter().any(|r| r.asset_id == *asset_id) { - removed_assets.push(asset_id.clone()); - } - } - - // Update current universe - self.current_universe.clear(); - for ranking in selected_assets { - let asset_id = ranking.asset_id.clone(); - self.current_universe.insert(asset_id, ranking); - } - - // Calculate performance metrics - let performance_metrics = self.calculate_performance_metrics()?; - self.performance_history.push(performance_metrics.clone()); - - // Create update event - let update = UniverseUpdate { - timestamp: start_time, - added_assets, - removed_assets, - ranking_changes, - selection_criteria_used: self.criteria.clone(), - update_reason: UniverseUpdateReason::Scheduled, - performance_metrics, - }; - - self.last_update = start_time; - self.update_count += 1; - - Ok(update) - } - - /// Score a single asset - fn score_asset(&mut self, asset: &AssetData) -> Result { - // Check cache first - if let Some((timestamp, cached_scores)) = self.scoring_cache.get(&asset.asset_id) { - if timestamp.elapsed().unwrap_or_default().as_secs() < 300 { - // 5 minute cache - return self.create_ranking_from_cache(asset, cached_scores); - } - } - - // Create asset metadata from asset data - let asset_metadata = AssetMetadata { - symbol: asset.symbol.clone(), - market_cap_usd: 1_000_000_000.0, // Default market cap - price: asset.price.to_f64(), - volume_24h: asset.volume.to_f64(), - volume: asset.volume.to_f64(), - spread: 0.001, // Default spread - depth: 100_000.0, // Default depth - momentum: 0.05, // Default momentum - reversal_risk: 0.1, // Default reversal risk - volatility: 0.15, // Default volatility - sector: asset.sector.clone(), - }; - - // Calculate fresh scores - let liquidity_score = self - .liquidity_scorer - .calculate_liquidity_score(&asset_metadata)?; - - let momentum_score = self - .momentum_ranker - .calculate_momentum_score(&asset_metadata)?; - - // Simplified scoring - in production would be more sophisticated - let composite_score = - liquidity_score.overall_score * 0.4 + momentum_score.overall_score * 0.6; - - let ranking = AssetRanking { - asset_id: asset.asset_id.clone(), - symbol: asset.symbol.clone().into(), - liquidity_rank: 0, // Would be calculated after sorting - momentum_rank: 0, // Would be calculated after sorting - volatility_rank: 0, - correlation_score: 0.5, // Default neutral correlation - composite_score, - percentile_rank: 0.0, // Would be calculated after sorting - sector: asset.sector.clone(), - market_cap_rank: 0, - is_selected: false, - selection_confidence: composite_score.min(1.0), - }; - - // Cache the scores - let mut scores = HashMap::new(); - scores.insert("liquidity".to_owned(), liquidity_score.overall_score); - scores.insert("momentum".to_owned(), momentum_score.overall_score); - scores.insert("composite".to_owned(), composite_score); - - self.scoring_cache - .insert(asset.asset_id.clone(), (SystemTime::now(), scores)); - - Ok(ranking) - } - - /// Create ranking from cached scores - fn create_ranking_from_cache( - &self, - asset: &AssetData, - cached_scores: &HashMap, - ) -> Result { - let composite_score = cached_scores.get("composite").cloned().unwrap_or(0.0); - - Ok(AssetRanking { - asset_id: asset.asset_id.clone(), - symbol: asset.symbol.clone().into(), - liquidity_rank: 0, - momentum_rank: 0, - volatility_rank: 0, - correlation_score: 0.5, - composite_score, - percentile_rank: 0.0, - sector: asset.sector.clone(), - market_cap_rank: 0, - is_selected: false, - selection_confidence: composite_score.min(1.0), - }) - } - - /// Apply selection criteria to ranked assets - fn apply_selection_criteria( - &self, - rankings: &[AssetRanking], - ) -> Result, MLError> { - let mut selected = Vec::new(); - let mut sector_counts: HashMap = HashMap::new(); - - for ranking in rankings { - // Check liquidity threshold - if ranking.selection_confidence < self.criteria.min_liquidity_score { - continue; - } - - // Check sector limits - let sector_count = sector_counts.get(&ranking.sector).cloned().unwrap_or(0); - if let Some(&limit) = self.criteria.sector_limits.get(&ranking.sector) { - if sector_count >= limit { - continue; - } - } - - // Check max assets limit - if selected.len() >= self.criteria.max_assets { - break; - } - - let mut selected_ranking = ranking.clone(); - selected_ranking.is_selected = true; - selected.push(selected_ranking); - - *sector_counts.entry(ranking.sector.clone()).or_insert(0) += 1; - } - - Ok(selected) - } - - /// Calculate performance metrics - fn calculate_performance_metrics(&self) -> Result { - let universe_assets: Vec<&AssetRanking> = self.current_universe.values().collect(); - - let total_universe_size = universe_assets.len(); - - let avg_liquidity_score = if !universe_assets.is_empty() { - universe_assets - .iter() - .map(|a| a.selection_confidence) - .sum::() - / total_universe_size as f64 - } else { - 0.0 - }; - - // Simplified metrics - in production would be more comprehensive - Ok(UniversePerformanceMetrics { - total_universe_size, - avg_liquidity_score, - avg_volatility: 0.15, // Production - sector_diversification: self.calculate_sector_diversification(&universe_assets), - correlation_efficiency: 0.7, // Production - turnover_rate: 0.1, // Production - selection_latency_ms: 50, // Production - }) - } - - /// Calculate sector diversification score - fn calculate_sector_diversification(&self, assets: &[&AssetRanking]) -> f64 { - if assets.is_empty() { - return 0.0; - } - - let mut sector_counts: HashMap = HashMap::new(); - for asset in assets { - *sector_counts.entry(asset.sector.clone()).or_insert(0) += 1; - } - - let _num_sectors = sector_counts.len() as f64; - let total_assets = assets.len() as f64; - - // Calculate Herfindahl-Hirschman Index for diversification - let hhi: f64 = sector_counts - .values() - .map(|&count| { - let proportion = count as f64 / total_assets; - proportion * proportion - }) - .sum(); - - // Convert to diversification score (lower HHI = higher diversification) - (1.0 - hhi).max(0.0) - } - - /// Get current universe - pub fn get_current_universe(&self) -> &HashMap { - &self.current_universe - } - - /// Get performance metrics - pub fn get_performance_metrics(&self) -> Option<&UniversePerformanceMetrics> { - self.performance_history.last() - } - - /// Update liquidity scores for all tracked assets - pub async fn update_liquidity_scores(&mut self) -> Result<(), MLError> { - // Production implementation - updates liquidity scores in background - tracing::debug!( - "Updating liquidity scores for {} assets", - self.current_universe.len() - ); - Ok(()) - } - - /// Update momentum rankings for all tracked assets - pub async fn update_momentum_rankings(&mut self) -> Result<(), MLError> { - // Production implementation - updates momentum rankings in background - tracing::debug!( - "Updating momentum rankings for {} assets", - self.current_universe.len() - ); - Ok(()) - } - - /// Update volatility clustering for all tracked assets - pub async fn update_volatility_clustering(&mut self) -> Result<(), MLError> { - // Production implementation - updates volatility clusters in background - tracing::debug!( - "Updating volatility clustering for {} assets", - self.current_universe.len() - ); - Ok(()) - } - - /// Generate a universe update event - pub fn generate_universe_update(&self) -> Result { - let added_assets: Vec = self.current_universe.keys().cloned().collect(); - let removed_assets = Vec::new(); // Would be populated with rejected assets - let ranking_changes = self.current_universe.clone(); - - Ok(UniverseUpdate { - added_assets, - removed_assets, - timestamp: SystemTime::now(), - ranking_changes, - selection_criteria_used: self.criteria.clone(), - update_reason: UniverseUpdateReason::Scheduled, - performance_metrics: self.performance_history.last().cloned().unwrap_or_default(), - }) - } -} +//! Universe Selection — thin facade re-exporting from ml-universe crate. +pub use ml_universe::*; diff --git a/crates/ml/src/universe/momentum.rs b/crates/ml/src/universe/momentum.rs deleted file mode 100644 index dc4ed7805..000000000 --- a/crates/ml/src/universe/momentum.rs +++ /dev/null @@ -1,61 +0,0 @@ -//! # Momentum Ranking Module -//! -//! ML-based momentum analysis for universe selection. -//! Implements multi-timeframe momentum scoring, cross-sectional ranking, and momentum regime detection. - -// Price imported from crate root (lib.rs) - -// DISABLED: Tests require MomentumRanker, Price, Volume, Utc, Duration types not available -// #[cfg(test)] -// mod tests { -// use super::*; -// use serde::{Serialize, Deserialize}; -// -// #[test] -// fn test_momentum_scoring() { -// let config = MomentumConfig::default(); -// let mut ranker = MomentumRanker::new(config)?; -// let asset_id = "TEST_ASSET".to_owned(); -// let data = create_test_price_data(); -// let score = ranker.score_momentum(asset_id, &data)?; -// assert!(score.overall_score >= 0.0 && score.overall_score <= 1.0); -// assert!(score.cross_sectional_rank >= 0.0 && score.cross_sectional_rank <= 1.0); -// } -// -// /// Price data structure for momentum analysis -// #[derive(Debug, Clone, Serialize, Deserialize)] -// struct PriceData { -// /// Timestamp in nanoseconds -// pub timestamp: u64, -// /// Current price -// pub price: Price, -// /// Trading volume -// pub volume: Volume, -// /// High price for the period -// pub high: Price, -// /// Low price for the period -// pub low: Price, -// /// Volume-weighted average price -// pub vwap: Price, -// } -// -// fn create_test_price_data() -> Vec { -// let start_price = 10000_u64; // $100.00 -// (0..100) -// .map(|i| { -// let trend = (i as f64 * 0.01).sin() * 0.02; // 2% volatility with trend -// let price = (start_price as f64 * (1.0 + trend)) as u64; -// PriceData { -// timestamp: (Utc::now() - Duration::days(100 - i)) -// .timestamp_nanos_opt() -// .unwrap_or(0) as u64, -// price: Price::from_f64(price as f64 / 100.0).unwrap_or_default(), -// volume: Volume::new((100_000 + i * 1000) as f64).unwrap_or_default(), -// high: Price::from_f64((price + 50) as f64 / 100.0).unwrap_or_default(), -// low: Price::from_f64((price - 50) as f64 / 100.0).unwrap_or_default(), -// vwap: Price::from_f64(price as f64 / 100.0).unwrap_or_default(), -// } -// }) -// .collect() -// } -// }