From 3a60f303e3d3922fc1ac6834fa1fb0a00b65ed5c Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 23 Feb 2026 14:36:36 +0100 Subject: [PATCH 1/7] =?UTF-8?q?refactor(ml):=20unify=20AssetClass=20?= =?UTF-8?q?=E2=80=94=20re-export=20from=20asset=5Fselection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove duplicate AssetClass enum from data_pipeline module and replace with a re-export from asset_selection, which is now the canonical location. This ensures data_pipeline::AssetClass and asset_selection::AssetClass are the same type, enabling direct comparison and preventing subtle type mismatch bugs. Co-Authored-By: Claude Opus 4.6 --- ml/src/data_pipeline/mod.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/ml/src/data_pipeline/mod.rs b/ml/src/data_pipeline/mod.rs index 78fbcdd3f..fcce3f44f 100644 --- a/ml/src/data_pipeline/mod.rs +++ b/ml/src/data_pipeline/mod.rs @@ -15,13 +15,8 @@ pub use prepared::{Batch, BatchIterator, PreparedDataset}; use chrono::NaiveDate; use serde::{Deserialize, Serialize}; -/// Asset class categorization -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum AssetClass { - Equity, - ETF, - Future, -} +// Re-export AssetClass from asset_selection (canonical location) +pub use crate::asset_selection::AssetClass; /// Bar size / time resolution #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -265,4 +260,11 @@ mod tests { let json = serde_json::to_string(&AssetClass::Equity).unwrap_or_default(); assert!(json.contains("Equity")); } + + #[test] + fn test_asset_class_is_same_type() { + let dp_future = AssetClass::Future; + let as_future = crate::asset_selection::AssetClass::Future; + assert_eq!(dp_future, as_future); + } } From 8ce2d53d213d0587b3f7a52141b3eeb02c070794 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 23 Feb 2026 14:41:22 +0100 Subject: [PATCH 2/7] feat(ml): add trading_symbol field to UniverseAsset for micro contract mapping Co-Authored-By: Claude Opus 4.6 --- ml/src/asset_selection/mod.rs | 51 +++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/ml/src/asset_selection/mod.rs b/ml/src/asset_selection/mod.rs index d3438bd4a..509e702bb 100644 --- a/ml/src/asset_selection/mod.rs +++ b/ml/src/asset_selection/mod.rs @@ -25,9 +25,12 @@ pub enum AssetClass { /// A single asset in the trading universe. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct UniverseAsset { - /// Ticker symbol (e.g. "AAPL"). + /// Ticker symbol (e.g. "ES.FUT") -- used for data/training. pub symbol: String, - /// MIC exchange code (e.g. "XNAS"). + /// Execution symbol (e.g. "MES.FUT") -- used for order routing. + /// When `None`, orders are routed to `symbol` directly. + pub trading_symbol: Option, + /// MIC exchange code (e.g. "GLBX.MDP3"). pub exchange: String, /// Asset classification. pub asset_class: AssetClass, @@ -39,6 +42,15 @@ pub struct UniverseAsset { pub enabled: bool, } +impl UniverseAsset { + /// The symbol to use for order execution. + /// Returns `trading_symbol` if set, otherwise `symbol`. + #[must_use] + pub fn execution_symbol(&self) -> &str { + self.trading_symbol.as_deref().unwrap_or(&self.symbol) + } +} + /// The full set of assets available for trading. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AssetUniverse { @@ -84,6 +96,7 @@ impl AssetUniverse { assets: vec![ UniverseAsset { symbol: "SPY".to_string(), + trading_symbol: None, exchange: "XNYS".to_string(), asset_class: AssetClass::ETF, sector: None, @@ -92,6 +105,7 @@ impl AssetUniverse { }, UniverseAsset { symbol: "QQQ".to_string(), + trading_symbol: None, exchange: "XNAS".to_string(), asset_class: AssetClass::ETF, sector: None, @@ -100,6 +114,7 @@ impl AssetUniverse { }, UniverseAsset { symbol: "AAPL".to_string(), + trading_symbol: None, exchange: "XNAS".to_string(), asset_class: AssetClass::Equity, sector: Some("Technology".to_string()), @@ -108,6 +123,7 @@ impl AssetUniverse { }, UniverseAsset { symbol: "MSFT".to_string(), + trading_symbol: None, exchange: "XNAS".to_string(), asset_class: AssetClass::Equity, sector: Some("Technology".to_string()), @@ -116,6 +132,7 @@ impl AssetUniverse { }, UniverseAsset { symbol: "NVDA".to_string(), + trading_symbol: None, exchange: "XNAS".to_string(), asset_class: AssetClass::Equity, sector: Some("Technology".to_string()), @@ -124,6 +141,7 @@ impl AssetUniverse { }, UniverseAsset { symbol: "AMZN".to_string(), + trading_symbol: None, exchange: "XNAS".to_string(), asset_class: AssetClass::Equity, sector: Some("Consumer".to_string()), @@ -132,6 +150,7 @@ impl AssetUniverse { }, UniverseAsset { symbol: "IWM".to_string(), + trading_symbol: None, exchange: "XNYS".to_string(), asset_class: AssetClass::ETF, sector: None, @@ -209,4 +228,32 @@ mod tests { serde_json::from_str(&json).unwrap_or_else(|_| AssetUniverse::new()); assert_eq!(restored.assets.len(), 7); } + + #[test] + fn test_trading_symbol_mapping() { + let asset = UniverseAsset { + symbol: "ES.FUT".to_string(), + trading_symbol: Some("MES.FUT".to_string()), + exchange: "GLBX.MDP3".to_string(), + asset_class: AssetClass::Future, + sector: None, + min_daily_volume: 1_000_000.0, + enabled: true, + }; + assert_eq!(asset.execution_symbol(), "MES.FUT"); + } + + #[test] + fn test_trading_symbol_none_falls_back() { + let asset = UniverseAsset { + symbol: "ZN.FUT".to_string(), + trading_symbol: None, + exchange: "GLBX.MDP3".to_string(), + asset_class: AssetClass::Future, + sector: None, + min_daily_volume: 500_000.0, + enabled: true, + }; + assert_eq!(asset.execution_symbol(), "ZN.FUT"); + } } From 6d844c009a6608beea357321758d55975b48ba4a Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 23 Feb 2026 14:50:40 +0100 Subject: [PATCH 3/7] feat(ml): add AssetUniverse::from_config() for TOML config loading Co-Authored-By: Claude Opus 4.6 --- ml/Cargo.toml | 1 + ml/src/asset_selection/mod.rs | 250 +++++++++++++++++++++++++++++++++- 2 files changed, 250 insertions(+), 1 deletion(-) diff --git a/ml/Cargo.toml b/ml/Cargo.toml index 5f20f090a..b82c19b65 100644 --- a/ml/Cargo.toml +++ b/ml/Cargo.toml @@ -47,6 +47,7 @@ clap.workspace = true # CLI argument parsing for train_tft binary serde.workspace = true serde_json.workspace = true serde_yaml = "0.9" # YAML serialization for hyperparameter exports +toml.workspace = true uuid.workspace = true thiserror.workspace = true anyhow.workspace = true diff --git a/ml/src/asset_selection/mod.rs b/ml/src/asset_selection/mod.rs index 509e702bb..c1ceef283 100644 --- a/ml/src/asset_selection/mod.rs +++ b/ml/src/asset_selection/mod.rs @@ -3,8 +3,12 @@ //! This module defines which assets are eligible for trading, including //! their classification, exchange, sector, and minimum volume requirements. +use std::path::Path; + use serde::{Deserialize, Serialize}; +use crate::MLError; + pub mod scorer; pub mod selector; @@ -58,6 +62,34 @@ pub struct AssetUniverse { pub assets: Vec, } +// --------------------------------------------------------------------------- +// TOML config deserialization +// --------------------------------------------------------------------------- + +#[derive(Deserialize)] +struct UniverseConfigFile { + #[allow(dead_code)] + universe: UniverseConfigMeta, + symbols: Vec, +} + +#[derive(Deserialize)] +struct UniverseConfigMeta { + #[allow(dead_code)] + name: String, + #[allow(dead_code)] + description: Option, +} + +#[derive(Deserialize)] +struct SymbolConfigEntry { + symbol: String, + trading_symbol: Option, + exchange: String, + asset_class: String, + min_daily_volume: Option, +} + impl AssetUniverse { /// Create an empty universe. #[must_use] @@ -65,6 +97,49 @@ impl AssetUniverse { Self { assets: Vec::new() } } + /// Load a universe from a TOML config file. + /// + /// # Errors + /// Returns `MLError::ConfigError` on I/O or parse failures. + pub fn from_config(path: &Path) -> Result { + let contents = std::fs::read_to_string(path).map_err(|e| MLError::ConfigError { + reason: format!("Failed to read universe config {}: {e}", path.display()), + })?; + + let config: UniverseConfigFile = + toml::from_str(&contents).map_err(|e| MLError::ConfigError { + reason: format!("Failed to parse universe config {}: {e}", path.display()), + })?; + + let assets = config + .symbols + .into_iter() + .map(|s| { + let asset_class = match s.asset_class.as_str() { + "Future" | "future" => AssetClass::Future, + "Equity" | "equity" => AssetClass::Equity, + "ETF" | "etf" => AssetClass::ETF, + other => { + return Err(MLError::ConfigError { + reason: format!("Unknown asset class: {other}"), + }) + } + }; + Ok(UniverseAsset { + symbol: s.symbol, + trading_symbol: s.trading_symbol, + exchange: s.exchange, + asset_class, + sector: None, + min_daily_volume: s.min_daily_volume.unwrap_or(0.0), + enabled: true, + }) + }) + .collect::, _>>()?; + + Ok(Self { assets }) + } + /// Return only enabled assets. #[must_use] pub fn eligible(&self) -> Vec<&UniverseAsset> { @@ -89,7 +164,7 @@ impl AssetUniverse { self.assets.iter().map(|a| a.symbol.as_str()).collect() } - /// Preset: US large-cap starter universe (7 highly liquid assets). + /// Preset: US large-cap starter universe (7 highly liquid equities/ETFs). #[must_use] pub fn us_starter() -> Self { Self { @@ -160,6 +235,54 @@ impl AssetUniverse { ], } } + + /// Preset: CME futures baseline universe (4 assets for limited capital). + /// + /// Training is done on full-size contracts (ES, NQ, ZN, 6E). + /// Execution routes to micro contracts where available (MES, MNQ, M6E). + #[must_use] + pub fn futures_baseline() -> Self { + Self { + assets: vec![ + UniverseAsset { + symbol: "ES.FUT".to_string(), + trading_symbol: Some("MES.FUT".to_string()), + exchange: "GLBX.MDP3".to_string(), + asset_class: AssetClass::Future, + sector: None, + min_daily_volume: 1_000_000.0, + enabled: true, + }, + UniverseAsset { + symbol: "NQ.FUT".to_string(), + trading_symbol: Some("MNQ.FUT".to_string()), + exchange: "GLBX.MDP3".to_string(), + asset_class: AssetClass::Future, + sector: None, + min_daily_volume: 500_000.0, + enabled: true, + }, + UniverseAsset { + symbol: "ZN.FUT".to_string(), + trading_symbol: None, + exchange: "GLBX.MDP3".to_string(), + asset_class: AssetClass::Future, + sector: None, + min_daily_volume: 500_000.0, + enabled: true, + }, + UniverseAsset { + symbol: "6E.FUT".to_string(), + trading_symbol: Some("M6E.FUT".to_string()), + exchange: "GLBX.MDP3".to_string(), + asset_class: AssetClass::Future, + sector: None, + min_daily_volume: 200_000.0, + enabled: true, + }, + ], + } + } } impl Default for AssetUniverse { @@ -256,4 +379,129 @@ mod tests { }; assert_eq!(asset.execution_symbol(), "ZN.FUT"); } + + #[test] + fn test_futures_baseline_universe() { + let universe = AssetUniverse::futures_baseline(); + assert_eq!(universe.assets.len(), 4); + assert_eq!(universe.eligible_count(), 4); + + // All are futures + for asset in &universe.assets { + assert_eq!(asset.asset_class, AssetClass::Future); + } + } + + #[test] + fn test_futures_baseline_symbols() { + let universe = AssetUniverse::futures_baseline(); + let symbols = universe.symbols(); + assert!(symbols.contains(&"ES.FUT")); + assert!(symbols.contains(&"NQ.FUT")); + assert!(symbols.contains(&"ZN.FUT")); + assert!(symbols.contains(&"6E.FUT")); + } + + #[test] + fn test_futures_baseline_micro_mapping() { + let universe = AssetUniverse::futures_baseline(); + let es = universe.find("ES.FUT"); + assert!(es.is_some()); + assert_eq!( + es.unwrap_or_else(|| panic!("ES.FUT not found")) + .execution_symbol(), + "MES.FUT" + ); + + let nq = universe.find("NQ.FUT"); + assert!(nq.is_some()); + assert_eq!( + nq.unwrap_or_else(|| panic!("NQ.FUT not found")) + .execution_symbol(), + "MNQ.FUT" + ); + + let zn = universe.find("ZN.FUT"); + assert!(zn.is_some()); + // ZN has no micro -- executes as ZN.FUT + assert_eq!( + zn.unwrap_or_else(|| panic!("ZN.FUT not found")) + .execution_symbol(), + "ZN.FUT" + ); + + let six_e = universe.find("6E.FUT"); + assert!(six_e.is_some()); + assert_eq!( + six_e + .unwrap_or_else(|| panic!("6E.FUT not found")) + .execution_symbol(), + "M6E.FUT" + ); + } + + #[test] + fn test_futures_baseline_serde_roundtrip() { + let universe = AssetUniverse::futures_baseline(); + let json = serde_json::to_string(&universe).unwrap_or_default(); + let restored: AssetUniverse = + serde_json::from_str(&json).unwrap_or_else(|_| AssetUniverse::new()); + assert_eq!(restored.assets.len(), 4); + // Verify trading_symbol survives roundtrip + let es = restored.find("ES.FUT"); + assert_eq!(es.map(|a| a.execution_symbol()), Some("MES.FUT")); + } + + #[test] + fn test_from_config_toml() { + let toml_content = r#" +[universe] +name = "test-universe" +description = "Test" + +[[symbols]] +symbol = "ES.FUT" +trading_symbol = "MES.FUT" +exchange = "GLBX.MDP3" +asset_class = "Future" +min_daily_volume = 1000000.0 + +[[symbols]] +symbol = "ZN.FUT" +exchange = "GLBX.MDP3" +asset_class = "Future" +min_daily_volume = 500000.0 +"#; + let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("tmpdir: {e}")); + let path = dir.path().join("universe.toml"); + std::fs::write(&path, toml_content).unwrap_or_else(|e| panic!("write: {e}")); + + let universe = AssetUniverse::from_config(&path); + assert!(universe.is_ok(), "from_config failed: {:?}", universe.err()); + let universe = universe.unwrap_or_else(|_| AssetUniverse::new()); + assert_eq!(universe.assets.len(), 2); + assert_eq!( + universe.find("ES.FUT").map(|a| a.execution_symbol()), + Some("MES.FUT") + ); + assert_eq!( + universe.find("ZN.FUT").map(|a| a.trading_symbol.as_deref()), + Some(None) + ); + } + + #[test] + fn test_from_config_missing_file() { + let result = AssetUniverse::from_config(std::path::Path::new("/nonexistent/path.toml")); + assert!(result.is_err()); + } + + #[test] + fn test_from_config_invalid_toml() { + let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("tmpdir: {e}")); + let path = dir.path().join("bad.toml"); + std::fs::write(&path, "not valid toml {{{{").unwrap_or_else(|e| panic!("write: {e}")); + let result = AssetUniverse::from_config(&path); + assert!(result.is_err()); + } } From 9d8622f41084e173a6387bcc71b96b9476829dec Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 23 Feb 2026 14:57:48 +0100 Subject: [PATCH 4/7] feat: add futures-baseline universe config (TOML) Co-Authored-By: Claude Opus 4.6 --- config/universe-futures-baseline.toml | 43 +++++++++++++++++++++++++++ ml/src/asset_selection/mod.rs | 27 +++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 config/universe-futures-baseline.toml diff --git a/config/universe-futures-baseline.toml b/config/universe-futures-baseline.toml new file mode 100644 index 000000000..a1d26223e --- /dev/null +++ b/config/universe-futures-baseline.toml @@ -0,0 +1,43 @@ +# Futures Baseline Universe +# +# 4 CME futures for initial trading with limited capital. +# Train on full-size contracts, execute on micro versions. +# Data window: 730 days (March 2024 → February 2026) +# +# Schema: OHLCV-1m via Databento (GLBX.MDP3 dataset) + +[universe] +name = "futures-baseline" +description = "CME futures baseline: 4 symbols, 730 days, OHLCV-1m" +date_range_start = "2024-03-01" +date_range_end = "2026-02-23" +bar_size = "1m" +databento_dataset = "GLBX.MDP3" +databento_schema = "ohlcv-1m" + +[[symbols]] +symbol = "ES.FUT" +trading_symbol = "MES.FUT" +exchange = "GLBX.MDP3" +asset_class = "Future" +min_daily_volume = 1000000.0 + +[[symbols]] +symbol = "NQ.FUT" +trading_symbol = "MNQ.FUT" +exchange = "GLBX.MDP3" +asset_class = "Future" +min_daily_volume = 500000.0 + +[[symbols]] +symbol = "ZN.FUT" +exchange = "GLBX.MDP3" +asset_class = "Future" +min_daily_volume = 500000.0 + +[[symbols]] +symbol = "6E.FUT" +trading_symbol = "M6E.FUT" +exchange = "GLBX.MDP3" +asset_class = "Future" +min_daily_volume = 200000.0 diff --git a/ml/src/asset_selection/mod.rs b/ml/src/asset_selection/mod.rs index c1ceef283..02d16ea4f 100644 --- a/ml/src/asset_selection/mod.rs +++ b/ml/src/asset_selection/mod.rs @@ -79,6 +79,16 @@ struct UniverseConfigMeta { name: String, #[allow(dead_code)] description: Option, + #[allow(dead_code)] + date_range_start: Option, + #[allow(dead_code)] + date_range_end: Option, + #[allow(dead_code)] + bar_size: Option, + #[allow(dead_code)] + databento_dataset: Option, + #[allow(dead_code)] + databento_schema: Option, } #[derive(Deserialize)] @@ -490,6 +500,23 @@ min_daily_volume = 500000.0 ); } + #[test] + fn test_load_futures_baseline_config() { + let config_path = std::path::Path::new("../config/universe-futures-baseline.toml"); + if !config_path.exists() { + // Skip in CI where config may not be at expected relative path + return; + } + let universe = AssetUniverse::from_config(config_path); + assert!(universe.is_ok(), "Failed to load config: {:?}", universe.err()); + let universe = universe.unwrap_or_else(|_| AssetUniverse::new()); + assert_eq!(universe.assets.len(), 4); + assert_eq!( + universe.find("ES.FUT").map(|a| a.execution_symbol()), + Some("MES.FUT") + ); + } + #[test] fn test_from_config_missing_file() { let result = AssetUniverse::from_config(std::path::Path::new("/nonexistent/path.toml")); From 7e96d6e303f617da7d91a7d948f902475a4815e8 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 23 Feb 2026 15:00:26 +0100 Subject: [PATCH 5/7] chore: add data/cache/ to .gitignore (downloaded market data) Co-Authored-By: Claude Opus 4.6 --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 6e65cca88..84e73a858 100644 --- a/.gitignore +++ b/.gitignore @@ -146,3 +146,6 @@ coordination/orchestration/* claude-flow # Removed Windows wrapper files per user request hive-mind-prompt-*.txt + +# Data cache (downloaded market data, not checked in) +data/cache/ From 323b77c82099130887c8ef8f710220626b424964 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 23 Feb 2026 15:05:10 +0100 Subject: [PATCH 6/7] =?UTF-8?q?feat(ml):=20add=20manifest=20generation=20t?= =?UTF-8?q?est=20=E2=80=94=20builds=20cache=20manifest=20from=20on-disk=20?= =?UTF-8?q?DBN=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.6 --- ml/src/data_pipeline/cache.rs | 86 ++++++++++++++++++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/ml/src/data_pipeline/cache.rs b/ml/src/data_pipeline/cache.rs index 08f4024cc..8205237bc 100644 --- a/ml/src/data_pipeline/cache.rs +++ b/ml/src/data_pipeline/cache.rs @@ -404,7 +404,91 @@ mod tests { } // ----------------------------------------------------------------------- - // 7. add_range keeps ranges sorted by start date + // 7. Build manifest from on-disk DBN files + // ----------------------------------------------------------------------- + #[test] + fn test_build_manifest_from_disk() { + use std::path::Path; + + let cache_dir = Path::new("../data/cache"); + if !cache_dir.exists() { + // Skip if data hasn't been moved yet + return; + } + + let mut manifest = CacheManifest::new(); + + // Scan each symbol directory + for symbol in &["ES.FUT", "NQ.FUT", "ZN.FUT", "6E.FUT"] { + let symbol_dir = cache_dir.join(symbol); + if !symbol_dir.exists() { + continue; + } + + let mut files: Vec<_> = std::fs::read_dir(&symbol_dir) + .into_iter() + .flatten() + .filter_map(|e| e.ok()) + .filter(|e| { + e.path() + .extension() + .map(|ext| ext == "dbn") + .unwrap_or(false) + }) + .collect(); + files.sort_by_key(|e| e.file_name()); + + for entry in &files { + let fname = entry.file_name(); + let fname_str = fname.to_string_lossy(); + // Parse date from filename: ohlcv-1m_2024-01-02.dbn + if let Some(date_str) = fname_str + .strip_prefix("ohlcv-1m_") + .and_then(|s| s.strip_suffix(".dbn")) + { + if let Ok(parsed_date) = + NaiveDate::parse_from_str(date_str, "%Y-%m-%d") + { + manifest.add_range( + symbol, + "GLBX.MDP3", + BarSize::OneMinute, + CachedRange { + start: parsed_date, + end: parsed_date, + bar_count: 390, // Approximate for 1m bars + file_path: entry + .path() + .strip_prefix(cache_dir) + .unwrap_or(&entry.path()) + .to_path_buf(), + cached_at: Utc::now(), + }, + ); + } + } + } + } + + // Verify we found data + assert!( + !manifest.symbols.is_empty(), + "No symbols found in cache" + ); + + // Save the manifest + let result = manifest.save(cache_dir); + assert!(result.is_ok(), "Failed to save manifest: {:?}", result.err()); + + // Verify it round-trips + let loaded = CacheManifest::load(cache_dir); + assert!(loaded.is_ok()); + let loaded = loaded.unwrap_or_default(); + assert_eq!(loaded.symbols.len(), manifest.symbols.len()); + } + + // ----------------------------------------------------------------------- + // 8. add_range keeps ranges sorted by start date // ----------------------------------------------------------------------- #[test] fn add_range_sorts_by_start() { From 236e1665cff4f74bd5483bc5ef597f3c4df3eb34 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 23 Feb 2026 15:17:58 +0100 Subject: [PATCH 7/7] fix(ml): address code review findings - Remove panic!() calls from test_futures_baseline_micro_mapping, use map()+Some() pattern consistent with rest of test suite - Make DatasetSpec::from_universe() accept a name parameter instead of hardcoding "futures-baseline" - Add doc comment to UniverseConfigMeta explaining dead_code fields Co-Authored-By: Claude Opus 4.6 --- ml/src/asset_selection/mod.rs | 38 ++++++----------- ml/src/data_pipeline/mod.rs | 77 +++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 25 deletions(-) diff --git a/ml/src/asset_selection/mod.rs b/ml/src/asset_selection/mod.rs index 02d16ea4f..8abebc510 100644 --- a/ml/src/asset_selection/mod.rs +++ b/ml/src/asset_selection/mod.rs @@ -73,6 +73,9 @@ struct UniverseConfigFile { symbols: Vec, } +/// Parsed metadata from the `[universe]` section of the TOML config. +/// Fields are currently used only for validation/documentation during parsing; +/// they will feed into `DatasetSpec` construction once download automation lands. #[derive(Deserialize)] struct UniverseConfigMeta { #[allow(dead_code)] @@ -415,38 +418,23 @@ mod tests { #[test] fn test_futures_baseline_micro_mapping() { let universe = AssetUniverse::futures_baseline(); - let es = universe.find("ES.FUT"); - assert!(es.is_some()); - assert_eq!( - es.unwrap_or_else(|| panic!("ES.FUT not found")) - .execution_symbol(), - "MES.FUT" - ); - let nq = universe.find("NQ.FUT"); - assert!(nq.is_some()); assert_eq!( - nq.unwrap_or_else(|| panic!("NQ.FUT not found")) - .execution_symbol(), - "MNQ.FUT" + universe.find("ES.FUT").map(|a| a.execution_symbol()), + Some("MES.FUT") + ); + assert_eq!( + universe.find("NQ.FUT").map(|a| a.execution_symbol()), + Some("MNQ.FUT") ); - - let zn = universe.find("ZN.FUT"); - assert!(zn.is_some()); // ZN has no micro -- executes as ZN.FUT assert_eq!( - zn.unwrap_or_else(|| panic!("ZN.FUT not found")) - .execution_symbol(), - "ZN.FUT" + universe.find("ZN.FUT").map(|a| a.execution_symbol()), + Some("ZN.FUT") ); - - let six_e = universe.find("6E.FUT"); - assert!(six_e.is_some()); assert_eq!( - six_e - .unwrap_or_else(|| panic!("6E.FUT not found")) - .execution_symbol(), - "M6E.FUT" + universe.find("6E.FUT").map(|a| a.execution_symbol()), + Some("M6E.FUT") ); } diff --git a/ml/src/data_pipeline/mod.rs b/ml/src/data_pipeline/mod.rs index fcce3f44f..4288b2228 100644 --- a/ml/src/data_pipeline/mod.rs +++ b/ml/src/data_pipeline/mod.rs @@ -166,6 +166,35 @@ impl DatasetSpec { } } + /// Create a spec from an asset universe and dataset mode. + /// + /// Only includes enabled assets from the universe. + #[must_use] + pub fn from_universe( + name: &str, + universe: &crate::asset_selection::AssetUniverse, + mode: DatasetMode, + ) -> Self { + let symbols = universe + .eligible() + .iter() + .map(|a| SymbolSpec { + symbol: a.symbol.clone(), + exchange: a.exchange.clone(), + asset_class: a.asset_class.clone(), + }) + .collect(); + + Self { + name: format!("{name}-{mode:?}"), + symbols, + date_range: mode.to_date_range(), + bar_size: BarSize::OneMinute, + split_ratio: SplitRatio::default(), + warmup_bars: 50, + } + } + /// Estimate total bars across all symbols pub fn estimated_total_bars(&self) -> usize { let trading_days = (self.date_range.calendar_days() as f64 * 252.0 / 365.0) as usize; @@ -267,4 +296,52 @@ mod tests { let as_future = crate::asset_selection::AssetClass::Future; assert_eq!(dp_future, as_future); } + + #[test] + fn test_dataset_spec_from_universe() { + let universe = crate::asset_selection::AssetUniverse::futures_baseline(); + let spec = DatasetSpec::from_universe("futures-baseline", &universe, DatasetMode::Dev); + + assert_eq!(spec.symbols.len(), 4); + assert!(spec.name.contains("Dev")); + assert_eq!(spec.bar_size, BarSize::OneMinute); + assert!(spec.split_ratio.is_valid()); + + // Verify symbols match universe + let symbol_names: Vec<&str> = spec.symbols.iter().map(|s| s.symbol.as_str()).collect(); + assert!(symbol_names.contains(&"ES.FUT")); + assert!(symbol_names.contains(&"NQ.FUT")); + assert!(symbol_names.contains(&"ZN.FUT")); + assert!(symbol_names.contains(&"6E.FUT")); + + // All should be Future class + for sym in &spec.symbols { + assert_eq!(sym.asset_class, AssetClass::Future); + } + } + + #[test] + fn test_dataset_spec_from_universe_modes() { + let universe = crate::asset_selection::AssetUniverse::futures_baseline(); + + let dev = DatasetSpec::from_universe("futures-baseline", &universe, DatasetMode::Dev); + let backtest = DatasetSpec::from_universe("futures-baseline", &universe, DatasetMode::Backtest); + let full = DatasetSpec::from_universe("futures-baseline", &universe, DatasetMode::Full); + + // Dev < Backtest < Full in estimated bars + assert!(dev.estimated_total_bars() < backtest.estimated_total_bars()); + assert!(backtest.estimated_total_bars() < full.estimated_total_bars()); + } + + #[test] + fn test_dataset_spec_from_universe_disabled_filtered() { + let mut universe = crate::asset_selection::AssetUniverse::futures_baseline(); + // Disable one asset + if let Some(first) = universe.assets.get_mut(0) { + first.enabled = false; + } + let spec = DatasetSpec::from_universe("futures-baseline", &universe, DatasetMode::Dev); + // Only 3 symbols because one was disabled + assert_eq!(spec.symbols.len(), 3); + } }