diff --git a/.gitignore b/.gitignore index 6e65cca88..84e73a858 100644 --- a/.gitignore +++ b/.gitignore @@ -146,3 +146,6 @@ coordination/orchestration/* claude-flow # Removed Windows wrapper files per user request hive-mind-prompt-*.txt + +# Data cache (downloaded market data, not checked in) +data/cache/ diff --git a/config/universe-futures-baseline.toml b/config/universe-futures-baseline.toml new file mode 100644 index 000000000..a1d26223e --- /dev/null +++ b/config/universe-futures-baseline.toml @@ -0,0 +1,43 @@ +# Futures Baseline Universe +# +# 4 CME futures for initial trading with limited capital. +# Train on full-size contracts, execute on micro versions. +# Data window: 730 days (March 2024 → February 2026) +# +# Schema: OHLCV-1m via Databento (GLBX.MDP3 dataset) + +[universe] +name = "futures-baseline" +description = "CME futures baseline: 4 symbols, 730 days, OHLCV-1m" +date_range_start = "2024-03-01" +date_range_end = "2026-02-23" +bar_size = "1m" +databento_dataset = "GLBX.MDP3" +databento_schema = "ohlcv-1m" + +[[symbols]] +symbol = "ES.FUT" +trading_symbol = "MES.FUT" +exchange = "GLBX.MDP3" +asset_class = "Future" +min_daily_volume = 1000000.0 + +[[symbols]] +symbol = "NQ.FUT" +trading_symbol = "MNQ.FUT" +exchange = "GLBX.MDP3" +asset_class = "Future" +min_daily_volume = 500000.0 + +[[symbols]] +symbol = "ZN.FUT" +exchange = "GLBX.MDP3" +asset_class = "Future" +min_daily_volume = 500000.0 + +[[symbols]] +symbol = "6E.FUT" +trading_symbol = "M6E.FUT" +exchange = "GLBX.MDP3" +asset_class = "Future" +min_daily_volume = 200000.0 diff --git a/ml/Cargo.toml b/ml/Cargo.toml index 5f20f090a..b82c19b65 100644 --- a/ml/Cargo.toml +++ b/ml/Cargo.toml @@ -47,6 +47,7 @@ clap.workspace = true # CLI argument parsing for train_tft binary serde.workspace = true serde_json.workspace = true serde_yaml = "0.9" # YAML serialization for hyperparameter exports +toml.workspace = true uuid.workspace = true thiserror.workspace = true anyhow.workspace = true diff --git a/ml/src/asset_selection/mod.rs b/ml/src/asset_selection/mod.rs index d3438bd4a..8abebc510 100644 --- a/ml/src/asset_selection/mod.rs +++ b/ml/src/asset_selection/mod.rs @@ -3,8 +3,12 @@ //! This module defines which assets are eligible for trading, including //! their classification, exchange, sector, and minimum volume requirements. +use std::path::Path; + use serde::{Deserialize, Serialize}; +use crate::MLError; + pub mod scorer; pub mod selector; @@ -25,9 +29,12 @@ pub enum AssetClass { /// A single asset in the trading universe. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct UniverseAsset { - /// Ticker symbol (e.g. "AAPL"). + /// Ticker symbol (e.g. "ES.FUT") -- used for data/training. pub symbol: String, - /// MIC exchange code (e.g. "XNAS"). + /// Execution symbol (e.g. "MES.FUT") -- used for order routing. + /// When `None`, orders are routed to `symbol` directly. + pub trading_symbol: Option, + /// MIC exchange code (e.g. "GLBX.MDP3"). pub exchange: String, /// Asset classification. pub asset_class: AssetClass, @@ -39,6 +46,15 @@ pub struct UniverseAsset { pub enabled: bool, } +impl UniverseAsset { + /// The symbol to use for order execution. + /// Returns `trading_symbol` if set, otherwise `symbol`. + #[must_use] + pub fn execution_symbol(&self) -> &str { + self.trading_symbol.as_deref().unwrap_or(&self.symbol) + } +} + /// The full set of assets available for trading. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AssetUniverse { @@ -46,6 +62,47 @@ pub struct AssetUniverse { pub assets: Vec, } +// --------------------------------------------------------------------------- +// TOML config deserialization +// --------------------------------------------------------------------------- + +#[derive(Deserialize)] +struct UniverseConfigFile { + #[allow(dead_code)] + universe: UniverseConfigMeta, + symbols: Vec, +} + +/// Parsed metadata from the `[universe]` section of the TOML config. +/// Fields are currently used only for validation/documentation during parsing; +/// they will feed into `DatasetSpec` construction once download automation lands. +#[derive(Deserialize)] +struct UniverseConfigMeta { + #[allow(dead_code)] + name: String, + #[allow(dead_code)] + description: Option, + #[allow(dead_code)] + date_range_start: Option, + #[allow(dead_code)] + date_range_end: Option, + #[allow(dead_code)] + bar_size: Option, + #[allow(dead_code)] + databento_dataset: Option, + #[allow(dead_code)] + databento_schema: Option, +} + +#[derive(Deserialize)] +struct SymbolConfigEntry { + symbol: String, + trading_symbol: Option, + exchange: String, + asset_class: String, + min_daily_volume: Option, +} + impl AssetUniverse { /// Create an empty universe. #[must_use] @@ -53,6 +110,49 @@ impl AssetUniverse { Self { assets: Vec::new() } } + /// Load a universe from a TOML config file. + /// + /// # Errors + /// Returns `MLError::ConfigError` on I/O or parse failures. + pub fn from_config(path: &Path) -> Result { + let contents = std::fs::read_to_string(path).map_err(|e| MLError::ConfigError { + reason: format!("Failed to read universe config {}: {e}", path.display()), + })?; + + let config: UniverseConfigFile = + toml::from_str(&contents).map_err(|e| MLError::ConfigError { + reason: format!("Failed to parse universe config {}: {e}", path.display()), + })?; + + let assets = config + .symbols + .into_iter() + .map(|s| { + let asset_class = match s.asset_class.as_str() { + "Future" | "future" => AssetClass::Future, + "Equity" | "equity" => AssetClass::Equity, + "ETF" | "etf" => AssetClass::ETF, + other => { + return Err(MLError::ConfigError { + reason: format!("Unknown asset class: {other}"), + }) + } + }; + Ok(UniverseAsset { + symbol: s.symbol, + trading_symbol: s.trading_symbol, + exchange: s.exchange, + asset_class, + sector: None, + min_daily_volume: s.min_daily_volume.unwrap_or(0.0), + enabled: true, + }) + }) + .collect::, _>>()?; + + Ok(Self { assets }) + } + /// Return only enabled assets. #[must_use] pub fn eligible(&self) -> Vec<&UniverseAsset> { @@ -77,13 +177,14 @@ impl AssetUniverse { self.assets.iter().map(|a| a.symbol.as_str()).collect() } - /// Preset: US large-cap starter universe (7 highly liquid assets). + /// Preset: US large-cap starter universe (7 highly liquid equities/ETFs). #[must_use] pub fn us_starter() -> Self { Self { assets: vec![ UniverseAsset { symbol: "SPY".to_string(), + trading_symbol: None, exchange: "XNYS".to_string(), asset_class: AssetClass::ETF, sector: None, @@ -92,6 +193,7 @@ impl AssetUniverse { }, UniverseAsset { symbol: "QQQ".to_string(), + trading_symbol: None, exchange: "XNAS".to_string(), asset_class: AssetClass::ETF, sector: None, @@ -100,6 +202,7 @@ impl AssetUniverse { }, UniverseAsset { symbol: "AAPL".to_string(), + trading_symbol: None, exchange: "XNAS".to_string(), asset_class: AssetClass::Equity, sector: Some("Technology".to_string()), @@ -108,6 +211,7 @@ impl AssetUniverse { }, UniverseAsset { symbol: "MSFT".to_string(), + trading_symbol: None, exchange: "XNAS".to_string(), asset_class: AssetClass::Equity, sector: Some("Technology".to_string()), @@ -116,6 +220,7 @@ impl AssetUniverse { }, UniverseAsset { symbol: "NVDA".to_string(), + trading_symbol: None, exchange: "XNAS".to_string(), asset_class: AssetClass::Equity, sector: Some("Technology".to_string()), @@ -124,6 +229,7 @@ impl AssetUniverse { }, UniverseAsset { symbol: "AMZN".to_string(), + trading_symbol: None, exchange: "XNAS".to_string(), asset_class: AssetClass::Equity, sector: Some("Consumer".to_string()), @@ -132,6 +238,7 @@ impl AssetUniverse { }, UniverseAsset { symbol: "IWM".to_string(), + trading_symbol: None, exchange: "XNYS".to_string(), asset_class: AssetClass::ETF, sector: None, @@ -141,6 +248,54 @@ impl AssetUniverse { ], } } + + /// Preset: CME futures baseline universe (4 assets for limited capital). + /// + /// Training is done on full-size contracts (ES, NQ, ZN, 6E). + /// Execution routes to micro contracts where available (MES, MNQ, M6E). + #[must_use] + pub fn futures_baseline() -> Self { + Self { + assets: vec![ + UniverseAsset { + symbol: "ES.FUT".to_string(), + trading_symbol: Some("MES.FUT".to_string()), + exchange: "GLBX.MDP3".to_string(), + asset_class: AssetClass::Future, + sector: None, + min_daily_volume: 1_000_000.0, + enabled: true, + }, + UniverseAsset { + symbol: "NQ.FUT".to_string(), + trading_symbol: Some("MNQ.FUT".to_string()), + exchange: "GLBX.MDP3".to_string(), + asset_class: AssetClass::Future, + sector: None, + min_daily_volume: 500_000.0, + enabled: true, + }, + UniverseAsset { + symbol: "ZN.FUT".to_string(), + trading_symbol: None, + exchange: "GLBX.MDP3".to_string(), + asset_class: AssetClass::Future, + sector: None, + min_daily_volume: 500_000.0, + enabled: true, + }, + UniverseAsset { + symbol: "6E.FUT".to_string(), + trading_symbol: Some("M6E.FUT".to_string()), + exchange: "GLBX.MDP3".to_string(), + asset_class: AssetClass::Future, + sector: None, + min_daily_volume: 200_000.0, + enabled: true, + }, + ], + } + } } impl Default for AssetUniverse { @@ -209,4 +364,159 @@ mod tests { serde_json::from_str(&json).unwrap_or_else(|_| AssetUniverse::new()); assert_eq!(restored.assets.len(), 7); } + + #[test] + fn test_trading_symbol_mapping() { + let asset = UniverseAsset { + symbol: "ES.FUT".to_string(), + trading_symbol: Some("MES.FUT".to_string()), + exchange: "GLBX.MDP3".to_string(), + asset_class: AssetClass::Future, + sector: None, + min_daily_volume: 1_000_000.0, + enabled: true, + }; + assert_eq!(asset.execution_symbol(), "MES.FUT"); + } + + #[test] + fn test_trading_symbol_none_falls_back() { + let asset = UniverseAsset { + symbol: "ZN.FUT".to_string(), + trading_symbol: None, + exchange: "GLBX.MDP3".to_string(), + asset_class: AssetClass::Future, + sector: None, + min_daily_volume: 500_000.0, + enabled: true, + }; + assert_eq!(asset.execution_symbol(), "ZN.FUT"); + } + + #[test] + fn test_futures_baseline_universe() { + let universe = AssetUniverse::futures_baseline(); + assert_eq!(universe.assets.len(), 4); + assert_eq!(universe.eligible_count(), 4); + + // All are futures + for asset in &universe.assets { + assert_eq!(asset.asset_class, AssetClass::Future); + } + } + + #[test] + fn test_futures_baseline_symbols() { + let universe = AssetUniverse::futures_baseline(); + let symbols = universe.symbols(); + assert!(symbols.contains(&"ES.FUT")); + assert!(symbols.contains(&"NQ.FUT")); + assert!(symbols.contains(&"ZN.FUT")); + assert!(symbols.contains(&"6E.FUT")); + } + + #[test] + fn test_futures_baseline_micro_mapping() { + let universe = AssetUniverse::futures_baseline(); + + assert_eq!( + universe.find("ES.FUT").map(|a| a.execution_symbol()), + Some("MES.FUT") + ); + assert_eq!( + universe.find("NQ.FUT").map(|a| a.execution_symbol()), + Some("MNQ.FUT") + ); + // ZN has no micro -- executes as ZN.FUT + assert_eq!( + universe.find("ZN.FUT").map(|a| a.execution_symbol()), + Some("ZN.FUT") + ); + assert_eq!( + universe.find("6E.FUT").map(|a| a.execution_symbol()), + Some("M6E.FUT") + ); + } + + #[test] + fn test_futures_baseline_serde_roundtrip() { + let universe = AssetUniverse::futures_baseline(); + let json = serde_json::to_string(&universe).unwrap_or_default(); + let restored: AssetUniverse = + serde_json::from_str(&json).unwrap_or_else(|_| AssetUniverse::new()); + assert_eq!(restored.assets.len(), 4); + // Verify trading_symbol survives roundtrip + let es = restored.find("ES.FUT"); + assert_eq!(es.map(|a| a.execution_symbol()), Some("MES.FUT")); + } + + #[test] + fn test_from_config_toml() { + let toml_content = r#" +[universe] +name = "test-universe" +description = "Test" + +[[symbols]] +symbol = "ES.FUT" +trading_symbol = "MES.FUT" +exchange = "GLBX.MDP3" +asset_class = "Future" +min_daily_volume = 1000000.0 + +[[symbols]] +symbol = "ZN.FUT" +exchange = "GLBX.MDP3" +asset_class = "Future" +min_daily_volume = 500000.0 +"#; + let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("tmpdir: {e}")); + let path = dir.path().join("universe.toml"); + std::fs::write(&path, toml_content).unwrap_or_else(|e| panic!("write: {e}")); + + let universe = AssetUniverse::from_config(&path); + assert!(universe.is_ok(), "from_config failed: {:?}", universe.err()); + let universe = universe.unwrap_or_else(|_| AssetUniverse::new()); + assert_eq!(universe.assets.len(), 2); + assert_eq!( + universe.find("ES.FUT").map(|a| a.execution_symbol()), + Some("MES.FUT") + ); + assert_eq!( + universe.find("ZN.FUT").map(|a| a.trading_symbol.as_deref()), + Some(None) + ); + } + + #[test] + fn test_load_futures_baseline_config() { + let config_path = std::path::Path::new("../config/universe-futures-baseline.toml"); + if !config_path.exists() { + // Skip in CI where config may not be at expected relative path + return; + } + let universe = AssetUniverse::from_config(config_path); + assert!(universe.is_ok(), "Failed to load config: {:?}", universe.err()); + let universe = universe.unwrap_or_else(|_| AssetUniverse::new()); + assert_eq!(universe.assets.len(), 4); + assert_eq!( + universe.find("ES.FUT").map(|a| a.execution_symbol()), + Some("MES.FUT") + ); + } + + #[test] + fn test_from_config_missing_file() { + let result = AssetUniverse::from_config(std::path::Path::new("/nonexistent/path.toml")); + assert!(result.is_err()); + } + + #[test] + fn test_from_config_invalid_toml() { + let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("tmpdir: {e}")); + let path = dir.path().join("bad.toml"); + std::fs::write(&path, "not valid toml {{{{").unwrap_or_else(|e| panic!("write: {e}")); + let result = AssetUniverse::from_config(&path); + assert!(result.is_err()); + } } diff --git a/ml/src/data_pipeline/cache.rs b/ml/src/data_pipeline/cache.rs index 08f4024cc..8205237bc 100644 --- a/ml/src/data_pipeline/cache.rs +++ b/ml/src/data_pipeline/cache.rs @@ -404,7 +404,91 @@ mod tests { } // ----------------------------------------------------------------------- - // 7. add_range keeps ranges sorted by start date + // 7. Build manifest from on-disk DBN files + // ----------------------------------------------------------------------- + #[test] + fn test_build_manifest_from_disk() { + use std::path::Path; + + let cache_dir = Path::new("../data/cache"); + if !cache_dir.exists() { + // Skip if data hasn't been moved yet + return; + } + + let mut manifest = CacheManifest::new(); + + // Scan each symbol directory + for symbol in &["ES.FUT", "NQ.FUT", "ZN.FUT", "6E.FUT"] { + let symbol_dir = cache_dir.join(symbol); + if !symbol_dir.exists() { + continue; + } + + let mut files: Vec<_> = std::fs::read_dir(&symbol_dir) + .into_iter() + .flatten() + .filter_map(|e| e.ok()) + .filter(|e| { + e.path() + .extension() + .map(|ext| ext == "dbn") + .unwrap_or(false) + }) + .collect(); + files.sort_by_key(|e| e.file_name()); + + for entry in &files { + let fname = entry.file_name(); + let fname_str = fname.to_string_lossy(); + // Parse date from filename: ohlcv-1m_2024-01-02.dbn + if let Some(date_str) = fname_str + .strip_prefix("ohlcv-1m_") + .and_then(|s| s.strip_suffix(".dbn")) + { + if let Ok(parsed_date) = + NaiveDate::parse_from_str(date_str, "%Y-%m-%d") + { + manifest.add_range( + symbol, + "GLBX.MDP3", + BarSize::OneMinute, + CachedRange { + start: parsed_date, + end: parsed_date, + bar_count: 390, // Approximate for 1m bars + file_path: entry + .path() + .strip_prefix(cache_dir) + .unwrap_or(&entry.path()) + .to_path_buf(), + cached_at: Utc::now(), + }, + ); + } + } + } + } + + // Verify we found data + assert!( + !manifest.symbols.is_empty(), + "No symbols found in cache" + ); + + // Save the manifest + let result = manifest.save(cache_dir); + assert!(result.is_ok(), "Failed to save manifest: {:?}", result.err()); + + // Verify it round-trips + let loaded = CacheManifest::load(cache_dir); + assert!(loaded.is_ok()); + let loaded = loaded.unwrap_or_default(); + assert_eq!(loaded.symbols.len(), manifest.symbols.len()); + } + + // ----------------------------------------------------------------------- + // 8. add_range keeps ranges sorted by start date // ----------------------------------------------------------------------- #[test] fn add_range_sorts_by_start() { diff --git a/ml/src/data_pipeline/mod.rs b/ml/src/data_pipeline/mod.rs index 78fbcdd3f..4288b2228 100644 --- a/ml/src/data_pipeline/mod.rs +++ b/ml/src/data_pipeline/mod.rs @@ -15,13 +15,8 @@ pub use prepared::{Batch, BatchIterator, PreparedDataset}; use chrono::NaiveDate; use serde::{Deserialize, Serialize}; -/// Asset class categorization -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum AssetClass { - Equity, - ETF, - Future, -} +// Re-export AssetClass from asset_selection (canonical location) +pub use crate::asset_selection::AssetClass; /// Bar size / time resolution #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -171,6 +166,35 @@ impl DatasetSpec { } } + /// Create a spec from an asset universe and dataset mode. + /// + /// Only includes enabled assets from the universe. + #[must_use] + pub fn from_universe( + name: &str, + universe: &crate::asset_selection::AssetUniverse, + mode: DatasetMode, + ) -> Self { + let symbols = universe + .eligible() + .iter() + .map(|a| SymbolSpec { + symbol: a.symbol.clone(), + exchange: a.exchange.clone(), + asset_class: a.asset_class.clone(), + }) + .collect(); + + Self { + name: format!("{name}-{mode:?}"), + symbols, + date_range: mode.to_date_range(), + bar_size: BarSize::OneMinute, + split_ratio: SplitRatio::default(), + warmup_bars: 50, + } + } + /// Estimate total bars across all symbols pub fn estimated_total_bars(&self) -> usize { let trading_days = (self.date_range.calendar_days() as f64 * 252.0 / 365.0) as usize; @@ -265,4 +289,59 @@ mod tests { let json = serde_json::to_string(&AssetClass::Equity).unwrap_or_default(); assert!(json.contains("Equity")); } + + #[test] + fn test_asset_class_is_same_type() { + let dp_future = AssetClass::Future; + let as_future = crate::asset_selection::AssetClass::Future; + assert_eq!(dp_future, as_future); + } + + #[test] + fn test_dataset_spec_from_universe() { + let universe = crate::asset_selection::AssetUniverse::futures_baseline(); + let spec = DatasetSpec::from_universe("futures-baseline", &universe, DatasetMode::Dev); + + assert_eq!(spec.symbols.len(), 4); + assert!(spec.name.contains("Dev")); + assert_eq!(spec.bar_size, BarSize::OneMinute); + assert!(spec.split_ratio.is_valid()); + + // Verify symbols match universe + let symbol_names: Vec<&str> = spec.symbols.iter().map(|s| s.symbol.as_str()).collect(); + assert!(symbol_names.contains(&"ES.FUT")); + assert!(symbol_names.contains(&"NQ.FUT")); + assert!(symbol_names.contains(&"ZN.FUT")); + assert!(symbol_names.contains(&"6E.FUT")); + + // All should be Future class + for sym in &spec.symbols { + assert_eq!(sym.asset_class, AssetClass::Future); + } + } + + #[test] + fn test_dataset_spec_from_universe_modes() { + let universe = crate::asset_selection::AssetUniverse::futures_baseline(); + + let dev = DatasetSpec::from_universe("futures-baseline", &universe, DatasetMode::Dev); + let backtest = DatasetSpec::from_universe("futures-baseline", &universe, DatasetMode::Backtest); + let full = DatasetSpec::from_universe("futures-baseline", &universe, DatasetMode::Full); + + // Dev < Backtest < Full in estimated bars + assert!(dev.estimated_total_bars() < backtest.estimated_total_bars()); + assert!(backtest.estimated_total_bars() < full.estimated_total_bars()); + } + + #[test] + fn test_dataset_spec_from_universe_disabled_filtered() { + let mut universe = crate::asset_selection::AssetUniverse::futures_baseline(); + // Disable one asset + if let Some(first) = universe.assets.get_mut(0) { + first.enabled = false; + } + let spec = DatasetSpec::from_universe("futures-baseline", &universe, DatasetMode::Dev); + // Only 3 symbols because one was disabled + assert_eq!(spec.symbols.len(), 3); + } }