gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feat/trading-universe-data-org'

Browse Source
This commit is contained in:
jgrusewski
2026-02-23 15:22:05 +01:00
parent 7b870726ab 236e1665cf
commit bb5286e910
6 changed files with 531 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -146,3 +146,6 @@ coordination/orchestration/*
claude-flow
# Removed Windows wrapper files per user request
hive-mind-prompt-*.txt
# Data cache (downloaded market data, not checked in)
data/cache/

43
config/universe-futures-baseline.toml Normal file
View File

@@ -0,0 +1,43 @@
# Futures Baseline Universe
#
# 4 CME futures for initial trading with limited capital.
# Train on full-size contracts, execute on micro versions.
# Data window: 730 days (March 2024 → February 2026)
#
# Schema: OHLCV-1m via Databento (GLBX.MDP3 dataset)
[universe]
name = "futures-baseline"
description = "CME futures baseline: 4 symbols, 730 days, OHLCV-1m"
date_range_start = "2024-03-01"
date_range_end = "2026-02-23"
bar_size = "1m"
databento_dataset = "GLBX.MDP3"
databento_schema = "ohlcv-1m"
[[symbols]]
symbol = "ES.FUT"
trading_symbol = "MES.FUT"
exchange = "GLBX.MDP3"
asset_class = "Future"
min_daily_volume = 1000000.0
[[symbols]]
symbol = "NQ.FUT"
trading_symbol = "MNQ.FUT"
exchange = "GLBX.MDP3"
asset_class = "Future"
min_daily_volume = 500000.0
[[symbols]]
symbol = "ZN.FUT"
exchange = "GLBX.MDP3"
asset_class = "Future"
min_daily_volume = 500000.0
[[symbols]]
symbol = "6E.FUT"
trading_symbol = "M6E.FUT"
exchange = "GLBX.MDP3"
asset_class = "Future"
min_daily_volume = 200000.0

1
ml/Cargo.toml
View File

@@ -47,6 +47,7 @@ clap.workspace = true # CLI argument parsing for train_tft binary
serde.workspace = true
serde_json.workspace = true
serde_yaml = "0.9" # YAML serialization for hyperparameter exports
toml.workspace = true
uuid.workspace = true
thiserror.workspace = true
anyhow.workspace = true

316
ml/src/asset_selection/mod.rs
View File

@@ -3,8 +3,12 @@
//! This module defines which assets are eligible for trading, including
//! their classification, exchange, sector, and minimum volume requirements.
use std::path::Path;
use serde::{Deserialize, Serialize};
use crate::MLError;
pub mod scorer;
pub mod selector;
@@ -25,9 +29,12 @@ pub enum AssetClass {
/// A single asset in the trading universe.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct UniverseAsset {
/// Ticker symbol (e.g. "AAPL").
/// Ticker symbol (e.g. "ES.FUT") -- used for data/training.
pub symbol: String,
/// MIC exchange code (e.g. "XNAS").
/// Execution symbol (e.g. "MES.FUT") -- used for order routing.
/// When `None`, orders are routed to `symbol` directly.
pub trading_symbol: Option<String>,
/// MIC exchange code (e.g. "GLBX.MDP3").
pub exchange: String,
/// Asset classification.
pub asset_class: AssetClass,
@@ -39,6 +46,15 @@ pub struct UniverseAsset {
pub enabled: bool,
}
impl UniverseAsset {
/// The symbol to use for order execution.
/// Returns `trading_symbol` if set, otherwise `symbol`.
#[must_use]
pub fn execution_symbol(&self) -> &str {
self.trading_symbol.as_deref().unwrap_or(&self.symbol)
}
}
/// The full set of assets available for trading.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AssetUniverse {
@@ -46,6 +62,47 @@ pub struct AssetUniverse {
pub assets: Vec<UniverseAsset>,
}
// ---------------------------------------------------------------------------
// TOML config deserialization
// ---------------------------------------------------------------------------
#[derive(Deserialize)]
struct UniverseConfigFile {
#[allow(dead_code)]
universe: UniverseConfigMeta,
symbols: Vec<SymbolConfigEntry>,
}
/// Parsed metadata from the `[universe]` section of the TOML config.
/// Fields are currently used only for validation/documentation during parsing;
/// they will feed into `DatasetSpec` construction once download automation lands.
#[derive(Deserialize)]
struct UniverseConfigMeta {
#[allow(dead_code)]
name: String,
#[allow(dead_code)]
description: Option<String>,
#[allow(dead_code)]
date_range_start: Option<String>,
#[allow(dead_code)]
date_range_end: Option<String>,
#[allow(dead_code)]
bar_size: Option<String>,
#[allow(dead_code)]
databento_dataset: Option<String>,
#[allow(dead_code)]
databento_schema: Option<String>,
}
#[derive(Deserialize)]
struct SymbolConfigEntry {
symbol: String,
trading_symbol: Option<String>,
exchange: String,
asset_class: String,
min_daily_volume: Option<f64>,
}
impl AssetUniverse {
/// Create an empty universe.
#[must_use]
@@ -53,6 +110,49 @@ impl AssetUniverse {
Self { assets: Vec::new() }
}
/// Load a universe from a TOML config file.
///
/// # Errors
/// Returns `MLError::ConfigError` on I/O or parse failures.
pub fn from_config(path: &Path) -> Result<Self, MLError> {
let contents = std::fs::read_to_string(path).map_err(|e| MLError::ConfigError {
reason: format!("Failed to read universe config {}: {e}", path.display()),
})?;
let config: UniverseConfigFile =
toml::from_str(&contents).map_err(|e| MLError::ConfigError {
reason: format!("Failed to parse universe config {}: {e}", path.display()),
})?;
let assets = config
.symbols
.into_iter()
.map(|s| {
let asset_class = match s.asset_class.as_str() {
"Future" | "future" => AssetClass::Future,
"Equity" | "equity" => AssetClass::Equity,
"ETF" | "etf" => AssetClass::ETF,
other => {
return Err(MLError::ConfigError {
reason: format!("Unknown asset class: {other}"),
})
}
};
Ok(UniverseAsset {
symbol: s.symbol,
trading_symbol: s.trading_symbol,
exchange: s.exchange,
asset_class,
sector: None,
min_daily_volume: s.min_daily_volume.unwrap_or(0.0),
enabled: true,
})
})
.collect::<Result<Vec<_>, _>>()?;
Ok(Self { assets })
}
/// Return only enabled assets.
#[must_use]
pub fn eligible(&self) -> Vec<&UniverseAsset> {
@@ -77,13 +177,14 @@ impl AssetUniverse {
self.assets.iter().map(|a| a.symbol.as_str()).collect()
}
/// Preset: US large-cap starter universe (7 highly liquid assets).
/// Preset: US large-cap starter universe (7 highly liquid equities/ETFs).
#[must_use]
pub fn us_starter() -> Self {
Self {
assets: vec![
UniverseAsset {
symbol: "SPY".to_string(),
trading_symbol: None,
exchange: "XNYS".to_string(),
asset_class: AssetClass::ETF,
sector: None,
@@ -92,6 +193,7 @@ impl AssetUniverse {
},
UniverseAsset {
symbol: "QQQ".to_string(),
trading_symbol: None,
exchange: "XNAS".to_string(),
asset_class: AssetClass::ETF,
sector: None,
@@ -100,6 +202,7 @@ impl AssetUniverse {
},
UniverseAsset {
symbol: "AAPL".to_string(),
trading_symbol: None,
exchange: "XNAS".to_string(),
asset_class: AssetClass::Equity,
sector: Some("Technology".to_string()),
@@ -108,6 +211,7 @@ impl AssetUniverse {
},
UniverseAsset {
symbol: "MSFT".to_string(),
trading_symbol: None,
exchange: "XNAS".to_string(),
asset_class: AssetClass::Equity,
sector: Some("Technology".to_string()),
@@ -116,6 +220,7 @@ impl AssetUniverse {
},
UniverseAsset {
symbol: "NVDA".to_string(),
trading_symbol: None,
exchange: "XNAS".to_string(),
asset_class: AssetClass::Equity,
sector: Some("Technology".to_string()),
@@ -124,6 +229,7 @@ impl AssetUniverse {
},
UniverseAsset {
symbol: "AMZN".to_string(),
trading_symbol: None,
exchange: "XNAS".to_string(),
asset_class: AssetClass::Equity,
sector: Some("Consumer".to_string()),
@@ -132,6 +238,7 @@ impl AssetUniverse {
},
UniverseAsset {
symbol: "IWM".to_string(),
trading_symbol: None,
exchange: "XNYS".to_string(),
asset_class: AssetClass::ETF,
sector: None,
@@ -141,6 +248,54 @@ impl AssetUniverse {
],
}
}
/// Preset: CME futures baseline universe (4 assets for limited capital).
///
/// Training is done on full-size contracts (ES, NQ, ZN, 6E).
/// Execution routes to micro contracts where available (MES, MNQ, M6E).
#[must_use]
pub fn futures_baseline() -> Self {
Self {
assets: vec![
UniverseAsset {
symbol: "ES.FUT".to_string(),
trading_symbol: Some("MES.FUT".to_string()),
exchange: "GLBX.MDP3".to_string(),
asset_class: AssetClass::Future,
sector: None,
min_daily_volume: 1_000_000.0,
enabled: true,
},
UniverseAsset {
symbol: "NQ.FUT".to_string(),
trading_symbol: Some("MNQ.FUT".to_string()),
exchange: "GLBX.MDP3".to_string(),
asset_class: AssetClass::Future,
sector: None,
min_daily_volume: 500_000.0,
enabled: true,
},
UniverseAsset {
symbol: "ZN.FUT".to_string(),
trading_symbol: None,
exchange: "GLBX.MDP3".to_string(),
asset_class: AssetClass::Future,
sector: None,
min_daily_volume: 500_000.0,
enabled: true,
},
UniverseAsset {
symbol: "6E.FUT".to_string(),
trading_symbol: Some("M6E.FUT".to_string()),
exchange: "GLBX.MDP3".to_string(),
asset_class: AssetClass::Future,
sector: None,
min_daily_volume: 200_000.0,
enabled: true,
},
],
}
}
}
impl Default for AssetUniverse {
@@ -209,4 +364,159 @@ mod tests {
serde_json::from_str(&json).unwrap_or_else(|_| AssetUniverse::new());
assert_eq!(restored.assets.len(), 7);
}
#[test]
fn test_trading_symbol_mapping() {
let asset = UniverseAsset {
symbol: "ES.FUT".to_string(),
trading_symbol: Some("MES.FUT".to_string()),
exchange: "GLBX.MDP3".to_string(),
asset_class: AssetClass::Future,
sector: None,
min_daily_volume: 1_000_000.0,
enabled: true,
};
assert_eq!(asset.execution_symbol(), "MES.FUT");
}
#[test]
fn test_trading_symbol_none_falls_back() {
let asset = UniverseAsset {
symbol: "ZN.FUT".to_string(),
trading_symbol: None,
exchange: "GLBX.MDP3".to_string(),
asset_class: AssetClass::Future,
sector: None,
min_daily_volume: 500_000.0,
enabled: true,
};
assert_eq!(asset.execution_symbol(), "ZN.FUT");
}
#[test]
fn test_futures_baseline_universe() {
let universe = AssetUniverse::futures_baseline();
assert_eq!(universe.assets.len(), 4);
assert_eq!(universe.eligible_count(), 4);
// All are futures
for asset in &universe.assets {
assert_eq!(asset.asset_class, AssetClass::Future);
}
}
#[test]
fn test_futures_baseline_symbols() {
let universe = AssetUniverse::futures_baseline();
let symbols = universe.symbols();
assert!(symbols.contains(&"ES.FUT"));
assert!(symbols.contains(&"NQ.FUT"));
assert!(symbols.contains(&"ZN.FUT"));
assert!(symbols.contains(&"6E.FUT"));
}
#[test]
fn test_futures_baseline_micro_mapping() {
let universe = AssetUniverse::futures_baseline();
assert_eq!(
universe.find("ES.FUT").map(|a| a.execution_symbol()),
Some("MES.FUT")
);
assert_eq!(
universe.find("NQ.FUT").map(|a| a.execution_symbol()),
Some("MNQ.FUT")
);
// ZN has no micro -- executes as ZN.FUT
assert_eq!(
universe.find("ZN.FUT").map(|a| a.execution_symbol()),
Some("ZN.FUT")
);
assert_eq!(
universe.find("6E.FUT").map(|a| a.execution_symbol()),
Some("M6E.FUT")
);
}
#[test]
fn test_futures_baseline_serde_roundtrip() {
let universe = AssetUniverse::futures_baseline();
let json = serde_json::to_string(&universe).unwrap_or_default();
let restored: AssetUniverse =
serde_json::from_str(&json).unwrap_or_else(|_| AssetUniverse::new());
assert_eq!(restored.assets.len(), 4);
// Verify trading_symbol survives roundtrip
let es = restored.find("ES.FUT");
assert_eq!(es.map(|a| a.execution_symbol()), Some("MES.FUT"));
}
#[test]
fn test_from_config_toml() {
let toml_content = r#"
[universe]
name = "test-universe"
description = "Test"
[[symbols]]
symbol = "ES.FUT"
trading_symbol = "MES.FUT"
exchange = "GLBX.MDP3"
asset_class = "Future"
min_daily_volume = 1000000.0
[[symbols]]
symbol = "ZN.FUT"
exchange = "GLBX.MDP3"
asset_class = "Future"
min_daily_volume = 500000.0
"#;
let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("tmpdir: {e}"));
let path = dir.path().join("universe.toml");
std::fs::write(&path, toml_content).unwrap_or_else(|e| panic!("write: {e}"));
let universe = AssetUniverse::from_config(&path);
assert!(universe.is_ok(), "from_config failed: {:?}", universe.err());
let universe = universe.unwrap_or_else(|_| AssetUniverse::new());
assert_eq!(universe.assets.len(), 2);
assert_eq!(
universe.find("ES.FUT").map(|a| a.execution_symbol()),
Some("MES.FUT")
);
assert_eq!(
universe.find("ZN.FUT").map(|a| a.trading_symbol.as_deref()),
Some(None)
);
}
#[test]
fn test_load_futures_baseline_config() {
let config_path = std::path::Path::new("../config/universe-futures-baseline.toml");
if !config_path.exists() {
// Skip in CI where config may not be at expected relative path
return;
}
let universe = AssetUniverse::from_config(config_path);
assert!(universe.is_ok(), "Failed to load config: {:?}", universe.err());
let universe = universe.unwrap_or_else(|_| AssetUniverse::new());
assert_eq!(universe.assets.len(), 4);
assert_eq!(
universe.find("ES.FUT").map(|a| a.execution_symbol()),
Some("MES.FUT")
);
}
#[test]
fn test_from_config_missing_file() {
let result = AssetUniverse::from_config(std::path::Path::new("/nonexistent/path.toml"));
assert!(result.is_err());
}
#[test]
fn test_from_config_invalid_toml() {
let dir = tempfile::tempdir().unwrap_or_else(|e| panic!("tmpdir: {e}"));
let path = dir.path().join("bad.toml");
std::fs::write(&path, "not valid toml {{{{").unwrap_or_else(|e| panic!("write: {e}"));
let result = AssetUniverse::from_config(&path);
assert!(result.is_err());
}
}

86
ml/src/data_pipeline/cache.rs
View File

@@ -404,7 +404,91 @@ mod tests {
}
// -----------------------------------------------------------------------
// 7. add_range keeps ranges sorted by start date
// 7. Build manifest from on-disk DBN files
// -----------------------------------------------------------------------
#[test]
fn test_build_manifest_from_disk() {
use std::path::Path;
let cache_dir = Path::new("../data/cache");
if !cache_dir.exists() {
// Skip if data hasn't been moved yet
return;
}
let mut manifest = CacheManifest::new();
// Scan each symbol directory
for symbol in &["ES.FUT", "NQ.FUT", "ZN.FUT", "6E.FUT"] {
let symbol_dir = cache_dir.join(symbol);
if !symbol_dir.exists() {
continue;
}
let mut files: Vec<_> = std::fs::read_dir(&symbol_dir)
.into_iter()
.flatten()
.filter_map(|e| e.ok())
.filter(|e| {
e.path()
.extension()
.map(|ext| ext == "dbn")
.unwrap_or(false)
})
.collect();
files.sort_by_key(|e| e.file_name());
for entry in &files {
let fname = entry.file_name();
let fname_str = fname.to_string_lossy();
// Parse date from filename: ohlcv-1m_2024-01-02.dbn
if let Some(date_str) = fname_str
.strip_prefix("ohlcv-1m_")
.and_then(|s| s.strip_suffix(".dbn"))
{
if let Ok(parsed_date) =
NaiveDate::parse_from_str(date_str, "%Y-%m-%d")
{
manifest.add_range(
symbol,
"GLBX.MDP3",
BarSize::OneMinute,
CachedRange {
start: parsed_date,
end: parsed_date,
bar_count: 390, // Approximate for 1m bars
file_path: entry
.path()
.strip_prefix(cache_dir)
.unwrap_or(&entry.path())
.to_path_buf(),
cached_at: Utc::now(),
},
);
}
}
}
}
// Verify we found data
assert!(
!manifest.symbols.is_empty(),
"No symbols found in cache"
);
// Save the manifest
let result = manifest.save(cache_dir);
assert!(result.is_ok(), "Failed to save manifest: {:?}", result.err());
// Verify it round-trips
let loaded = CacheManifest::load(cache_dir);
assert!(loaded.is_ok());
let loaded = loaded.unwrap_or_default();
assert_eq!(loaded.symbols.len(), manifest.symbols.len());
}
// -----------------------------------------------------------------------
// 8. add_range keeps ranges sorted by start date
// -----------------------------------------------------------------------
#[test]
fn add_range_sorts_by_start() {

93
ml/src/data_pipeline/mod.rs
View File

@@ -15,13 +15,8 @@ pub use prepared::{Batch, BatchIterator, PreparedDataset};
use chrono::NaiveDate;
use serde::{Deserialize, Serialize};
/// Asset class categorization
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum AssetClass {
Equity,
ETF,
Future,
}
// Re-export AssetClass from asset_selection (canonical location)
pub use crate::asset_selection::AssetClass;
/// Bar size / time resolution
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -171,6 +166,35 @@ impl DatasetSpec {
}
}
/// Create a spec from an asset universe and dataset mode.
///
/// Only includes enabled assets from the universe.
#[must_use]
pub fn from_universe(
name: &str,
universe: &crate::asset_selection::AssetUniverse,
mode: DatasetMode,
) -> Self {
let symbols = universe
.eligible()
.iter()
.map(|a| SymbolSpec {
symbol: a.symbol.clone(),
exchange: a.exchange.clone(),
asset_class: a.asset_class.clone(),
})
.collect();
Self {
name: format!("{name}-{mode:?}"),
symbols,
date_range: mode.to_date_range(),
bar_size: BarSize::OneMinute,
split_ratio: SplitRatio::default(),
warmup_bars: 50,
}
}
/// Estimate total bars across all symbols
pub fn estimated_total_bars(&self) -> usize {
let trading_days = (self.date_range.calendar_days() as f64 * 252.0 / 365.0) as usize;
@@ -265,4 +289,59 @@ mod tests {
let json = serde_json::to_string(&AssetClass::Equity).unwrap_or_default();
assert!(json.contains("Equity"));
}
#[test]
fn test_asset_class_is_same_type() {
let dp_future = AssetClass::Future;
let as_future = crate::asset_selection::AssetClass::Future;
assert_eq!(dp_future, as_future);
}
#[test]
fn test_dataset_spec_from_universe() {
let universe = crate::asset_selection::AssetUniverse::futures_baseline();
let spec = DatasetSpec::from_universe("futures-baseline", &universe, DatasetMode::Dev);
assert_eq!(spec.symbols.len(), 4);
assert!(spec.name.contains("Dev"));
assert_eq!(spec.bar_size, BarSize::OneMinute);
assert!(spec.split_ratio.is_valid());
// Verify symbols match universe
let symbol_names: Vec<&str> = spec.symbols.iter().map(|s| s.symbol.as_str()).collect();
assert!(symbol_names.contains(&"ES.FUT"));
assert!(symbol_names.contains(&"NQ.FUT"));
assert!(symbol_names.contains(&"ZN.FUT"));
assert!(symbol_names.contains(&"6E.FUT"));
// All should be Future class
for sym in &spec.symbols {
assert_eq!(sym.asset_class, AssetClass::Future);
}
}
#[test]
fn test_dataset_spec_from_universe_modes() {
let universe = crate::asset_selection::AssetUniverse::futures_baseline();
let dev = DatasetSpec::from_universe("futures-baseline", &universe, DatasetMode::Dev);
let backtest = DatasetSpec::from_universe("futures-baseline", &universe, DatasetMode::Backtest);
let full = DatasetSpec::from_universe("futures-baseline", &universe, DatasetMode::Full);
// Dev < Backtest < Full in estimated bars
assert!(dev.estimated_total_bars() < backtest.estimated_total_bars());
assert!(backtest.estimated_total_bars() < full.estimated_total_bars());
}
#[test]
fn test_dataset_spec_from_universe_disabled_filtered() {
let mut universe = crate::asset_selection::AssetUniverse::futures_baseline();
// Disable one asset
if let Some(first) = universe.assets.get_mut(0) {
first.enabled = false;
}
let spec = DatasetSpec::from_universe("futures-baseline", &universe, DatasetMode::Dev);
// Only 3 symbols because one was disabled
assert_eq!(spec.symbols.len(), 3);
}
}