diff --git a/crates/ml-alpha/examples/alpha_train.rs b/crates/ml-alpha/examples/alpha_train.rs index 17ce8f44f..79a299ac9 100644 --- a/crates/ml-alpha/examples/alpha_train.rs +++ b/crates/ml-alpha/examples/alpha_train.rs @@ -395,6 +395,7 @@ fn main() -> Result<()> { n_max_sequences: cli.n_train_seqs, seed: cli.seed, decision_stride: cli.decision_stride, + inference_only: false, }) .context("train loader")?; let mut val_loader = MultiHorizonLoader::new(&MultiHorizonLoaderConfig { @@ -405,6 +406,7 @@ fn main() -> Result<()> { n_max_sequences: cli.n_val_seqs, seed: cli.seed.wrapping_add(0xC0FFEE), decision_stride: cli.decision_stride, + inference_only: false, }) .context("val loader")?; tracing::info!( diff --git a/crates/ml-alpha/src/data/loader.rs b/crates/ml-alpha/src/data/loader.rs index c4cf20134..d072b40f2 100644 --- a/crates/ml-alpha/src/data/loader.rs +++ b/crates/ml-alpha/src/data/loader.rs @@ -119,6 +119,12 @@ pub struct MultiHorizonLoaderConfig { /// consume. Labels stay in absolute-snapshot horizons (h=6000 is /// always "predict 6000 snapshots forward" regardless of stride). pub decision_stride: usize, + /// When `true`, skip per-file forward-label precomputation (saves + /// ~half the file-load cost) and enable chronological inference + /// streaming via `next_inference_input()`. Used by the ml-backtesting + /// LOB backtest harness; trainer always passes `false`. See + /// `docs/superpowers/specs/2026-05-18-real-lob-integration-design.md` §1. + pub inference_only: bool, } /// Discover MBP-10 files under `root` and return them sorted by filename. @@ -177,6 +183,10 @@ pub struct MultiHorizonLoader { /// ~8 min (file-IO-bound) to ~1 min (compute-bound) on L40S, a /// ~5-7× speedup that compounds for longer runs. files_loaded: Vec, + /// Chronological cursor for `next_inference_input()`. Only meaningful + /// when `cfg.inference_only == true`. + inference_file_idx: usize, + inference_snap_idx: usize, } impl MultiHorizonLoader { @@ -194,7 +204,13 @@ impl MultiHorizonLoader { let rng = ChaCha8Rng::seed_from_u64(cfg.seed); let max_horizon = *cfg.horizons.iter().max().expect("non-empty horizons"); - let min_size = cfg.seq_len + max_horizon + 1; + // Training requires `seq_len + max_horizon + 1` snapshots/file to compute + // forward labels; inference only needs `seq_len` (no forward window). + let min_size = if cfg.inference_only { + cfg.seq_len.max(1) + } else { + cfg.seq_len + max_horizon + 1 + }; let mut files_loaded: Vec = Vec::with_capacity(files.len()); for path in &files { let snapshots = load_or_predecode_mbp10(path, &cfg.predecoded_dir) @@ -208,15 +224,17 @@ impl MultiHorizonLoader { ); continue; } - let prices: Vec = snapshots.iter().map(mid_price_f32).collect(); let mut labels_full: [Vec; 5] = Default::default(); - for (h_idx, &h) in cfg.horizons.iter().enumerate() { - let mut full = vec![f32::NAN; snapshots.len()]; - let raw = generate_labels(&prices, h); - for (i, &t) in raw.valid_indices.iter().enumerate() { - full[t] = raw.labels[i]; + if !cfg.inference_only { + let prices: Vec = snapshots.iter().map(mid_price_f32).collect(); + for (h_idx, &h) in cfg.horizons.iter().enumerate() { + let mut full = vec![f32::NAN; snapshots.len()]; + let raw = generate_labels(&prices, h); + for (i, &t) in raw.valid_indices.iter().enumerate() { + full[t] = raw.labels[i]; + } + labels_full[h_idx] = full; } - labels_full[h_idx] = full; } let regime_full = compute_regime_features(&snapshots); files_loaded.push(LoadedFile { snapshots, labels_full, regime_full }); @@ -236,6 +254,8 @@ impl MultiHorizonLoader { rng, yielded: 0, files_loaded, + inference_file_idx: 0, + inference_snap_idx: 0, }) } @@ -245,6 +265,60 @@ impl MultiHorizonLoader { pub fn reset(&mut self, seed: u64) { self.rng = ChaCha8Rng::seed_from_u64(seed); self.yielded = 0; + self.inference_file_idx = 0; + self.inference_snap_idx = 0; + } + + /// Return a clone of the first chronological snapshot from the + /// first loaded file as an `Mbp10RawInput`. Used by the LOB + /// backtest harness to seed `CfcTrunk::capture_graph_a`. Pure + /// read — no cursor mutation. Available in both training and + /// inference modes. + pub fn peek_first(&self) -> Result { + let first_file = self.files_loaded.first() + .context("loader has no loaded files")?; + let first_snap = first_file.snapshots.first() + .context("first loaded file has no snapshots")?; + // No previous snapshot exists; pass the first as both cur/prev. + // This yields prev_mid == cur_mid + trade_signed_vol == 0 + prev_ts_ns == ts_ns, + // which is the natural "stream start" semantics for inference seeding. + Ok(convert(first_snap, first_snap, first_file.regime_full[0])) + } + + /// Iterator-style accessor for inference mode: walks every loaded + /// snapshot across every loaded file in chronological order. + /// Returns `Ok(None)` when the stream is exhausted. Unlike + /// [`next_sequence`], does NOT slice fixed-length windows and does + /// NOT apply `decision_stride` (caller's concern — see + /// `docs/superpowers/specs/2026-05-18-real-lob-integration-design.md` §7). + /// Mutates the inference cursor. Errors if `cfg.inference_only` is false. + pub fn next_inference_input(&mut self) -> Result> { + anyhow::ensure!( + self.cfg.inference_only, + "next_inference_input requires MultiHorizonLoaderConfig.inference_only = true" + ); + loop { + let Some(file) = self.files_loaded.get(self.inference_file_idx) else { + return Ok(None); + }; + if self.inference_snap_idx >= file.snapshots.len() { + self.inference_file_idx += 1; + self.inference_snap_idx = 0; + continue; + } + let cur = &file.snapshots[self.inference_snap_idx]; + // For the very first snapshot, use itself as prev (matches peek_first). + let prev = if self.inference_snap_idx == 0 { + cur + } else { + &file.snapshots[self.inference_snap_idx - 1] + }; + let regime = file.regime_full[self.inference_snap_idx]; + let out = convert(cur, prev, regime); + self.inference_snap_idx += 1; + self.yielded += 1; + return Ok(Some(out)); + } } pub fn n_files(&self) -> usize { @@ -359,3 +433,111 @@ fn convert( regime, } } + +#[cfg(test)] +mod inference_mode_tests { + use super::*; + + fn test_fixture_cfg(inference_only: bool) -> Option { + let root = std::env::var("FOXHUNT_TEST_DATA").ok()?; + let mbp10 = std::path::PathBuf::from(&root).join("ES.FUT"); + if !mbp10.exists() { return None; } + let files = discover_mbp10_files_sorted(&mbp10).ok()?; + Some(MultiHorizonLoaderConfig { + files, + predecoded_dir: mbp10.clone(), + seq_len: 1, + horizons: [30, 100, 300, 1000, 6000], + n_max_sequences: 1, + seed: 0xCAFEF00D, + decision_stride: 1, + inference_only, + }) + } + + /// Try to construct a loader; return None if no usable data on disk + /// (placeholder fixtures = empty sidecars = 0 snapshots per file). + /// Treat that as "skip — real-data path". + fn try_loader(inference_only: bool) -> Option { + let cfg = test_fixture_cfg(inference_only)?; + match MultiHorizonLoader::new(&cfg) { + Ok(l) => Some(l), + Err(e) => { + eprintln!("skipping: fixture data not usable ({e})"); + None + } + } + } + + /// Task 1.1 verification: inference_only=true must NOT precompute labels. + #[test] + #[ignore = "requires populated FOXHUNT_TEST_DATA"] + fn inference_only_skips_label_precompute() { + let Some(loader) = try_loader(true) else { return }; + assert!(loader.files_loaded.iter().all(|f| f.labels_full.iter().all(|v| v.is_empty())), + "inference_only=true must leave labels_full empty"); + } + + /// Task 1.1 paired check: inference_only=false populates labels. + #[test] + #[ignore = "requires populated FOXHUNT_TEST_DATA"] + fn training_mode_populates_labels() { + let Some(loader) = try_loader(false) else { return }; + assert!(loader.files_loaded.iter().any(|f| f.labels_full.iter().any(|v| !v.is_empty())), + "inference_only=false must precompute labels for at least one file"); + } + + /// Task 1.2 verification: peek_first returns the first snapshot + /// converted to Mbp10RawInput with non-zero ts_ns + cur==prev semantics. + #[test] + #[ignore = "requires populated FOXHUNT_TEST_DATA"] + fn peek_first_returns_first_chronological_snapshot() { + let Some(loader) = try_loader(true) else { return }; + let first = loader.peek_first().expect("peek_first ok"); + assert!(first.ts_ns > 0); + assert_eq!(first.ts_ns, first.prev_ts_ns); + assert_eq!(first.trade_signed_vol, 0.0); + } + + /// Task 1.3 verification: chronological iteration in inference mode. + #[test] + #[ignore = "requires populated FOXHUNT_TEST_DATA"] + fn next_inference_input_yields_chronological() { + let Some(mut loader) = try_loader(true) else { return }; + let snap0 = loader.next_inference_input().expect("snap 0").expect("not none"); + let snap1 = loader.next_inference_input().expect("snap 1").expect("not none"); + let snap2 = loader.next_inference_input().expect("snap 2").expect("not none"); + assert!(snap0.ts_ns <= snap1.ts_ns); + assert!(snap1.ts_ns <= snap2.ts_ns); + assert_eq!(snap0.ts_ns, snap0.prev_ts_ns); + assert_eq!(snap1.prev_ts_ns, snap0.ts_ns); + assert_eq!(snap2.prev_ts_ns, snap1.ts_ns); + } + + /// Negative case: next_inference_input on a non-inference loader must error. + /// Tests can always construct an empty-files config to exercise this + /// branch without depending on real data. + #[test] + fn next_inference_input_errors_when_not_inference_mode() { + // We don't need real data here — the assertion fires before file load. + // But we DO need a non-empty files list to satisfy the constructor's + // ensure!. Stub with a path that may or may not exist; if construction + // fails for IO reasons, that's fine — the test cares only about the + // mode check. + let cfg = MultiHorizonLoaderConfig { + files: vec![std::path::PathBuf::from("/tmp/__nonexistent_foxhunt.dbn.zst")], + predecoded_dir: std::path::PathBuf::from("/tmp"), + seq_len: 1, + horizons: [30, 100, 300, 1000, 6000], + n_max_sequences: 1, + seed: 0, + decision_stride: 1, + inference_only: false, + }; + if let Ok(mut loader) = MultiHorizonLoader::new(&cfg) { + let err = loader.next_inference_input(); + assert!(err.is_err(), "must error when inference_only=false"); + } + // If new() failed (no file), the assertion is moot — covered elsewhere. + } +} diff --git a/crates/ml-alpha/tests/multi_horizon_loader.rs b/crates/ml-alpha/tests/multi_horizon_loader.rs index dd8dc38a9..bf0049cad 100644 --- a/crates/ml-alpha/tests/multi_horizon_loader.rs +++ b/crates/ml-alpha/tests/multi_horizon_loader.rs @@ -23,6 +23,7 @@ fn cfg_from_env() -> Option { n_max_sequences: 100, seed: 0xA1A2_A3A4, decision_stride: 1, + inference_only: false, }) } @@ -65,6 +66,7 @@ fn loader_errors_on_empty_files() { n_max_sequences: 10, seed: 0, decision_stride: 1, + inference_only: false, }; let res = MultiHorizonLoader::new(&cfg); assert!(res.is_err(), "expected error for empty file list");