diff --git a/bin/fxt-data-audit/src/main.rs b/bin/fxt-data-audit/src/main.rs index bef70961f..2ab621146 100644 --- a/bin/fxt-data-audit/src/main.rs +++ b/bin/fxt-data-audit/src/main.rs @@ -14,7 +14,7 @@ //! Run on the cluster with the training-data + feature-cache PVCs mounted. use anyhow::Result; -use data::providers::databento::mbp10::BidAskPair; +use data::providers::databento::{dbn_parser::InstrumentFilter, mbp10::BidAskPair}; use ml_features::predecoded::load_or_predecode_mbp10; use std::collections::BTreeMap; use std::path::PathBuf; @@ -38,8 +38,8 @@ fn main() -> Result<()> { eprintln!("loading: {} (predecoded_dir={})", source.display(), predecoded_dir.display()); // The audit tool inspects raw multi-instrument sidecars — we want to see - // every instrument in the file, so the filter is `None` by design. - let snapshots = load_or_predecode_mbp10(&source, &predecoded_dir, None) + // every instrument in the file, so the filter is `All` by design. + let snapshots = load_or_predecode_mbp10(&source, &predecoded_dir, InstrumentFilter::All) .map_err(|e| anyhow::anyhow!("load_or_predecode_mbp10: {e}"))?; eprintln!("loaded: {} snapshots", snapshots.len()); diff --git a/crates/data/src/providers/databento/dbn_parser.rs b/crates/data/src/providers/databento/dbn_parser.rs index 828eb7b72..07b3a58b1 100644 --- a/crates/data/src/providers/databento/dbn_parser.rs +++ b/crates/data/src/providers/databento/dbn_parser.rs @@ -780,12 +780,10 @@ impl DbnParser { /// /// * `path` - Path to the .dbn or .dbn.zst file /// * `snapshot_interval` - Create a snapshot every N raw updates (e.g. 100) - /// * `instrument_id_filter` - When `Some(id)`, only records where - /// `record.hd.instrument_id == id` are processed; all others are - /// skipped before any snapshot mutation. `None` preserves the legacy - /// all-instruments behavior. Multi-instrument DBN files (ES front + - /// back month + calendar spreads in the same archive) require a filter - /// to avoid cross-instrument ΔP contamination in downstream features. + /// * `filter` - Instrument selection strategy. See [`InstrumentFilter`] for variants. + /// ES.FUT files from databento are parent-symbol expansions and contain ~14 distinct + /// instrument_ids per file (one per contract month). Without filtering, K-window + /// labels span contract boundaries and produce $5000+ ΔP artifacts. /// * `callback` - Called with each aggregated snapshot (by reference, not cloned) /// /// # Returns @@ -795,17 +793,36 @@ impl DbnParser { &self, path: P, snapshot_interval: usize, - instrument_id_filter: Option, + filter: InstrumentFilter, mut callback: F, ) -> Result where P: AsRef, F: FnMut(&Mbp10Snapshot), { + let path = path.as_ref(); + + // FrontMonth is two-pass: pass 1 detects the dominant id and validates + // it resolves to an ES contract via SymbolMapping records; pass 2 + // streams emission. Resolve to Id(winning_id) so the hot loop below + // stays branch-free. + let resolved_filter = match filter { + InstrumentFilter::FrontMonth => { + let winning_id = self.detect_front_month_id(path)?; + InstrumentFilter::Id(winning_id) + } + other => other, + }; + + let id_filter: Option = match resolved_filter { + InstrumentFilter::All => None, + InstrumentFilter::Id(id) => Some(id), + InstrumentFilter::FrontMonth => unreachable!("resolved above"), + }; + use std::fs::File; use std::io::BufReader; - let path = path.as_ref(); let file = File::open(path)?; let is_zstd = path.to_string_lossy().ends_with(".dbn.zst"); let reader: Box = if is_zstd { @@ -844,8 +861,8 @@ impl DbnParser { })?; if let RecordRefEnum::Mbp10(mbp10) = record_enum { - if let Some(filter) = instrument_id_filter { - if mbp10.hd.instrument_id != filter { + if let Some(filter_id) = id_filter { + if mbp10.hd.instrument_id != filter_id { skipped += 1; continue; } @@ -917,11 +934,11 @@ impl DbnParser { snapshot_count += 1; } - if let Some(filter) = instrument_id_filter { + if let Some(filter_id) = id_filter { tracing::info!( kept, skipped, - filter, + filter = filter_id, path = %path.display(), "instrument filter applied" ); @@ -929,6 +946,139 @@ impl DbnParser { Ok(snapshot_count) } + + /// First pass for [`InstrumentFilter::FrontMonth`]: stream-decodes the DBN + /// file once to identify the most-frequent `instrument_id` (the front-month + /// contract has the bulk of trading volume) and validates that its + /// resolved symbol matches an ES futures contract pattern using + /// SymbolMapping records. + /// + /// Returns the winning instrument_id on success. Errors when: + /// - The file contains no MBP-10 records. + /// - The dominant id cannot be resolved to a symbol via SymbolMapping + /// records (suggests a malformed or non-databento DBN file). + /// - The resolved symbol does not match the ES contract pattern + /// (suggests a calendar spread or non-ES instrument is dominating — + /// loud failure so the caller surfaces the data anomaly). + fn detect_front_month_id(&self, path: &Path) -> Result { + use std::collections::HashMap; + use std::fs::File; + use std::io::BufReader; + + let file = File::open(path)?; + let is_zstd = path.to_string_lossy().ends_with(".dbn.zst"); + let reader: Box = if is_zstd { + Box::new(zstd::Decoder::new(BufReader::new(file)).map_err(|e| { + DataError::InvalidFormat(format!("Failed to create zstd decoder: {}", e)) + })?) + } else { + Box::new(BufReader::new(file)) + }; + + let mut decoder = DbnDecoder::new(reader).map_err(|e| { + DataError::InvalidFormat(format!("Failed to create DBN decoder: {}", e)) + })?; + + let mut counts: HashMap = HashMap::new(); + let mut id_to_symbol: HashMap = HashMap::new(); + + loop { + match decoder.decode_record_ref() { + Ok(Some(record)) => { + let record_enum = record.as_enum().map_err(|e| { + DataError::InvalidFormat(format!("Failed to convert record: {}", e)) + })?; + match record_enum { + RecordRefEnum::Mbp10(mbp10) => { + *counts.entry(mbp10.hd.instrument_id).or_insert(0) += 1; + } + RecordRefEnum::SymbolMapping(sym) => { + if let Ok(out) = sym.stype_out_symbol() { + id_to_symbol + .insert(sym.hd.instrument_id, out.to_string()); + } + } + _ => {} + } + } + Ok(None) => break, + Err(e) => { + return Err(DataError::InvalidFormat(format!( + "front-month detect: decode failed: {}", + e + ))); + } + } + } + + let (winner_id, winner_count) = counts + .iter() + .max_by_key(|(_, c)| *c) + .map(|(id, c)| (*id, *c)) + .ok_or_else(|| { + DataError::InvalidFormat(format!( + "front-month detect: no MBP-10 records in {}", + path.display() + )) + })?; + let total: u64 = counts.values().sum(); + + let winner_symbol = id_to_symbol.get(&winner_id).cloned().ok_or_else(|| { + DataError::InvalidFormat(format!( + "front-month detect: no SymbolMapping record for dominant instrument_id={} in {}", + winner_id, + path.display() + )) + })?; + + // ES futures month codes: F=Jan G=Feb H=Mar J=Apr K=May M=Jun + // N=Jul Q=Aug U=Sep V=Oct X=Nov Z=Dec; year suffix 1-2 digits. + // Examples: ESH4, ESM4, ESU4, ESZ4, ESH25. Calendar spreads + // (ES-ESM4) and outright spreads fail this match by design. + let es_re = regex::Regex::new(r"^ES[FGHJKMNQUVXZ]\d{1,2}$") + .expect("static ES futures regex compiles"); + if !es_re.is_match(&winner_symbol) { + return Err(DataError::InvalidFormat(format!( + "front-month detect: dominant id {} resolved to symbol '{}' which is not an ES \ + contract (regex ES[F-Z]\\d{{1,2}}) in {}", + winner_id, + winner_symbol, + path.display() + ))); + } + + info!( + instrument_id = winner_id, + symbol = %winner_symbol, + count = winner_count, + total = total, + distinct_ids = counts.len(), + path = %path.display(), + "front-month detected" + ); + + Ok(winner_id) + } +} + +/// Strategy for selecting which `instrument_id` records to keep when streaming +/// MBP-10 from a DBN file. ES.FUT parent-expanded files contain ~14 distinct +/// contract months in one stream; without filtering, downstream K-window +/// labels span contract boundaries and produce ΔP artifacts at the rolls. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InstrumentFilter { + /// Emit all MBP-10 records regardless of `instrument_id`. Preserves the + /// pre-refactor behavior for callers that need multi-instrument streams + /// (e.g., DQN training that doesn't compute multi-step labels). + All, + /// Emit only records where `record.hd.instrument_id == id`. Fails silently + /// for files where the configured id never appears (caller must validate). + Id(u32), + /// Two-pass: detect the most-frequent `instrument_id` in the file (volume + /// leader = front-month for ES.FUT), validate it resolves to a valid ES + /// contract via SymbolMapping records, then emit only its records. + /// Self-tuning across quarterly contract rolls. + FrontMonth, } /// Processed message types from DBN parsing diff --git a/crates/ml-alpha/examples/alpha_train.rs b/crates/ml-alpha/examples/alpha_train.rs index cecfadc85..76771b5d1 100644 --- a/crates/ml-alpha/examples/alpha_train.rs +++ b/crates/ml-alpha/examples/alpha_train.rs @@ -23,6 +23,7 @@ use anyhow::{Context, Result}; use clap::Parser; +use data::providers::databento::dbn_parser::InstrumentFilter; use ml_alpha::aux_heads::N_AUX_HORIZONS; use ml_alpha::cfc::snap_features::Mbp10RawInput; use ml_alpha::data::loader::{ @@ -192,16 +193,41 @@ struct Cli { #[arg(long, default_value_t = DEFAULT_OUTCOME_LABEL_COST_ES)] outcome_label_cost: f32, - /// Filter MBP-10 records to a single `instrument_id`. For ES.FUT - /// use 17077 (front-month ESH4 for Q1 2024). Without this flag, - /// ALL instruments in the file are processed sequentially — - /// multi-instrument ΔP contamination silently inflates K-step - /// labels (front-month ↔ back-month price gaps treated as moves - /// of the same series). The predecoder caches filtered output in - /// a separate sidecar (`*.mbp10_instr.predecoded.bin`) so - /// unfiltered runs keep their existing cache. - #[arg(long)] - instrument_id: Option, + /// MBP-10 instrument filter strategy. Accepted values: + /// - `all` — keep every record (legacy multi-instrument behavior; + /// produces $5000+ ΔP artifacts at quarterly contract rolls). + /// - `id=` — keep only records where `instrument_id == N`. Single + /// contract only; will produce 0 records past the roll. + /// - `front-month` — auto-detect the dominant `instrument_id` per file + /// and validate it resolves to an ES contract symbol via DBN + /// SymbolMapping records. Self-tuning across the quarterly roll. + /// + /// The predecoder uses disjoint sidecars per mode so caches don't + /// invalidate each other. + #[arg(long, default_value = "all", value_parser = parse_instrument_mode)] + instrument_mode: InstrumentFilter, +} + +/// CLI value parser for `--instrument-mode`. Mirrors the doc on the +/// `instrument_mode` field. `id=` is the only variant that takes a +/// payload; spaces aren't supported. +fn parse_instrument_mode(s: &str) -> Result { + let trimmed = s.trim(); + if trimmed.eq_ignore_ascii_case("all") { + return Ok(InstrumentFilter::All); + } + if trimmed.eq_ignore_ascii_case("front-month") || trimmed.eq_ignore_ascii_case("front_month") { + return Ok(InstrumentFilter::FrontMonth); + } + if let Some(num) = trimmed.strip_prefix("id=") { + let id: u32 = num.parse().map_err(|e| { + format!("instrument-mode: expected `id=` but failed to parse '{num}': {e}") + })?; + return Ok(InstrumentFilter::Id(id)); + } + Err(format!( + "instrument-mode: expected `all`, `front-month`, or `id=`; got '{s}'" + )) } #[derive(Serialize, serde::Deserialize, Default)] @@ -472,7 +498,7 @@ fn main() -> Result<()> { seed: cli.seed, inference_only: false, outcome_label_cost: cli.outcome_label_cost, - instrument_id_filter: cli.instrument_id, + instrument_filter: cli.instrument_mode, }) .context("train loader")?; let mut val_loader = MultiHorizonLoader::new(&MultiHorizonLoaderConfig { @@ -484,7 +510,7 @@ fn main() -> Result<()> { seed: cli.seed.wrapping_add(0xC0FFEE), inference_only: false, outcome_label_cost: cli.outcome_label_cost, - instrument_id_filter: cli.instrument_id, + instrument_filter: cli.instrument_mode, }) .context("val loader")?; tracing::info!( diff --git a/crates/ml-alpha/src/data/loader.rs b/crates/ml-alpha/src/data/loader.rs index 2c144af73..80c6de515 100644 --- a/crates/ml-alpha/src/data/loader.rs +++ b/crates/ml-alpha/src/data/loader.rs @@ -15,6 +15,10 @@ use std::path::PathBuf; use anyhow::{Context, Result}; use data::providers::databento::mbp10::{BidAskPair, Mbp10Snapshot}; use ml_features::predecoded::load_or_predecode_mbp10; + +/// Re-export so downstream crates (ml-backtesting, tests) can construct a +/// `MultiHorizonLoaderConfig` without taking a direct dependency on `data`. +pub use data::providers::databento::dbn_parser::InstrumentFilter; use rand::{Rng, SeedableRng}; use rand_chacha::ChaCha8Rng; @@ -139,18 +143,18 @@ pub struct MultiHorizonLoaderConfig { /// the aux head learn a cost-free outcome and rarely matches the /// inference-time trading objective. pub outcome_label_cost: f32, - /// When `Some(id)`, the underlying MBP-10 predecoder keeps only the - /// records whose `record.hd.instrument_id == id`. `None` preserves - /// the legacy all-records behavior. + /// Selects which `instrument_id` records the MBP-10 predecoder keeps. + /// See [`InstrumentFilter`] for variants. /// - /// ES.FUT DBN archives are multi-instrument: the same file carries - /// the front-month contract (e.g. `ESH4` for Q1 2024, instrument_id - /// 17077), the back month (`ESM4`, 5602), plus calendar spreads / - /// related futures. Without a filter the loader interleaves all of - /// them and computes ΔP across instruments — which is meaningless - /// and silently inflates K-step labels. For ES set this to the - /// front-month id; for single-instrument datasets leave it `None`. - pub instrument_id_filter: Option, + /// ES.FUT DBN archives are parent-symbol expansions: the same file + /// carries the front-month contract (e.g. `ESH4` for Q1 2024, + /// instrument_id 17077), back months (`ESM4`, etc.), plus calendar + /// spreads. Without a filter the loader interleaves all of them and + /// computes ΔP across instruments — silently inflating K-step labels + /// by thousands of dollars at the contract boundaries. For ES, + /// `FrontMonth` auto-detects the dominant id per file (handles the + /// quarterly roll); single-instrument datasets can use `All`. + pub instrument_filter: InstrumentFilter, } /// Default round-trip cost (price units) baked into D-style outcome labels @@ -287,9 +291,9 @@ impl MultiHorizonLoader { }; let mut files_loaded: Vec = Vec::with_capacity(files.len()); for path in &files { - let snapshots = load_or_predecode_mbp10(path, &cfg.predecoded_dir, cfg.instrument_id_filter) + let snapshots = load_or_predecode_mbp10(path, &cfg.predecoded_dir, cfg.instrument_filter) .with_context(|| format!( - "load mbp10 {} (filter={:?})", path.display(), cfg.instrument_id_filter + "load mbp10 {} (filter={:?})", path.display(), cfg.instrument_filter ))?; if snapshots.len() < min_size { tracing::warn!( @@ -697,7 +701,7 @@ mod inference_mode_tests { seed: 0xCAFEF00D, inference_only, outcome_label_cost: DEFAULT_OUTCOME_LABEL_COST_ES, - instrument_id_filter: None, + instrument_filter: InstrumentFilter::All, }) } @@ -779,7 +783,7 @@ mod inference_mode_tests { seed: 0, inference_only: false, outcome_label_cost: DEFAULT_OUTCOME_LABEL_COST_ES, - instrument_id_filter: None, + instrument_filter: InstrumentFilter::All, }; if let Ok(mut loader) = MultiHorizonLoader::new(&cfg) { let err = loader.next_inference_input(); diff --git a/crates/ml-alpha/tests/multi_horizon_loader.rs b/crates/ml-alpha/tests/multi_horizon_loader.rs index 675f038fb..f1fc559fd 100644 --- a/crates/ml-alpha/tests/multi_horizon_loader.rs +++ b/crates/ml-alpha/tests/multi_horizon_loader.rs @@ -2,6 +2,7 @@ //! //! Real-data path is `--ignored`; runs only with FOXHUNT_TEST_DATA set. +use data::providers::databento::dbn_parser::InstrumentFilter; use ml_alpha::data::loader::{ discover_mbp10_files_sorted, MultiHorizonLoader, MultiHorizonLoaderConfig, DEFAULT_OUTCOME_LABEL_COST_ES, @@ -26,7 +27,7 @@ fn cfg_from_env() -> Option { seed: 0xA1A2_A3A4, inference_only: false, outcome_label_cost: DEFAULT_OUTCOME_LABEL_COST_ES, - instrument_id_filter: None, + instrument_filter: InstrumentFilter::All, }) } @@ -70,7 +71,7 @@ fn loader_errors_on_empty_files() { seed: 0, inference_only: false, outcome_label_cost: DEFAULT_OUTCOME_LABEL_COST_ES, - instrument_id_filter: None, + instrument_filter: InstrumentFilter::All, }; let res = MultiHorizonLoader::new(&cfg); assert!(res.is_err(), "expected error for empty file list"); diff --git a/crates/ml-backtesting/src/harness.rs b/crates/ml-backtesting/src/harness.rs index 856339a89..c4c9cf6bb 100644 --- a/crates/ml-backtesting/src/harness.rs +++ b/crates/ml-backtesting/src/harness.rs @@ -19,7 +19,7 @@ use anyhow::{Context, Result}; use ml_alpha::cfc::snap_features::Mbp10RawInput; use ml_alpha::data::loader::{ - discover_mbp10_files_sorted, MultiHorizonLoader, MultiHorizonLoaderConfig, + discover_mbp10_files_sorted, InstrumentFilter, MultiHorizonLoader, MultiHorizonLoaderConfig, }; use ml_alpha::pinned_mem::MappedF32Buffer; use ml_alpha::trainer::perception::PerceptionTrainer; @@ -260,7 +260,7 @@ impl BacktestHarness { // No instrument filter at the harness level — backtest sidecars // are still the legacy all-records files. A follow-up that plumbs // a CLI flag through `BacktestHarnessConfig` can lift this. - instrument_id_filter: None, + instrument_filter: InstrumentFilter::All, }; let loader = MultiHorizonLoader::new(&loader_cfg)?; diff --git a/crates/ml-backtesting/tests/ring3_replay.rs b/crates/ml-backtesting/tests/ring3_replay.rs index 3a5530531..9cd575dd8 100644 --- a/crates/ml-backtesting/tests/ring3_replay.rs +++ b/crates/ml-backtesting/tests/ring3_replay.rs @@ -25,7 +25,7 @@ use anyhow::Result; use ml_alpha::data::loader::{ - discover_mbp10_files_sorted, MultiHorizonLoader, MultiHorizonLoaderConfig, + discover_mbp10_files_sorted, InstrumentFilter, MultiHorizonLoader, MultiHorizonLoaderConfig, }; use ml_backtesting::sim::LobSimCuda; use ml_core::device::MlDevice; @@ -49,7 +49,7 @@ fn try_loader() -> Option { seed: 0xCAFE_F00D, inference_only: true, outcome_label_cost: ml_alpha::data::loader::DEFAULT_OUTCOME_LABEL_COST_ES, - instrument_id_filter: None, + instrument_filter: InstrumentFilter::All, }; match MultiHorizonLoader::new(&cfg) { Ok(l) => Some(l), diff --git a/crates/ml-backtesting/tests/trainer_parity.rs b/crates/ml-backtesting/tests/trainer_parity.rs index 1355cfc56..4ec6680ec 100644 --- a/crates/ml-backtesting/tests/trainer_parity.rs +++ b/crates/ml-backtesting/tests/trainer_parity.rs @@ -16,7 +16,7 @@ use anyhow::Result; use ml_alpha::data::loader::{ - discover_mbp10_files_sorted, MultiHorizonLoader, MultiHorizonLoaderConfig, + discover_mbp10_files_sorted, InstrumentFilter, MultiHorizonLoader, MultiHorizonLoaderConfig, }; use std::path::PathBuf; @@ -36,7 +36,7 @@ fn try_loader(inference_only: bool) -> Option { seed: 0xCAFEF00D, inference_only, outcome_label_cost: ml_alpha::data::loader::DEFAULT_OUTCOME_LABEL_COST_ES, - instrument_id_filter: None, + instrument_filter: InstrumentFilter::All, }; match MultiHorizonLoader::new(&cfg) { Ok(l) => Some(l), diff --git a/crates/ml-features/src/mbp10_loader.rs b/crates/ml-features/src/mbp10_loader.rs index bd93cf707..6e0fe775b 100644 --- a/crates/ml-features/src/mbp10_loader.rs +++ b/crates/ml-features/src/mbp10_loader.rs @@ -26,7 +26,10 @@ //! } //! ``` -use data::providers::databento::{dbn_parser::DbnParser, mbp10::{BidAskPair, Mbp10Snapshot}}; +use data::providers::databento::{ + dbn_parser::{DbnParser, InstrumentFilter}, + mbp10::{BidAskPair, Mbp10Snapshot}, +}; use std::path::Path; use chrono::{DateTime, Utc}; @@ -98,7 +101,7 @@ pub fn compute_ofi_from_file(file_path: &Path) -> Result, MLError> let mut features = Vec::new(); let snapshot_count = parser - .parse_mbp10_streaming(file_path, 100, None, |snapshot| { + .parse_mbp10_streaming(file_path, 100, InstrumentFilter::All, |snapshot| { if let Ok(f) = calculator.calculate(snapshot) { features.push(f.to_array()); } // Skip failed calculations (e.g. first snapshot with no prev) @@ -159,7 +162,7 @@ pub fn compute_ofi_with_trades( let mut trade_cursor: usize = 0; let snapshot_count = parser - .parse_mbp10_streaming(mbp10_file, 100, None, |snapshot| { + .parse_mbp10_streaming(mbp10_file, 100, InstrumentFilter::All, |snapshot| { // Feed all trades with timestamp <= this snapshot's timestamp. // This ensures VPIN/Kyle's Lambda/trade_imbalance are populated // before the OFI calculator processes the snapshot. diff --git a/crates/ml-features/src/predecoded.rs b/crates/ml-features/src/predecoded.rs index 4ea3aa54a..789beb7f1 100644 --- a/crates/ml-features/src/predecoded.rs +++ b/crates/ml-features/src/predecoded.rs @@ -19,7 +19,7 @@ use std::path::{Path, PathBuf}; use std::time::SystemTime; use bincode; -use data::providers::databento::dbn_parser::DbnParser; +use data::providers::databento::dbn_parser::{DbnParser, InstrumentFilter}; use data::providers::databento::mbp10::Mbp10Snapshot; use serde::{Deserialize, Serialize}; @@ -162,27 +162,29 @@ pub fn load_or_predecode_trades( /// `Vec`, persists a sidecar, returns the snapshots. On hit: /// deserializes the sidecar (skips zstd). /// -/// # `instrument_id_filter` +/// # `filter` /// -/// When `Some(id)`, only DBN records whose `record.hd.instrument_id == id` -/// are kept; all others are dropped at decode time. This is required for -/// multi-instrument DBN files (e.g. ES.FUT archives that contain front -/// month + back month + calendar spreads in one stream) — without it, -/// downstream features (ΔP, Mid, K-step labels) span across instruments -/// and produce garbage. +/// Selects which `instrument_id` records to keep. ES.FUT parent-expanded +/// DBN files contain ~14 distinct contract months; without filtering, +/// downstream K-window labels span contract boundaries and produce $5000+ +/// ΔP artifacts at the rolls. /// -/// The sidecar filename embeds the filter id (`.mbp10_instr.predecoded.bin`) -/// so filtered and unfiltered runs use disjoint cache files and never -/// invalidate each other. `None` keeps the legacy `.mbp10.predecoded.bin` -/// name so existing sidecars on disk remain valid for unfiltered callers. +/// The sidecar filename embeds the filter spec so disjoint runs use disjoint +/// caches and never invalidate each other: +/// - `InstrumentFilter::All` → `.mbp10.predecoded.bin` +/// - `InstrumentFilter::Id(N)` → `.mbp10_instr.predecoded.bin` +/// - `InstrumentFilter::FrontMonth` → `.mbp10_front_month.predecoded.bin` pub fn load_or_predecode_mbp10( source: &Path, predecoded_dir: &Path, - instrument_id_filter: Option, + filter: InstrumentFilter, ) -> Result, MLError> { - let sidecar = match instrument_id_filter { - Some(filter) => sidecar_path(source, predecoded_dir, &format!("mbp10_instr{}", filter)), - None => sidecar_path(source, predecoded_dir, "mbp10"), + let sidecar = match filter { + InstrumentFilter::All => sidecar_path(source, predecoded_dir, "mbp10"), + InstrumentFilter::Id(id) => { + sidecar_path(source, predecoded_dir, &format!("mbp10_instr{}", id)) + } + InstrumentFilter::FrontMonth => sidecar_path(source, predecoded_dir, "mbp10_front_month"), }; match try_read_sidecar::>(source, &sidecar) { Ok(Some(snapshots)) => { @@ -206,7 +208,7 @@ pub fn load_or_predecode_mbp10( .map_err(|e| MLError::InsufficientData(format!("DbnParser init failed: {e}")))?; let mut snapshots = Vec::new(); parser - .parse_mbp10_streaming(source, 100, instrument_id_filter, |snap| { + .parse_mbp10_streaming(source, 100, filter, |snap| { snapshots.push(snap.clone()); }) .map_err(|e| MLError::InsufficientData(format!("parse_mbp10_streaming failed: {e}")))?; diff --git a/crates/ml/examples/alpha_dqn_h600_smoke.rs b/crates/ml/examples/alpha_dqn_h600_smoke.rs index 352b7641d..e391f19a4 100644 --- a/crates/ml/examples/alpha_dqn_h600_smoke.rs +++ b/crates/ml/examples/alpha_dqn_h600_smoke.rs @@ -172,7 +172,10 @@ impl Drop for MappedF32 { } } -use data::providers::databento::{dbn_parser::DbnParser, mbp10::Mbp10Snapshot}; +use data::providers::databento::{ + dbn_parser::{DbnParser, InstrumentFilter}, + mbp10::Mbp10Snapshot, +}; use ml::cuda_pipeline::alpha_isv_slots::{ ACTION_ENTROPY_EMA_INDEX, ALPHA_ISV_BLOCK_LO, EARLY_Q_MOVEMENT_EMA_INDEX, Q_SPREAD_EMA_INDEX, RANDOM_BASELINE_MEAN_INDEX, RANDOM_BASELINE_STD_INDEX, @@ -391,7 +394,7 @@ fn load_snapshots( let mut rows: Vec = Vec::with_capacity(max_snapshots); 'files: for file in &files { let mut hit_limit = false; - let result = parser.parse_mbp10_streaming(file, snapshot_interval, None, |snap: &Mbp10Snapshot| { + let result = parser.parse_mbp10_streaming(file, snapshot_interval, InstrumentFilter::All, |snap: &Mbp10Snapshot| { if rows.len() >= max_snapshots { hit_limit = true; return; diff --git a/crates/ml/examples/alpha_fit_fill_model.rs b/crates/ml/examples/alpha_fit_fill_model.rs index d80904bf5..444fdbce7 100644 --- a/crates/ml/examples/alpha_fit_fill_model.rs +++ b/crates/ml/examples/alpha_fit_fill_model.rs @@ -40,7 +40,7 @@ use clap::Parser; use tracing::{info, warn}; use data::providers::databento::{ - dbn_parser::DbnParser, + dbn_parser::{DbnParser, InstrumentFilter}, mbp10::Mbp10Snapshot, }; @@ -163,7 +163,7 @@ fn main() -> Result<()> { let result = parser.parse_mbp10_streaming( file, cli.snapshot_interval, - None, + InstrumentFilter::All, |snap: &Mbp10Snapshot| { if n_processed >= limit { hit_limit = true; diff --git a/crates/ml/examples/alpha_random_baseline.rs b/crates/ml/examples/alpha_random_baseline.rs index 930174f7a..08f321815 100644 --- a/crates/ml/examples/alpha_random_baseline.rs +++ b/crates/ml/examples/alpha_random_baseline.rs @@ -35,7 +35,7 @@ use clap::Parser; use tracing::{info, warn}; use data::providers::databento::{ - dbn_parser::DbnParser, + dbn_parser::{DbnParser, InstrumentFilter}, mbp10::Mbp10Snapshot, }; use ml::env::action_space::N_ACTIONS; @@ -168,7 +168,7 @@ fn main() -> Result<()> { 'files: for file in &files { let mut hit_limit = false; let result = - parser.parse_mbp10_streaming(file, cli.snapshot_interval, None, |snap: &Mbp10Snapshot| { + parser.parse_mbp10_streaming(file, cli.snapshot_interval, InstrumentFilter::All, |snap: &Mbp10Snapshot| { if rows.len() >= cli.max_snapshots { hit_limit = true; return; diff --git a/crates/ml/examples/precompute_features.rs b/crates/ml/examples/precompute_features.rs index ff5bc9ea1..a37ad71d4 100644 --- a/crates/ml/examples/precompute_features.rs +++ b/crates/ml/examples/precompute_features.rs @@ -580,7 +580,7 @@ async fn main() -> Result<()> { // precompute_features is the multi-symbol feature-cache // builder; it intentionally consumes every instrument in // each file. No filter. - match ml::features::predecoded::load_or_predecode_mbp10(file, &predecoded_dir, None) { + match ml::features::predecoded::load_or_predecode_mbp10(file, &predecoded_dir, data::providers::databento::dbn_parser::InstrumentFilter::All) { Ok(snapshots) => { info!( " {} -> {} snapshots", diff --git a/crates/ml/src/trainers/dqn/data_loading.rs b/crates/ml/src/trainers/dqn/data_loading.rs index 2594c234b..ae5681aad 100644 --- a/crates/ml/src/trainers/dqn/data_loading.rs +++ b/crates/ml/src/trainers/dqn/data_loading.rs @@ -314,7 +314,7 @@ impl DQNTrainer { if !self.hyperparams.mbp10_data_dir.is_empty() { use crate::features::ofi_calculator::OFICalculator; use crate::features::trades_loader::get_trades_for_bar; - use data::providers::databento::dbn_parser::DbnParser; + use data::providers::databento::dbn_parser::{DbnParser, InstrumentFilter}; let mbp10_dir = Path::new(&self.hyperparams.mbp10_data_dir); if mbp10_dir.exists() { @@ -336,10 +336,10 @@ impl DQNTrainer { Err(e) => { warn!(" DBN parser init failed: {e}"); return None; } }; let mut snapshots = Vec::new(); - // DQN data loading predates the instrument-filter - // refactor; keep legacy all-instruments behavior - // until the DQN trainer needs filtering. - match parser.parse_mbp10_streaming(file, 100, None, |snap| { + // DQN data loading keeps legacy all-instruments + // behavior; switch to FrontMonth when DQN training + // moves to multi-quarter walk-forward. + match parser.parse_mbp10_streaming(file, 100, InstrumentFilter::All, |snap| { snapshots.push(snap.clone()); }) { Ok(_) => { diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index 49bf4ad1d..f5499edc0 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -18735,3 +18735,18 @@ multi-instrument data; if/when ES.FUT calendar spreads become a training signal DQN, the call site should be updated to forward an instrument_id from config. **Wire-up impact:** None (semantic preservation only). + +## 2026-05-22 — chore(dqn): migrate to InstrumentFilter::All after enum refactor + +**Branch:** `ml-alpha-phase-a`. **Trigger:** `parse_mbp10_streaming` filter parameter +changed from `Option` to `InstrumentFilter` enum to support per-file front-month +auto-detection across quarterly ES contract rolls (smoke `alpha-perception-k54wd` +failed because Q2-Q9 contain different `instrument_id`s than Q1 — see +`docs/superpowers/plans/2026-05-22-loader-instrument-filter-fix.md`). + +**Change:** `crates/ml/src/trainers/dqn/data_loading.rs:342` passes +`InstrumentFilter::All` to preserve the same legacy semantics as the previous +`None`. The comment is updated to point at the future migration target +(`FrontMonth` once DQN training moves to multi-quarter walk-forward). + +**Wire-up impact:** None (semantic preservation only). diff --git a/infra/k8s/argo/alpha-perception-template.yaml b/infra/k8s/argo/alpha-perception-template.yaml index c777ecdf5..c2d1e0383 100644 --- a/infra/k8s/argo/alpha-perception-template.yaml +++ b/infra/k8s/argo/alpha-perception-template.yaml @@ -66,8 +66,8 @@ spec: value: "1" - name: auto-horizon-weights value: "false" - - name: instrument-id - value: "" + - name: instrument-mode + value: "all" - name: early-stop-metric value: "mean_auc" - name: early-stop-patience @@ -447,8 +447,8 @@ spec: if [ "{{workflow.parameters.auto-horizon-weights}}" = "true" ]; then EXTRA_FLAGS="$EXTRA_FLAGS --auto-horizon-weights" fi - if [ -n "{{workflow.parameters.instrument-id}}" ]; then - EXTRA_FLAGS="$EXTRA_FLAGS --instrument-id {{workflow.parameters.instrument-id}}" + if [ -n "{{workflow.parameters.instrument-mode}}" ]; then + EXTRA_FLAGS="$EXTRA_FLAGS --instrument-mode {{workflow.parameters.instrument-mode}}" fi TRACE_FLAG="" diff --git a/scripts/argo-alpha-perception.sh b/scripts/argo-alpha-perception.sh index 692704f4a..28a2faf25 100755 --- a/scripts/argo-alpha-perception.sh +++ b/scripts/argo-alpha-perception.sh @@ -35,7 +35,7 @@ CV_FOLD=0 CV_N_FOLDS=1 CV_TRAIN_WINDOW=0 DECISION_STRIDE=1 -INSTRUMENT_ID="" +INSTRUMENT_MODE="all" WATCH=false usage() { @@ -56,6 +56,7 @@ Usage: $0 [OPTIONS] --cv-n-folds Total CV folds (default: $CV_N_FOLDS) --cv-train-window Files per train window (default: $CV_TRAIN_WINDOW = auto) --decision-stride Snapshot stride for sequence sampling (default: $DECISION_STRIDE) + --instrument-mode MBP-10 filter: all | front-month | id= (default: $INSTRUMENT_MODE) --watch Follow logs via argo watch EOF } @@ -81,7 +82,7 @@ while [[ $# -gt 0 ]]; do --cv-n-folds) CV_N_FOLDS="$2"; shift 2 ;; --cv-train-window) CV_TRAIN_WINDOW="$2"; shift 2 ;; --decision-stride) DECISION_STRIDE="$2"; shift 2 ;; - --instrument-id) INSTRUMENT_ID="$2"; shift 2 ;; + --instrument-mode) INSTRUMENT_MODE="$2"; shift 2 ;; --watch) WATCH=true; shift ;; -h|--help) usage; exit 0 ;; *) echo "Unknown option: $1"; usage; exit 1 ;; @@ -153,5 +154,5 @@ argo submit -n foxhunt --from=wftmpl/alpha-perception \ -p cv-n-folds="$CV_N_FOLDS" \ -p cv-train-window="$CV_TRAIN_WINDOW" \ -p decision-stride="$DECISION_STRIDE" \ - -p instrument-id="$INSTRUMENT_ID" \ + -p instrument-mode="$INSTRUMENT_MODE" \ $WATCH_FLAG