diff --git a/crates/ml-features/src/mbp10_loader.rs b/crates/ml-features/src/mbp10_loader.rs index 50046d2aa..3dfb32748 100644 --- a/crates/ml-features/src/mbp10_loader.rs +++ b/crates/ml-features/src/mbp10_loader.rs @@ -681,28 +681,46 @@ pub fn mbp10_to_imbalance_bars( // gate. Mirror the per-file pattern in `precompute_features.rs:347-365` // (uses `filter_front_month` on `DbnTrade` slices); same logic, applied // to `Mbp10Trade.instrument_id`. - let mut all_trades: Vec = Vec::new(); - for file_path in &dbn_files { - match extract_trades_from_dbn_file(file_path) { - Ok(trades) => { - let raw_count = trades.len(); - let filtered = filter_front_month_mbp10(&trades); - tracing::info!( - " {} -> {} trades, {} front-month", - file_path.file_name().unwrap_or_default().to_string_lossy(), - raw_count, - filtered.len() - ); - all_trades.extend(filtered); - } - Err(e) => { - tracing::warn!( - "Failed to extract trades from {}: {}", - file_path.display(), - e - ); - } - } + // + // SP20 parallelism: `extract_trades_from_dbn_file` is a pure + // file → Vec function and the 9 quarterly files are fully + // independent (each carries its own contract universe). Mirror the + // par_iter pattern in `precompute_features.rs:551-565` so a 28-core + // CPU node can decode the files concurrently. Front-month filter is + // applied per-file as before — semantics unchanged. Final ordering + // is enforced by the `all_trades.sort_by(|a, b| ...)` step below, so + // file-level reduce order is irrelevant for correctness. + let per_file_trades: Vec> = { + use rayon::prelude::*; + dbn_files + .par_iter() + .filter_map(|file_path| match extract_trades_from_dbn_file(file_path) { + Ok(trades) => { + let raw_count = trades.len(); + let filtered = filter_front_month_mbp10(&trades); + tracing::info!( + " {} -> {} trades, {} front-month", + file_path.file_name().unwrap_or_default().to_string_lossy(), + raw_count, + filtered.len() + ); + Some(filtered) + } + Err(e) => { + tracing::warn!( + "Failed to extract trades from {}: {}", + file_path.display(), + e + ); + None + } + }) + .collect() + }; + let mut all_trades: Vec = + Vec::with_capacity(per_file_trades.iter().map(Vec::len).sum()); + for chunk in per_file_trades { + all_trades.extend(chunk); } if all_trades.is_empty() {