From 6beedca11051fd772a4ea6921e7d5353f5cf74d7 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Thu, 12 Mar 2026 20:54:44 +0100 Subject: [PATCH] perf(ofi): replace O(n) linear scan with binary search in get_snapshots_for_timestamp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The function was doing a linear scan through 14.5M sorted MBP-10 snapshots for each of 1.12M OHLCV bars, resulting in ~16.2 trillion comparisons. Replaced with partition_point (binary search) for O(log n) per lookup, reducing total comparisons to ~27M — a ~600,000x improvement. This was the root cause of OFI computation taking 30+ minutes during hyperopt data loading on H100. Co-Authored-By: Claude Opus 4.6 --- crates/ml-features/src/mbp10_loader.rs | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/crates/ml-features/src/mbp10_loader.rs b/crates/ml-features/src/mbp10_loader.rs index 08273ffc4..b49decd36 100644 --- a/crates/ml-features/src/mbp10_loader.rs +++ b/crates/ml-features/src/mbp10_loader.rs @@ -363,22 +363,14 @@ pub fn get_snapshots_for_timestamp( return &[]; } - // Find the first snapshot with timestamp >= target_ts - let mut start_idx = None; - for (i, snap) in snapshots.iter().enumerate() { - if snap.timestamp >= target_ts { - start_idx = Some(i); - break; - } - } - - match start_idx { - Some(idx) => { - let end_idx = (idx + window_size).min(snapshots.len()); - &snapshots[idx..end_idx] - } - None => &[], // Target timestamp is after all snapshots + // Binary search for the first snapshot with timestamp >= target_ts + // Snapshots are pre-sorted by timestamp (from DBN file ordering) + let idx = snapshots.partition_point(|s| s.timestamp < target_ts); + if idx >= snapshots.len() { + return &[]; // Target timestamp is after all snapshots } + let end_idx = (idx + window_size).min(snapshots.len()); + &snapshots[idx..end_idx] } /// Get the most recent N snapshots ending at the given index