From a60bb562b026ccb81eac2653784191d4d4366479 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 8 Apr 2026 21:31:46 +0200 Subject: [PATCH] =?UTF-8?q?test(data):=20per-file=20vs=20global=20front-mo?= =?UTF-8?q?nth=20filter=20=E2=80=94=20documents=20quarterly=20roll=20bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test proves that global filter_front_month on merged trades drops entire quarters (Q1 lost when Q2 has higher volume). Per-file filtering keeps both quarters. This is the regression test for the bug that produced "Need at least 18 months of data" on H100. Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/ml-features/src/trades_loader.rs | 42 +++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/crates/ml-features/src/trades_loader.rs b/crates/ml-features/src/trades_loader.rs index f86cc05dd..aa1934451 100644 --- a/crates/ml-features/src/trades_loader.rs +++ b/crates/ml-features/src/trades_loader.rs @@ -322,4 +322,46 @@ mod tests { let filtered = super::filter_front_month(&[]); assert!(filtered.is_empty()); } + + #[test] + fn test_filter_front_month_per_file_pattern() { + // Simulates quarterly roll: Q1 has front-month id=100, Q2 has id=200. + // If you filter globally, only one quarter survives (the one with more volume). + // Correct behavior: filter PER FILE, then concatenate both quarters. + let q1_trades = vec![ + DbnTrade { timestamp: 1000, price: 5100.0, volume: 200, is_buy: true, instrument_id: 100 }, + DbnTrade { timestamp: 2000, price: 5101.0, volume: 100, is_buy: false, instrument_id: 100 }, + DbnTrade { timestamp: 1500, price: 5160.0, volume: 30, is_buy: true, instrument_id: 101 }, // back month + ]; + let q2_trades = vec![ + DbnTrade { timestamp: 5000, price: 5200.0, volume: 300, is_buy: true, instrument_id: 200 }, + DbnTrade { timestamp: 6000, price: 5201.0, volume: 150, is_buy: false, instrument_id: 200 }, + DbnTrade { timestamp: 5500, price: 5260.0, volume: 20, is_buy: true, instrument_id: 201 }, // back month + ]; + + // Per-file filtering (CORRECT): each file keeps its own front month + let q1_filtered = super::filter_front_month(&q1_trades); + let q2_filtered = super::filter_front_month(&q2_trades); + assert_eq!(q1_filtered.len(), 2, "Q1 should keep id=100 (2 trades)"); + assert_eq!(q2_filtered.len(), 2, "Q2 should keep id=200 (2 trades)"); + assert!(q1_filtered.iter().all(|t| t.instrument_id == 100)); + assert!(q2_filtered.iter().all(|t| t.instrument_id == 200)); + let combined_len = q1_filtered.len() + q2_filtered.len(); + assert_eq!(combined_len, 4, "Per-file: 4 trades from both quarters"); + + // Global filtering (WRONG): merges all, picks single highest-volume id + let mut all_trades = q1_trades.clone(); + all_trades.extend(q2_trades.clone()); + let global_filtered = super::filter_front_month(&all_trades); + // id=200 has 450 volume vs id=100's 300 → global keeps only Q2 + assert!(global_filtered.iter().all(|t| t.instrument_id == 200), + "Global filter incorrectly drops Q1 entirely"); + assert_eq!(global_filtered.len(), 2, "Global filter loses Q1 data"); + + // This test documents WHY per-file filtering is required: + // global loses 50% of the data (one quarter). + assert!(combined_len > global_filtered.len(), + "Per-file ({combined_len}) must produce more data than global ({})", + global_filtered.len()); + } }