From dd8d731dcf171e9c186618ab03659cbe8c93e8b6 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 18 May 2026 10:09:49 +0200 Subject: [PATCH] test(ml-backtesting): Ring 3 production-day replay validation (C17) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the spec §8 Ring 3 deferral. Two integration tests against FOXHUNT_TEST_DATA real MBP-10 day files: buy_and_hold_full_day Walk the first .dbn.zst from open to close; submit a market buy of 1 lot at the first event, run the full step-loop (apply_snapshot + step_resting_orders) over every subsequent event, then close with a market sell at the last event. Assert realized P&L matches (close_mid - open_mid) within ±2 ticks per spec slippage budget. Proves the book-walking + position accounting + step-loop orchestration are wire-level correct against real data, not just hand-crafted JSON fixtures. walk_book_one_full_day Same fixture, no trades — just iterates every event through apply_snapshot + step_resting_orders and asserts the book invariants (bid/ask monotonicity, no-crossed-book) hold at every 10_000-event checkpoint across the full day. Stress test of the kernel against real market microstructure (gaps, locked markets, regime shifts). Both tests skip gracefully when FOXHUNT_TEST_DATA is absent OR the predecoded sidecars are empty (the current 40-byte placeholder fixtures in test_data/futures-baseline/ES.FUT/*.predecoded.bin will trigger the skip path; populating those with a real Databento decode makes both tests fire end-to-end). Ring 3 is operational, not CI-mandatory. Co-Authored-By: Claude Opus 4.7 --- crates/ml-backtesting/tests/ring3_replay.rs | 180 ++++++++++++++++++++ 1 file changed, 180 insertions(+) create mode 100644 crates/ml-backtesting/tests/ring3_replay.rs diff --git a/crates/ml-backtesting/tests/ring3_replay.rs b/crates/ml-backtesting/tests/ring3_replay.rs new file mode 100644 index 000000000..669e5b4aa --- /dev/null +++ b/crates/ml-backtesting/tests/ring3_replay.rs @@ -0,0 +1,180 @@ +//! Ring 3 — production-day replay validation. See real-LOB spec §8 Ring 3. +//! +//! Two scenarios: +//! +//! buy_and_hold_full_day +//! Walk a populated MBP-10 fixture from first event to last, submit +//! a market buy of 1 lot at open, close with a market sell at the +//! last event. Assert simulated realized P&L matches +//! (close_mid - open_mid) within a slippage budget (a few ticks). +//! This proves the book-walking + position accounting + step-loop +//! orchestration are wire-level correct against real data, not +//! just hand-crafted JSON fixtures. +//! +//! walk_book_one_full_day +//! Same fixture; no trades. Just iterates every event through +//! apply_snapshot + step_resting_orders and asserts the LOB never +//! violates the bid/ask monotonicity + no-crossed-book invariants +//! at any point in the day. A stress test of the kernel against +//! real market microstructure (gaps, locked markets, etc). +//! +//! Both skip gracefully if FOXHUNT_TEST_DATA lacks populated predecoded +//! sidecars — the placeholder-empty 40-byte sidecars committed in the +//! test fixture tree won't drive these tests, but a real Databento +//! decode will. The Ring 3 gate is operational, not CI-mandatory. + +use anyhow::Result; +use ml_alpha::data::loader::{ + discover_mbp10_files_sorted, MultiHorizonLoader, MultiHorizonLoaderConfig, +}; +use ml_backtesting::sim::LobSimCuda; +use ml_core::device::MlDevice; +use std::path::PathBuf; + +fn try_loader() -> Option { + let root = std::env::var("FOXHUNT_TEST_DATA").ok()?; + let mbp10 = PathBuf::from(&root).join("ES.FUT"); + if !mbp10.exists() { + return None; + } + let files = discover_mbp10_files_sorted(&mbp10).ok()?; + // Take just the first file to keep the test bounded. + let files = files.into_iter().take(1).collect::>(); + let cfg = MultiHorizonLoaderConfig { + files, + predecoded_dir: mbp10.clone(), + seq_len: 1, + horizons: [30, 100, 300, 1000, 6000], + n_max_sequences: 0, + seed: 0xCAFE_F00D, + decision_stride: 1, + inference_only: true, + }; + match MultiHorizonLoader::new(&cfg) { + Ok(l) => Some(l), + Err(e) => { + eprintln!("skipping: fixture data not usable ({e})"); + None + } + } +} + +fn try_device() -> Option { + match MlDevice::cuda(0) { + Ok(d) => Some(d), + Err(e) => { + eprintln!("skipping: cuda device unavailable ({e})"); + None + } + } +} + +fn mid_of(raw: &ml_alpha::cfc::snap_features::Mbp10RawInput) -> f32 { + (raw.bid_px[0] + raw.ask_px[0]) * 0.5 +} + +#[test] +#[ignore = "requires populated FOXHUNT_TEST_DATA"] +fn buy_and_hold_full_day() -> Result<()> { + let Some(mut loader) = try_loader() else { return Ok(()); }; + let Some(dev) = try_device() else { return Ok(()); }; + let mut sim = LobSimCuda::new(1, &dev)?; + + // Snapshot the OPENING price as soon as we see the first event. + let first = match loader.next_inference_input()? { + Some(r) => r, + None => { + eprintln!("skipping: empty stream"); + return Ok(()); + } + }; + sim.apply_snapshot(&first.bid_px, &first.bid_sz, &first.ask_px, &first.ask_sz)?; + let open_mid = mid_of(&first); + // Aggressive buy 1 lot at the open's best ask. + sim.submit_market(0, 0 /*buy*/, 1, first.ts_ns)?; + + let pos_after_open = sim.read_pos(0)?; + assert_eq!( + pos_after_open.position_lots, 1, + "open buy didn't fill (likely fixture data has zero ask depth)" + ); + let entry_vwap = pos_after_open.vwap_entry; + + // Walk every subsequent event through book_update + resting_orders. + let mut last_raw = first; + let mut event_count = 1u64; + while let Some(raw) = loader.next_inference_input()? { + sim.apply_snapshot(&raw.bid_px, &raw.bid_sz, &raw.ask_px, &raw.ask_sz)?; + sim.step_resting_orders(raw.ts_ns, raw.trade_signed_vol)?; + event_count += 1; + last_raw = raw; + } + + // Close at the last event: aggressive sell 1 lot. + sim.submit_market(0, 1 /*sell*/, 1, last_raw.ts_ns)?; + let final_pos = sim.read_pos(0)?; + assert_eq!( + final_pos.position_lots, 0, + "close sell didn't fully unwind position" + ); + + let close_mid = mid_of(&last_raw); + let expected_pnl_price_units = close_mid - open_mid; + let realized = final_pos.realized_pnl; + let drift_ticks = (realized - expected_pnl_price_units).abs() / 0.25; + + eprintln!( + "ring3 buy_and_hold: events={} entry_vwap={:.2} open_mid={:.2} close_mid={:.2} expected_pnl={:.4} realized={:.4} drift_ticks={:.2}", + event_count, entry_vwap, open_mid, close_mid, expected_pnl_price_units, realized, drift_ticks + ); + + // Slippage budget: ±2 ticks (= 0.5 price units) per spec §8 Ring 3. + // The simulator's market-order path walks the book at both ends, so + // the difference vs mid-to-mid pricing accumulates the open-side + // half-spread + the close-side half-spread, plus any in-day book + // movement we observed via realized_pnl bookkeeping. + assert!( + drift_ticks <= 2.0, + "realized P&L drift {drift_ticks:.2} ticks > 2-tick budget — sim books vs hand-computed mid diverge" + ); + Ok(()) +} + +#[test] +#[ignore = "requires populated FOXHUNT_TEST_DATA"] +fn walk_book_one_full_day() -> Result<()> { + let Some(mut loader) = try_loader() else { return Ok(()); }; + let Some(dev) = try_device() else { return Ok(()); }; + let mut sim = LobSimCuda::new(1, &dev)?; + + let mut event_count = 0u64; + while let Some(raw) = loader.next_inference_input()? { + sim.apply_snapshot(&raw.bid_px, &raw.bid_sz, &raw.ask_px, &raw.ask_sz)?; + sim.step_resting_orders(raw.ts_ns, raw.trade_signed_vol)?; + event_count += 1; + + // Spot-check invariants every 10_000 events to bound test time. + if event_count % 10_000 == 0 { + let books = sim.read_books()?; + for (b, book) in books.iter().enumerate() { + for k in 1..10 { + assert!( + book.bid_px[k] <= book.bid_px[k - 1] + 1e-4, + "event {event_count} backtest {b}: bid monotonicity violated at level {k}" + ); + assert!( + book.ask_px[k] >= book.ask_px[k - 1] - 1e-4, + "event {event_count} backtest {b}: ask monotonicity violated at level {k}" + ); + } + assert!( + book.ask_px[0] > book.bid_px[0], + "event {event_count} backtest {b}: crossed book" + ); + } + } + } + eprintln!("ring3 walk_book: walked {event_count} events without invariant violation"); + assert!(event_count > 0, "fixture produced no events"); + Ok(()) +}