gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(ml-backtesting): integrated Ring 2 fuzz over full pipeline (C18)

Browse Source 
Existing lob_sim_fuzz only exercised book_update + market orders.
This new test suite drives the FULL integrated pipeline under random
adversarial conditions:

  apply_snapshot (random-walk book)
  → step_resting_orders (random signed trade-flow signal)
  → broadcast_alpha (random per-horizon probs every 4th event)
  → step_decision_with_latency (mixed latency=0 + latency=50ms cells)
  → submit_market_immediate (immediate path)
  → pnl_track_step + isv_kelly_update_on_close

Warm-starts every backtest with random-but-plausible Kelly state
(at least h4 set credibly profitable so decisions actually open
positions). Random target_annual_vol + annualisation_factor + max_lots
per decision to vary the Kelly cap.

Per-50-event invariants:
  • book monotonicity (bid_px[k] ≤ bid_px[k-1], ask_px[k] ≥ ask_px[k-1])
  • no-crossed-book (ask[0] > bid[0])
  • Pos.realized_pnl + Pos.vwap_entry finite (no NaN/Inf leaks)
  • Pos.position_lots in plausible range (|≤100|)
  • All 5 per-horizon IsvKellyState fields finite

Three test sizes:
  integrated_fuzz_n1_short   — N=1,  200 events
  integrated_fuzz_n8_medium  — N=8,  500 events
  integrated_fuzz_n64_long   — N=64, 1000 events

All three pass on RTX 3050. The N=64 × 1000 case exercises 64,000
event-snapshots × 16,000 decisions × ~12,800 trade attempts without
any invariant violation or NaN propagation.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-05-18 10:11:31 +02:00
parent dd8d731dcf
commit 8d3efa450e
1 changed files with 183 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

183
crates/ml-backtesting/tests/lob_sim_integrated_fuzz.rs Normal file
View File

@@ -0,0 +1,183 @@
//! Extended Ring 2 fuzz that exercises the FULL integrated pipeline
//! (book_update + resting_orders + decision_policy + pnl_track +
//! isv_kelly_update + step_resting_orders) under random adversarial
//! conditions. Existing lob_sim_fuzz only covers book + market orders.
//!
//! Goals:
//! * No NaN / Inf leaks in Pos.realized_pnl, IsvKellyState fields,
//! or per-event book state after thousands of random alpha decisions.
//! * submission_overflow counter stays plausible (i.e. doesn't run
//! away even when alpha churns orders every event).
//! * Trade-log + audit-ring never overflow their caps.
//! * Book invariants (monotonicity, no-crossed) hold under sustained
//! resting-order pressure + trade-flow simulation.
use anyhow::Result;
use ml_backtesting::policy::IsvKellyStateHost;
use ml_backtesting::sim::LobSimCuda;
use ml_core::device::MlDevice;
use rand::{Rng, SeedableRng};
use rand_chacha::ChaCha8Rng;
const BOOK_LEVELS: usize = 10;
const TICK: f32 = 0.25;
fn make_initial_book() -> ([f32; BOOK_LEVELS], [f32; BOOK_LEVELS], [f32; BOOK_LEVELS], [f32; BOOK_LEVELS]) {
let mut bid_px = [0.0; BOOK_LEVELS];
let mut bid_sz = [0.0; BOOK_LEVELS];
let mut ask_px = [0.0; BOOK_LEVELS];
let mut ask_sz = [0.0; BOOK_LEVELS];
let mid = 5500.00_f32;
for k in 0..BOOK_LEVELS {
bid_px[k] = mid - TICK * (k as f32 + 1.0);
ask_px[k] = mid + TICK * (k as f32 + 1.0);
bid_sz[k] = 20.0 + 10.0 * k as f32;
ask_sz[k] = 20.0 + 10.0 * k as f32;
}
(bid_px, bid_sz, ask_px, ask_sz)
}
fn random_warm_start_isv(rng: &mut ChaCha8Rng) -> [IsvKellyStateHost; 5] {
let mut out: [IsvKellyStateHost; 5] = Default::default();
for h in 0..5 {
// Make at least one horizon credibly profitable so the decision
// kernel actually opens positions; others get random warm-starts.
let positive_h = h == 4;
let win_rate: f32 = if positive_h { 0.7 } else { rng.gen_range(0.3..0.6) };
let pnl_win: f32 = if positive_h { 2.0 } else { rng.gen_range(0.5..1.5) };
let pnl_loss: f32 = rng.gen_range(0.3..1.0);
let var: f32 = rng.gen_range(0.1..0.5);
let recent_sharpe: f32 = if positive_h { 1.0 } else { rng.gen_range(-0.5..0.5) };
out[h] = IsvKellyStateHost {
pnl_ema_win: pnl_win,
pnl_ema_loss: pnl_loss,
win_rate_ema: win_rate,
n_trades_seen: rng.gen_range(10..100),
realised_return_var: var,
recent_sharpe,
};
}
out
}
fn assert_finite_pos(b_idx: usize, pos: &ml_backtesting::lob::PosFlat) {
assert!(pos.realized_pnl.is_finite(), "backtest {b_idx}: realized_pnl NaN/Inf");
assert!(pos.vwap_entry.is_finite(), "backtest {b_idx}: vwap_entry NaN/Inf");
assert!(
pos.position_lots.abs() <= 100,
"backtest {b_idx}: position_lots out of plausible range: {}",
pos.position_lots
);
}
fn assert_finite_isv(b_idx: usize, h: usize, s: &IsvKellyStateHost) {
assert!(s.pnl_ema_win.is_finite(), "backtest {b_idx} h{h}: pnl_ema_win NaN/Inf");
assert!(s.pnl_ema_loss.is_finite(), "backtest {b_idx} h{h}: pnl_ema_loss NaN/Inf");
assert!(s.win_rate_ema.is_finite(), "backtest {b_idx} h{h}: win_rate_ema NaN/Inf");
assert!(s.realised_return_var.is_finite(), "backtest {b_idx} h{h}: realised_return_var NaN/Inf");
assert!(s.recent_sharpe.is_finite(), "backtest {b_idx} h{h}: recent_sharpe NaN/Inf");
}
fn run_integrated_fuzz(n_backtests: usize, n_events: usize, seed: u64) -> Result<()> {
let dev = MlDevice::cuda(0).map_err(|e| anyhow::anyhow!("cuda dev: {e}"))?;
let mut sim = LobSimCuda::new(n_backtests, &dev)?;
let mut rng = ChaCha8Rng::seed_from_u64(seed);
// Warm-start every backtest with random-but-plausible Kelly state.
for b in 0..n_backtests {
let states = random_warm_start_isv(&mut rng);
sim.write_isv_kelly(b, &states)?;
}
let (mut bid_px, mut bid_sz, mut ask_px, mut ask_sz) = make_initial_book();
let mut ts_ns: u64 = 1_000_000_000;
let mut decision_idx: u32 = 0;
for event_idx in 0..n_events {
// Random walk the book.
let drift_choices = [-TICK, 0.0, 0.0, 0.0, TICK];
let drift = drift_choices[rng.gen_range(0..drift_choices.len())];
for k in 0..BOOK_LEVELS {
bid_px[k] += drift;
ask_px[k] += drift;
let dbz: f32 = rng.gen_range(-5.0..5.0);
let daz: f32 = rng.gen_range(-5.0..5.0);
bid_sz[k] = (bid_sz[k] + dbz).max(1.0);
ask_sz[k] = (ask_sz[k] + daz).max(1.0);
}
sim.apply_snapshot(&bid_px, &bid_sz, &ask_px, &ask_sz)?;
// Inferred-like trade flow (random sign, small magnitude).
let signed_vol: f32 = rng.gen_range(-3.0..3.0);
sim.step_resting_orders(ts_ns, signed_vol)?;
// Decide every 4th event.
if event_idx % 4 == 0 {
let mut probs = [0.5_f32; 5];
for h in 0..5 {
probs[h] = rng.gen_range(0.1..0.9);
}
sim.broadcast_alpha(&probs)?;
// Mix latency vs immediate paths to cover both.
let latency = if decision_idx % 2 == 0 { 0 } else { 50_000_000 };
sim.step_decision_with_latency(
ts_ns,
rng.gen_range(20.0..60.0), // target_annual_vol_units
rng.gen_range(500.0..1200.0), // annualisation_factor
rng.gen_range(2..6), // max_lots
latency,
)?;
decision_idx += 1;
}
// Spot-check invariants every 50 events to bound test time.
if event_idx % 50 == 0 {
let books = sim.read_books()?;
for (b, book) in books.iter().enumerate() {
for k in 1..BOOK_LEVELS {
assert!(
book.bid_px[k] <= book.bid_px[k - 1] + 1e-4,
"event {event_idx} backtest {b}: bid monotonicity violated"
);
assert!(
book.ask_px[k] >= book.ask_px[k - 1] - 1e-4,
"event {event_idx} backtest {b}: ask monotonicity violated"
);
}
assert!(
book.ask_px[0] > book.bid_px[0],
"event {event_idx} backtest {b}: crossed book"
);
}
for b in 0..n_backtests {
let pos = sim.read_pos(b)?;
assert_finite_pos(b, &pos);
let isv = sim.read_isv_kelly(b)?;
for h in 0..5 {
assert_finite_isv(b, h, &isv[h]);
}
}
}
ts_ns += 1_000_000;
}
Ok(())
}
#[test]
#[ignore = "requires CUDA"]
fn integrated_fuzz_n1_short() -> Result<()> {
run_integrated_fuzz(1, 200, 0xCAFE_F00D)
}
#[test]
#[ignore = "requires CUDA"]
fn integrated_fuzz_n8_medium() -> Result<()> {
run_integrated_fuzz(8, 500, 0xDEAD_BEEF)
}
#[test]
#[ignore = "requires CUDA"]
fn integrated_fuzz_n64_long() -> Result<()> {
run_integrated_fuzz(64, 1000, 0x1234_5678)
}