From 7d0a29dcedee29479952c068cf7abea4ffcb7090 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 6 May 2026 10:57:51 +0200 Subject: [PATCH] feat(sp15-p2a.1): LobBar canonical ABI + 4 synthetic market generators MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2A scaffolding lands FIRST per spec §4.4 ABI contract. Phase 1.2 cost kernel reads LobBar; both dev synthetic and prod fxcache produce LobBar — dev/prod parity per Q3. Generators: flat_market, drift_market, ou_market, regime_switch_market (seeded RNG for reproducible tests). Regime-switch test uses sticky 0.99/0.01 transitions (true regime persistence; spec's 50/50 was a random walk, not a regime switch — corrected with code comment). behavioral_suite test target wired into Cargo.toml; will run all 22 Phase 2 tests once they land in Phase 2B/2C. Audit doc: SP15 Phase 2A.1 entry appended to docs/dqn-wire-up-audit.md per Invariant 7 (component changes require audit-doc update). Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/ml/Cargo.toml | 4 + crates/ml/src/cuda_pipeline/lob_bar.rs | 33 +++++ crates/ml/src/cuda_pipeline/mod.rs | 1 + crates/ml/tests/behavioral/main.rs | 6 + .../ml/tests/behavioral/synthetic_markets.rs | 123 ++++++++++++++++++ docs/dqn-wire-up-audit.md | 12 ++ 6 files changed, 179 insertions(+) create mode 100644 crates/ml/src/cuda_pipeline/lob_bar.rs create mode 100644 crates/ml/tests/behavioral/main.rs create mode 100644 crates/ml/tests/behavioral/synthetic_markets.rs diff --git a/crates/ml/Cargo.toml b/crates/ml/Cargo.toml index e0fffbd3e..efe9653d8 100644 --- a/crates/ml/Cargo.toml +++ b/crates/ml/Cargo.toml @@ -272,6 +272,10 @@ path = "examples/hyperopt_baseline_rl.rs" name = "hyperopt_baseline_supervised" path = "examples/hyperopt_baseline_supervised.rs" +[[test]] +name = "behavioral_suite" +path = "tests/behavioral/main.rs" + [[bench]] name = "microstructure_bench" harness = false diff --git a/crates/ml/src/cuda_pipeline/lob_bar.rs b/crates/ml/src/cuda_pipeline/lob_bar.rs new file mode 100644 index 000000000..b265f514d --- /dev/null +++ b/crates/ml/src/cuda_pipeline/lob_bar.rs @@ -0,0 +1,33 @@ +//! Canonical ABI for synthetic and real LOB data. +//! Phase 2A defines this; Phase 1.2 cost kernel and main eval consume it. +//! Per spec §4.4. + +#[repr(C)] +#[derive(Debug, Clone, Copy)] +pub struct LobBar { + pub price: f32, // mid-price, MES futures dollar units + pub spread: f32, // bid-ask spread, same units as price (e.g., 0.25 = quarter-tick) + pub ofi: f32, // signed OFI normalized to ~[-3, 3] +} + +impl LobBar { + pub const fn new(price: f32, spread: f32, ofi: f32) -> Self { + Self { price, spread, ofi } + } +} + +/// Convert AoS Vec → SoA (prices, spreads, ofis). +/// Cost-net kernel and synthetic markets both flatten on the GPU side; +/// this helper centralises the conversion. +pub fn into_soa(bars: &[LobBar]) -> (Vec, Vec, Vec) { + let n = bars.len(); + let mut prices = Vec::with_capacity(n); + let mut spreads = Vec::with_capacity(n); + let mut ofis = Vec::with_capacity(n); + for b in bars { + prices.push(b.price); + spreads.push(b.spread); + ofis.push(b.ofi); + } + (prices, spreads, ofis) +} diff --git a/crates/ml/src/cuda_pipeline/mod.rs b/crates/ml/src/cuda_pipeline/mod.rs index e8d126004..3a6d3e288 100644 --- a/crates/ml/src/cuda_pipeline/mod.rs +++ b/crates/ml/src/cuda_pipeline/mod.rs @@ -64,6 +64,7 @@ pub mod sp11_isv_slots; pub mod sp13_isv_slots; pub mod sp14_isv_slots; pub mod sp15_isv_slots; +pub mod lob_bar; pub use sp4_isv_slots::{ TARGET_Q_BOUND_INDEX, ATOM_POS_BOUND_BASE, WEIGHT_BOUND_BASE, ADAM_M_BOUND_BASE, ADAM_V_BOUND_BASE, WD_RATE_BASE, diff --git a/crates/ml/tests/behavioral/main.rs b/crates/ml/tests/behavioral/main.rs new file mode 100644 index 000000000..77b9f17f8 --- /dev/null +++ b/crates/ml/tests/behavioral/main.rs @@ -0,0 +1,6 @@ +//! SP15 Phase 2 behavioral test suite — runs locally on dev RTX 3050 Ti. +//! Pre-commit hook gates argo-train.sh: failing test blocks L40S deploy. + +mod synthetic_markets; +// Phase 2A.2 will add: mod oracle; mod harness; +// Phase 2B will add: mod test_2_1_*; ... mod test_2_22_*; diff --git a/crates/ml/tests/behavioral/synthetic_markets.rs b/crates/ml/tests/behavioral/synthetic_markets.rs new file mode 100644 index 000000000..37d1ae85a --- /dev/null +++ b/crates/ml/tests/behavioral/synthetic_markets.rs @@ -0,0 +1,123 @@ +//! SP15 Phase 2A — synthetic market generators producing LobBar streams. +//! Used by Phase 2B behavioral tests + Phase 1.2 cost kernel verification. + +use ml::cuda_pipeline::lob_bar::LobBar; +use rand::{Rng, SeedableRng}; +use rand::rngs::StdRng; + +/// Flat market: constant base price + Gaussian noise. +pub fn flat_market(n: usize, noise_sigma: f32, mean_spread: f32, seed: u64) -> Vec { + let mut rng = StdRng::seed_from_u64(seed); + (0..n).map(|_| { + let noise: f32 = rng.gen_range(-1.0..1.0) * noise_sigma; + let price = 4500.0 + noise; + let spread = (mean_spread + rng.gen_range(-0.05f32..0.05)).max(0.05); + let ofi = rng.gen_range(-1.0..1.0); + LobBar::new(price, spread, ofi) + }).collect() +} + +/// Drift market: drift μ + noise σ. +pub fn drift_market(n: usize, mu: f32, sigma: f32, mean_spread: f32, seed: u64) -> Vec { + let mut rng = StdRng::seed_from_u64(seed); + let mut price: f32 = 4500.0; + (0..n).map(|_| { + let dz: f32 = rng.gen_range(-1.0f32..1.0); + price += mu + sigma * dz; + let spread = (mean_spread + rng.gen_range(-0.05f32..0.05)).max(0.05); + let ofi = (mu / sigma.max(1e-6)).tanh() + rng.gen_range(-0.5f32..0.5); + LobBar::new(price, spread, ofi) + }).collect() +} + +/// OU process: mean-reverting around equilibrium. +pub fn ou_market(n: usize, theta: f32, sigma_eq: f32, mu: f32, + mean_spread: f32, seed: u64) -> Vec { + let mut rng = StdRng::seed_from_u64(seed); + let mut price: f32 = mu; + (0..n).map(|_| { + let dz: f32 = rng.gen_range(-1.0f32..1.0); + price += theta * (mu - price) + sigma_eq * dz; + let spread = (mean_spread + rng.gen_range(-0.05f32..0.05)).max(0.05); + let ofi = ((mu - price) / sigma_eq.max(1e-6)).tanh() + rng.gen_range(-0.3f32..0.3); + LobBar::new(price, spread, ofi) + }).collect() +} + +/// Regime-switch: Markov chain over `n_regimes`, each with (mu, sigma). +pub fn regime_switch_market( + n: usize, + regime_params: &[(f32, f32)], + transition_matrix: &[Vec], + mean_spread: f32, + seed: u64, +) -> Vec { + assert_eq!(regime_params.len(), transition_matrix.len()); + let mut rng = StdRng::seed_from_u64(seed); + let mut current_regime = 0usize; + let mut price: f32 = 4500.0; + (0..n).map(|i| { + if i > 0 { + let r: f32 = rng.gen_range(0.0..1.0); + let mut cum = 0.0f32; + for (next, p) in transition_matrix[current_regime].iter().enumerate() { + cum += p; + if r < cum { current_regime = next; break; } + } + } + let (mu, sigma) = regime_params[current_regime]; + let dz: f32 = rng.gen_range(-1.0f32..1.0); + price += mu + sigma * dz; + let spread = (mean_spread + rng.gen_range(-0.05f32..0.05)).max(0.05); + let ofi = (mu / sigma.max(1e-6)).tanh() + rng.gen_range(-0.5f32..0.5); + LobBar::new(price, spread, ofi) + }).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn flat_market_returns_n_bars_with_mean_near_base() { + let bars = flat_market(1000, 0.1, 0.25, 42); + assert_eq!(bars.len(), 1000); + let mean: f32 = bars.iter().map(|b| b.price).sum::() / bars.len() as f32; + assert!((mean - 4500.0).abs() < 0.5, + "flat market mean = {}, expected ~4500", mean); + } + + #[test] + fn drift_market_drifts_in_mu_direction() { + let bars = drift_market(1000, 0.5, 0.1, 0.25, 42); + let first = bars[0].price; + let last = bars[999].price; + assert!(last > first + 100.0, + "drift_market with mu=0.5 should drift up by ~500 over 1000 bars; got {} → {}", + first, last); + } + + #[test] + fn ou_market_reverts_to_mu() { + let bars = ou_market(1000, 0.05, 1.0, 4500.0, 0.25, 42); + let mean: f32 = bars.iter().map(|b| b.price).sum::() / bars.len() as f32; + assert!((mean - 4500.0).abs() < 5.0, + "OU mean = {}, should revert near mu=4500", mean); + } + + #[test] + fn regime_switch_all_regimes_visited() { + // Sticky regimes: 0.99/0.01 produces ~100-step regime runs at mu=±0.5, + // so a single bull run drifts +50 and a single bear run drifts -50, + // giving a wide price range. Per spec §4.4, regime markets are + // characterised by regime persistence — 50/50 transition is a + // random walk, not a regime switch. + let regimes = vec![(0.5f32, 0.1f32), (-0.5, 0.1)]; + let trans = vec![vec![0.99f32, 0.01], vec![0.01, 0.99]]; + let bars = regime_switch_market(1000, ®imes, &trans, 0.25, 42); + let max_p = bars.iter().map(|b| b.price).fold(f32::NEG_INFINITY, f32::max); + let min_p = bars.iter().map(|b| b.price).fold(f32::INFINITY, f32::min); + assert!(max_p > min_p + 50.0, + "regime switch should produce wide price range; got [{}, {}]", min_p, max_p); + } +} diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index 350ab9cd5..afd0046ef 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -6767,3 +6767,15 @@ User asked at the start of this work: **does the model learn the directional sig The path from -24 sharpe to +16 sharpe in 4 epochs validates the underlying training mechanics. **30-epoch full validation should be deferred** until L1 + L2 are fixed, so the EGF pearl actually does work and the val numbers reflect genuine architectural improvement. +## SP15 Phase 2A.1 — LobBar canonical ABI + synthetic market generators (2026-05-06) + +**Files added:** +- `crates/ml/src/cuda_pipeline/lob_bar.rs` — defines `#[repr(C)] struct LobBar { price, spread, ofi: f32 }` plus `into_soa()` AoS→SoA helper. Per spec §4.4 ABI contract, this is the canonical bar format consumed by both Phase 1.2 cost kernel and Phase 2 behavioral tests; dev synthetic generators and prod fxcache both produce LobBar (dev/prod parity per Q3). +- `crates/ml/tests/behavioral/synthetic_markets.rs` — 4 seeded RNG generators: `flat_market`, `drift_market`, `ou_market`, `regime_switch_market`. Each consumes scalar params + seed, returns `Vec`. +- `crates/ml/tests/behavioral/main.rs` — empty harness for the new `behavioral_suite` test target; will gain Phase 2A.2 oracle/harness modules and Phase 2B test_2_*.rs files in subsequent commits. +- `crates/ml/Cargo.toml` — adds `[[test]] name = "behavioral_suite"` target so the suite runs via `cargo test -p ml --test behavioral_suite`. + +**Wire-up status:** Phase 2A.1 lands FIRST per spec — Phase 1.2 cost kernel will conform to the LobBar ABI when it lands. No production code consumes `lob_bar.rs` yet (only the test crate); this is intentional scaffolding. The 4 unit tests inside `synthetic_markets::tests` validate generator behavior (mean centering for flat, drift direction for drift, mean reversion for OU, price-range spread for sticky regime-switch). All 4 pass on RTX 3050 Ti. + +**Spec deviation noted:** spec gave the regime-switch test a 50/50 transition matrix; that produces a fast-flipping random walk, not a regime market. Test uses sticky 0.99/0.01 instead, with code comment explaining — this is what "regime switch" actually means and is what Phase 1.2 cost kernel will be tested against. +