diff --git a/crates/ml/examples/evaluate_baseline.rs b/crates/ml/examples/evaluate_baseline.rs index 4bcf92a2f..9cd73270e 100644 --- a/crates/ml/examples/evaluate_baseline.rs +++ b/crates/ml/examples/evaluate_baseline.rs @@ -268,6 +268,38 @@ struct Args { /// Only applies to the DQN pure-CUDA forward path (--gpu-eval). #[arg(long, default_value_t = false)] cuda_graphs: bool, + + // ------------------------------------------------------------------ + // Surrogate-noise / pooled-metrics flags (Task 0.16) + // ------------------------------------------------------------------ + + /// Surrogate-noise mode. `off` = normal eval. `random` = replace model + /// actions with random actions sampled from --surrogate-marginals (or + /// uniform if not provided). Forces the CPU DQN path so surrogate + /// sampling can override the action-selection step. + #[arg(long, default_value = "off")] + surrogate_mode: String, + + /// Seed for `--surrogate-mode=random`. + #[arg(long, default_value_t = 0)] + surrogate_seed: u64, + + /// Path to JSON file with action marginals (output of + /// `--emit-action-marginals`). Required when + /// `--surrogate-mode=random`. Falls back to UNIFORM random actions + /// with a warning if omitted. + #[arg(long)] + surrogate_marginals: Option, + + /// Emit action marginal distribution to stdout as a single JSON line + /// prefixed `ACTION_MARGINALS:` for machine parsing. + #[arg(long, default_value_t = false)] + emit_action_marginals: bool, + + /// Emit single pooled Sharpe ratio (across all folds) as a single line + /// prefixed `POOLED_SHARPE:` for machine parsing. + #[arg(long, default_value_t = false)] + emit_pooled_sharpe: bool, } // --------------------------------------------------------------------------- @@ -687,6 +719,110 @@ fn compute_metrics(returns: &[f64], bars_per_year: f64) -> ComputedMetrics { } } +// --------------------------------------------------------------------------- +// Surrogate-noise helpers (Task 0.16) +// --------------------------------------------------------------------------- + +/// Surrogate action sampler used when `--surrogate-mode=random`. +/// +/// Samples a flat action index from either a provided marginal distribution +/// (1D probabilities over `num_actions`) or uniform if no marginals were +/// supplied. Uses a seeded `StdRng` for reproducibility. +struct SurrogateSampler { + rng: rand::rngs::StdRng, + /// Cumulative distribution over `num_actions` (last entry = 1.0). `None` + /// falls back to uniform sampling. + cdf: Option>, + num_actions: usize, +} + +impl SurrogateSampler { + fn new(seed: u64, marginals: Option<&[f64]>, num_actions: usize) -> Self { + use rand::SeedableRng; + let rng = rand::rngs::StdRng::seed_from_u64(seed); + let cdf = marginals.and_then(|m| { + if m.is_empty() { + return None; + } + let sum: f64 = m.iter().sum(); + if !(sum > 1e-12) { + return None; + } + let mut c = Vec::with_capacity(m.len()); + let mut acc = 0.0; + for &p in m { + acc += p.max(0.0) / sum; + c.push(acc); + } + // Force last entry to exactly 1.0 to avoid floating drift + if let Some(last) = c.last_mut() { *last = 1.0; } + Some(c) + }); + Self { rng, cdf, num_actions } + } + + /// Sample a single action index in `0..num_actions`. + fn sample(&mut self) -> usize { + use rand::Rng; + match &self.cdf { + Some(cdf) => { + let u: f64 = self.rng.gen(); + // partition_point handles zero-mass actions correctly (consecutive + // equal CDF entries collapse to a single index). binary_search_by + // returns an arbitrary index on equality, which can land on a + // zero-probability entry when several CDF bins share the same value. + // We want the first index where `cdf[i] >= u`, i.e. the first index + // where the predicate `cdf[i] < u` is FALSE. + let idx = cdf.partition_point(|&p| p < u); + idx.min(cdf.len().saturating_sub(1)) + } + None => { + if self.num_actions == 0 { 0 } else { self.rng.gen_range(0..self.num_actions) } + } + } + } +} + +/// Read marginals from a JSON file emitted by `--emit-action-marginals`. +/// +/// Accepts two shapes: +/// 1. `{"flat": [p0, p1, ...]}` — flat distribution over num_actions entries +/// 2. `[p0, p1, ...]` — bare array of probabilities +/// +/// Returns the flat probability vector. +fn load_surrogate_marginals(path: &Path) -> Result> { + let contents = std::fs::read_to_string(path) + .with_context(|| format!("reading surrogate marginals: {}", path.display()))?; + let val: Value = serde_json::from_str(&contents) + .with_context(|| format!("parsing surrogate marginals JSON: {}", path.display()))?; + if let Some(arr) = val.as_array() { + let out: Vec = arr.iter().filter_map(|v| v.as_f64()).collect(); + if !out.is_empty() { return Ok(out); } + } + if let Some(obj) = val.as_object() { + if let Some(flat) = obj.get("flat").and_then(|v| v.as_array()) { + let out: Vec = flat.iter().filter_map(|v| v.as_f64()).collect(); + if !out.is_empty() { return Ok(out); } + } + } + anyhow::bail!( + "marginals JSON at {} must be either a bare array of floats or have a \"flat\" array field", + path.display() + ); +} + +/// Compute pooled Sharpe across concatenated per-fold returns. +/// +/// Uses the same annualization factor as the per-fold `compute_metrics` path. +fn compute_pooled_sharpe(all_returns: &[f64], bars_per_year: f64) -> f64 { + if all_returns.is_empty() { return 0.0; } + let n = all_returns.len() as f64; + let mean = all_returns.iter().sum::() / n; + let var = all_returns.iter().map(|&r| (r - mean).powi(2)).sum::() / n; + let std = var.sqrt(); + if std > 1e-12 { (mean / std) * bars_per_year.sqrt() } else { 0.0 } +} + // --------------------------------------------------------------------------- // GPU-Batched Inference Helpers // --------------------------------------------------------------------------- @@ -849,6 +985,8 @@ fn evaluate_dqn_fold( models_dir: &Path, args: &Args, hp: &Option, + surrogate: Option<&mut SurrogateSampler>, + flat_action_counts: &mut [usize], ) -> Result<(Vec, [usize; 3])> { // Prefer `_best` checkpoint; fall back to highest `_epoch{N}` if training // early-stopped without marking a "best" (e.g. validation loss plateaued). @@ -963,22 +1101,59 @@ fn evaluate_dqn_fold( eval_bars, ); + // Surrogate-noise mode bypasses the model: we still need to build the + // state for feature-vector validation, but we skip the GPU forward pass. + let use_surrogate = surrogate.is_some(); + // Move the surrogate out of `Option<&mut T>` into a local for the loop + // without tripping the borrow checker's "captured twice" check. + let mut surrogate_local = surrogate; + for bar_idx in 0..eval_bars { - // Build single state vector with current portfolio features - let flat_state = build_chunk_states( - test_features, test_bars, bar_idx, bar_idx + 1, - feature_dim, &portfolio, args.tick_size, args.spread_ticks, - ); + let action_idx = if let Some(sampler) = surrogate_local.as_deref_mut() { + // Random-action surrogate — bypass model entirely + sampler.sample() + } else { + // Build single state vector with current portfolio features + let flat_state = build_chunk_states( + test_features, test_bars, bar_idx, bar_idx + 1, + feature_dim, &portfolio, args.tick_size, args.spread_ticks, + ); - // Single-bar GPU forward pass + hierarchical softmax selection (B1) - let state_tensor = GpuTensor::from_host(&flat_state, vec![1, feature_dim], &stream) - .map_err(|e| anyhow::anyhow!("Failed to create DQN state tensor for bar {}: {}", bar_idx, e))?; + // Single-bar GPU forward pass + hierarchical softmax selection (B1) + let state_tensor = GpuTensor::from_host(&flat_state, vec![1, feature_dim], &stream) + .map_err(|e| anyhow::anyhow!("Failed to create DQN state tensor for bar {}: {}", bar_idx, e))?; + + let action_tensor = dqn.batch_hierarchical_softmax_actions(&state_tensor, eval_softmax_temp) + .with_context(|| format!("DQN softmax action selection failed for bar {}", bar_idx))?; + let action_host = action_tensor.to_host(&stream) + .map_err(|e| anyhow::anyhow!("Action extraction: {e}"))?; + action_host.first().copied().unwrap_or(0.0) as usize + }; + + // Track flat-index count for action-marginals emission. + // + // Fix 1 (code-quality review): hard-fail on out-of-range instead of + // silently dropping. A silent drop corrupts the surrogate marginals + // because some bins never receive counts. The binary's `--num-actions` + // is semantically ambiguous in this codebase (historically "5 exposure + // levels", today a 4-branch factored policy producing dir*27+mag*9+ + // ord*3+urg = 81 flat actions, with the CPU eval path in this file + // currently reading only the direction branch). Rather than paper over + // the mismatch, the gate demands the caller pass `--num-actions` that + // matches the model's actual flat action space. + if action_idx >= flat_action_counts.len() { + anyhow::bail!( + "Action index {} exceeds --num-actions={}. Pass --num-actions \ + matching the model's flat action space (typically 81 for the \ + 4-branch factored DQN: dir*27 + mag*9 + ord*3 + urg). \ + Silently dropping out-of-range indices would corrupt \ + --emit-action-marginals and --surrogate-mode=random sampling.", + action_idx, + flat_action_counts.len(), + ); + } + flat_action_counts[action_idx] = flat_action_counts[action_idx].saturating_add(1); - let action_tensor = dqn.batch_hierarchical_softmax_actions(&state_tensor, eval_softmax_temp) - .with_context(|| format!("DQN softmax action selection failed for bar {}", bar_idx))?; - let action_host = action_tensor.to_host(&stream) - .map_err(|e| anyhow::anyhow!("Action extraction: {e}"))?; - let action_idx = action_host.first().copied().unwrap_or(0.0) as usize; let action_indices = vec![action_idx]; // Simulate trade and update portfolio state (B3: state synced per bar) @@ -988,6 +1163,10 @@ fn evaluate_dqn_fold( ); } + if use_surrogate { + info!(" [DQN] Surrogate-noise mode: {} bars used random actions", eval_bars); + } + Ok((returns, action_counts)) } @@ -1807,7 +1986,57 @@ fn main() -> Result<()> { metrics_server::start_metrics_server(9094); tm::set_active_workers(1.0); - let args = Args::parse(); + let mut args = Args::parse(); + + // ── Surrogate-noise mode setup (Task 0.16) ────────────────────────── + // + // When --surrogate-mode=random is requested, we force the CPU DQN path + // so the surrogate sampler can override action selection. GPU path is + // not supported (would require invasive kernel changes that defeat the + // point of a bypass-the-model sanity check). + let surrogate_enabled = args.surrogate_mode == "random"; + if surrogate_enabled && args.gpu_eval { + warn!( + " [SURROGATE] --surrogate-mode=random forces CPU DQN path (--gpu-eval disabled)" + ); + args.gpu_eval = false; + } + // Fix 1 (code-quality review): the binary's --num-actions default (5) is + // historically tied to the old "5 exposure levels" semantics and predates + // the 4-branch factored policy (81 flat actions). When marginals collection + // or random-surrogate sampling is requested, warn loudly so callers don't + // silently undersize flat_action_counts. The CPU DQN path ALSO hard-fails + // on out-of-range action_idx (see Fix 1 at the collection site) — this + // warning is a pre-flight signal rather than a safety net. + if (surrogate_enabled || args.emit_action_marginals) && args.num_actions == 5 { + warn!( + " [SURROGATE] --num-actions=5 (default). If the trained model uses \ + a different flat action space (typical: 81 for 4-branch factored \ + DQN), pass --num-actions explicitly. The CPU DQN path will \ + hard-fail rather than silently drop out-of-range indices." + ); + } + let surrogate_marginals: Option> = if surrogate_enabled { + match &args.surrogate_marginals { + Some(p) => { + let m = load_surrogate_marginals(p) + .with_context(|| format!("loading {}", p.display()))?; + info!( + " [SURROGATE] Loaded {}-dim action marginals from {}", + m.len(), p.display() + ); + Some(m) + } + None => { + warn!( + " [SURROGATE] --surrogate-mode=random without --surrogate-marginals: \ + falling back to UNIFORM random actions across {} actions", + args.num_actions + ); + None + } + } + } else { None }; let eval_dqn = args.model == "dqn" || args.model == "both"; let eval_ppo = args.model == "ppo" || args.model == "both"; @@ -1894,6 +2123,32 @@ fn main() -> Result<()> { info!("Step 3/5: Evaluating models on test data..."); let mut all_fold_metrics: Vec = Vec::new(); let mut all_action_counts: Vec<[usize; 3]> = Vec::new(); + // Flat per-action-index counts for --emit-action-marginals (CPU DQN path). + // GPU paths are excluded because they collapse actions to a different shape + // (5 exposure scores) before argmax; mixing them would yield a misleading + // histogram. When only GPU paths run, the emitted marginals will be + // all-zeros with a warning. + let mut flat_action_counts: Vec = vec![0; args.num_actions]; + // Fix 3 (code-quality review): distinguish "stub because CPU DQN path + // never ran" (GPU took over, or wrong --model) from "CPU DQN path ran + // but counted zero actions" (empty fold). Set to true inside the CPU + // DQN Ok(...) branch below. + let mut cpu_dqn_ran = false; + // All per-bar returns concatenated across folds for --emit-pooled-sharpe. + // Populated only by the CPU DQN path (the only path that returns raw + // per-bar returns). When GPU path is used, this stays empty and pooled + // Sharpe is approximated from the fold-level Sharpe ratios. + let mut pooled_returns: Vec = Vec::new(); + + // Initialise surrogate sampler once (state carries across folds so a + // single seed produces a deterministic sequence over the full backtest). + let mut surrogate_sampler: Option = if surrogate_enabled { + Some(SurrogateSampler::new( + args.surrogate_seed, + surrogate_marginals.as_deref(), + args.num_actions, + )) + } else { None }; for window in &windows { info!( @@ -2055,8 +2310,16 @@ fn main() -> Result<()> { &args.models_dir, &args, &hp, + surrogate_sampler.as_mut(), + flat_action_counts.as_mut_slice(), ) { Ok((returns, action_counts)) => { + // Fix 3: mark that the CPU DQN path actually completed + // a fold so the stub-marker logic can distinguish this + // from "GPU took over" / "model != dqn" / "zero bars". + cpu_dqn_ran = true; + // Accumulate concatenated returns for pooled-Sharpe + pooled_returns.extend(returns.iter().copied()); let fold_metrics = compute_metrics(&returns, args.bars_per_year); let fold_str = window.fold.to_string(); tm::set_epoch("dqn", &fold_str, window.fold as f64); @@ -2340,6 +2603,78 @@ fn main() -> Result<()> { info!(" Report saved to: {}", args.output.display()); info!(" Total fold evaluations: {}", report.folds.len()); + // ── Task 0.16: machine-parseable output lines ─────────────────────── + // + // These lines are emitted to stdout (not stderr / tracing) so the + // surrogate-noise smoke test can parse them via a subprocess. The + // "ACTION_MARGINALS:" / "POOLED_SHARPE:" prefixes are load-bearing — + // the consumer matches them exactly. + if args.emit_action_marginals { + let total: usize = flat_action_counts.iter().sum(); + if total == 0 { + // Fix 3 (code-quality review): disambiguate the stub marker. A zero + // total can mean any of three distinct things, each needing a + // different caller response: + // * `stub_no_dqn_eval` — --model != dqn (no DQN ran at all) + // * `stub_cpu_path_not_exercised` — GPU path handled all folds, or + // --model=dqn but every fold + // errored before the CPU branch + // set `cpu_dqn_ran = true` + // * `stub_zero_bars` — CPU path ran but counted no + // actions (empty fold) + let marker = if !eval_dqn { + "stub_no_dqn_eval" + } else if !cpu_dqn_ran { + "stub_cpu_path_not_exercised" + } else { + "stub_zero_bars" + }; + println!("ACTION_MARGINALS: {}", marker); + warn!( + " [SURROGATE] --emit-action-marginals: emitted {} marker. \ + eval_dqn={} cpu_dqn_ran={} total_samples={}", + marker, eval_dqn, cpu_dqn_ran, total, + ); + } else { + let flat: Vec = flat_action_counts + .iter() + .map(|&c| c as f64 / total as f64) + .collect(); + // Fix 6 (code-quality review): include a "schema" field so the + // consumer can validate the payload shape and refuse silently + // mismatched future versions. + let json = serde_json::json!({ + "schema": "action_marginals_v1", + "flat": flat, + "num_actions": args.num_actions, + "total_samples": total, + }); + match serde_json::to_string(&json) { + Ok(s) => println!("ACTION_MARGINALS: {}", s), + Err(e) => warn!(" [SURROGATE] Failed to serialize action marginals: {}", e), + } + } + } + + if args.emit_pooled_sharpe { + let pooled = if !pooled_returns.is_empty() { + compute_pooled_sharpe(&pooled_returns, args.bars_per_year) + } else { + // Fallback: mean of fold-level Sharpes (unpooled). The smoke + // test prefers the true pooled value; this fallback avoids + // silently emitting zero when the CPU path wasn't exercised. + let sharpes: Vec = report.folds.iter().map(|f| f.sharpe_ratio).collect(); + if sharpes.is_empty() { 0.0 } else { + warn!( + " [SURROGATE] --emit-pooled-sharpe: pooled_returns empty \ + (GPU path only) — falling back to mean of fold Sharpes" + ); + sharpes.iter().sum::() / sharpes.len() as f64 + } + }; + println!("POOLED_SHARPE: {}", pooled); + } + tm::set_active_workers(0.0); // Push final metrics to pushgateway so they persist after pod termination diff --git a/crates/ml/src/trainers/dqn/smoke_tests/mod.rs b/crates/ml/src/trainers/dqn/smoke_tests/mod.rs index 58a5a16aa..2e58cd4a6 100644 --- a/crates/ml/src/trainers/dqn/smoke_tests/mod.rs +++ b/crates/ml/src/trainers/dqn/smoke_tests/mod.rs @@ -34,3 +34,5 @@ mod controller_activity; mod multi_fold_convergence; #[cfg(test)] mod reward_component_audit; +#[cfg(test)] +mod surrogate_noise_check; diff --git a/crates/ml/src/trainers/dqn/smoke_tests/surrogate_noise_check.rs b/crates/ml/src/trainers/dqn/smoke_tests/surrogate_noise_check.rs new file mode 100644 index 000000000..02308c4b7 --- /dev/null +++ b/crates/ml/src/trainers/dqn/smoke_tests/surrogate_noise_check.rs @@ -0,0 +1,312 @@ +//! Smoke test: surrogate-noise false-positive detector (spec §4.1, Task 0.16). +//! +//! Mandatory gate. Runs a trained model against N=30 random-action surrogate +//! backtests; trained pooled Sharpe must exceed the 95th percentile of the +//! surrogate distribution. Protects against look-ahead / leakage bugs. +//! +//! The test drives the `evaluate_baseline` example binary as a subprocess: +//! 1. Trained run with `--emit-action-marginals --emit-pooled-sharpe` to +//! capture both the marginal distribution and the pooled Sharpe. +//! 2. N=30 surrogate runs with `--surrogate-mode=random --surrogate-seed=N` +//! using the marginals collected above, each emitting `POOLED_SHARPE:`. +//! +//! The 95th percentile of the surrogate pooled Sharpes is the significance +//! threshold: trained Sharpe must exceed it to PASS. +//! +//! ## Requirements +//! +//! - A trained checkpoint at `/workspace/output/dqn_fold0_best.safetensors` +//! (Phase 3 deliverable). Until this exists the test asserts-out with a +//! clear message; it is `#[ignore]`'d so regular `cargo test` skips it. +//! - `FOXHUNT_TEST_DATA` pointing at a futures-baseline data root. +//! - A compiled `evaluate_baseline` example binary at +//! `target/release/examples/evaluate_baseline` (or the test falls back to +//! `cargo run --release --example evaluate_baseline -- …`). +//! +//! Run: +//! ```bash +//! FOXHUNT_TEST_DATA=test_data/futures-baseline \ +//! cargo test -p ml --release --lib -- surrogate_noise_check --ignored --nocapture +//! ``` + +use super::helpers::{test_data_dir, workspace_root}; +use anyhow::{anyhow, Context, Result}; +use std::path::{Path, PathBuf}; +use std::process::Command; + +/// Default checkpoint location produced by Phase 3 training. +const DEFAULT_CHECKPOINT: &str = "/workspace/output/dqn_fold0_best.safetensors"; + +/// Models dir passed to `evaluate_baseline`. The checkpoint's parent is used +/// because `evaluate_baseline` reads `/dqn_fold{N}_best.safetensors`. +fn models_dir_for_checkpoint(ckpt: &Path) -> PathBuf { + ckpt.parent().map(std::path::Path::to_path_buf).unwrap_or_else(|| PathBuf::from(".")) +} + +#[test] +#[ignore] // Requires a trained checkpoint and substantial runtime (~30 subprocess calls). +fn test_surrogate_noise_check() -> Result<()> { + let data_dir = test_data_dir().expect("FOXHUNT_TEST_DATA or test_data/ must exist"); + + // Allow overriding the checkpoint path for local development; default to + // the Phase 3 production path. + let ckpt_path = std::env::var("FOXHUNT_SURROGATE_CKPT") + .unwrap_or_else(|_| DEFAULT_CHECKPOINT.to_owned()); + let ckpt = Path::new(&ckpt_path); + assert!( + ckpt.exists(), + "surrogate-noise test requires a trained checkpoint at {:?}. \ + Produce one via Phase 3 training or set FOXHUNT_SURROGATE_CKPT to a \ + local checkpoint.", + ckpt + ); + + // 1. Trained backtest: capture pooled Sharpe + action marginals + let (trained_sharpe, marginals_json) = + run_trained_backtest_pooled(&data_dir, ckpt)?; + println!("[SURROGATE] trained pooled Sharpe = {:.3}", trained_sharpe); + + // 2. Write marginals to a temp file for surrogate runs + let marginals_path = workspace_root() + .join("target") + .join("smoke_surrogate_marginals.json"); + if let Some(parent) = marginals_path.parent() { + std::fs::create_dir_all(parent).context("create marginals dir")?; + } + std::fs::write(&marginals_path, &marginals_json) + .context("writing marginals JSON for surrogate runs")?; + + // 3. N=30 surrogate backtests + // + // Fix 4 (code-quality review): decorrelate seeds via a multiplicative hash. + // Sequential small integers (0, 1, 2, …) fed into `StdRng::seed_from_u64` + // produce streams whose initial bits are highly correlated — surrogate + // distributions collapse toward the same trajectory and the p95 threshold + // becomes non-representative. Multiplying by the golden-ratio constant + // spreads the seeds evenly across the 64-bit space. + // + // Fix 5 (code-quality review): emit per-iteration progress so `--nocapture` + // output is usable for triage even when a run hangs or diverges. + const SEED_MULT: u64 = 0x9E37_79B9_7F4A_7C15; // golden-ratio constant + let mut surrogate_sharpes: Vec = Vec::with_capacity(30); + for i in 0..30_u64 { + // Multiplicative hash to decorrelate sequential indices — consecutive + // seeds with StdRng share initial bits. + let seed = SEED_MULT.wrapping_mul(i.wrapping_add(1)); + let sh = run_surrogate_backtest_pooled(&data_dir, ckpt, &marginals_path, seed)?; + println!( + "[SURROGATE] seed {}/30 (hash={:#x}) sharpe={:.3}", + i + 1, + seed, + sh, + ); + surrogate_sharpes.push(sh); + } + surrogate_sharpes.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let n = surrogate_sharpes.len(); + let p95_idx = ((n as f64) * 0.95) as usize; + let p95 = *surrogate_sharpes + .get(p95_idx.min(n.saturating_sub(1))) + .ok_or_else(|| anyhow!("surrogate_sharpes empty"))?; + let median = *surrogate_sharpes + .get(n / 2) + .ok_or_else(|| anyhow!("surrogate_sharpes empty"))?; + let min = *surrogate_sharpes + .first() + .ok_or_else(|| anyhow!("surrogate_sharpes empty"))?; + let max = *surrogate_sharpes + .last() + .ok_or_else(|| anyhow!("surrogate_sharpes empty"))?; + + println!( + "[SURROGATE] N={} random-action pooled Sharpe 95th %ile = {:.3}", + n, p95 + ); + println!( + "[SURROGATE] distribution: min={:.3} median={:.3} max={:.3}", + min, median, max + ); + + assert!( + trained_sharpe > p95, + "SURROGATE-NOISE FAILURE: trained Sharpe {:.3} is not beyond the 95th \ + percentile of random-action Sharpes ({:.3}). The apparent edge may be \ + measurement noise, data leakage, or look-ahead bias. DO NOT deploy.", + trained_sharpe, p95 + ); + Ok(()) +} + +/// Locate a compiled `evaluate_baseline` example binary, or fall back to +/// `cargo run --release --example evaluate_baseline`. +/// +/// Cargo examples are emitted at `target/release/examples/evaluate_baseline` +/// when built with `cargo build --release --example evaluate_baseline -p ml`. +/// Using the compiled binary directly avoids cargo's build-lock on every call +/// (which would block multiple smoke tests running in parallel) and shaves +/// ~2s of cargo overhead per surrogate iteration. +fn evaluate_baseline_command() -> Command { + let root = workspace_root(); + let compiled = root.join("target").join("release").join("examples").join("evaluate_baseline"); + if compiled.exists() { + Command::new(compiled) + } else { + let mut c = Command::new("cargo"); + c.current_dir(&root) + .args(["run", "--release", "-p", "ml", "--example", "evaluate_baseline", "--"]); + c + } +} + +/// Run a trained-model backtest and return `(pooled_sharpe, action_marginals_json)`. +/// +/// The action-marginals JSON is the exact `{"flat": [...], ...}` payload +/// emitted by `--emit-action-marginals` — it can be written to disk as-is +/// and fed back into surrogate runs via `--surrogate-marginals`. +fn run_trained_backtest_pooled(data: &str, ckpt: &Path) -> Result<(f64, String)> { + let models_dir = models_dir_for_checkpoint(ckpt); + let report_path = workspace_root() + .join("target") + .join("smoke_surrogate_trained_report.json"); + if let Some(parent) = report_path.parent() { + std::fs::create_dir_all(parent).ok(); + } + + let mut cmd = evaluate_baseline_command(); + cmd.env("SQLX_OFFLINE", "true") + .env("CUBLAS_WORKSPACE_CONFIG", ":4096:8") + .env("FOXHUNT_TEST_DATA", data) + .args([ + "--model", "dqn", + "--data-dir", data, + "--models-dir", models_dir.to_str().unwrap_or(""), + "--symbol", "ES.FUT", + "--output", report_path.to_str().unwrap_or(""), + // Force CPU path to populate flat action counts + pooled returns + "--no-gpu-eval", + "--emit-action-marginals", + "--emit-pooled-sharpe", + ]); + + let output = cmd.output().context("spawning evaluate_baseline (trained)")?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + anyhow::bail!( + "evaluate_baseline (trained) exited {:?}:\nstderr:\n{}", + output.status.code(), + stderr.lines().rev().take(50).collect::>().iter().rev().cloned().collect::>().join("\n"), + ); + } + let stdout = String::from_utf8_lossy(&output.stdout); + let sharpe = parse_pooled_sharpe(&stdout) + .context("parsing POOLED_SHARPE from trained run")?; + let marginals = parse_action_marginals(&stdout) + .context("parsing ACTION_MARGINALS from trained run")?; + Ok((sharpe, marginals)) +} + +/// Run a random-action surrogate backtest and return pooled Sharpe. +fn run_surrogate_backtest_pooled( + data: &str, + ckpt: &Path, + marginals_path: &Path, + seed: u64, +) -> Result { + let models_dir = models_dir_for_checkpoint(ckpt); + let report_path = workspace_root() + .join("target") + .join(format!("smoke_surrogate_report_seed{}.json", seed)); + if let Some(parent) = report_path.parent() { + std::fs::create_dir_all(parent).ok(); + } + + let mut cmd = evaluate_baseline_command(); + cmd.env("SQLX_OFFLINE", "true") + .env("CUBLAS_WORKSPACE_CONFIG", ":4096:8") + .env("FOXHUNT_TEST_DATA", data) + .args([ + "--model", "dqn", + "--data-dir", data, + "--models-dir", models_dir.to_str().unwrap_or(""), + "--symbol", "ES.FUT", + "--output", report_path.to_str().unwrap_or(""), + "--surrogate-mode", "random", + "--surrogate-seed", &seed.to_string(), + "--surrogate-marginals", marginals_path.to_str().unwrap_or(""), + "--emit-pooled-sharpe", + ]); + + let output = cmd.output().context("spawning evaluate_baseline (surrogate)")?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + anyhow::bail!( + "evaluate_baseline (surrogate seed={}) exited {:?}:\nstderr:\n{}", + seed, + output.status.code(), + stderr.lines().rev().take(30).collect::>().iter().rev().cloned().collect::>().join("\n"), + ); + } + let stdout = String::from_utf8_lossy(&output.stdout); + parse_pooled_sharpe(&stdout) + .with_context(|| format!("parsing POOLED_SHARPE from surrogate seed={}", seed)) +} + +/// Extract the single `POOLED_SHARPE: ` line from stdout. +fn parse_pooled_sharpe(stdout: &str) -> Result { + for line in stdout.lines() { + if let Some(rest) = line.strip_prefix("POOLED_SHARPE:") { + let val: f64 = rest.trim().parse() + .with_context(|| format!("parsing pooled Sharpe value: {:?}", rest.trim()))?; + return Ok(val); + } + } + anyhow::bail!("no POOLED_SHARPE: line found in evaluate_baseline stdout"); +} + +/// Extract the JSON payload following `ACTION_MARGINALS:` prefix. +/// +/// Fix 3 (code-quality review): accept any `stub_` marker with a clear +/// bail message that names the marker, rather than hard-coding a single +/// `stub_gpu_only_no_flat_counts` case. The emitter distinguishes between +/// `stub_no_dqn_eval`, `stub_cpu_path_not_exercised`, and `stub_zero_bars` — +/// each implies a different caller fix. +/// +/// Fix 6 (code-quality review): verify the payload carries the expected +/// `schema: "action_marginals_v1"` field. Missing schema is accepted with a +/// warning for forward-compat with older emitters; a mismatched schema is a +/// hard error (prevents silent breakage if the emitter rev-bumps). +fn parse_action_marginals(stdout: &str) -> Result { + for line in stdout.lines() { + if let Some(rest) = line.strip_prefix("ACTION_MARGINALS:") { + let payload = rest.trim(); + if payload.starts_with("stub_") { + anyhow::bail!( + "evaluate_baseline emitted stub marginals marker {:?}. \ + Surrogate test requires real action marginals — pass \ + --no-gpu-eval and ensure --model=dqn so the CPU DQN \ + path is exercised and counts actions.", + payload + ); + } + // Sanity: must be valid JSON + let parsed: serde_json::Value = serde_json::from_str(payload) + .context("ACTION_MARGINALS payload is not valid JSON")?; + match parsed.get("schema").and_then(|v| v.as_str()) { + Some("action_marginals_v1") => {} + Some(other) => anyhow::bail!( + "ACTION_MARGINALS schema mismatch: expected \ + \"action_marginals_v1\", got {:?}. Update the consumer \ + to handle the new schema before proceeding.", + other + ), + None => eprintln!( + "[SURROGATE] WARN: ACTION_MARGINALS payload missing \ + \"schema\" field — accepting for forward-compat with \ + pre-v1 emitters." + ), + } + return Ok(payload.to_owned()); + } + } + anyhow::bail!("no ACTION_MARGINALS: line found in evaluate_baseline stdout"); +}