gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(policy-quality): Task 0.16 surrogate_noise_check smoke (mandatory gate)

Browse Source 
Adds evaluate_baseline CLI args:
  --surrogate-mode=off|random
  --surrogate-seed=<u64>
  --surrogate-marginals=<path.json>
  --emit-action-marginals
  --emit-pooled-sharpe

Random-action surrogate samples actions from marginal distribution (or
uniform fallback) using a seeded RNG, bypassing the model entirely.
Surrogate mode forces the CPU DQN path so action selection can be
overridden (GPU path would need invasive kernel changes and defeats
the bypass-the-model sanity check).

Flat action-index counts are tracked in the CPU DQN path only; the
ACTION_MARGINALS: line emits those as a JSON distribution, or emits
an honest stub marker when only the GPU path was exercised. Pooled
Sharpe is computed from concatenated per-fold returns (CPU path) or
falls back to mean-of-fold-Sharpes with a warning.

Smoke test runs 30 surrogate seeds + 1 trained run via evaluate_baseline
subprocess, asserts trained pooled Sharpe exceeds the 95th percentile of
surrogate Sharpes. Test is #[ignore]'d and requires a trained checkpoint
at /workspace/output/dqn_fold0_best.safetensors (Phase 3 deliverable);
FOXHUNT_SURROGATE_CKPT env var overrides for local testing. Uses the
compiled target/release/examples/evaluate_baseline if present, otherwise
falls back to cargo run.

Will pass once Phase 3 produces a checkpoint.
This commit is contained in:
jgrusewski
2026-04-21 23:12:12 +02:00
parent bf8415c5d6
commit f4ea0f6d13
3 changed files with 663 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

363
crates/ml/examples/evaluate_baseline.rs
View File

@@ -268,6 +268,38 @@ struct Args {
/// Only applies to the DQN pure-CUDA forward path (--gpu-eval).
#[arg(long, default_value_t = false)]
cuda_graphs: bool,
// ------------------------------------------------------------------
// Surrogate-noise / pooled-metrics flags (Task 0.16)
// ------------------------------------------------------------------
/// Surrogate-noise mode. `off` = normal eval. `random` = replace model
/// actions with random actions sampled from --surrogate-marginals (or
/// uniform if not provided). Forces the CPU DQN path so surrogate
/// sampling can override the action-selection step.
#[arg(long, default_value = "off")]
surrogate_mode: String,
/// Seed for `--surrogate-mode=random`.
#[arg(long, default_value_t = 0)]
surrogate_seed: u64,
/// Path to JSON file with action marginals (output of
/// `--emit-action-marginals`). Required when
/// `--surrogate-mode=random`. Falls back to UNIFORM random actions
/// with a warning if omitted.
#[arg(long)]
surrogate_marginals: Option<PathBuf>,
/// Emit action marginal distribution to stdout as a single JSON line
/// prefixed `ACTION_MARGINALS:` for machine parsing.
#[arg(long, default_value_t = false)]
emit_action_marginals: bool,
/// Emit single pooled Sharpe ratio (across all folds) as a single line
/// prefixed `POOLED_SHARPE:` for machine parsing.
#[arg(long, default_value_t = false)]
emit_pooled_sharpe: bool,
}
// ---------------------------------------------------------------------------
@@ -687,6 +719,110 @@ fn compute_metrics(returns: &[f64], bars_per_year: f64) -> ComputedMetrics {
}
}
// ---------------------------------------------------------------------------
// Surrogate-noise helpers (Task 0.16)
// ---------------------------------------------------------------------------
/// Surrogate action sampler used when `--surrogate-mode=random`.
///
/// Samples a flat action index from either a provided marginal distribution
/// (1D probabilities over `num_actions`) or uniform if no marginals were
/// supplied. Uses a seeded `StdRng` for reproducibility.
struct SurrogateSampler {
rng: rand::rngs::StdRng,
/// Cumulative distribution over `num_actions` (last entry = 1.0). `None`
/// falls back to uniform sampling.
cdf: Option<Vec<f64>>,
num_actions: usize,
}
impl SurrogateSampler {
fn new(seed: u64, marginals: Option<&[f64]>, num_actions: usize) -> Self {
use rand::SeedableRng;
let rng = rand::rngs::StdRng::seed_from_u64(seed);
let cdf = marginals.and_then(|m| {
if m.is_empty() {
return None;
}
let sum: f64 = m.iter().sum();
if !(sum > 1e-12) {
return None;
}
let mut c = Vec::with_capacity(m.len());
let mut acc = 0.0;
for &p in m {
acc += p.max(0.0) / sum;
c.push(acc);
}
// Force last entry to exactly 1.0 to avoid floating drift
if let Some(last) = c.last_mut() { *last = 1.0; }
Some(c)
});
Self { rng, cdf, num_actions }
}
/// Sample a single action index in `0..num_actions`.
fn sample(&mut self) -> usize {
use rand::Rng;
match &self.cdf {
Some(cdf) => {
let u: f64 = self.rng.gen();
// partition_point handles zero-mass actions correctly (consecutive
// equal CDF entries collapse to a single index). binary_search_by
// returns an arbitrary index on equality, which can land on a
// zero-probability entry when several CDF bins share the same value.
// We want the first index where `cdf[i] >= u`, i.e. the first index
// where the predicate `cdf[i] < u` is FALSE.
let idx = cdf.partition_point(|&p| p < u);
idx.min(cdf.len().saturating_sub(1))
}
None => {
if self.num_actions == 0 { 0 } else { self.rng.gen_range(0..self.num_actions) }
}
}
}
}
/// Read marginals from a JSON file emitted by `--emit-action-marginals`.
///
/// Accepts two shapes:
/// 1. `{"flat": [p0, p1, ...]}` — flat distribution over num_actions entries
/// 2. `[p0, p1, ...]` — bare array of probabilities
///
/// Returns the flat probability vector.
fn load_surrogate_marginals(path: &Path) -> Result<Vec<f64>> {
let contents = std::fs::read_to_string(path)
.with_context(|| format!("reading surrogate marginals: {}", path.display()))?;
let val: Value = serde_json::from_str(&contents)
.with_context(|| format!("parsing surrogate marginals JSON: {}", path.display()))?;
if let Some(arr) = val.as_array() {
let out: Vec<f64> = arr.iter().filter_map(|v| v.as_f64()).collect();
if !out.is_empty() { return Ok(out); }
}
if let Some(obj) = val.as_object() {
if let Some(flat) = obj.get("flat").and_then(|v| v.as_array()) {
let out: Vec<f64> = flat.iter().filter_map(|v| v.as_f64()).collect();
if !out.is_empty() { return Ok(out); }
}
}
anyhow::bail!(
"marginals JSON at {} must be either a bare array of floats or have a \"flat\" array field",
path.display()
);
}
/// Compute pooled Sharpe across concatenated per-fold returns.
///
/// Uses the same annualization factor as the per-fold `compute_metrics` path.
fn compute_pooled_sharpe(all_returns: &[f64], bars_per_year: f64) -> f64 {
if all_returns.is_empty() { return 0.0; }
let n = all_returns.len() as f64;
let mean = all_returns.iter().sum::<f64>() / n;
let var = all_returns.iter().map(|&r| (r - mean).powi(2)).sum::<f64>() / n;
let std = var.sqrt();
if std > 1e-12 { (mean / std) * bars_per_year.sqrt() } else { 0.0 }
}
// ---------------------------------------------------------------------------
// GPU-Batched Inference Helpers
// ---------------------------------------------------------------------------
@@ -849,6 +985,8 @@ fn evaluate_dqn_fold(
models_dir: &Path,
args: &Args,
hp: &Option<Value>,
surrogate: Option<&mut SurrogateSampler>,
flat_action_counts: &mut [usize],
) -> Result<(Vec<f64>, [usize; 3])> {
// Prefer `_best` checkpoint; fall back to highest `_epoch{N}` if training
// early-stopped without marking a "best" (e.g. validation loss plateaued).
@@ -963,22 +1101,59 @@ fn evaluate_dqn_fold(
eval_bars,
);
// Surrogate-noise mode bypasses the model: we still need to build the
// state for feature-vector validation, but we skip the GPU forward pass.
let use_surrogate = surrogate.is_some();
// Move the surrogate out of `Option<&mut T>` into a local for the loop
// without tripping the borrow checker's "captured twice" check.
let mut surrogate_local = surrogate;
for bar_idx in 0..eval_bars {
// Build single state vector with current portfolio features
let flat_state = build_chunk_states(
test_features, test_bars, bar_idx, bar_idx + 1,
feature_dim, &portfolio, args.tick_size, args.spread_ticks,
);
let action_idx = if let Some(sampler) = surrogate_local.as_deref_mut() {
// Random-action surrogate — bypass model entirely
sampler.sample()
} else {
// Build single state vector with current portfolio features
let flat_state = build_chunk_states(
test_features, test_bars, bar_idx, bar_idx + 1,
feature_dim, &portfolio, args.tick_size, args.spread_ticks,
);
// Single-bar GPU forward pass + hierarchical softmax selection (B1)
let state_tensor = GpuTensor::from_host(&flat_state, vec![1, feature_dim], &stream)
.map_err(|e| anyhow::anyhow!("Failed to create DQN state tensor for bar {}: {}", bar_idx, e))?;
// Single-bar GPU forward pass + hierarchical softmax selection (B1)
let state_tensor = GpuTensor::from_host(&flat_state, vec![1, feature_dim], &stream)
.map_err(|e| anyhow::anyhow!("Failed to create DQN state tensor for bar {}: {}", bar_idx, e))?;
let action_tensor = dqn.batch_hierarchical_softmax_actions(&state_tensor, eval_softmax_temp)
.with_context(|| format!("DQN softmax action selection failed for bar {}", bar_idx))?;
let action_host = action_tensor.to_host(&stream)
.map_err(|e| anyhow::anyhow!("Action extraction: {e}"))?;
action_host.first().copied().unwrap_or(0.0) as usize
};
// Track flat-index count for action-marginals emission.
//
// Fix 1 (code-quality review): hard-fail on out-of-range instead of
// silently dropping. A silent drop corrupts the surrogate marginals
// because some bins never receive counts. The binary's `--num-actions`
// is semantically ambiguous in this codebase (historically "5 exposure
// levels", today a 4-branch factored policy producing dir*27+mag*9+
// ord*3+urg = 81 flat actions, with the CPU eval path in this file
// currently reading only the direction branch). Rather than paper over
// the mismatch, the gate demands the caller pass `--num-actions` that
// matches the model's actual flat action space.
if action_idx >= flat_action_counts.len() {
anyhow::bail!(
"Action index {} exceeds --num-actions={}. Pass --num-actions \
matching the model's flat action space (typically 81 for the \
4-branch factored DQN: dir*27 + mag*9 + ord*3 + urg). \
Silently dropping out-of-range indices would corrupt \
--emit-action-marginals and --surrogate-mode=random sampling.",
action_idx,
flat_action_counts.len(),
);
}
flat_action_counts[action_idx] = flat_action_counts[action_idx].saturating_add(1);
let action_tensor = dqn.batch_hierarchical_softmax_actions(&state_tensor, eval_softmax_temp)
.with_context(|| format!("DQN softmax action selection failed for bar {}", bar_idx))?;
let action_host = action_tensor.to_host(&stream)
.map_err(|e| anyhow::anyhow!("Action extraction: {e}"))?;
let action_idx = action_host.first().copied().unwrap_or(0.0) as usize;
let action_indices = vec![action_idx];
// Simulate trade and update portfolio state (B3: state synced per bar)
@@ -988,6 +1163,10 @@ fn evaluate_dqn_fold(
);
}
if use_surrogate {
info!(" [DQN] Surrogate-noise mode: {} bars used random actions", eval_bars);
}
Ok((returns, action_counts))
}
@@ -1807,7 +1986,57 @@ fn main() -> Result<()> {
metrics_server::start_metrics_server(9094);
tm::set_active_workers(1.0);
let args = Args::parse();
let mut args = Args::parse();
// ── Surrogate-noise mode setup (Task 0.16) ──────────────────────────
//
// When --surrogate-mode=random is requested, we force the CPU DQN path
// so the surrogate sampler can override action selection. GPU path is
// not supported (would require invasive kernel changes that defeat the
// point of a bypass-the-model sanity check).
let surrogate_enabled = args.surrogate_mode == "random";
if surrogate_enabled && args.gpu_eval {
warn!(
" [SURROGATE] --surrogate-mode=random forces CPU DQN path (--gpu-eval disabled)"
);
args.gpu_eval = false;
}
// Fix 1 (code-quality review): the binary's --num-actions default (5) is
// historically tied to the old "5 exposure levels" semantics and predates
// the 4-branch factored policy (81 flat actions). When marginals collection
// or random-surrogate sampling is requested, warn loudly so callers don't
// silently undersize flat_action_counts. The CPU DQN path ALSO hard-fails
// on out-of-range action_idx (see Fix 1 at the collection site) — this
// warning is a pre-flight signal rather than a safety net.
if (surrogate_enabled || args.emit_action_marginals) && args.num_actions == 5 {
warn!(
" [SURROGATE] --num-actions=5 (default). If the trained model uses \
a different flat action space (typical: 81 for 4-branch factored \
DQN), pass --num-actions explicitly. The CPU DQN path will \
hard-fail rather than silently drop out-of-range indices."
);
}
let surrogate_marginals: Option<Vec<f64>> = if surrogate_enabled {
match &args.surrogate_marginals {
Some(p) => {
let m = load_surrogate_marginals(p)
.with_context(|| format!("loading {}", p.display()))?;
info!(
" [SURROGATE] Loaded {}-dim action marginals from {}",
m.len(), p.display()
);
Some(m)
}
None => {
warn!(
" [SURROGATE] --surrogate-mode=random without --surrogate-marginals: \
falling back to UNIFORM random actions across {} actions",
args.num_actions
);
None
}
}
} else { None };
let eval_dqn = args.model == "dqn" || args.model == "both";
let eval_ppo = args.model == "ppo" || args.model == "both";
@@ -1894,6 +2123,32 @@ fn main() -> Result<()> {
info!("Step 3/5: Evaluating models on test data...");
let mut all_fold_metrics: Vec<FoldMetrics> = Vec::new();
let mut all_action_counts: Vec<[usize; 3]> = Vec::new();
// Flat per-action-index counts for --emit-action-marginals (CPU DQN path).
// GPU paths are excluded because they collapse actions to a different shape
// (5 exposure scores) before argmax; mixing them would yield a misleading
// histogram. When only GPU paths run, the emitted marginals will be
// all-zeros with a warning.
let mut flat_action_counts: Vec<usize> = vec![0; args.num_actions];
// Fix 3 (code-quality review): distinguish "stub because CPU DQN path
// never ran" (GPU took over, or wrong --model) from "CPU DQN path ran
// but counted zero actions" (empty fold). Set to true inside the CPU
// DQN Ok(...) branch below.
let mut cpu_dqn_ran = false;
// All per-bar returns concatenated across folds for --emit-pooled-sharpe.
// Populated only by the CPU DQN path (the only path that returns raw
// per-bar returns). When GPU path is used, this stays empty and pooled
// Sharpe is approximated from the fold-level Sharpe ratios.
let mut pooled_returns: Vec<f64> = Vec::new();
// Initialise surrogate sampler once (state carries across folds so a
// single seed produces a deterministic sequence over the full backtest).
let mut surrogate_sampler: Option<SurrogateSampler> = if surrogate_enabled {
Some(SurrogateSampler::new(
args.surrogate_seed,
surrogate_marginals.as_deref(),
args.num_actions,
))
} else { None };
for window in &windows {
info!(
@@ -2055,8 +2310,16 @@ fn main() -> Result<()> {
&args.models_dir,
&args,
&hp,
surrogate_sampler.as_mut(),
flat_action_counts.as_mut_slice(),
) {
Ok((returns, action_counts)) => {
// Fix 3: mark that the CPU DQN path actually completed
// a fold so the stub-marker logic can distinguish this
// from "GPU took over" / "model != dqn" / "zero bars".
cpu_dqn_ran = true;
// Accumulate concatenated returns for pooled-Sharpe
pooled_returns.extend(returns.iter().copied());
let fold_metrics = compute_metrics(&returns, args.bars_per_year);
let fold_str = window.fold.to_string();
tm::set_epoch("dqn", &fold_str, window.fold as f64);
@@ -2340,6 +2603,78 @@ fn main() -> Result<()> {
info!(" Report saved to: {}", args.output.display());
info!(" Total fold evaluations: {}", report.folds.len());
// ── Task 0.16: machine-parseable output lines ───────────────────────
//
// These lines are emitted to stdout (not stderr / tracing) so the
// surrogate-noise smoke test can parse them via a subprocess. The
// "ACTION_MARGINALS:" / "POOLED_SHARPE:" prefixes are load-bearing —
// the consumer matches them exactly.
if args.emit_action_marginals {
let total: usize = flat_action_counts.iter().sum();
if total == 0 {
// Fix 3 (code-quality review): disambiguate the stub marker. A zero
// total can mean any of three distinct things, each needing a
// different caller response:
// * `stub_no_dqn_eval` — --model != dqn (no DQN ran at all)
// * `stub_cpu_path_not_exercised` — GPU path handled all folds, or
// --model=dqn but every fold
// errored before the CPU branch
// set `cpu_dqn_ran = true`
// * `stub_zero_bars` — CPU path ran but counted no
// actions (empty fold)
let marker = if !eval_dqn {
"stub_no_dqn_eval"
} else if !cpu_dqn_ran {
"stub_cpu_path_not_exercised"
} else {
"stub_zero_bars"
};
println!("ACTION_MARGINALS: {}", marker);
warn!(
" [SURROGATE] --emit-action-marginals: emitted {} marker. \
eval_dqn={} cpu_dqn_ran={} total_samples={}",
marker, eval_dqn, cpu_dqn_ran, total,
);
} else {
let flat: Vec<f64> = flat_action_counts
.iter()
.map(|&c| c as f64 / total as f64)
.collect();
// Fix 6 (code-quality review): include a "schema" field so the
// consumer can validate the payload shape and refuse silently
// mismatched future versions.
let json = serde_json::json!({
"schema": "action_marginals_v1",
"flat": flat,
"num_actions": args.num_actions,
"total_samples": total,
});
match serde_json::to_string(&json) {
Ok(s) => println!("ACTION_MARGINALS: {}", s),
Err(e) => warn!(" [SURROGATE] Failed to serialize action marginals: {}", e),
}
}
}
if args.emit_pooled_sharpe {
let pooled = if !pooled_returns.is_empty() {
compute_pooled_sharpe(&pooled_returns, args.bars_per_year)
} else {
// Fallback: mean of fold-level Sharpes (unpooled). The smoke
// test prefers the true pooled value; this fallback avoids
// silently emitting zero when the CPU path wasn't exercised.
let sharpes: Vec<f64> = report.folds.iter().map(|f| f.sharpe_ratio).collect();
if sharpes.is_empty() { 0.0 } else {
warn!(
" [SURROGATE] --emit-pooled-sharpe: pooled_returns empty \
(GPU path only) — falling back to mean of fold Sharpes"
);
sharpes.iter().sum::<f64>() / sharpes.len() as f64
}
};
println!("POOLED_SHARPE: {}", pooled);
}
tm::set_active_workers(0.0);
// Push final metrics to pushgateway so they persist after pod termination

2
crates/ml/src/trainers/dqn/smoke_tests/mod.rs
View File

@@ -34,3 +34,5 @@ mod controller_activity;
mod multi_fold_convergence;
#[cfg(test)]
mod reward_component_audit;
#[cfg(test)]
mod surrogate_noise_check;

312
crates/ml/src/trainers/dqn/smoke_tests/surrogate_noise_check.rs Normal file
View File

@@ -0,0 +1,312 @@
//! Smoke test: surrogate-noise false-positive detector (spec §4.1, Task 0.16).
//!
//! Mandatory gate. Runs a trained model against N=30 random-action surrogate
//! backtests; trained pooled Sharpe must exceed the 95th percentile of the
//! surrogate distribution. Protects against look-ahead / leakage bugs.
//!
//! The test drives the `evaluate_baseline` example binary as a subprocess:
//! 1. Trained run with `--emit-action-marginals --emit-pooled-sharpe` to
//! capture both the marginal distribution and the pooled Sharpe.
//! 2. N=30 surrogate runs with `--surrogate-mode=random --surrogate-seed=N`
//! using the marginals collected above, each emitting `POOLED_SHARPE:`.
//!
//! The 95th percentile of the surrogate pooled Sharpes is the significance
//! threshold: trained Sharpe must exceed it to PASS.
//!
//! ## Requirements
//!
//! - A trained checkpoint at `/workspace/output/dqn_fold0_best.safetensors`
//! (Phase 3 deliverable). Until this exists the test asserts-out with a
//! clear message; it is `#[ignore]`'d so regular `cargo test` skips it.
//! - `FOXHUNT_TEST_DATA` pointing at a futures-baseline data root.
//! - A compiled `evaluate_baseline` example binary at
//! `target/release/examples/evaluate_baseline` (or the test falls back to
//! `cargo run --release --example evaluate_baseline -- …`).
//!
//! Run:
//! ```bash
//! FOXHUNT_TEST_DATA=test_data/futures-baseline \
//! cargo test -p ml --release --lib -- surrogate_noise_check --ignored --nocapture
//! ```
use super::helpers::{test_data_dir, workspace_root};
use anyhow::{anyhow, Context, Result};
use std::path::{Path, PathBuf};
use std::process::Command;
/// Default checkpoint location produced by Phase 3 training.
const DEFAULT_CHECKPOINT: &str = "/workspace/output/dqn_fold0_best.safetensors";
/// Models dir passed to `evaluate_baseline`. The checkpoint's parent is used
/// because `evaluate_baseline` reads `<models_dir>/dqn_fold{N}_best.safetensors`.
fn models_dir_for_checkpoint(ckpt: &Path) -> PathBuf {
ckpt.parent().map(std::path::Path::to_path_buf).unwrap_or_else(|| PathBuf::from("."))
}
#[test]
#[ignore] // Requires a trained checkpoint and substantial runtime (~30 subprocess calls).
fn test_surrogate_noise_check() -> Result<()> {
let data_dir = test_data_dir().expect("FOXHUNT_TEST_DATA or test_data/ must exist");
// Allow overriding the checkpoint path for local development; default to
// the Phase 3 production path.
let ckpt_path = std::env::var("FOXHUNT_SURROGATE_CKPT")
.unwrap_or_else(|_| DEFAULT_CHECKPOINT.to_owned());
let ckpt = Path::new(&ckpt_path);
assert!(
ckpt.exists(),
"surrogate-noise test requires a trained checkpoint at {:?}. \
Produce one via Phase 3 training or set FOXHUNT_SURROGATE_CKPT to a \
local checkpoint.",
ckpt
);
// 1. Trained backtest: capture pooled Sharpe + action marginals
let (trained_sharpe, marginals_json) =
run_trained_backtest_pooled(&data_dir, ckpt)?;
println!("[SURROGATE] trained pooled Sharpe = {:.3}", trained_sharpe);
// 2. Write marginals to a temp file for surrogate runs
let marginals_path = workspace_root()
.join("target")
.join("smoke_surrogate_marginals.json");
if let Some(parent) = marginals_path.parent() {
std::fs::create_dir_all(parent).context("create marginals dir")?;
}
std::fs::write(&marginals_path, &marginals_json)
.context("writing marginals JSON for surrogate runs")?;
// 3. N=30 surrogate backtests
//
// Fix 4 (code-quality review): decorrelate seeds via a multiplicative hash.
// Sequential small integers (0, 1, 2, …) fed into `StdRng::seed_from_u64`
// produce streams whose initial bits are highly correlated — surrogate
// distributions collapse toward the same trajectory and the p95 threshold
// becomes non-representative. Multiplying by the golden-ratio constant
// spreads the seeds evenly across the 64-bit space.
//
// Fix 5 (code-quality review): emit per-iteration progress so `--nocapture`
// output is usable for triage even when a run hangs or diverges.
const SEED_MULT: u64 = 0x9E37_79B9_7F4A_7C15; // golden-ratio constant
let mut surrogate_sharpes: Vec<f64> = Vec::with_capacity(30);
for i in 0..30_u64 {
// Multiplicative hash to decorrelate sequential indices — consecutive
// seeds with StdRng share initial bits.
let seed = SEED_MULT.wrapping_mul(i.wrapping_add(1));
let sh = run_surrogate_backtest_pooled(&data_dir, ckpt, &marginals_path, seed)?;
println!(
"[SURROGATE] seed {}/30 (hash={:#x}) sharpe={:.3}",
i + 1,
seed,
sh,
);
surrogate_sharpes.push(sh);
}
surrogate_sharpes.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let n = surrogate_sharpes.len();
let p95_idx = ((n as f64) * 0.95) as usize;
let p95 = *surrogate_sharpes
.get(p95_idx.min(n.saturating_sub(1)))
.ok_or_else(|| anyhow!("surrogate_sharpes empty"))?;
let median = *surrogate_sharpes
.get(n / 2)
.ok_or_else(|| anyhow!("surrogate_sharpes empty"))?;
let min = *surrogate_sharpes
.first()
.ok_or_else(|| anyhow!("surrogate_sharpes empty"))?;
let max = *surrogate_sharpes
.last()
.ok_or_else(|| anyhow!("surrogate_sharpes empty"))?;
println!(
"[SURROGATE] N={} random-action pooled Sharpe 95th %ile = {:.3}",
n, p95
);
println!(
"[SURROGATE] distribution: min={:.3} median={:.3} max={:.3}",
min, median, max
);
assert!(
trained_sharpe > p95,
"SURROGATE-NOISE FAILURE: trained Sharpe {:.3} is not beyond the 95th \
percentile of random-action Sharpes ({:.3}). The apparent edge may be \
measurement noise, data leakage, or look-ahead bias. DO NOT deploy.",
trained_sharpe, p95
);
Ok(())
}
/// Locate a compiled `evaluate_baseline` example binary, or fall back to
/// `cargo run --release --example evaluate_baseline`.
///
/// Cargo examples are emitted at `target/release/examples/evaluate_baseline`
/// when built with `cargo build --release --example evaluate_baseline -p ml`.
/// Using the compiled binary directly avoids cargo's build-lock on every call
/// (which would block multiple smoke tests running in parallel) and shaves
/// ~2s of cargo overhead per surrogate iteration.
fn evaluate_baseline_command() -> Command {
let root = workspace_root();
let compiled = root.join("target").join("release").join("examples").join("evaluate_baseline");
if compiled.exists() {
Command::new(compiled)
} else {
let mut c = Command::new("cargo");
c.current_dir(&root)
.args(["run", "--release", "-p", "ml", "--example", "evaluate_baseline", "--"]);
c
}
}
/// Run a trained-model backtest and return `(pooled_sharpe, action_marginals_json)`.
///
/// The action-marginals JSON is the exact `{"flat": [...], ...}` payload
/// emitted by `--emit-action-marginals` — it can be written to disk as-is
/// and fed back into surrogate runs via `--surrogate-marginals`.
fn run_trained_backtest_pooled(data: &str, ckpt: &Path) -> Result<(f64, String)> {
let models_dir = models_dir_for_checkpoint(ckpt);
let report_path = workspace_root()
.join("target")
.join("smoke_surrogate_trained_report.json");
if let Some(parent) = report_path.parent() {
std::fs::create_dir_all(parent).ok();
}
let mut cmd = evaluate_baseline_command();
cmd.env("SQLX_OFFLINE", "true")
.env("CUBLAS_WORKSPACE_CONFIG", ":4096:8")
.env("FOXHUNT_TEST_DATA", data)
.args([
"--model", "dqn",
"--data-dir", data,
"--models-dir", models_dir.to_str().unwrap_or(""),
"--symbol", "ES.FUT",
"--output", report_path.to_str().unwrap_or(""),
// Force CPU path to populate flat action counts + pooled returns
"--no-gpu-eval",
"--emit-action-marginals",
"--emit-pooled-sharpe",
]);
let output = cmd.output().context("spawning evaluate_baseline (trained)")?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
anyhow::bail!(
"evaluate_baseline (trained) exited {:?}:\nstderr:\n{}",
output.status.code(),
stderr.lines().rev().take(50).collect::<Vec<_>>().iter().rev().cloned().collect::<Vec<_>>().join("\n"),
);
}
let stdout = String::from_utf8_lossy(&output.stdout);
let sharpe = parse_pooled_sharpe(&stdout)
.context("parsing POOLED_SHARPE from trained run")?;
let marginals = parse_action_marginals(&stdout)
.context("parsing ACTION_MARGINALS from trained run")?;
Ok((sharpe, marginals))
}
/// Run a random-action surrogate backtest and return pooled Sharpe.
fn run_surrogate_backtest_pooled(
data: &str,
ckpt: &Path,
marginals_path: &Path,
seed: u64,
) -> Result<f64> {
let models_dir = models_dir_for_checkpoint(ckpt);
let report_path = workspace_root()
.join("target")
.join(format!("smoke_surrogate_report_seed{}.json", seed));
if let Some(parent) = report_path.parent() {
std::fs::create_dir_all(parent).ok();
}
let mut cmd = evaluate_baseline_command();
cmd.env("SQLX_OFFLINE", "true")
.env("CUBLAS_WORKSPACE_CONFIG", ":4096:8")
.env("FOXHUNT_TEST_DATA", data)
.args([
"--model", "dqn",
"--data-dir", data,
"--models-dir", models_dir.to_str().unwrap_or(""),
"--symbol", "ES.FUT",
"--output", report_path.to_str().unwrap_or(""),
"--surrogate-mode", "random",
"--surrogate-seed", &seed.to_string(),
"--surrogate-marginals", marginals_path.to_str().unwrap_or(""),
"--emit-pooled-sharpe",
]);
let output = cmd.output().context("spawning evaluate_baseline (surrogate)")?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
anyhow::bail!(
"evaluate_baseline (surrogate seed={}) exited {:?}:\nstderr:\n{}",
seed,
output.status.code(),
stderr.lines().rev().take(30).collect::<Vec<_>>().iter().rev().cloned().collect::<Vec<_>>().join("\n"),
);
}
let stdout = String::from_utf8_lossy(&output.stdout);
parse_pooled_sharpe(&stdout)
.with_context(|| format!("parsing POOLED_SHARPE from surrogate seed={}", seed))
}
/// Extract the single `POOLED_SHARPE: <float>` line from stdout.
fn parse_pooled_sharpe(stdout: &str) -> Result<f64> {
for line in stdout.lines() {
if let Some(rest) = line.strip_prefix("POOLED_SHARPE:") {
let val: f64 = rest.trim().parse()
.with_context(|| format!("parsing pooled Sharpe value: {:?}", rest.trim()))?;
return Ok(val);
}
}
anyhow::bail!("no POOLED_SHARPE: line found in evaluate_baseline stdout");
}
/// Extract the JSON payload following `ACTION_MARGINALS:` prefix.
///
/// Fix 3 (code-quality review): accept any `stub_<name>` marker with a clear
/// bail message that names the marker, rather than hard-coding a single
/// `stub_gpu_only_no_flat_counts` case. The emitter distinguishes between
/// `stub_no_dqn_eval`, `stub_cpu_path_not_exercised`, and `stub_zero_bars` —
/// each implies a different caller fix.
///
/// Fix 6 (code-quality review): verify the payload carries the expected
/// `schema: "action_marginals_v1"` field. Missing schema is accepted with a
/// warning for forward-compat with older emitters; a mismatched schema is a
/// hard error (prevents silent breakage if the emitter rev-bumps).
fn parse_action_marginals(stdout: &str) -> Result<String> {
for line in stdout.lines() {
if let Some(rest) = line.strip_prefix("ACTION_MARGINALS:") {
let payload = rest.trim();
if payload.starts_with("stub_") {
anyhow::bail!(
"evaluate_baseline emitted stub marginals marker {:?}. \
Surrogate test requires real action marginals — pass \
--no-gpu-eval and ensure --model=dqn so the CPU DQN \
path is exercised and counts actions.",
payload
);
}
// Sanity: must be valid JSON
let parsed: serde_json::Value = serde_json::from_str(payload)
.context("ACTION_MARGINALS payload is not valid JSON")?;
match parsed.get("schema").and_then(|v| v.as_str()) {
Some("action_marginals_v1") => {}
Some(other) => anyhow::bail!(
"ACTION_MARGINALS schema mismatch: expected \
\"action_marginals_v1\", got {:?}. Update the consumer \
to handle the new schema before proceeding.",
other
),
None => eprintln!(
"[SURROGATE] WARN: ACTION_MARGINALS payload missing \
\"schema\" field — accepting for forward-compat with \
pre-v1 emitters."
),
}
return Ok(payload.to_owned());
}
}
anyhow::bail!("no ACTION_MARGINALS: line found in evaluate_baseline stdout");
}