diff --git a/config/ml/sweep_smoke.yaml b/config/ml/sweep_smoke.yaml index 6549b8ce7..884db2662 100644 --- a/config/ml/sweep_smoke.yaml +++ b/config/ml/sweep_smoke.yaml @@ -25,7 +25,12 @@ base: # dbd500ecf is the post-trunk-grows training checkpoint. checkpoint: /feature-cache/alpha-perception-runs/dbd500ecf/trunk_best_h6000.bin sim_variants: - - { name: t30c1l200, threshold: 0.30, cost_per_lot_per_side: 0.125, latency_ns: 200000000 } + # Threshold-tuning pass: threshold=0.0 captures the full max_conviction + # distribution into convictions.bin + conviction_percentiles.json. + # Use the resulting p60-p95 values to populate the real deployability + # sweep's threshold axis. Cost stays at 1-tick realistic anchor so the + # observed Sharpe is net-of-cost too — useful as the no-gate floor. + - { name: t0c1l200, threshold: 0.0, cost_per_lot_per_side: 0.125, latency_ns: 200000000 } cells: - { name: smoke } diff --git a/crates/ml-backtesting/src/harness.rs b/crates/ml-backtesting/src/harness.rs index 5405127ea..d2f87efd6 100644 --- a/crates/ml-backtesting/src/harness.rs +++ b/crates/ml-backtesting/src/harness.rs @@ -101,6 +101,12 @@ pub struct BacktestHarness { /// Per-cell cumulative P&L curve in USD, sampled once per event. /// `pnl_curves[b][i]` = realized_pnl USD for backtest b after event i. pnl_curves: Vec>, + /// Side-channel log of max_conviction per decision (one value per + /// stride boundary, NOT per event). Shared across all backtests in + /// a batched cell because broadcast_alpha gives every backtest the + /// same probs. Used by the threshold-tuning step to compute p60-p95 + /// absolute values: percentiles of this Vec → calibrated thresholds. + conviction_log: Vec, } impl BacktestHarness { @@ -189,6 +195,9 @@ impl BacktestHarness { decision_count: 0, event_count: 0, pnl_curves, + // Pre-size for ~2.5M decisions (one full quarter at stride=4). + // Auto-grows past this; pre-allocation just avoids re-allocs. + conviction_log: Vec::with_capacity(3_000_000), }) } @@ -235,6 +244,15 @@ impl BacktestHarness { let last_probs: [f32; N_HORIZONS] = probs_all[last_probs_start..] .try_into() .context("slice last K probs")?; + // Side-channel: record this decision's max_conviction for + // the threshold-tuning percentile computation. Doing it + // BEFORE broadcast/step so the log captures every decision + // attempt, including those the threshold gate would skip. + let max_conv = last_probs.iter() + .map(|p| ((p - 0.5).abs() * 2.0).min(1.0).max(0.0)) + .fold(0.0_f32, f32::max); + self.conviction_log.push(max_conv); + self.sim.broadcast_alpha(&last_probs)?; self.sim.step_decision_with_latency(raw.ts_ns, &self.sim_config)?; self.decision_count += 1; @@ -273,6 +291,53 @@ impl BacktestHarness { pub fn write_artifacts(&self, out_dir: &std::path::Path) -> Result<()> { std::fs::create_dir_all(out_dir) .with_context(|| format!("create out dir {}", out_dir.display()))?; + + // Write the threshold-tuning side-channel ONCE per cell (shared + // across all backtests in a batched cell because broadcast_alpha + // gives every backtest the same probs). Raw convictions.bin + // (little-endian f32) + conviction_percentiles.json with the + // pre-computed p60/p70/p80/p90/p95 values. + if !self.conviction_log.is_empty() { + let convictions_path = out_dir.join("convictions.bin"); + let mut bytes = Vec::with_capacity(self.conviction_log.len() * 4); + for v in &self.conviction_log { + bytes.extend_from_slice(&v.to_le_bytes()); + } + std::fs::write(&convictions_path, &bytes) + .with_context(|| format!("write {}", convictions_path.display()))?; + + let mut sorted = self.conviction_log.clone(); + sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + let pct = |q: f32| -> f32 { + let idx = ((sorted.len() - 1) as f32 * q).round() as usize; + sorted[idx] + }; + let pcts = serde_json::json!({ + "n_decisions": sorted.len(), + "min": sorted.first().copied().unwrap_or(0.0), + "max": sorted.last().copied().unwrap_or(0.0), + "mean": sorted.iter().sum::() / sorted.len() as f32, + "p10": pct(0.10), + "p25": pct(0.25), + "p50": pct(0.50), + "p60": pct(0.60), + "p70": pct(0.70), + "p80": pct(0.80), + "p90": pct(0.90), + "p95": pct(0.95), + "p99": pct(0.99), + }); + let pcts_path = out_dir.join("conviction_percentiles.json"); + std::fs::write(&pcts_path, serde_json::to_string_pretty(&pcts)?) + .with_context(|| format!("write {}", pcts_path.display()))?; + eprintln!( + "convictions: n={} min={:.4} p60={:.4} p70={:.4} p80={:.4} p90={:.4} p95={:.4} max={:.4}", + sorted.len(), + sorted.first().copied().unwrap_or(0.0), + pct(0.60), pct(0.70), pct(0.80), pct(0.90), pct(0.95), + sorted.last().copied().unwrap_or(0.0), + ); + } if let Some(names) = self.cfg.variant_names.as_ref() { anyhow::ensure!( names.len() == self.cfg.n_parallel,