gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(ml-backtesting): conviction_log side-channel + threshold-tuning smoke

Browse Source 
P4 plumbed the threshold-gate kernel side but deferred the side-channel
that captures observed max_conviction per decision. Wire it now so the
threshold pre-registration step (spec §3.4) can compute the calibrated
p60-p95 absolute threshold values from a real model-on-data run.

Harness changes:
- BacktestHarness gains conviction_log: Vec<f32>. Per decision, computes
  max_h |alpha[h] - 0.5| * 2 from the SAME probs that go into broadcast_
  alpha (same value the threshold gate would compare against), pushes
  to the log. One shared vec — batched cells broadcast the same probs
  to every backtest, so per-backtest is redundant.
- write_artifacts emits convictions.bin (raw little-endian f32) +
  conviction_percentiles.json with pre-computed p10/p25/p50/p60/p70/
  p80/p90/p95/p99 + mean/min/max. Also eprintln-prints the summary
  line for at-a-glance log inspection.

Smoke YAML switched to the threshold-tuning configuration: threshold=0
(no gate, full distribution captured), cost=0.125 (1-tick realistic
anchor so the observed Sharpe is the no-gate net-of-cost floor for
the sweep's deployability story).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-05-19 19:21:29 +02:00
parent 7b96268efc
commit 81decf40f8
2 changed files with 71 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
config/ml/sweep_smoke.yaml
View File

@@ -25,7 +25,12 @@ base:
# dbd500ecf is the post-trunk-grows training checkpoint.
checkpoint: /feature-cache/alpha-perception-runs/dbd500ecf/trunk_best_h6000.bin
sim_variants:
- { name: t30c1l200, threshold: 0.30, cost_per_lot_per_side: 0.125, latency_ns: 200000000 }
# Threshold-tuning pass: threshold=0.0 captures the full max_conviction
# distribution into convictions.bin + conviction_percentiles.json.
# Use the resulting p60-p95 values to populate the real deployability
# sweep's threshold axis. Cost stays at 1-tick realistic anchor so the
# observed Sharpe is net-of-cost too — useful as the no-gate floor.
- { name: t0c1l200, threshold: 0.0, cost_per_lot_per_side: 0.125, latency_ns: 200000000 }
cells:
- { name: smoke }

65
crates/ml-backtesting/src/harness.rs
View File

@@ -101,6 +101,12 @@ pub struct BacktestHarness {
/// Per-cell cumulative P&L curve in USD, sampled once per event.
/// `pnl_curves[b][i]` = realized_pnl USD for backtest b after event i.
pnl_curves: Vec<Vec<f32>>,
/// Side-channel log of max_conviction per decision (one value per
/// stride boundary, NOT per event). Shared across all backtests in
/// a batched cell because broadcast_alpha gives every backtest the
/// same probs. Used by the threshold-tuning step to compute p60-p95
/// absolute values: percentiles of this Vec → calibrated thresholds.
conviction_log: Vec<f32>,
}
impl BacktestHarness {
@@ -189,6 +195,9 @@ impl BacktestHarness {
decision_count: 0,
event_count: 0,
pnl_curves,
// Pre-size for ~2.5M decisions (one full quarter at stride=4).
// Auto-grows past this; pre-allocation just avoids re-allocs.
conviction_log: Vec::with_capacity(3_000_000),
})
}
@@ -235,6 +244,15 @@ impl BacktestHarness {
let last_probs: [f32; N_HORIZONS] = probs_all[last_probs_start..]
.try_into()
.context("slice last K probs")?;
// Side-channel: record this decision's max_conviction for
// the threshold-tuning percentile computation. Doing it
// BEFORE broadcast/step so the log captures every decision
// attempt, including those the threshold gate would skip.
let max_conv = last_probs.iter()
.map(|p| ((p - 0.5).abs() * 2.0).min(1.0).max(0.0))
.fold(0.0_f32, f32::max);
self.conviction_log.push(max_conv);
self.sim.broadcast_alpha(&last_probs)?;
self.sim.step_decision_with_latency(raw.ts_ns, &self.sim_config)?;
self.decision_count += 1;
@@ -273,6 +291,53 @@ impl BacktestHarness {
pub fn write_artifacts(&self, out_dir: &std::path::Path) -> Result<()> {
std::fs::create_dir_all(out_dir)
.with_context(|| format!("create out dir {}", out_dir.display()))?;
// Write the threshold-tuning side-channel ONCE per cell (shared
// across all backtests in a batched cell because broadcast_alpha
// gives every backtest the same probs). Raw convictions.bin
// (little-endian f32) + conviction_percentiles.json with the
// pre-computed p60/p70/p80/p90/p95 values.
if !self.conviction_log.is_empty() {
let convictions_path = out_dir.join("convictions.bin");
let mut bytes = Vec::with_capacity(self.conviction_log.len() * 4);
for v in &self.conviction_log {
bytes.extend_from_slice(&v.to_le_bytes());
}
std::fs::write(&convictions_path, &bytes)
.with_context(|| format!("write {}", convictions_path.display()))?;
let mut sorted = self.conviction_log.clone();
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
let pct = |q: f32| -> f32 {
let idx = ((sorted.len() - 1) as f32 * q).round() as usize;
sorted[idx]
};
let pcts = serde_json::json!({
"n_decisions": sorted.len(),
"min": sorted.first().copied().unwrap_or(0.0),
"max": sorted.last().copied().unwrap_or(0.0),
"mean": sorted.iter().sum::<f32>() / sorted.len() as f32,
"p10": pct(0.10),
"p25": pct(0.25),
"p50": pct(0.50),
"p60": pct(0.60),
"p70": pct(0.70),
"p80": pct(0.80),
"p90": pct(0.90),
"p95": pct(0.95),
"p99": pct(0.99),
});
let pcts_path = out_dir.join("conviction_percentiles.json");
std::fs::write(&pcts_path, serde_json::to_string_pretty(&pcts)?)
.with_context(|| format!("write {}", pcts_path.display()))?;
eprintln!(
"convictions: n={} min={:.4} p60={:.4} p70={:.4} p80={:.4} p90={:.4} p95={:.4} max={:.4}",
sorted.len(),
sorted.first().copied().unwrap_or(0.0),
pct(0.60), pct(0.70), pct(0.80), pct(0.90), pct(0.95),
sorted.last().copied().unwrap_or(0.0),
);
}
if let Some(names) = self.cfg.variant_names.as_ref() {
anyhow::ensure!(
names.len() == self.cfg.n_parallel,