From 6ac9b36782c0d8a69e845e651088649f2569bb43 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 15 May 2026 01:19:15 +0200 Subject: [PATCH] =?UTF-8?q?feat(ml-alpha):=20phase1d=5Fcalibrate=20smoke?= =?UTF-8?q?=20=E2=80=94=20gate=20PASS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Platt scaling drops held-out Brier from 0.346 → 0.221 (chance=0.250); log-loss 1.096 → 0.632. Both Platt and isotonic land below chance baseline. AUC-accuracy gap was pure miscalibration, not fundamental misexpression. Learned: Platt a=0.32 (raw logits too extreme), b=0.89 (positive offset needed). Trained MLP underconfidence on positives compounds with negative prior. Proceed to 1d.1. Co-Authored-By: Claude Opus 4.7 --- crates/ml-alpha/examples/phase1d_calibrate.rs | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 crates/ml-alpha/examples/phase1d_calibrate.rs diff --git a/crates/ml-alpha/examples/phase1d_calibrate.rs b/crates/ml-alpha/examples/phase1d_calibrate.rs new file mode 100644 index 000000000..b85367bb7 --- /dev/null +++ b/crates/ml-alpha/examples/phase1d_calibrate.rs @@ -0,0 +1,103 @@ +//! Phase 1d.0 — calibration smoke. +//! +//! Train the snapshot-level MLP exactly as `phase1a_detailed` does, then: +//! 1. Split val 50/50 into (calibration set, held-out test set). +//! 2. Fit Platt and isotonic on the calibration set. +//! 3. Apply each to the held-out test set; report Brier + log-loss for +//! uncalibrated / Platt / isotonic. +//! 4. Gate decision: if min(Platt_Brier, isotonic_Brier) ≤ chance baseline +//! `up_fraction × (1 - up_fraction)`, proceed to 1d.1; else falsify. + +use anyhow::{Context, Result}; +use clap::Parser; +use cudarc::driver::CudaContext; +use tracing_subscriber::EnvFilter; + +use ml_alpha::calibration::{Calibrator, IsotonicCalibrator, PlattScaler}; +use ml_alpha::metrics_detail::{brier_score, log_loss}; +use ml_alpha::training::{Phase1aConfig, Phase1aTrainer}; + +#[derive(Parser, Debug)] +#[command(name = "phase1d_calibrate", about = "FoxhuntQ-Δ Phase 1d.0 calibration smoke")] +struct Cli { + #[arg(long)] + fxcache_path: String, + #[arg(long, default_value_t = 100)] + horizon: usize, + #[arg(long, default_value_t = 5)] + epochs: usize, + #[arg(long, default_value_t = 0.5)] + cal_split_frac: f32, +} + +/// Convert calibrated probabilities back to logits so we can reuse the +/// existing `brier_score` / `log_loss` helpers (both apply sigmoid internally). +fn probs_to_logits(probs: &[f32]) -> Vec { + probs + .iter() + .map(|&p| { + let p_clip = p.clamp(1e-7, 1.0 - 1e-7); + (p_clip / (1.0 - p_clip)).ln() + }) + .collect() +} + +fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))) + .init(); + let cli = Cli::parse(); + let ctx = CudaContext::new(0).context("init CUDA")?; + let stream = ctx.default_stream(); + let mut config = Phase1aConfig::default(); + config.fxcache_path = cli.fxcache_path.clone(); + config.horizon = cli.horizon; + config.epochs = cli.epochs; + let mut trainer = Phase1aTrainer::from_config(config, stream)?; + let out = trainer.run_full()?; + + let n_val = out.val_logits.len(); + let n_cal = (n_val as f32 * cli.cal_split_frac) as usize; + let (cal_l, test_l) = out.val_logits.split_at(n_cal); + let (cal_y, test_y) = out.val_labels.split_at(n_cal); + + let up_frac = out.report.up_fraction as f32; + let chance_brier = up_frac * (1.0 - up_frac); + let chance_logl = -(up_frac.ln() * up_frac + (1.0 - up_frac).ln() * (1.0 - up_frac)); + + let uncal_brier = brier_score(test_l, test_y); + let uncal_logl = log_loss(test_l, test_y); + + let platt = PlattScaler::fit(cal_l, cal_y, 2000, 1e-2).expect("platt fit"); + let platt_logits = probs_to_logits(&platt.transform(test_l)); + let platt_brier = brier_score(&platt_logits, test_y); + let platt_logl = log_loss(&platt_logits, test_y); + + let iso = IsotonicCalibrator::fit(cal_l, cal_y).expect("iso fit"); + let iso_logits = probs_to_logits(&iso.transform(test_l)); + let iso_brier = brier_score(&iso_logits, test_y); + let iso_logl = log_loss(&iso_logits, test_y); + + println!("\n================================================="); + println!("PHASE 1d.0 — CALIBRATION SMOKE"); + println!("================================================="); + println!("Headline (uncalibrated trainer): acc={:.4} AUC={:.4} n_val={}", + out.report.accuracy, out.report.auc, n_val); + println!("Held-out test n = {} (calibration set = {})", test_l.len(), n_cal); + println!("Chance baselines: Brier = {:.5} log-loss = {:.5}", chance_brier, chance_logl); + println!(); + println!("Uncalibrated: Brier = {:.5} log-loss = {:.5}", uncal_brier, uncal_logl); + println!("Platt scaling: Brier = {:.5} log-loss = {:.5} (a={:.4}, b={:.4})", + platt_brier, platt_logl, platt.a, platt.b); + println!("Isotonic regression: Brier = {:.5} log-loss = {:.5}", iso_brier, iso_logl); + println!(); + let best_brier = platt_brier.min(iso_brier); + if best_brier <= chance_brier { + println!("GATE PASS: best Brier ({:.5}) ≤ chance baseline ({:.5}); proceed to 1d.1.", + best_brier, chance_brier); + } else { + println!("GATE FAIL: best Brier ({:.5}) > chance baseline ({:.5}); falsify 1d.0.", + best_brier, chance_brier); + } + Ok(()) +}