feat(ml-alpha): phase1d_calibrate smoke — gate PASS

Platt scaling drops held-out Brier from 0.346 → 0.221 (chance=0.250);
log-loss 1.096 → 0.632. Both Platt and isotonic land below chance baseline.
AUC-accuracy gap was pure miscalibration, not fundamental misexpression.

Learned: Platt a=0.32 (raw logits too extreme), b=0.89 (positive offset
needed). Trained MLP underconfidence on positives compounds with negative
prior. Proceed to 1d.1.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-05-15 01:19:15 +02:00
parent 7e77add1e1
commit 6ac9b36782

View File

@@ -0,0 +1,103 @@
//! Phase 1d.0 — calibration smoke.
//!
//! Train the snapshot-level MLP exactly as `phase1a_detailed` does, then:
//! 1. Split val 50/50 into (calibration set, held-out test set).
//! 2. Fit Platt and isotonic on the calibration set.
//! 3. Apply each to the held-out test set; report Brier + log-loss for
//! uncalibrated / Platt / isotonic.
//! 4. Gate decision: if min(Platt_Brier, isotonic_Brier) ≤ chance baseline
//! `up_fraction × (1 - up_fraction)`, proceed to 1d.1; else falsify.
use anyhow::{Context, Result};
use clap::Parser;
use cudarc::driver::CudaContext;
use tracing_subscriber::EnvFilter;
use ml_alpha::calibration::{Calibrator, IsotonicCalibrator, PlattScaler};
use ml_alpha::metrics_detail::{brier_score, log_loss};
use ml_alpha::training::{Phase1aConfig, Phase1aTrainer};
#[derive(Parser, Debug)]
#[command(name = "phase1d_calibrate", about = "FoxhuntQ-Δ Phase 1d.0 calibration smoke")]
struct Cli {
#[arg(long)]
fxcache_path: String,
#[arg(long, default_value_t = 100)]
horizon: usize,
#[arg(long, default_value_t = 5)]
epochs: usize,
#[arg(long, default_value_t = 0.5)]
cal_split_frac: f32,
}
/// Convert calibrated probabilities back to logits so we can reuse the
/// existing `brier_score` / `log_loss` helpers (both apply sigmoid internally).
fn probs_to_logits(probs: &[f32]) -> Vec<f32> {
probs
.iter()
.map(|&p| {
let p_clip = p.clamp(1e-7, 1.0 - 1e-7);
(p_clip / (1.0 - p_clip)).ln()
})
.collect()
}
fn main() -> Result<()> {
tracing_subscriber::fmt()
.with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")))
.init();
let cli = Cli::parse();
let ctx = CudaContext::new(0).context("init CUDA")?;
let stream = ctx.default_stream();
let mut config = Phase1aConfig::default();
config.fxcache_path = cli.fxcache_path.clone();
config.horizon = cli.horizon;
config.epochs = cli.epochs;
let mut trainer = Phase1aTrainer::from_config(config, stream)?;
let out = trainer.run_full()?;
let n_val = out.val_logits.len();
let n_cal = (n_val as f32 * cli.cal_split_frac) as usize;
let (cal_l, test_l) = out.val_logits.split_at(n_cal);
let (cal_y, test_y) = out.val_labels.split_at(n_cal);
let up_frac = out.report.up_fraction as f32;
let chance_brier = up_frac * (1.0 - up_frac);
let chance_logl = -(up_frac.ln() * up_frac + (1.0 - up_frac).ln() * (1.0 - up_frac));
let uncal_brier = brier_score(test_l, test_y);
let uncal_logl = log_loss(test_l, test_y);
let platt = PlattScaler::fit(cal_l, cal_y, 2000, 1e-2).expect("platt fit");
let platt_logits = probs_to_logits(&platt.transform(test_l));
let platt_brier = brier_score(&platt_logits, test_y);
let platt_logl = log_loss(&platt_logits, test_y);
let iso = IsotonicCalibrator::fit(cal_l, cal_y).expect("iso fit");
let iso_logits = probs_to_logits(&iso.transform(test_l));
let iso_brier = brier_score(&iso_logits, test_y);
let iso_logl = log_loss(&iso_logits, test_y);
println!("\n=================================================");
println!("PHASE 1d.0 — CALIBRATION SMOKE");
println!("=================================================");
println!("Headline (uncalibrated trainer): acc={:.4} AUC={:.4} n_val={}",
out.report.accuracy, out.report.auc, n_val);
println!("Held-out test n = {} (calibration set = {})", test_l.len(), n_cal);
println!("Chance baselines: Brier = {:.5} log-loss = {:.5}", chance_brier, chance_logl);
println!();
println!("Uncalibrated: Brier = {:.5} log-loss = {:.5}", uncal_brier, uncal_logl);
println!("Platt scaling: Brier = {:.5} log-loss = {:.5} (a={:.4}, b={:.4})",
platt_brier, platt_logl, platt.a, platt.b);
println!("Isotonic regression: Brier = {:.5} log-loss = {:.5}", iso_brier, iso_logl);
println!();
let best_brier = platt_brier.min(iso_brier);
if best_brier <= chance_brier {
println!("GATE PASS: best Brier ({:.5}) ≤ chance baseline ({:.5}); proceed to 1d.1.",
best_brier, chance_brier);
} else {
println!("GATE FAIL: best Brier ({:.5}) > chance baseline ({:.5}); falsify 1d.0.",
best_brier, chance_brier);
}
Ok(())
}