feat(ml-alpha): phase1d_calibrate smoke — gate PASS
Platt scaling drops held-out Brier from 0.346 → 0.221 (chance=0.250); log-loss 1.096 → 0.632. Both Platt and isotonic land below chance baseline. AUC-accuracy gap was pure miscalibration, not fundamental misexpression. Learned: Platt a=0.32 (raw logits too extreme), b=0.89 (positive offset needed). Trained MLP underconfidence on positives compounds with negative prior. Proceed to 1d.1. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
103
crates/ml-alpha/examples/phase1d_calibrate.rs
Normal file
103
crates/ml-alpha/examples/phase1d_calibrate.rs
Normal file
@@ -0,0 +1,103 @@
|
||||
//! Phase 1d.0 — calibration smoke.
|
||||
//!
|
||||
//! Train the snapshot-level MLP exactly as `phase1a_detailed` does, then:
|
||||
//! 1. Split val 50/50 into (calibration set, held-out test set).
|
||||
//! 2. Fit Platt and isotonic on the calibration set.
|
||||
//! 3. Apply each to the held-out test set; report Brier + log-loss for
|
||||
//! uncalibrated / Platt / isotonic.
|
||||
//! 4. Gate decision: if min(Platt_Brier, isotonic_Brier) ≤ chance baseline
|
||||
//! `up_fraction × (1 - up_fraction)`, proceed to 1d.1; else falsify.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use clap::Parser;
|
||||
use cudarc::driver::CudaContext;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
use ml_alpha::calibration::{Calibrator, IsotonicCalibrator, PlattScaler};
|
||||
use ml_alpha::metrics_detail::{brier_score, log_loss};
|
||||
use ml_alpha::training::{Phase1aConfig, Phase1aTrainer};
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(name = "phase1d_calibrate", about = "FoxhuntQ-Δ Phase 1d.0 calibration smoke")]
|
||||
struct Cli {
|
||||
#[arg(long)]
|
||||
fxcache_path: String,
|
||||
#[arg(long, default_value_t = 100)]
|
||||
horizon: usize,
|
||||
#[arg(long, default_value_t = 5)]
|
||||
epochs: usize,
|
||||
#[arg(long, default_value_t = 0.5)]
|
||||
cal_split_frac: f32,
|
||||
}
|
||||
|
||||
/// Convert calibrated probabilities back to logits so we can reuse the
|
||||
/// existing `brier_score` / `log_loss` helpers (both apply sigmoid internally).
|
||||
fn probs_to_logits(probs: &[f32]) -> Vec<f32> {
|
||||
probs
|
||||
.iter()
|
||||
.map(|&p| {
|
||||
let p_clip = p.clamp(1e-7, 1.0 - 1e-7);
|
||||
(p_clip / (1.0 - p_clip)).ln()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")))
|
||||
.init();
|
||||
let cli = Cli::parse();
|
||||
let ctx = CudaContext::new(0).context("init CUDA")?;
|
||||
let stream = ctx.default_stream();
|
||||
let mut config = Phase1aConfig::default();
|
||||
config.fxcache_path = cli.fxcache_path.clone();
|
||||
config.horizon = cli.horizon;
|
||||
config.epochs = cli.epochs;
|
||||
let mut trainer = Phase1aTrainer::from_config(config, stream)?;
|
||||
let out = trainer.run_full()?;
|
||||
|
||||
let n_val = out.val_logits.len();
|
||||
let n_cal = (n_val as f32 * cli.cal_split_frac) as usize;
|
||||
let (cal_l, test_l) = out.val_logits.split_at(n_cal);
|
||||
let (cal_y, test_y) = out.val_labels.split_at(n_cal);
|
||||
|
||||
let up_frac = out.report.up_fraction as f32;
|
||||
let chance_brier = up_frac * (1.0 - up_frac);
|
||||
let chance_logl = -(up_frac.ln() * up_frac + (1.0 - up_frac).ln() * (1.0 - up_frac));
|
||||
|
||||
let uncal_brier = brier_score(test_l, test_y);
|
||||
let uncal_logl = log_loss(test_l, test_y);
|
||||
|
||||
let platt = PlattScaler::fit(cal_l, cal_y, 2000, 1e-2).expect("platt fit");
|
||||
let platt_logits = probs_to_logits(&platt.transform(test_l));
|
||||
let platt_brier = brier_score(&platt_logits, test_y);
|
||||
let platt_logl = log_loss(&platt_logits, test_y);
|
||||
|
||||
let iso = IsotonicCalibrator::fit(cal_l, cal_y).expect("iso fit");
|
||||
let iso_logits = probs_to_logits(&iso.transform(test_l));
|
||||
let iso_brier = brier_score(&iso_logits, test_y);
|
||||
let iso_logl = log_loss(&iso_logits, test_y);
|
||||
|
||||
println!("\n=================================================");
|
||||
println!("PHASE 1d.0 — CALIBRATION SMOKE");
|
||||
println!("=================================================");
|
||||
println!("Headline (uncalibrated trainer): acc={:.4} AUC={:.4} n_val={}",
|
||||
out.report.accuracy, out.report.auc, n_val);
|
||||
println!("Held-out test n = {} (calibration set = {})", test_l.len(), n_cal);
|
||||
println!("Chance baselines: Brier = {:.5} log-loss = {:.5}", chance_brier, chance_logl);
|
||||
println!();
|
||||
println!("Uncalibrated: Brier = {:.5} log-loss = {:.5}", uncal_brier, uncal_logl);
|
||||
println!("Platt scaling: Brier = {:.5} log-loss = {:.5} (a={:.4}, b={:.4})",
|
||||
platt_brier, platt_logl, platt.a, platt.b);
|
||||
println!("Isotonic regression: Brier = {:.5} log-loss = {:.5}", iso_brier, iso_logl);
|
||||
println!();
|
||||
let best_brier = platt_brier.min(iso_brier);
|
||||
if best_brier <= chance_brier {
|
||||
println!("GATE PASS: best Brier ({:.5}) ≤ chance baseline ({:.5}); proceed to 1d.1.",
|
||||
best_brier, chance_brier);
|
||||
} else {
|
||||
println!("GATE FAIL: best Brier ({:.5}) > chance baseline ({:.5}); falsify 1d.0.",
|
||||
best_brier, chance_brier);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user