feat(ml-alpha): Phase 1d.2 smoke + post-hoc Platt/Isotonic calibration

Extends phase1d_long_horizon with a 50/50 val split (cal/test halves):
fit Platt and Isotonic on cal, evaluate on held-out test. Reports both
uncalibrated AND calibrated metrics (accuracy, AUC, Brier, log-loss).

Hypothesis: the K=6000 Mamba result (AUC=0.66 / acc=0.62 from the
uncalibrated 4cf9499b5 smoke) has a 4-point AUC-accuracy gap which
mirrors the Phase 1c pattern; Platt should compress that gap and lift
accuracy further without retraining.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-05-15 09:06:43 +02:00
parent 4cf9499b58
commit d57026b0ba

View File

@@ -30,11 +30,13 @@ use rand_chacha::ChaCha8Rng;
use std::sync::Arc;
use tracing_subscriber::EnvFilter;
use ml_alpha::calibration::{Calibrator, IsotonicCalibrator, PlattScaler};
use ml_alpha::eval::{accuracy_from_logits, auc_from_logits};
use ml_alpha::fxcache_reader::{FxCacheReader, COL_RAW_CLOSE, FEAT_DIM};
use ml_alpha::mamba2_block::{
Mamba2AdamW, Mamba2AdamWConfig, Mamba2Block, Mamba2BlockConfig,
};
use ml_alpha::metrics_detail::{brier_score, log_loss};
use ml_alpha::multi_horizon_labels::generate_labels;
use ml_core::cuda_autograd::gpu_tensor::GpuTensor;
@@ -59,6 +61,9 @@ struct Cli {
/// Subsample stride for train sequences (1 = use every starting position).
#[arg(long, default_value_t = 4)] train_stride: usize,
#[arg(long, default_value_t = 42)] seed: u64,
/// Fraction of val to use as calibration set (rest is held-out test).
/// 0 disables Platt/isotonic post-hoc calibration.
#[arg(long, default_value_t = 0.5)] cal_frac: f32,
}
fn main() -> Result<()> {
@@ -230,13 +235,77 @@ fn main() -> Result<()> {
let labels_u8: Vec<u8> = val_ys.iter().map(|&y| if y > 0.5 { 1 } else { 0 }).collect();
let acc = accuracy_from_logits(&val_logits, &labels_u8);
let auc = auc_from_logits(&val_logits, &labels_u8);
let uncal_brier = brier_score(&val_logits, &val_ys);
let uncal_logl = log_loss(&val_logits, &val_ys);
let up_frac = val_ys.iter().sum::<f32>() / val_ys.len() as f32;
let chance_brier = up_frac * (1.0 - up_frac);
let chance_logl = -(up_frac.ln() * up_frac + (1.0 - up_frac).ln() * (1.0 - up_frac));
println!();
println!("===========================================================");
println!("PHASE 1d.2 — MAMBA2 MULTI-MINUTE SMOKE (K={})", cli.horizon);
println!("===========================================================");
println!("val sequences: {}", val_logits.len());
println!("val accuracy: {:.4}", acc);
println!("val AUC: {:.4}", auc);
println!("up_fraction: {:.4} (chance baselines: Brier={:.5} log-loss={:.5})",
up_frac, chance_brier, chance_logl);
println!("--- Uncalibrated ---");
println!(" accuracy: {:.4}", acc);
println!(" AUC: {:.4}", auc);
println!(" Brier: {:.5}", uncal_brier);
println!(" log-loss: {:.5}", uncal_logl);
// ── Post-hoc Platt + Isotonic calibration on a held-out cal split ──
if cli.cal_frac > 0.0 && cli.cal_frac < 1.0 {
let n_val = val_logits.len();
let n_cal = (n_val as f32 * cli.cal_frac) as usize;
let (cal_l, test_l) = val_logits.split_at(n_cal);
let (cal_y, test_y) = val_ys.split_at(n_cal);
let test_labels_u8: Vec<u8> = test_y.iter().map(|&y| if y > 0.5 { 1 } else { 0 }).collect();
// Headline on test half BEFORE calibration (baseline for comparison).
let test_acc_raw = accuracy_from_logits(test_l, &test_labels_u8);
let test_auc_raw = auc_from_logits(test_l, &test_labels_u8);
let test_brier_raw = brier_score(test_l, test_y);
let test_logl_raw = log_loss(test_l, test_y);
// Fit + apply Platt.
let platt = PlattScaler::fit(cal_l, cal_y, 2000, 1e-2)
.map_err(|e| anyhow::anyhow!("platt fit: {e}"))?;
let platt_probs = platt.transform(test_l);
// accuracy_from_logits expects logits; convert calibrated probs back to logits.
let to_logits = |probs: &[f32]| -> Vec<f32> {
probs.iter().map(|&p| {
let pc = p.clamp(1e-7, 1.0 - 1e-7);
(pc / (1.0 - pc)).ln()
}).collect()
};
let platt_logits = to_logits(&platt_probs);
let platt_acc = accuracy_from_logits(&platt_logits, &test_labels_u8);
let platt_auc = auc_from_logits(&platt_logits, &test_labels_u8);
let platt_brier = brier_score(&platt_logits, test_y);
let platt_logl = log_loss(&platt_logits, test_y);
// Fit + apply Isotonic.
let iso = IsotonicCalibrator::fit(cal_l, cal_y)
.map_err(|e| anyhow::anyhow!("iso fit: {e}"))?;
let iso_probs = iso.transform(test_l);
let iso_logits = to_logits(&iso_probs);
let iso_acc = accuracy_from_logits(&iso_logits, &test_labels_u8);
let iso_auc = auc_from_logits(&iso_logits, &test_labels_u8);
let iso_brier = brier_score(&iso_logits, test_y);
let iso_logl = log_loss(&iso_logits, test_y);
println!("--- Held-out test (cal_frac={:.2}, n_cal={}, n_test={}) ---",
cli.cal_frac, n_cal, test_l.len());
println!(" Uncalibrated: acc={:.4} AUC={:.4} Brier={:.5} log-loss={:.5}",
test_acc_raw, test_auc_raw, test_brier_raw, test_logl_raw);
println!(" Platt: acc={:.4} AUC={:.4} Brier={:.5} log-loss={:.5} (a={:.4} b={:.4})",
platt_acc, platt_auc, platt_brier, platt_logl, platt.a, platt.b);
println!(" Isotonic: acc={:.4} AUC={:.4} Brier={:.5} log-loss={:.5}",
iso_acc, iso_auc, iso_brier, iso_logl);
}
println!();
if auc > 0.55 {
println!("GATE PASS: AUC > 0.55 at K={}; multi-minute alpha confirmed.", cli.horizon);
println!(" Two-head FoxhuntQ-Δ architecture validated.");