diff --git a/crates/ml-alpha/examples/phase1d_long_horizon.rs b/crates/ml-alpha/examples/phase1d_long_horizon.rs index 285edfc4d..32c40c1cc 100644 --- a/crates/ml-alpha/examples/phase1d_long_horizon.rs +++ b/crates/ml-alpha/examples/phase1d_long_horizon.rs @@ -30,11 +30,13 @@ use rand_chacha::ChaCha8Rng; use std::sync::Arc; use tracing_subscriber::EnvFilter; +use ml_alpha::calibration::{Calibrator, IsotonicCalibrator, PlattScaler}; use ml_alpha::eval::{accuracy_from_logits, auc_from_logits}; use ml_alpha::fxcache_reader::{FxCacheReader, COL_RAW_CLOSE, FEAT_DIM}; use ml_alpha::mamba2_block::{ Mamba2AdamW, Mamba2AdamWConfig, Mamba2Block, Mamba2BlockConfig, }; +use ml_alpha::metrics_detail::{brier_score, log_loss}; use ml_alpha::multi_horizon_labels::generate_labels; use ml_core::cuda_autograd::gpu_tensor::GpuTensor; @@ -59,6 +61,9 @@ struct Cli { /// Subsample stride for train sequences (1 = use every starting position). #[arg(long, default_value_t = 4)] train_stride: usize, #[arg(long, default_value_t = 42)] seed: u64, + /// Fraction of val to use as calibration set (rest is held-out test). + /// 0 disables Platt/isotonic post-hoc calibration. + #[arg(long, default_value_t = 0.5)] cal_frac: f32, } fn main() -> Result<()> { @@ -230,13 +235,77 @@ fn main() -> Result<()> { let labels_u8: Vec = val_ys.iter().map(|&y| if y > 0.5 { 1 } else { 0 }).collect(); let acc = accuracy_from_logits(&val_logits, &labels_u8); let auc = auc_from_logits(&val_logits, &labels_u8); + let uncal_brier = brier_score(&val_logits, &val_ys); + let uncal_logl = log_loss(&val_logits, &val_ys); + let up_frac = val_ys.iter().sum::() / val_ys.len() as f32; + let chance_brier = up_frac * (1.0 - up_frac); + let chance_logl = -(up_frac.ln() * up_frac + (1.0 - up_frac).ln() * (1.0 - up_frac)); + println!(); println!("==========================================================="); println!("PHASE 1d.2 — MAMBA2 MULTI-MINUTE SMOKE (K={})", cli.horizon); println!("==========================================================="); println!("val sequences: {}", val_logits.len()); - println!("val accuracy: {:.4}", acc); - println!("val AUC: {:.4}", auc); + println!("up_fraction: {:.4} (chance baselines: Brier={:.5} log-loss={:.5})", + up_frac, chance_brier, chance_logl); + println!("--- Uncalibrated ---"); + println!(" accuracy: {:.4}", acc); + println!(" AUC: {:.4}", auc); + println!(" Brier: {:.5}", uncal_brier); + println!(" log-loss: {:.5}", uncal_logl); + + // ── Post-hoc Platt + Isotonic calibration on a held-out cal split ── + if cli.cal_frac > 0.0 && cli.cal_frac < 1.0 { + let n_val = val_logits.len(); + let n_cal = (n_val as f32 * cli.cal_frac) as usize; + let (cal_l, test_l) = val_logits.split_at(n_cal); + let (cal_y, test_y) = val_ys.split_at(n_cal); + let test_labels_u8: Vec = test_y.iter().map(|&y| if y > 0.5 { 1 } else { 0 }).collect(); + + // Headline on test half BEFORE calibration (baseline for comparison). + let test_acc_raw = accuracy_from_logits(test_l, &test_labels_u8); + let test_auc_raw = auc_from_logits(test_l, &test_labels_u8); + let test_brier_raw = brier_score(test_l, test_y); + let test_logl_raw = log_loss(test_l, test_y); + + // Fit + apply Platt. + let platt = PlattScaler::fit(cal_l, cal_y, 2000, 1e-2) + .map_err(|e| anyhow::anyhow!("platt fit: {e}"))?; + let platt_probs = platt.transform(test_l); + // accuracy_from_logits expects logits; convert calibrated probs back to logits. + let to_logits = |probs: &[f32]| -> Vec { + probs.iter().map(|&p| { + let pc = p.clamp(1e-7, 1.0 - 1e-7); + (pc / (1.0 - pc)).ln() + }).collect() + }; + let platt_logits = to_logits(&platt_probs); + let platt_acc = accuracy_from_logits(&platt_logits, &test_labels_u8); + let platt_auc = auc_from_logits(&platt_logits, &test_labels_u8); + let platt_brier = brier_score(&platt_logits, test_y); + let platt_logl = log_loss(&platt_logits, test_y); + + // Fit + apply Isotonic. + let iso = IsotonicCalibrator::fit(cal_l, cal_y) + .map_err(|e| anyhow::anyhow!("iso fit: {e}"))?; + let iso_probs = iso.transform(test_l); + let iso_logits = to_logits(&iso_probs); + let iso_acc = accuracy_from_logits(&iso_logits, &test_labels_u8); + let iso_auc = auc_from_logits(&iso_logits, &test_labels_u8); + let iso_brier = brier_score(&iso_logits, test_y); + let iso_logl = log_loss(&iso_logits, test_y); + + println!("--- Held-out test (cal_frac={:.2}, n_cal={}, n_test={}) ---", + cli.cal_frac, n_cal, test_l.len()); + println!(" Uncalibrated: acc={:.4} AUC={:.4} Brier={:.5} log-loss={:.5}", + test_acc_raw, test_auc_raw, test_brier_raw, test_logl_raw); + println!(" Platt: acc={:.4} AUC={:.4} Brier={:.5} log-loss={:.5} (a={:.4} b={:.4})", + platt_acc, platt_auc, platt_brier, platt_logl, platt.a, platt.b); + println!(" Isotonic: acc={:.4} AUC={:.4} Brier={:.5} log-loss={:.5}", + iso_acc, iso_auc, iso_brier, iso_logl); + } + + println!(); if auc > 0.55 { println!("GATE PASS: AUC > 0.55 at K={}; multi-minute alpha confirmed.", cli.horizon); println!(" Two-head FoxhuntQ-Δ architecture validated.");