feat(ml-alpha): Phase 1d.2 smoke + post-hoc Platt/Isotonic calibration
Extends phase1d_long_horizon with a 50/50 val split (cal/test halves):
fit Platt and Isotonic on cal, evaluate on held-out test. Reports both
uncalibrated AND calibrated metrics (accuracy, AUC, Brier, log-loss).
Hypothesis: the K=6000 Mamba result (AUC=0.66 / acc=0.62 from the
uncalibrated 4cf9499b5 smoke) has a 4-point AUC-accuracy gap which
mirrors the Phase 1c pattern; Platt should compress that gap and lift
accuracy further without retraining.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -30,11 +30,13 @@ use rand_chacha::ChaCha8Rng;
|
||||
use std::sync::Arc;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
use ml_alpha::calibration::{Calibrator, IsotonicCalibrator, PlattScaler};
|
||||
use ml_alpha::eval::{accuracy_from_logits, auc_from_logits};
|
||||
use ml_alpha::fxcache_reader::{FxCacheReader, COL_RAW_CLOSE, FEAT_DIM};
|
||||
use ml_alpha::mamba2_block::{
|
||||
Mamba2AdamW, Mamba2AdamWConfig, Mamba2Block, Mamba2BlockConfig,
|
||||
};
|
||||
use ml_alpha::metrics_detail::{brier_score, log_loss};
|
||||
use ml_alpha::multi_horizon_labels::generate_labels;
|
||||
|
||||
use ml_core::cuda_autograd::gpu_tensor::GpuTensor;
|
||||
@@ -59,6 +61,9 @@ struct Cli {
|
||||
/// Subsample stride for train sequences (1 = use every starting position).
|
||||
#[arg(long, default_value_t = 4)] train_stride: usize,
|
||||
#[arg(long, default_value_t = 42)] seed: u64,
|
||||
/// Fraction of val to use as calibration set (rest is held-out test).
|
||||
/// 0 disables Platt/isotonic post-hoc calibration.
|
||||
#[arg(long, default_value_t = 0.5)] cal_frac: f32,
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
@@ -230,13 +235,77 @@ fn main() -> Result<()> {
|
||||
let labels_u8: Vec<u8> = val_ys.iter().map(|&y| if y > 0.5 { 1 } else { 0 }).collect();
|
||||
let acc = accuracy_from_logits(&val_logits, &labels_u8);
|
||||
let auc = auc_from_logits(&val_logits, &labels_u8);
|
||||
let uncal_brier = brier_score(&val_logits, &val_ys);
|
||||
let uncal_logl = log_loss(&val_logits, &val_ys);
|
||||
let up_frac = val_ys.iter().sum::<f32>() / val_ys.len() as f32;
|
||||
let chance_brier = up_frac * (1.0 - up_frac);
|
||||
let chance_logl = -(up_frac.ln() * up_frac + (1.0 - up_frac).ln() * (1.0 - up_frac));
|
||||
|
||||
println!();
|
||||
println!("===========================================================");
|
||||
println!("PHASE 1d.2 — MAMBA2 MULTI-MINUTE SMOKE (K={})", cli.horizon);
|
||||
println!("===========================================================");
|
||||
println!("val sequences: {}", val_logits.len());
|
||||
println!("val accuracy: {:.4}", acc);
|
||||
println!("val AUC: {:.4}", auc);
|
||||
println!("up_fraction: {:.4} (chance baselines: Brier={:.5} log-loss={:.5})",
|
||||
up_frac, chance_brier, chance_logl);
|
||||
println!("--- Uncalibrated ---");
|
||||
println!(" accuracy: {:.4}", acc);
|
||||
println!(" AUC: {:.4}", auc);
|
||||
println!(" Brier: {:.5}", uncal_brier);
|
||||
println!(" log-loss: {:.5}", uncal_logl);
|
||||
|
||||
// ── Post-hoc Platt + Isotonic calibration on a held-out cal split ──
|
||||
if cli.cal_frac > 0.0 && cli.cal_frac < 1.0 {
|
||||
let n_val = val_logits.len();
|
||||
let n_cal = (n_val as f32 * cli.cal_frac) as usize;
|
||||
let (cal_l, test_l) = val_logits.split_at(n_cal);
|
||||
let (cal_y, test_y) = val_ys.split_at(n_cal);
|
||||
let test_labels_u8: Vec<u8> = test_y.iter().map(|&y| if y > 0.5 { 1 } else { 0 }).collect();
|
||||
|
||||
// Headline on test half BEFORE calibration (baseline for comparison).
|
||||
let test_acc_raw = accuracy_from_logits(test_l, &test_labels_u8);
|
||||
let test_auc_raw = auc_from_logits(test_l, &test_labels_u8);
|
||||
let test_brier_raw = brier_score(test_l, test_y);
|
||||
let test_logl_raw = log_loss(test_l, test_y);
|
||||
|
||||
// Fit + apply Platt.
|
||||
let platt = PlattScaler::fit(cal_l, cal_y, 2000, 1e-2)
|
||||
.map_err(|e| anyhow::anyhow!("platt fit: {e}"))?;
|
||||
let platt_probs = platt.transform(test_l);
|
||||
// accuracy_from_logits expects logits; convert calibrated probs back to logits.
|
||||
let to_logits = |probs: &[f32]| -> Vec<f32> {
|
||||
probs.iter().map(|&p| {
|
||||
let pc = p.clamp(1e-7, 1.0 - 1e-7);
|
||||
(pc / (1.0 - pc)).ln()
|
||||
}).collect()
|
||||
};
|
||||
let platt_logits = to_logits(&platt_probs);
|
||||
let platt_acc = accuracy_from_logits(&platt_logits, &test_labels_u8);
|
||||
let platt_auc = auc_from_logits(&platt_logits, &test_labels_u8);
|
||||
let platt_brier = brier_score(&platt_logits, test_y);
|
||||
let platt_logl = log_loss(&platt_logits, test_y);
|
||||
|
||||
// Fit + apply Isotonic.
|
||||
let iso = IsotonicCalibrator::fit(cal_l, cal_y)
|
||||
.map_err(|e| anyhow::anyhow!("iso fit: {e}"))?;
|
||||
let iso_probs = iso.transform(test_l);
|
||||
let iso_logits = to_logits(&iso_probs);
|
||||
let iso_acc = accuracy_from_logits(&iso_logits, &test_labels_u8);
|
||||
let iso_auc = auc_from_logits(&iso_logits, &test_labels_u8);
|
||||
let iso_brier = brier_score(&iso_logits, test_y);
|
||||
let iso_logl = log_loss(&iso_logits, test_y);
|
||||
|
||||
println!("--- Held-out test (cal_frac={:.2}, n_cal={}, n_test={}) ---",
|
||||
cli.cal_frac, n_cal, test_l.len());
|
||||
println!(" Uncalibrated: acc={:.4} AUC={:.4} Brier={:.5} log-loss={:.5}",
|
||||
test_acc_raw, test_auc_raw, test_brier_raw, test_logl_raw);
|
||||
println!(" Platt: acc={:.4} AUC={:.4} Brier={:.5} log-loss={:.5} (a={:.4} b={:.4})",
|
||||
platt_acc, platt_auc, platt_brier, platt_logl, platt.a, platt.b);
|
||||
println!(" Isotonic: acc={:.4} AUC={:.4} Brier={:.5} log-loss={:.5}",
|
||||
iso_acc, iso_auc, iso_brier, iso_logl);
|
||||
}
|
||||
|
||||
println!();
|
||||
if auc > 0.55 {
|
||||
println!("GATE PASS: AUC > 0.55 at K={}; multi-minute alpha confirmed.", cli.horizon);
|
||||
println!(" Two-head FoxhuntQ-Δ architecture validated.");
|
||||
|
||||
Reference in New Issue
Block a user