diff --git a/crates/ml-alpha/examples/alpha_gate.rs b/crates/ml-alpha/examples/alpha_gate.rs new file mode 100644 index 000000000..ed18ca033 --- /dev/null +++ b/crates/ml-alpha/examples/alpha_gate.rs @@ -0,0 +1,94 @@ +//! Validation gate runner — reads two `alpha_train_summary.json` +//! artifacts (one CfC, one Mamba2 baseline), emits `phase_a_gate.json`, +//! exits 0 on PASS, 1 on FAIL. +//! +//! Both summaries must be generated against the SAME data window +//! (train + val quarters identical) for a meaningful comparison. +//! Run via: +//! alpha_gate \ +//! --cfc-summary artifacts/cfc/alpha_train_summary.json \ +//! --mamba2-summary artifacts/mamba2/alpha_train_summary.json \ +//! --tolerance 0.01 \ +//! --out artifacts/phase_a_gate.json + +use anyhow::{Context, Result}; +use clap::Parser; +use ml_alpha::gate::cfc_vs_mamba2::{ + gate_verdict, load_summary, GateReport, GateVerdict, HORIZONS, +}; +use serde::Serialize; +use std::path::PathBuf; + +#[derive(Parser)] +#[command(name = "alpha_gate")] +struct Cli { + #[arg(long)] + cfc_summary: PathBuf, + + #[arg(long)] + mamba2_summary: PathBuf, + + #[arg(long, default_value_t = 0.01)] + tolerance: f32, + + #[arg(long)] + out: PathBuf, +} + +#[derive(Serialize)] +struct GateOutput { + report: GateReport, + verdict: GateVerdict, +} + +fn main() -> Result<()> { + tracing_subscriber::fmt::init(); + let cli = Cli::parse(); + + let cfc = load_summary(&cli.cfc_summary).context("load CfC summary")?; + let mamba2 = load_summary(&cli.mamba2_summary).context("load Mamba2 summary")?; + + anyhow::ensure!( + cfc.horizons == mamba2.horizons, + "horizon mismatch: CfC={:?}, Mamba2={:?}", + cfc.horizons, + mamba2.horizons + ); + + let report = GateReport { + horizons: HORIZONS, + cfc_auc: cfc.final_val_auc, + mamba2_auc: mamba2.final_val_auc, + tolerance: cli.tolerance, + }; + let verdict = gate_verdict(&report); + + if let Some(parent) = cli.out.parent() { + std::fs::create_dir_all(parent).with_context(|| format!("mkdir {}", parent.display()))?; + } + let output = GateOutput { report: report.clone(), verdict: verdict.clone() }; + std::fs::write( + &cli.out, + serde_json::to_vec_pretty(&output).context("serialize gate output")?, + ) + .with_context(|| format!("write {}", cli.out.display()))?; + + if verdict.pass { + tracing::info!( + cfc_auc = ?report.cfc_auc, + mamba2_auc = ?report.mamba2_auc, + deltas = ?verdict.deltas, + "CfC-vs-Mamba2 gate: PASS" + ); + Ok(()) + } else { + tracing::error!( + failing_horizons = ?verdict.failing_horizons, + cfc_auc = ?report.cfc_auc, + mamba2_auc = ?report.mamba2_auc, + deltas = ?verdict.deltas, + "CfC-vs-Mamba2 gate: FAIL" + ); + std::process::exit(1); + } +} diff --git a/crates/ml-alpha/src/gate/cfc_vs_mamba2.rs b/crates/ml-alpha/src/gate/cfc_vs_mamba2.rs new file mode 100644 index 000000000..b6920da8e --- /dev/null +++ b/crates/ml-alpha/src/gate/cfc_vs_mamba2.rs @@ -0,0 +1,143 @@ +//! CfC-vs-Mamba2 validation gate logic. +//! +//! Per spec Section 4 (with 2026-05-16 stacked amendment): +//! "stacked AUC ≥ Mamba2-only baseline AUC at every horizon" +//! +//! This module owns the verdict semantics. The binary +//! `examples/alpha_gate.rs` orchestrates running both trainers and +//! consuming this module's verdict. + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use std::path::Path; + +pub const HORIZONS: [usize; 5] = [30, 100, 300, 1000, 6000]; + +/// Summary emitted by `alpha_train` (one per backbone). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AlphaTrainSummary { + pub epochs: usize, + pub n_hid: usize, + pub seq_len: usize, + pub horizons: [usize; 5], + pub final_train_loss: f32, + pub final_val_loss: f32, + pub final_val_auc: [f32; 5], + pub n_train_seqs_consumed: usize, + pub n_val_seqs_consumed: usize, +} + +pub fn load_summary(path: &Path) -> Result { + let bytes = std::fs::read(path).with_context(|| format!("read {}", path.display()))?; + serde_json::from_slice(&bytes).with_context(|| format!("parse {}", path.display())) +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GateReport { + pub horizons: [usize; 5], + pub cfc_auc: [f32; 5], + pub mamba2_auc: [f32; 5], + pub tolerance: f32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GateVerdict { + pub pass: bool, + pub failing_horizons: Vec, + pub deltas: [f32; 5], +} + +impl GateVerdict { + pub fn is_pass(&self) -> bool { self.pass } + pub fn failing_horizons(&self) -> Vec { self.failing_horizons.clone() } +} + +/// Verdict: CfC AUC must be ≥ Mamba2 AUC − tolerance at every horizon. +pub fn gate_verdict(r: &GateReport) -> GateVerdict { + let mut deltas = [0f32; 5]; + let mut failing = Vec::new(); + let mut pass = true; + for i in 0..5 { + let delta = r.cfc_auc[i] - r.mamba2_auc[i]; + deltas[i] = delta; + if delta < -r.tolerance { + pass = false; + failing.push(r.horizons[i]); + } + } + GateVerdict { pass, failing_horizons: failing, deltas } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gate_passes_when_cfc_meets_or_exceeds_mamba2() { + let report = GateReport { + horizons: HORIZONS, + cfc_auc: [0.59, 0.62, 0.65, 0.68, 0.71], + mamba2_auc: [0.58, 0.61, 0.64, 0.67, 0.70], + tolerance: 0.01, + }; + let v = gate_verdict(&report); + assert!(v.is_pass()); + assert!(v.failing_horizons().is_empty()); + } + + #[test] + fn gate_passes_within_tolerance() { + // CfC 0.005 below Mamba2 at every horizon, tolerance 0.01 -> PASS. + let report = GateReport { + horizons: HORIZONS, + cfc_auc: [0.575, 0.605, 0.635, 0.665, 0.695], + mamba2_auc: [0.58, 0.61, 0.64, 0.67, 0.70], + tolerance: 0.01, + }; + let v = gate_verdict(&report); + assert!(v.is_pass(), "CfC within tolerance should PASS"); + } + + #[test] + fn gate_fails_when_cfc_lags_at_long_horizon() { + let report = GateReport { + horizons: HORIZONS, + cfc_auc: [0.59, 0.62, 0.65, 0.62, 0.63], + mamba2_auc: [0.58, 0.61, 0.64, 0.67, 0.70], + tolerance: 0.01, + }; + let v = gate_verdict(&report); + assert!(!v.is_pass()); + assert_eq!(v.failing_horizons(), vec![1000, 6000]); + } + + #[test] + fn gate_fails_when_cfc_lags_at_short_horizon() { + let report = GateReport { + horizons: HORIZONS, + cfc_auc: [0.50, 0.62, 0.65, 0.68, 0.71], + mamba2_auc: [0.58, 0.61, 0.64, 0.67, 0.70], + tolerance: 0.01, + }; + let v = gate_verdict(&report); + assert!(!v.is_pass()); + assert_eq!(v.failing_horizons(), vec![30]); + } + + #[test] + fn deltas_sign_tracks_relative_perf() { + let report = GateReport { + horizons: HORIZONS, + cfc_auc: [0.70, 0.50, 0.65, 0.60, 0.80], + mamba2_auc: [0.60, 0.60, 0.65, 0.70, 0.70], + tolerance: 0.01, + }; + let v = gate_verdict(&report); + // CfC ahead at h=30, 6000; behind at 100, 1000; tied at 300 + assert!(v.deltas[0] > 0.0); + assert!(v.deltas[1] < 0.0); + assert!((v.deltas[2]).abs() < 1e-6); + assert!(v.deltas[3] < 0.0); + assert!(v.deltas[4] > 0.0); + } +} diff --git a/crates/ml-alpha/src/gate/mod.rs b/crates/ml-alpha/src/gate/mod.rs index 4fd0a4399..5bd924ffd 100644 --- a/crates/ml-alpha/src/gate/mod.rs +++ b/crates/ml-alpha/src/gate/mod.rs @@ -5,4 +5,4 @@ //! framing to "stacked AUC ≥ Mamba2-only baseline AUC"). pub mod auc; -// pub mod cfc_vs_mamba2; — Task 17 will land this +pub mod cfc_vs_mamba2;