feat(ml-alpha): CfC-vs-Mamba2 gate verdict + alpha_gate binary
GateReport + GateVerdict + load_summary in src/gate/cfc_vs_mamba2.rs. Verdict criterion per spec Section 4 (with 2026-05-16 stacked amendment): CfC AUC >= Mamba2 AUC - tolerance at every horizon. Default tolerance 0.01. alpha_gate binary takes two AlphaTrainSummary JSON paths (one per backbone, generated by alpha_train), runs the verdict, emits phase_a_gate.json with the full report + verdict, exits 0/1. Tests (5/5 lib unit): - PASS when CfC >= Mamba2 everywhere - PASS within tolerance (CfC 0.005 below) - FAIL at long horizon (h=1000, 6000) - FAIL at short horizon (h=30) - delta signs correctly track relative performance For the cluster gate run (Task 18), the Mamba2 baseline summary is generated by an existing/separate Mamba2 trainer pass on the same data window. Apples-to-apples comparison requires identical train + val seeds and quarters. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
94
crates/ml-alpha/examples/alpha_gate.rs
Normal file
94
crates/ml-alpha/examples/alpha_gate.rs
Normal file
@@ -0,0 +1,94 @@
|
||||
//! Validation gate runner — reads two `alpha_train_summary.json`
|
||||
//! artifacts (one CfC, one Mamba2 baseline), emits `phase_a_gate.json`,
|
||||
//! exits 0 on PASS, 1 on FAIL.
|
||||
//!
|
||||
//! Both summaries must be generated against the SAME data window
|
||||
//! (train + val quarters identical) for a meaningful comparison.
|
||||
//! Run via:
|
||||
//! alpha_gate \
|
||||
//! --cfc-summary artifacts/cfc/alpha_train_summary.json \
|
||||
//! --mamba2-summary artifacts/mamba2/alpha_train_summary.json \
|
||||
//! --tolerance 0.01 \
|
||||
//! --out artifacts/phase_a_gate.json
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use clap::Parser;
|
||||
use ml_alpha::gate::cfc_vs_mamba2::{
|
||||
gate_verdict, load_summary, GateReport, GateVerdict, HORIZONS,
|
||||
};
|
||||
use serde::Serialize;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "alpha_gate")]
|
||||
struct Cli {
|
||||
#[arg(long)]
|
||||
cfc_summary: PathBuf,
|
||||
|
||||
#[arg(long)]
|
||||
mamba2_summary: PathBuf,
|
||||
|
||||
#[arg(long, default_value_t = 0.01)]
|
||||
tolerance: f32,
|
||||
|
||||
#[arg(long)]
|
||||
out: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct GateOutput {
|
||||
report: GateReport,
|
||||
verdict: GateVerdict,
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
tracing_subscriber::fmt::init();
|
||||
let cli = Cli::parse();
|
||||
|
||||
let cfc = load_summary(&cli.cfc_summary).context("load CfC summary")?;
|
||||
let mamba2 = load_summary(&cli.mamba2_summary).context("load Mamba2 summary")?;
|
||||
|
||||
anyhow::ensure!(
|
||||
cfc.horizons == mamba2.horizons,
|
||||
"horizon mismatch: CfC={:?}, Mamba2={:?}",
|
||||
cfc.horizons,
|
||||
mamba2.horizons
|
||||
);
|
||||
|
||||
let report = GateReport {
|
||||
horizons: HORIZONS,
|
||||
cfc_auc: cfc.final_val_auc,
|
||||
mamba2_auc: mamba2.final_val_auc,
|
||||
tolerance: cli.tolerance,
|
||||
};
|
||||
let verdict = gate_verdict(&report);
|
||||
|
||||
if let Some(parent) = cli.out.parent() {
|
||||
std::fs::create_dir_all(parent).with_context(|| format!("mkdir {}", parent.display()))?;
|
||||
}
|
||||
let output = GateOutput { report: report.clone(), verdict: verdict.clone() };
|
||||
std::fs::write(
|
||||
&cli.out,
|
||||
serde_json::to_vec_pretty(&output).context("serialize gate output")?,
|
||||
)
|
||||
.with_context(|| format!("write {}", cli.out.display()))?;
|
||||
|
||||
if verdict.pass {
|
||||
tracing::info!(
|
||||
cfc_auc = ?report.cfc_auc,
|
||||
mamba2_auc = ?report.mamba2_auc,
|
||||
deltas = ?verdict.deltas,
|
||||
"CfC-vs-Mamba2 gate: PASS"
|
||||
);
|
||||
Ok(())
|
||||
} else {
|
||||
tracing::error!(
|
||||
failing_horizons = ?verdict.failing_horizons,
|
||||
cfc_auc = ?report.cfc_auc,
|
||||
mamba2_auc = ?report.mamba2_auc,
|
||||
deltas = ?verdict.deltas,
|
||||
"CfC-vs-Mamba2 gate: FAIL"
|
||||
);
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
143
crates/ml-alpha/src/gate/cfc_vs_mamba2.rs
Normal file
143
crates/ml-alpha/src/gate/cfc_vs_mamba2.rs
Normal file
@@ -0,0 +1,143 @@
|
||||
//! CfC-vs-Mamba2 validation gate logic.
|
||||
//!
|
||||
//! Per spec Section 4 (with 2026-05-16 stacked amendment):
|
||||
//! "stacked AUC ≥ Mamba2-only baseline AUC at every horizon"
|
||||
//!
|
||||
//! This module owns the verdict semantics. The binary
|
||||
//! `examples/alpha_gate.rs` orchestrates running both trainers and
|
||||
//! consuming this module's verdict.
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::Path;
|
||||
|
||||
pub const HORIZONS: [usize; 5] = [30, 100, 300, 1000, 6000];
|
||||
|
||||
/// Summary emitted by `alpha_train` (one per backbone).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AlphaTrainSummary {
|
||||
pub epochs: usize,
|
||||
pub n_hid: usize,
|
||||
pub seq_len: usize,
|
||||
pub horizons: [usize; 5],
|
||||
pub final_train_loss: f32,
|
||||
pub final_val_loss: f32,
|
||||
pub final_val_auc: [f32; 5],
|
||||
pub n_train_seqs_consumed: usize,
|
||||
pub n_val_seqs_consumed: usize,
|
||||
}
|
||||
|
||||
pub fn load_summary(path: &Path) -> Result<AlphaTrainSummary> {
|
||||
let bytes = std::fs::read(path).with_context(|| format!("read {}", path.display()))?;
|
||||
serde_json::from_slice(&bytes).with_context(|| format!("parse {}", path.display()))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GateReport {
|
||||
pub horizons: [usize; 5],
|
||||
pub cfc_auc: [f32; 5],
|
||||
pub mamba2_auc: [f32; 5],
|
||||
pub tolerance: f32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GateVerdict {
|
||||
pub pass: bool,
|
||||
pub failing_horizons: Vec<usize>,
|
||||
pub deltas: [f32; 5],
|
||||
}
|
||||
|
||||
impl GateVerdict {
|
||||
pub fn is_pass(&self) -> bool { self.pass }
|
||||
pub fn failing_horizons(&self) -> Vec<usize> { self.failing_horizons.clone() }
|
||||
}
|
||||
|
||||
/// Verdict: CfC AUC must be ≥ Mamba2 AUC − tolerance at every horizon.
|
||||
pub fn gate_verdict(r: &GateReport) -> GateVerdict {
|
||||
let mut deltas = [0f32; 5];
|
||||
let mut failing = Vec::new();
|
||||
let mut pass = true;
|
||||
for i in 0..5 {
|
||||
let delta = r.cfc_auc[i] - r.mamba2_auc[i];
|
||||
deltas[i] = delta;
|
||||
if delta < -r.tolerance {
|
||||
pass = false;
|
||||
failing.push(r.horizons[i]);
|
||||
}
|
||||
}
|
||||
GateVerdict { pass, failing_horizons: failing, deltas }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn gate_passes_when_cfc_meets_or_exceeds_mamba2() {
|
||||
let report = GateReport {
|
||||
horizons: HORIZONS,
|
||||
cfc_auc: [0.59, 0.62, 0.65, 0.68, 0.71],
|
||||
mamba2_auc: [0.58, 0.61, 0.64, 0.67, 0.70],
|
||||
tolerance: 0.01,
|
||||
};
|
||||
let v = gate_verdict(&report);
|
||||
assert!(v.is_pass());
|
||||
assert!(v.failing_horizons().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_passes_within_tolerance() {
|
||||
// CfC 0.005 below Mamba2 at every horizon, tolerance 0.01 -> PASS.
|
||||
let report = GateReport {
|
||||
horizons: HORIZONS,
|
||||
cfc_auc: [0.575, 0.605, 0.635, 0.665, 0.695],
|
||||
mamba2_auc: [0.58, 0.61, 0.64, 0.67, 0.70],
|
||||
tolerance: 0.01,
|
||||
};
|
||||
let v = gate_verdict(&report);
|
||||
assert!(v.is_pass(), "CfC within tolerance should PASS");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_fails_when_cfc_lags_at_long_horizon() {
|
||||
let report = GateReport {
|
||||
horizons: HORIZONS,
|
||||
cfc_auc: [0.59, 0.62, 0.65, 0.62, 0.63],
|
||||
mamba2_auc: [0.58, 0.61, 0.64, 0.67, 0.70],
|
||||
tolerance: 0.01,
|
||||
};
|
||||
let v = gate_verdict(&report);
|
||||
assert!(!v.is_pass());
|
||||
assert_eq!(v.failing_horizons(), vec![1000, 6000]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gate_fails_when_cfc_lags_at_short_horizon() {
|
||||
let report = GateReport {
|
||||
horizons: HORIZONS,
|
||||
cfc_auc: [0.50, 0.62, 0.65, 0.68, 0.71],
|
||||
mamba2_auc: [0.58, 0.61, 0.64, 0.67, 0.70],
|
||||
tolerance: 0.01,
|
||||
};
|
||||
let v = gate_verdict(&report);
|
||||
assert!(!v.is_pass());
|
||||
assert_eq!(v.failing_horizons(), vec![30]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deltas_sign_tracks_relative_perf() {
|
||||
let report = GateReport {
|
||||
horizons: HORIZONS,
|
||||
cfc_auc: [0.70, 0.50, 0.65, 0.60, 0.80],
|
||||
mamba2_auc: [0.60, 0.60, 0.65, 0.70, 0.70],
|
||||
tolerance: 0.01,
|
||||
};
|
||||
let v = gate_verdict(&report);
|
||||
// CfC ahead at h=30, 6000; behind at 100, 1000; tied at 300
|
||||
assert!(v.deltas[0] > 0.0);
|
||||
assert!(v.deltas[1] < 0.0);
|
||||
assert!((v.deltas[2]).abs() < 1e-6);
|
||||
assert!(v.deltas[3] < 0.0);
|
||||
assert!(v.deltas[4] > 0.0);
|
||||
}
|
||||
}
|
||||
@@ -5,4 +5,4 @@
|
||||
//! framing to "stacked AUC ≥ Mamba2-only baseline AUC").
|
||||
|
||||
pub mod auc;
|
||||
// pub mod cfc_vs_mamba2; — Task 17 will land this
|
||||
pub mod cfc_vs_mamba2;
|
||||
|
||||
Reference in New Issue
Block a user