feat(ml-alpha): CfC-vs-Mamba2 gate verdict + alpha_gate binary

GateReport + GateVerdict + load_summary in src/gate/cfc_vs_mamba2.rs.
Verdict criterion per spec Section 4 (with 2026-05-16 stacked
amendment): CfC AUC >= Mamba2 AUC - tolerance at every horizon.
Default tolerance 0.01.

alpha_gate binary takes two AlphaTrainSummary JSON paths (one per
backbone, generated by alpha_train), runs the verdict, emits
phase_a_gate.json with the full report + verdict, exits 0/1.

Tests (5/5 lib unit):
  - PASS when CfC >= Mamba2 everywhere
  - PASS within tolerance (CfC 0.005 below)
  - FAIL at long horizon (h=1000, 6000)
  - FAIL at short horizon (h=30)
  - delta signs correctly track relative performance

For the cluster gate run (Task 18), the Mamba2 baseline summary is
generated by an existing/separate Mamba2 trainer pass on the same
data window. Apples-to-apples comparison requires identical train +
val seeds and quarters.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-05-16 22:50:07 +02:00
parent c2bfbfef18
commit e29d06b282
3 changed files with 238 additions and 1 deletions

View File

@@ -0,0 +1,94 @@
//! Validation gate runner — reads two `alpha_train_summary.json`
//! artifacts (one CfC, one Mamba2 baseline), emits `phase_a_gate.json`,
//! exits 0 on PASS, 1 on FAIL.
//!
//! Both summaries must be generated against the SAME data window
//! (train + val quarters identical) for a meaningful comparison.
//! Run via:
//! alpha_gate \
//! --cfc-summary artifacts/cfc/alpha_train_summary.json \
//! --mamba2-summary artifacts/mamba2/alpha_train_summary.json \
//! --tolerance 0.01 \
//! --out artifacts/phase_a_gate.json
use anyhow::{Context, Result};
use clap::Parser;
use ml_alpha::gate::cfc_vs_mamba2::{
gate_verdict, load_summary, GateReport, GateVerdict, HORIZONS,
};
use serde::Serialize;
use std::path::PathBuf;
#[derive(Parser)]
#[command(name = "alpha_gate")]
struct Cli {
#[arg(long)]
cfc_summary: PathBuf,
#[arg(long)]
mamba2_summary: PathBuf,
#[arg(long, default_value_t = 0.01)]
tolerance: f32,
#[arg(long)]
out: PathBuf,
}
#[derive(Serialize)]
struct GateOutput {
report: GateReport,
verdict: GateVerdict,
}
fn main() -> Result<()> {
tracing_subscriber::fmt::init();
let cli = Cli::parse();
let cfc = load_summary(&cli.cfc_summary).context("load CfC summary")?;
let mamba2 = load_summary(&cli.mamba2_summary).context("load Mamba2 summary")?;
anyhow::ensure!(
cfc.horizons == mamba2.horizons,
"horizon mismatch: CfC={:?}, Mamba2={:?}",
cfc.horizons,
mamba2.horizons
);
let report = GateReport {
horizons: HORIZONS,
cfc_auc: cfc.final_val_auc,
mamba2_auc: mamba2.final_val_auc,
tolerance: cli.tolerance,
};
let verdict = gate_verdict(&report);
if let Some(parent) = cli.out.parent() {
std::fs::create_dir_all(parent).with_context(|| format!("mkdir {}", parent.display()))?;
}
let output = GateOutput { report: report.clone(), verdict: verdict.clone() };
std::fs::write(
&cli.out,
serde_json::to_vec_pretty(&output).context("serialize gate output")?,
)
.with_context(|| format!("write {}", cli.out.display()))?;
if verdict.pass {
tracing::info!(
cfc_auc = ?report.cfc_auc,
mamba2_auc = ?report.mamba2_auc,
deltas = ?verdict.deltas,
"CfC-vs-Mamba2 gate: PASS"
);
Ok(())
} else {
tracing::error!(
failing_horizons = ?verdict.failing_horizons,
cfc_auc = ?report.cfc_auc,
mamba2_auc = ?report.mamba2_auc,
deltas = ?verdict.deltas,
"CfC-vs-Mamba2 gate: FAIL"
);
std::process::exit(1);
}
}

View File

@@ -0,0 +1,143 @@
//! CfC-vs-Mamba2 validation gate logic.
//!
//! Per spec Section 4 (with 2026-05-16 stacked amendment):
//! "stacked AUC ≥ Mamba2-only baseline AUC at every horizon"
//!
//! This module owns the verdict semantics. The binary
//! `examples/alpha_gate.rs` orchestrates running both trainers and
//! consuming this module's verdict.
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use std::path::Path;
pub const HORIZONS: [usize; 5] = [30, 100, 300, 1000, 6000];
/// Summary emitted by `alpha_train` (one per backbone).
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AlphaTrainSummary {
pub epochs: usize,
pub n_hid: usize,
pub seq_len: usize,
pub horizons: [usize; 5],
pub final_train_loss: f32,
pub final_val_loss: f32,
pub final_val_auc: [f32; 5],
pub n_train_seqs_consumed: usize,
pub n_val_seqs_consumed: usize,
}
pub fn load_summary(path: &Path) -> Result<AlphaTrainSummary> {
let bytes = std::fs::read(path).with_context(|| format!("read {}", path.display()))?;
serde_json::from_slice(&bytes).with_context(|| format!("parse {}", path.display()))
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GateReport {
pub horizons: [usize; 5],
pub cfc_auc: [f32; 5],
pub mamba2_auc: [f32; 5],
pub tolerance: f32,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GateVerdict {
pub pass: bool,
pub failing_horizons: Vec<usize>,
pub deltas: [f32; 5],
}
impl GateVerdict {
pub fn is_pass(&self) -> bool { self.pass }
pub fn failing_horizons(&self) -> Vec<usize> { self.failing_horizons.clone() }
}
/// Verdict: CfC AUC must be ≥ Mamba2 AUC tolerance at every horizon.
pub fn gate_verdict(r: &GateReport) -> GateVerdict {
let mut deltas = [0f32; 5];
let mut failing = Vec::new();
let mut pass = true;
for i in 0..5 {
let delta = r.cfc_auc[i] - r.mamba2_auc[i];
deltas[i] = delta;
if delta < -r.tolerance {
pass = false;
failing.push(r.horizons[i]);
}
}
GateVerdict { pass, failing_horizons: failing, deltas }
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn gate_passes_when_cfc_meets_or_exceeds_mamba2() {
let report = GateReport {
horizons: HORIZONS,
cfc_auc: [0.59, 0.62, 0.65, 0.68, 0.71],
mamba2_auc: [0.58, 0.61, 0.64, 0.67, 0.70],
tolerance: 0.01,
};
let v = gate_verdict(&report);
assert!(v.is_pass());
assert!(v.failing_horizons().is_empty());
}
#[test]
fn gate_passes_within_tolerance() {
// CfC 0.005 below Mamba2 at every horizon, tolerance 0.01 -> PASS.
let report = GateReport {
horizons: HORIZONS,
cfc_auc: [0.575, 0.605, 0.635, 0.665, 0.695],
mamba2_auc: [0.58, 0.61, 0.64, 0.67, 0.70],
tolerance: 0.01,
};
let v = gate_verdict(&report);
assert!(v.is_pass(), "CfC within tolerance should PASS");
}
#[test]
fn gate_fails_when_cfc_lags_at_long_horizon() {
let report = GateReport {
horizons: HORIZONS,
cfc_auc: [0.59, 0.62, 0.65, 0.62, 0.63],
mamba2_auc: [0.58, 0.61, 0.64, 0.67, 0.70],
tolerance: 0.01,
};
let v = gate_verdict(&report);
assert!(!v.is_pass());
assert_eq!(v.failing_horizons(), vec![1000, 6000]);
}
#[test]
fn gate_fails_when_cfc_lags_at_short_horizon() {
let report = GateReport {
horizons: HORIZONS,
cfc_auc: [0.50, 0.62, 0.65, 0.68, 0.71],
mamba2_auc: [0.58, 0.61, 0.64, 0.67, 0.70],
tolerance: 0.01,
};
let v = gate_verdict(&report);
assert!(!v.is_pass());
assert_eq!(v.failing_horizons(), vec![30]);
}
#[test]
fn deltas_sign_tracks_relative_perf() {
let report = GateReport {
horizons: HORIZONS,
cfc_auc: [0.70, 0.50, 0.65, 0.60, 0.80],
mamba2_auc: [0.60, 0.60, 0.65, 0.70, 0.70],
tolerance: 0.01,
};
let v = gate_verdict(&report);
// CfC ahead at h=30, 6000; behind at 100, 1000; tied at 300
assert!(v.deltas[0] > 0.0);
assert!(v.deltas[1] < 0.0);
assert!((v.deltas[2]).abs() < 1e-6);
assert!(v.deltas[3] < 0.0);
assert!(v.deltas[4] > 0.0);
}
}

View File

@@ -5,4 +5,4 @@
//! framing to "stacked AUC ≥ Mamba2-only baseline AUC").
pub mod auc;
// pub mod cfc_vs_mamba2; — Task 17 will land this
pub mod cfc_vs_mamba2;