feat(ml-alpha): Phase 1d.2 smoke + Block-S stratified accuracy diagnostic

After calibration, also stratify val accuracy across the 6 Block-S
features (time_since_trade, time_since_snap, book_event_rate, spread_bps,
L1_imbalance, micro_mid_drift) by sampling each val sequence's END BAR
feature value, then running `metrics_detail::stratified_accuracy` per
column with 5 quintile bins.

Tells us whether the K=6000 Mamba alpha concentrates in specific book
regimes (justifying an explicit regime head per Phase 1d.3) or is
uniform across regimes (allowing direct backtest in Phase 1d.4). The
Phase 1c stateless MLP showed strong stratification (spread-Q4 hit
0.752 acc on 76K samples while middle quintiles fell below 0.50);
this run tests whether the Mamba inherits or transcends that pattern.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-05-15 09:13:14 +02:00
parent d57026b0ba
commit 635e2c8b48

View File

@@ -36,7 +36,7 @@ use ml_alpha::fxcache_reader::{FxCacheReader, COL_RAW_CLOSE, FEAT_DIM};
use ml_alpha::mamba2_block::{
Mamba2AdamW, Mamba2AdamWConfig, Mamba2Block, Mamba2BlockConfig,
};
use ml_alpha::metrics_detail::{brier_score, log_loss};
use ml_alpha::metrics_detail::{brier_score, log_loss, stratified_accuracy};
use ml_alpha::multi_horizon_labels::generate_labels;
use ml_core::cuda_autograd::gpu_tensor::GpuTensor;
@@ -305,6 +305,50 @@ fn main() -> Result<()> {
iso_acc, iso_auc, iso_brier, iso_logl);
}
// ── Stratified accuracy on Block-S features (regime diagnostic) ──
//
// Block-S sits at cols 75..81 of the 81-dim snapshot row. For each
// val sequence, gather the END-BAR's feature value, then stratify
// val accuracy across quintiles of that feature. This tells us
// whether Mamba's K=6000 alpha is regime-conditional (concentrated
// in spread-Q4 / micro-drift-Q4 like the Phase 1c stateless MLP) or
// uniform — which decides whether we need an explicit regime head
// before the backtest.
const BLOCK_S_OFF: usize = 75;
const BLOCK_S_NAMES: [&str; 6] = [
"time_since_trade_s",
"time_since_snap_s",
"book_event_rate_per_s",
"spread_bps",
"L1_imbalance",
"micro_mid_drift",
];
let labels_u8_full: Vec<u8> =
val_ys.iter().map(|&y| if y > 0.5 { 1 } else { 0 }).collect();
let val_labels_f32: Vec<f32> = labels_u8_full.iter().map(|&y| y as f32).collect();
println!();
println!("--- Stratified val accuracy across Block-S features (5 quintiles) ---");
for (col_off, name) in BLOCK_S_NAMES.iter().enumerate() {
let global_col = BLOCK_S_OFF + col_off;
let feat: Vec<f32> = val_pos.iter()
.map(|&p| {
let end_bar = labels.valid_indices[p];
feature_matrix[end_bar * alpha_dim + global_col]
})
.collect();
let strats = stratified_accuracy(&val_logits, &val_labels_f32, &feat, 5);
println!();
println!(" ▸ feature: {}", name);
println!(" {:>4} {:>14} {:>14} {:>10} {:>10} {:>12}",
"k", "feat_lo", "feat_hi", "n", "accuracy", "observed");
for s in &strats {
println!(
" {:>4} {:>14.4} {:>14.4} {:>10} {:>10.4} {:>12.4}",
s.stratum, s.feature_lo, s.feature_hi, s.n, s.accuracy, s.observed_pos_rate
);
}
}
println!();
if auc > 0.55 {
println!("GATE PASS: AUC > 0.55 at K={}; multi-minute alpha confirmed.", cli.horizon);