feat(ml-alpha): Phase 1d.2 smoke + Block-S stratified accuracy diagnostic
After calibration, also stratify val accuracy across the 6 Block-S features (time_since_trade, time_since_snap, book_event_rate, spread_bps, L1_imbalance, micro_mid_drift) by sampling each val sequence's END BAR feature value, then running `metrics_detail::stratified_accuracy` per column with 5 quintile bins. Tells us whether the K=6000 Mamba alpha concentrates in specific book regimes (justifying an explicit regime head per Phase 1d.3) or is uniform across regimes (allowing direct backtest in Phase 1d.4). The Phase 1c stateless MLP showed strong stratification (spread-Q4 hit 0.752 acc on 76K samples while middle quintiles fell below 0.50); this run tests whether the Mamba inherits or transcends that pattern. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -36,7 +36,7 @@ use ml_alpha::fxcache_reader::{FxCacheReader, COL_RAW_CLOSE, FEAT_DIM};
|
||||
use ml_alpha::mamba2_block::{
|
||||
Mamba2AdamW, Mamba2AdamWConfig, Mamba2Block, Mamba2BlockConfig,
|
||||
};
|
||||
use ml_alpha::metrics_detail::{brier_score, log_loss};
|
||||
use ml_alpha::metrics_detail::{brier_score, log_loss, stratified_accuracy};
|
||||
use ml_alpha::multi_horizon_labels::generate_labels;
|
||||
|
||||
use ml_core::cuda_autograd::gpu_tensor::GpuTensor;
|
||||
@@ -305,6 +305,50 @@ fn main() -> Result<()> {
|
||||
iso_acc, iso_auc, iso_brier, iso_logl);
|
||||
}
|
||||
|
||||
// ── Stratified accuracy on Block-S features (regime diagnostic) ──
|
||||
//
|
||||
// Block-S sits at cols 75..81 of the 81-dim snapshot row. For each
|
||||
// val sequence, gather the END-BAR's feature value, then stratify
|
||||
// val accuracy across quintiles of that feature. This tells us
|
||||
// whether Mamba's K=6000 alpha is regime-conditional (concentrated
|
||||
// in spread-Q4 / micro-drift-Q4 like the Phase 1c stateless MLP) or
|
||||
// uniform — which decides whether we need an explicit regime head
|
||||
// before the backtest.
|
||||
const BLOCK_S_OFF: usize = 75;
|
||||
const BLOCK_S_NAMES: [&str; 6] = [
|
||||
"time_since_trade_s",
|
||||
"time_since_snap_s",
|
||||
"book_event_rate_per_s",
|
||||
"spread_bps",
|
||||
"L1_imbalance",
|
||||
"micro_mid_drift",
|
||||
];
|
||||
let labels_u8_full: Vec<u8> =
|
||||
val_ys.iter().map(|&y| if y > 0.5 { 1 } else { 0 }).collect();
|
||||
let val_labels_f32: Vec<f32> = labels_u8_full.iter().map(|&y| y as f32).collect();
|
||||
println!();
|
||||
println!("--- Stratified val accuracy across Block-S features (5 quintiles) ---");
|
||||
for (col_off, name) in BLOCK_S_NAMES.iter().enumerate() {
|
||||
let global_col = BLOCK_S_OFF + col_off;
|
||||
let feat: Vec<f32> = val_pos.iter()
|
||||
.map(|&p| {
|
||||
let end_bar = labels.valid_indices[p];
|
||||
feature_matrix[end_bar * alpha_dim + global_col]
|
||||
})
|
||||
.collect();
|
||||
let strats = stratified_accuracy(&val_logits, &val_labels_f32, &feat, 5);
|
||||
println!();
|
||||
println!(" ▸ feature: {}", name);
|
||||
println!(" {:>4} {:>14} {:>14} {:>10} {:>10} {:>12}",
|
||||
"k", "feat_lo", "feat_hi", "n", "accuracy", "observed");
|
||||
for s in &strats {
|
||||
println!(
|
||||
" {:>4} {:>14.4} {:>14.4} {:>10} {:>10.4} {:>12.4}",
|
||||
s.stratum, s.feature_lo, s.feature_hi, s.n, s.accuracy, s.observed_pos_rate
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
println!();
|
||||
if auc > 0.55 {
|
||||
println!("GATE PASS: AUC > 0.55 at K={}; multi-minute alpha confirmed.", cli.horizon);
|
||||
|
||||
Reference in New Issue
Block a user