diff --git a/crates/ml-alpha/src/cfc/snap_features.rs b/crates/ml-alpha/src/cfc/snap_features.rs index 033147a5f..87a78aea3 100644 --- a/crates/ml-alpha/src/cfc/snap_features.rs +++ b/crates/ml-alpha/src/cfc/snap_features.rs @@ -22,16 +22,19 @@ pub const FEATURE_DIM: usize = 40; /// Number of regime-context features pre-computed by the loader and /// piped into snap_features slots `out[20..20+REGIME_DIM]`. These give -/// the model multi-minute trend / volatility / liquidity context that -/// is structurally unreachable inside the K-snapshot BPTT window. +/// the model microstructure-burst (~2.6ms) and 1-2-second-clustering +/// (~100ms) trend / volatility / liquidity context that is structurally +/// unreachable inside the K-snapshot BPTT window. (NOTE: these are NOT +/// macro-minute regimes — see `loader.rs` ALPHA_* constants for the +/// empirically-derived half-lives at the actual 74µs median Δt.) /// /// Slot assignments (loader computes via EMA cascade): -/// regime[0] = (mid - ema_mid_med) / sqrt(ema_rv_med + eps) z-score, ~35s -/// regime[1] = (mid - ema_mid_slow) / sqrt(ema_rv_slow + eps) z-score, ~6min -/// regime[2] = (ema_mid_med - ema_mid_slow) / sqrt(ema_rv_slow + eps) trend signal -/// regime[3] = log1p(ema_rv_slow * 1e4) log-compressed slow vol -/// regime[4] = log1p(ema_spread_in_ticks_med) liquidity regime -/// regime[5] = log1p(ema_trades_per_snap_med * 100) activity regime +/// regime[0] = (mid - ema_mid_fast) / sqrt(ema_rv_fast + eps) z-score, ~2.6ms half-life +/// regime[1] = (mid - ema_mid_med) / sqrt(ema_rv_med + eps) z-score, ~100ms half-life +/// regime[2] = (ema_mid_fast - ema_mid_med) / sqrt(ema_rv_med + eps) trend signal +/// regime[3] = log1p(ema_rv_med * 1e4) log-compressed med vol +/// regime[4] = log1p(ema_spread_in_ticks_fast) liquidity regime +/// regime[5] = log1p(ema_trades_per_snap_fast * 100) activity regime pub const REGIME_DIM: usize = 6; /// One raw MBP-10 snapshot in struct-of-arrays form. Phase A loader diff --git a/crates/ml-alpha/src/data/loader.rs b/crates/ml-alpha/src/data/loader.rs index 2e625d6cc..fdf149031 100644 --- a/crates/ml-alpha/src/data/loader.rs +++ b/crates/ml-alpha/src/data/loader.rs @@ -22,8 +22,24 @@ use crate::cfc::snap_features::{Mbp10RawInput, ES_TICK_SIZE, REGIME_DIM}; use crate::multi_horizon_labels::generate_labels; /// EMA smoothing factors. Half-life ≈ ln(2)/α snapshots. -const ALPHA_MED: f32 = 0.02; // half-life ≈ 35 snapshots ≈ 9s @ 250ms -const ALPHA_SLOW: f32 = 0.0005; // half-life ≈ 1400 snapshots ≈ 6 min +/// +/// Empirical ES MBP-10 median inter-event Δt = 74µs (NOT 250ms as the +/// previous comments claimed — mean Δt is 55ms but the distribution is +/// extremely heavy-tailed so the median dominates EMA dynamics). At that +/// rate the half-lives in wall-clock are roughly: +/// +/// * `ALPHA_FAST_2P6MS` (α=0.02) → ~35 snapshots × 74µs ≈ 2.6ms +/// — microstructure-burst scale +/// * `ALPHA_MED_100MS` (α=0.0005) → ~1400 snapshots × 74µs ≈ 100ms +/// — 1-2-second clustering scale +/// +/// Values preserved (validated by all prior training); the names are +/// updated to reflect what they actually do. True macro-regime EMAs +/// (multi-second / multi-minute half-lives) would need α ~5e-6 or +/// smaller and are deliberately NOT added here — that's a separate +/// feature-engineering decision. +const ALPHA_FAST_2P6MS: f32 = 0.02; +const ALPHA_MED_100MS: f32 = 0.0005; const REGIME_EPS: f32 = 1e-6; /// Compute per-snapshot regime features via a single forward EMA pass @@ -39,10 +55,10 @@ fn compute_regime_features(snapshots: &[Mbp10Snapshot]) -> Vec<[f32; REGIME_DIM] let mut out: Vec<[f32; REGIME_DIM]> = vec![[0.0; REGIME_DIM]; n]; if n == 0 { return out; } + let mut ema_mid_fast = f32::NAN; let mut ema_mid_med = f32::NAN; - let mut ema_mid_slow = f32::NAN; - let mut ema_mid2_med = f32::NAN; // E[mid²] for var estimate - let mut ema_mid2_slow = f32::NAN; + let mut ema_mid2_fast = f32::NAN; // E[mid²] for var estimate + let mut ema_mid2_med = f32::NAN; let mut ema_spread = f32::NAN; // ticks let mut ema_trades = f32::NAN; // per-snapshot delta count let mut prev_trade_count: u32 = 0; @@ -57,30 +73,30 @@ fn compute_regime_features(snapshots: &[Mbp10Snapshot]) -> Vec<[f32; REGIME_DIM] prev_trade_count = snap.trade_count; // Update EMAs with sentinel-bootstrap. - if ema_mid_med.is_nan() { - ema_mid_med = mid; ema_mid_slow = mid; - ema_mid2_med = mid * mid; ema_mid2_slow = mid * mid; + if ema_mid_fast.is_nan() { + ema_mid_fast = mid; ema_mid_med = mid; + ema_mid2_fast = mid * mid; ema_mid2_med = mid * mid; ema_spread = spread_t; ema_trades = trades as f32; } else { - ema_mid_med = (1.0 - ALPHA_MED) * ema_mid_med + ALPHA_MED * mid; - ema_mid_slow = (1.0 - ALPHA_SLOW) * ema_mid_slow + ALPHA_SLOW * mid; - ema_mid2_med = (1.0 - ALPHA_MED) * ema_mid2_med + ALPHA_MED * mid * mid; - ema_mid2_slow = (1.0 - ALPHA_SLOW) * ema_mid2_slow + ALPHA_SLOW * mid * mid; - ema_spread = (1.0 - ALPHA_MED) * ema_spread + ALPHA_MED * spread_t; - ema_trades = (1.0 - ALPHA_MED) * ema_trades + ALPHA_MED * trades as f32; + ema_mid_fast = (1.0 - ALPHA_FAST_2P6MS) * ema_mid_fast + ALPHA_FAST_2P6MS * mid; + ema_mid_med = (1.0 - ALPHA_MED_100MS) * ema_mid_med + ALPHA_MED_100MS * mid; + ema_mid2_fast = (1.0 - ALPHA_FAST_2P6MS) * ema_mid2_fast + ALPHA_FAST_2P6MS * mid * mid; + ema_mid2_med = (1.0 - ALPHA_MED_100MS) * ema_mid2_med + ALPHA_MED_100MS * mid * mid; + ema_spread = (1.0 - ALPHA_FAST_2P6MS) * ema_spread + ALPHA_FAST_2P6MS * spread_t; + ema_trades = (1.0 - ALPHA_FAST_2P6MS) * ema_trades + ALPHA_FAST_2P6MS * trades as f32; } + let var_fast = (ema_mid2_fast - ema_mid_fast * ema_mid_fast).max(0.0); let var_med = (ema_mid2_med - ema_mid_med * ema_mid_med ).max(0.0); - let var_slow = (ema_mid2_slow - ema_mid_slow * ema_mid_slow).max(0.0); + let std_fast = (var_fast + REGIME_EPS).sqrt(); let std_med = (var_med + REGIME_EPS).sqrt(); - let std_slow = (var_slow + REGIME_EPS).sqrt(); - out[k][0] = (mid - ema_mid_med) / std_med; // z-score med - out[k][1] = (mid - ema_mid_slow) / std_slow; // z-score slow - out[k][2] = (ema_mid_med - ema_mid_slow) / std_slow; // trend signal - out[k][3] = (1.0 + var_slow.sqrt() * 1e4).ln(); // log-compressed slow vol - out[k][4] = (1.0 + ema_spread).ln(); // liquidity regime - out[k][5] = (1.0 + ema_trades * 100.0).ln(); // activity regime + out[k][0] = (mid - ema_mid_fast) / std_fast; // z-score fast (~2.6ms) + out[k][1] = (mid - ema_mid_med) / std_med; // z-score med (~100ms) + out[k][2] = (ema_mid_fast - ema_mid_med) / std_med; // trend signal + out[k][3] = (1.0 + var_med.sqrt() * 1e4).ln(); // log-compressed med vol + out[k][4] = (1.0 + ema_spread).ln(); // liquidity regime + out[k][5] = (1.0 + ema_trades * 100.0).ln(); // activity regime } out }