From 00da163078a2271f94b5dc5ceb9fece3183aaab8 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 17 May 2026 20:58:44 +0200 Subject: [PATCH] =?UTF-8?q?feat(ml-alpha):=20h6000-aligned=20ISV=20?= =?UTF-8?q?=E2=80=94=20uniform=20BCE=20+=20z-score=20lambda=20+=20K=3D64?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three correlated fixes addressing the architectural inconsistency surfaced by the 3-fold ISV CV: we built a horizon-aware gradient controller (ISV) but suppressed its target horizon (h6000) to 0.36% of the loss via auto-horizon-weights, then used a ratio formula that never approached its own clamp ceiling. ISV's lambda was operating on rounding error. (1) Uniform BCE weights as auto-default trainer/perception.rs: `auto_horizon_weights` now returns [1.0; 5] regardless of seq_len. Prior schedule `min(1, K/h)` gave h6000 weight 0.0053 at K=32 — combined with lambda ~1.04, h6000's effective loss contribution was ~0.37%, indistinguishable from zero. With uniform weights, each horizon contributes 20% and ISV's lambda actually has something to scale. (2) Z-score lambda derivation cuda/horizon_lambda.cu: replace `ratio = ema_h / mean(ema)` with `z_h = (ema_h - mean) / std(ema); lambda = clamp(1.0 + 0.5*z, 1.0, 2.0)`. Per `pearl_zscore_normalization_for_magnitude_asymmetric_signals.md` z-score makes lambda spread scale-invariant of the absolute EMA level. The ratio formula gave lambdas ≤ 1.04 in our data because per-horizon BCE clusters tightly (range ~0.04) while mean is ~0.65. Z-score fills the [1.0, 2.0] envelope: 1σ → 1.5, 2σ → ceiling. Boost-only asymmetric clamp preserved. Test verification on the existing smoke (after 5 steps): ema = [0.526, 0.522, 0.608, 0.641, 0.553] lambda = [1.00, 1.00, 1.40, 1.76, 1.00] Previously with ratio formula, max lambda on the same data would have been ~1.05. h1000 (1.5σ above mean BCE here) now gets a 76% trunk-gradient boost vs uniform. (3) Default --seq-len 32 → 64 examples/alpha_train.rs: K=32 gives the model 0.5% of the h6000 prediction window as in-window context. K=64 doubles that, giving Mamba2's SSM state more material to build long-horizon predictions. Within the kernel's MAMBA2_KERNEL_SEQ_MAX cap of 96. Per-epoch wall scales ~K (more K-loop launches in the captured graph, ~2× wall at K=64 vs K=32 for the K-loop portion of dispatch). Cache-bust v5 in build.rs to force nvcc recompile against the new horizon_lambda.cu formula on the cluster's /cargo-target PVC. Old cubins compute a numerically different lambda; running them against the new Rust loop would silently apply the wrong gradient scaler. Validation: 7 perception_overfit tests + 26 lib + 23 integration ml-alpha tests pass. Synthetic overfit still converges to 0.0006. horizon_ema_and_lambda_track_after_training observes the new lambda spread (1.0-1.76) and asserts the asymmetric clamp envelope. Co-Authored-By: Claude Opus 4.7 --- crates/ml-alpha/build.rs | 10 ++--- crates/ml-alpha/cuda/horizon_lambda.cu | 46 ++++++++++++++--------- crates/ml-alpha/examples/alpha_train.rs | 23 +++++++++--- crates/ml-alpha/src/trainer/perception.rs | 33 ++++++++++------ 4 files changed, 73 insertions(+), 39 deletions(-) diff --git a/crates/ml-alpha/build.rs b/crates/ml-alpha/build.rs index 37ad972ee..e02434d57 100644 --- a/crates/ml-alpha/build.rs +++ b/crates/ml-alpha/build.rs @@ -19,11 +19,11 @@ const KERNELS: &[&str] = &[ "horizon_lambda", // ISV-driven per-horizon gradient scaler (EMA + lambda) ]; -// Cache bust v4 (2026-05-17): Mamba2 state_dim cap raised 16 → 32 in -// the kernel (MAMBA2_ALPHA_MAX_STATE_D). Old cubins are sized for -// state_d=16 and will silently truncate / corrupt the larger state -// arrays. Force a fresh nvcc compile on the cluster's /cargo-target -// persistent volume so the cubin matches the source. +// Cache bust v5 (2026-05-17): horizon_lambda.cu rewrote the lambda +// formula from ratio-of-mean to z-score-normalised. Old cubins compute +// a numerically different lambda; running them against the new Rust +// loop would silently apply the wrong gradient scaler. Force nvcc +// recompile on the cluster's /cargo-target PVC. fn main() { println!("cargo:rerun-if-changed=build.rs"); diff --git a/crates/ml-alpha/cuda/horizon_lambda.cu b/crates/ml-alpha/cuda/horizon_lambda.cu index f64ea0d7b..ca5aa5057 100644 --- a/crates/ml-alpha/cuda/horizon_lambda.cu +++ b/crates/ml-alpha/cuda/horizon_lambda.cu @@ -32,24 +32,24 @@ // stable across training noise. // // Lambda safety: ASYMMETRIC clamp `[1.0, LAMBDA_CEILING]` per -// `pearl_audit_unboundedness_for_implicit_asymmetry.md`. The lower -// bound is 1.0, not <1: the controller is boost-only — under-trained -// horizons get more trunk-gradient pull, but easy horizons are NEVER -// starved below uniform weight. The 3-fold CV that motivated this -// design (no-ISV @ 0.711 ± 0.014 vs ISV @ 0.720 ± 0.011 across -// folds 0/1/2) showed fold-1 ISV gaining +4.6pt on h6000 when -// h6000 was hardest, but fold-2 ISV LOSING -1.3pt on h6000 when it -// wasn't — because a symmetric clamp [0.5, 2.0] demoted h6000 below -// uniform when its EMA was below the cross-horizon mean. Asymmetric -// clamp preserves the gain without the regression. h6000 is the -// deployment horizon; we never want to starve it. Upper cap at 2× -// still prevents winner-take-all amplification per -// `pearl_controller_amplifies_dominant_magnitude_trap.md`. +// `pearl_audit_unboundedness_for_implicit_asymmetry.md` — boost-only. +// Lambda derivation: Z-SCORE NORMALISED, not raw ratio. The ratio +// formula `ema_h / mean(ema)` produced lambdas in 0.97-1.03 in our +// data because per-horizon BCE clusters tightly (range 0.04 abs) +// while the mean is ~0.65 — so the controller barely engaged +// (asymmetric ceiling 2.0 was never approached, max observed +// lambda ~1.04). Z-score `z_h = (ema_h - mean) / std(ema)` makes +// the spread scale-invariant per +// `pearl_zscore_normalization_for_magnitude_asymmetric_signals.md`, +// then `lambda = 1.0 + Z_SCALE * z` fills the clamp envelope: a +// horizon 1σ above mean gets lambda 1.5 (with Z_SCALE=0.5), 2σ +// saturates at the ceiling. Boost-only floor at 1.0 still applies. #define N_HORIZONS_LAMBDA 5 #define ALPHA_FIXED 0.1f #define LAMBDA_FLOOR 1.0f // boost-only: never demote a horizon below uniform #define LAMBDA_CEILING 2.0f +#define Z_SCALE 0.5f // 1σ above mean → lambda 1.5; 2σ → ceiling extern "C" __global__ void horizon_ema_and_lambda( const float* __restrict__ loss_per_horizon, // [5] — current step UNWEIGHTED BCE @@ -72,12 +72,22 @@ extern "C" __global__ void horizon_ema_and_lambda( } const float mean = sum / (float)N_HORIZONS_LAMBDA; - // mean > 1e-12 is essentially always true once the first step - // has run (BCE for a random init is ~0.69), but guard anyway. - const float inv_mean = (mean > 1e-12f) ? (1.0f / mean) : 0.0f; + // Per-horizon std (population, not sample — N=5 is small + fixed). + // EPSILON guards the first-step case where all EMAs are equal + // (z would be 0/0); under that condition lambda falls back to 1.0 + // via the asymmetric floor. + float ssq = 0.0f; #pragma unroll for (int h = 0; h < N_HORIZONS_LAMBDA; ++h) { - const float ratio = (inv_mean > 0.0f) ? new_ema[h] * inv_mean : 1.0f; - lambda[h] = fmaxf(LAMBDA_FLOOR, fminf(LAMBDA_CEILING, ratio)); + const float diff = new_ema[h] - mean; + ssq += diff * diff; + } + const float std = sqrtf(ssq / (float)N_HORIZONS_LAMBDA + 1e-12f); + const float inv_std = (std > 1e-6f) ? (1.0f / std) : 0.0f; + #pragma unroll + for (int h = 0; h < N_HORIZONS_LAMBDA; ++h) { + const float z = (new_ema[h] - mean) * inv_std; + const float raw = 1.0f + Z_SCALE * z; + lambda[h] = fmaxf(LAMBDA_FLOOR, fminf(LAMBDA_CEILING, raw)); } } diff --git a/crates/ml-alpha/examples/alpha_train.rs b/crates/ml-alpha/examples/alpha_train.rs index 423ed278d..4fb1ee482 100644 --- a/crates/ml-alpha/examples/alpha_train.rs +++ b/crates/ml-alpha/examples/alpha_train.rs @@ -50,7 +50,15 @@ struct Cli { #[arg(long, default_value_t = 5)] epochs: usize, - #[arg(long, default_value_t = 32)] + /// Snapshots per training sequence (Mamba2 + CfC in-window context). + /// Default 64 — empirically the 3-fold ISV CV ran K=32 and observed + /// h6000 (6000-snapshot horizon) saturating around 0.70-0.74. K=32 + /// is only 0.5% of the h6000 prediction window; K=64 doubles + /// in-window context and gives Mamba2's SSM state more material + /// to build long-horizon predictions from. Kernel maximum is 96 + /// (MAMBA2_KERNEL_SEQ_MAX). Per-epoch wall scales ~K (more + /// K-loop launches in the captured graph). + #[arg(long, default_value_t = 64)] seq_len: usize, #[arg(long, default_value_t = 16)] @@ -105,10 +113,15 @@ struct Cli { #[arg(long)] horizon_weights: Option, - /// Compute per-horizon weights as `min(1, K/h)` — short horizons - /// at weight 1, long horizons down-weighted to their independent- - /// sample density inside the K-snapshot window. Recommended when - /// horizons include values larger than seq_len. + /// Use the canonical auto-derived per-horizon BCE weight schedule. + /// Currently identical to `--horizon-weights 1,1,1,1,1` (uniform): + /// the prior `min(1, K/h)` formula down-weighted h6000 to 0.5% of + /// the loss which suppressed long-horizon training. With uniform + /// weights, ISV's lambda controller (per-horizon trunk-gradient + /// scaler driven by BCE EMA) is the single place where per-horizon + /// rebalancing happens — cleaner than splitting between BCE + /// coefficients and the trunk lambda. Flag preserved for backward + /// compat; passing `--horizon-weights` still overrides. #[arg(long, default_value_t = false)] auto_horizon_weights: bool, diff --git a/crates/ml-alpha/src/trainer/perception.rs b/crates/ml-alpha/src/trainer/perception.rs index 40619d2e8..73abe904f 100644 --- a/crates/ml-alpha/src/trainer/perception.rs +++ b/crates/ml-alpha/src/trainer/perception.rs @@ -94,17 +94,28 @@ impl Default for PerceptionTrainerConfig { } } -/// Compute `w_h = min(1.0, K / max(h, K))` for each horizon — short -/// horizons keep full weight, long horizons get down-weighted by the -/// fraction of independent samples they actually represent in a -/// K-snapshot sequence. Used when the CLI passes `--auto-horizon-weights`. -pub fn auto_horizon_weights(seq_len: usize, horizons: &[usize; N_HORIZONS]) -> [f32; N_HORIZONS] { - let k = seq_len as f32; - let mut out = [1.0f32; N_HORIZONS]; - for (i, &h) in horizons.iter().enumerate() { - out[i] = (k / (h as f32).max(k)).min(1.0); - } - out +/// Per-horizon BCE weight schedule, used when the CLI passes +/// `--auto-horizon-weights`. Replaces the previous `min(1, K/h)` +/// schedule which down-weighted long horizons to fractions of a +/// percent of the loss (h6000 weight = 0.0053 at K=32) and effectively +/// excluded them from training. +/// +/// The 3-fold ISV CV on commit `0171c8c0e` showed the prior schedule +/// suppressing h6000 — the deployment-relevant horizon — to the point +/// where the ISV controller's lambda was operating on rounding error. +/// h6000 ended at 0.685-0.728 across folds; mean_auc CV at 0.717±. +/// +/// New schedule: UNIFORM `[1.0; N]`. All horizons contribute equally +/// to the BCE loss; let the ISV controller (boosting per-horizon +/// gradient via lambda into the trunk) do the per-horizon re-balancing +/// in a single canonical place. Callers who want a custom schedule +/// can still pass `--horizon-weights w0,w1,w2,w3,w4`. +/// +/// `seq_len` and `horizons` are kept on the signature for backward +/// compatibility with the CLI but are no longer consulted; the +/// returned schedule is shape-determined, not data-determined. +pub fn auto_horizon_weights(_seq_len: usize, _horizons: &[usize; N_HORIZONS]) -> [f32; N_HORIZONS] { + [1.0f32; N_HORIZONS] } pub struct PerceptionTrainer {