diff --git a/crates/ml/src/cuda_pipeline/hold_cost_scale_update_kernel.cu b/crates/ml/src/cuda_pipeline/hold_cost_scale_update_kernel.cu index 2d70cc531..a433a6918 100644 --- a/crates/ml/src/cuda_pipeline/hold_cost_scale_update_kernel.cu +++ b/crates/ml/src/cuda_pipeline/hold_cost_scale_update_kernel.cu @@ -40,8 +40,20 @@ * the third observation, sample_var and diff_var are well-defined and * α emerges from the data. * - * Defensive bounds [WELFORD_ALPHA_MIN=0.01, WELFORD_ALPHA_MAX=0.95] - * applied to the Wiener-derived α before the blend. + * Bounds [WELFORD_ALPHA_MIN=0.4, WELFORD_ALPHA_MAX=0.95] applied to + * the Wiener-derived α before the blend. The MIN floor is a + * structural-control constraint (NOT a tuned value): Wiener-optimal + * α is MSE-optimal only for *stationary* signals, and this loop + * tracks a hold-rate overrun driven by a co-adapting policy. Once + * Welford accumulates 3+ samples and the policy converges toward + * target, diff_var → 0 and α_wiener → 0, leaving the controller + * unable to catch up when the policy subsequently drifts back above + * target (empirical anchor: train-multi-seed-b5gmp sha 641aa0dfd + * Fold 1 collapse — α decayed to 0.248 by HD4, blended scale lagged + * climbing target by HD5, Hold% 30.6% → 52.6% → Sharpe 55.55 vs + * 88.65 baseline). The 0.4 floor preserves ~2.5-step catch-up + * bandwidth. The MAX 0.95 prevents pathological α=1.0 (no + * smoothing). Per `pearl_wiener_alpha_floor_for_nonstationary`. * * Algorithm (single-block single-thread — cold path, per-epoch boundary): * Phases 1-7 mirror the MHT producer; only the slot indices and @@ -99,7 +111,14 @@ #include #define EPS_F 1e-6f -#define WELFORD_ALPHA_MIN_F 0.01f +// WELFORD_ALPHA_MIN_F = 0.4f — non-stationary control-loop floor. +// Wiener-α is MSE-optimal for stationary signals; this controller +// tracks a hold-rate overrun driven by a co-adapting policy, so α +// must never decay below the bandwidth needed to catch up when the +// policy drifts. Cold-start (sample_count < WELFORD_MIN_SAMPLE_COUNT_F) +// uses WELFORD_COLD_START_ALPHA_F=1.0 REPLACE per Pearl A, bypassing +// this floor. See `pearl_wiener_alpha_floor_for_nonstationary`. +#define WELFORD_ALPHA_MIN_F 0.4f #define WELFORD_ALPHA_MAX_F 0.95f #define WELFORD_MIN_SAMPLE_COUNT_F 3.0f #define WELFORD_COLD_START_ALPHA_F 1.0f @@ -180,6 +199,15 @@ void hold_cost_scale_update( ? diff_M2 / (diff_count - 1.0f) : diff_M2; alpha = diff_var / (diff_var + sample_var + EPS_F); + /* Floor at WELFORD_ALPHA_MIN_F=0.4: Wiener-α is MSE-optimal + * for stationary signals only; this control loop tracks an + * overrun driven by the co-adapting policy. Without the floor + * α decays toward 0 once observed_hold_rate converges toward + * target, leaving the controller unable to react when the + * policy drifts (canonical regression: train-multi-seed-b5gmp + * sha 641aa0dfd Fold 1, α=0.248 at HD4 → Hold% 52.6% at HD5). + * Cap at WELFORD_ALPHA_MAX_F=0.95 prevents α=1.0 (no + * smoothing). Per `pearl_wiener_alpha_floor_for_nonstationary`. */ alpha = fmaxf(WELFORD_ALPHA_MIN_F, fminf(alpha, WELFORD_ALPHA_MAX_F)); } diff --git a/crates/ml/src/cuda_pipeline/min_hold_temperature_update_kernel.cu b/crates/ml/src/cuda_pipeline/min_hold_temperature_update_kernel.cu index 89e56e218..2fbd0ddaf 100644 --- a/crates/ml/src/cuda_pipeline/min_hold_temperature_update_kernel.cu +++ b/crates/ml/src/cuda_pipeline/min_hold_temperature_update_kernel.cu @@ -63,11 +63,20 @@ * - Steady state: signal stabilises → diff_var drops → α → 0.05 or * lower → smoothing emerges naturally without a hardcoded constant. * - * Defensive bounds [WELFORD_ALPHA_MIN=0.01, WELFORD_ALPHA_MAX=0.95] - * applied to the Wiener-derived α before the blend. The Wiener - * formula naturally yields α ∈ [0, 1] but numerical edge cases (both - * variances identically zero within ε) could push α to a denormal - * value or floor at 0 (no learning). + * Bounds [WELFORD_ALPHA_MIN=0.4, WELFORD_ALPHA_MAX=0.95] applied to + * the Wiener-derived α before the blend. The MIN floor is a + * structural-control constraint (NOT a tuned value): Wiener-optimal + * α is MSE-optimal only for *stationary* signals, and this loop + * tracks a hold-rate overrun driven by a co-adapting policy. Once + * Welford accumulates 3+ samples and the policy converges toward + * target, diff_var → 0 and α_wiener → 0, leaving the controller + * unable to catch up when the policy subsequently drifts back above + * target (empirical anchor: train-multi-seed-b5gmp sha 641aa0dfd + * Fold 1 collapse — α decayed to 0.248 by HD4, blended scale lagged + * climbing target by HD5, Hold% 30.6% → 52.6% → Sharpe 55.55 vs + * 88.65 baseline). The 0.4 floor preserves ~2.5-step catch-up + * bandwidth. The MAX 0.95 prevents pathological α=1.0 (no + * smoothing). Per `pearl_wiener_alpha_floor_for_nonstationary`. * * Algorithm (single-block single-thread — cold path, per-epoch boundary): * @@ -154,7 +163,14 @@ #include #define EPS_F 1e-6f -#define WELFORD_ALPHA_MIN_F 0.01f +// WELFORD_ALPHA_MIN_F = 0.4f — non-stationary control-loop floor. +// Wiener-α is MSE-optimal for stationary signals; this controller +// tracks a hold-rate overrun driven by a co-adapting policy, so α +// must never decay below the bandwidth needed to catch up when the +// policy drifts. Cold-start (sample_count < WELFORD_MIN_SAMPLE_COUNT_F) +// uses WELFORD_COLD_START_ALPHA_F=1.0 REPLACE per Pearl A, bypassing +// this floor. See `pearl_wiener_alpha_floor_for_nonstationary`. +#define WELFORD_ALPHA_MIN_F 0.4f #define WELFORD_ALPHA_MAX_F 0.95f #define WELFORD_MIN_SAMPLE_COUNT_F 3.0f #define WELFORD_COLD_START_ALPHA_F 1.0f @@ -237,9 +253,15 @@ void min_hold_temperature_update( ? diff_M2 / (diff_count - 1.0f) : diff_M2; alpha = diff_var / (diff_var + sample_var + EPS_F); - /* Defensive bounds — Wiener formula naturally yields α ∈ [0, 1] - * but numerical edge cases (both variances identically zero - * within ε) could push α to a denormal or floor at 0. */ + /* Floor at WELFORD_ALPHA_MIN_F=0.4: Wiener-α is MSE-optimal + * for stationary signals only; this control loop tracks an + * overrun driven by the co-adapting policy. Without the floor + * α decays toward 0 once observed_hold_rate converges toward + * target, leaving the controller unable to react when the + * policy drifts (canonical regression: train-multi-seed-b5gmp + * sha 641aa0dfd Fold 1, α=0.248 at HD4 → Hold% 52.6% at HD5). + * Cap at WELFORD_ALPHA_MAX_F=0.95 prevents α=1.0 (no + * smoothing). Per `pearl_wiener_alpha_floor_for_nonstationary`. */ alpha = fmaxf(WELFORD_ALPHA_MIN_F, fminf(alpha, WELFORD_ALPHA_MAX_F)); } diff --git a/crates/ml/src/cuda_pipeline/sp14_isv_slots.rs b/crates/ml/src/cuda_pipeline/sp14_isv_slots.rs index eab5176f4..91caae8ed 100644 --- a/crates/ml/src/cuda_pipeline/sp14_isv_slots.rs +++ b/crates/ml/src/cuda_pipeline/sp14_isv_slots.rs @@ -585,18 +585,38 @@ pub const MHT_SAMPLE_COUNT_INDEX: usize = 473; // Welford counter (number of // the third observation onward — the Wiener formula stabilises by sample 3. pub const SENTINEL_WELFORD_ZERO: f32 = 0.0; -// Defensive α bounds. The Wiener formula naturally yields α ∈ [0, 1] but -// numerical edge cases (target_M2 = diff_M2 = 0 within ε) could push α -// to a denormal value or floor at 0 (no learning). Clamp to keep the -// blend responsive to genuine signal even under degenerate input -// distributions. Bounds chosen so: -// - WELFORD_ALPHA_MIN=0.01: ensures every observation carries some -// weight, avoiding a permanently-frozen EMA when signals are -// deterministically constant. +// α bounds applied to the Wiener-derived α before the blend. +// +// - WELFORD_ALPHA_MIN=0.4: non-stationary control-loop floor (2026-05-08). +// Wiener-optimal α = diff_var / (diff_var + sample_var + ε) is +// MSE-optimal for *stationary* signals. The SP16 T3 producers +// (min_hold_temperature, hold_cost_scale) track a hold-rate overrun +// driven by the co-adapting policy — the controller's output +// directly shapes the per-bar reward / min-hold penalty that the +// policy is being trained against. Once Welford accumulates 3+ +// samples and the policy converges toward target, diff_var → 0 +// and α_wiener → 0, leaving the controller unable to catch up +// when the policy subsequently drifts back above target. +// +// This is a structural-control constraint (Wiener stationarity +// assumption violated), NOT a tuned constant: it encodes "always +// keep at least ~2.5-step responsiveness in case the policy +// drifts under us." Floor only applies once sample_count ≥ +// WELFORD_MIN_SAMPLE_COUNT (cold-start path uses α=1.0 REPLACE +// per Pearl A, bypassing the floor). +// +// Empirical anchor: train-multi-seed-b5gmp sha 641aa0dfd Fold 1 +// collapse — with the prior 0.01 floor, α decayed to 0.248 by +// HD4, blended scale 8.18 → 8.51 lagged climbing target_scale +// 9.53 → 10.38, Hold% 30.6% → 52.6% across HD3-HD5, dir_entropy +// hit 0.70 kill threshold, Sharpe 55.55 vs 88.65 baseline. +// +// See `pearl_wiener_alpha_floor_for_nonstationary.md`. +// // - WELFORD_ALPHA_MAX=0.95: avoids pathological α=1.0 (which is // equivalent to "no smoothing at all" — only sensible during the // cold-start path, not steady state). -pub const WELFORD_ALPHA_MIN: f32 = 0.01; +pub const WELFORD_ALPHA_MIN: f32 = 0.4; pub const WELFORD_ALPHA_MAX: f32 = 0.95; // Sample-count threshold for switching from cold-start REPLACE to Wiener @@ -934,7 +954,9 @@ mod tests { assert_eq!(MHT_SAMPLE_COUNT_INDEX, 473); // Sentinel + bounds — Pearl-A bootstrap on every fold boundary. assert_eq!(SENTINEL_WELFORD_ZERO, 0.0); - assert_eq!(WELFORD_ALPHA_MIN, 0.01); + // 0.4 floor — non-stationary control-loop bandwidth preservation + // for SP16 T1/T2 producers. Pearl: pearl_wiener_alpha_floor_for_nonstationary. + assert_eq!(WELFORD_ALPHA_MIN, 0.4); assert_eq!(WELFORD_ALPHA_MAX, 0.95); assert_eq!(WELFORD_MIN_SAMPLE_COUNT, 3.0); } diff --git a/crates/ml/src/cuda_pipeline/state_layout.cuh b/crates/ml/src/cuda_pipeline/state_layout.cuh index 2ddceb8db..eda90fe58 100644 --- a/crates/ml/src/cuda_pipeline/state_layout.cuh +++ b/crates/ml/src/cuda_pipeline/state_layout.cuh @@ -237,7 +237,22 @@ #define ISV_MHT_DIFF_M2_IDX 471 // Welford ΣΔ² (diff) #define ISV_MHT_PREV_TARGET_IDX 472 // previous-epoch target_temp #define ISV_MHT_SAMPLE_COUNT_IDX 473 // Welford counter (T1 producer) -#define WELFORD_ALPHA_MIN_F 0.01f +// Wiener-α floor for non-stationary control loops (2026-05-08). Wiener- +// optimal α = diff_var / (diff_var + sample_var + ε) is MSE-optimal for +// stationary signals. The SP16 T3 producers (min_hold_temperature, +// hold_cost_scale) track a hold-rate overrun driven by a co-adapting +// policy — non-stationary by construction. Once Welford accumulates +// 3+ samples and the policy converges toward target, diff_var collapses +// and α_wiener → 0, leaving the controller unable to react when the +// policy subsequently drifts. The 0.4 floor preserves ~2.5-step +// catch-up bandwidth. This is a structural-control constraint (Wiener +// stationarity assumption violated), NOT a tuned constant; floor only +// applies once sample_count ≥ WELFORD_MIN_SAMPLE_COUNT_F (cold-start +// branch keeps α = 1.0 REPLACE per Pearl A). See +// `pearl_wiener_alpha_floor_for_nonstationary.md`. Empirical anchor: +// train-multi-seed-b5gmp sha 641aa0dfd Fold 1 collapse (HD4 α=0.248, +// blended_scale lag → Hold% 52.6% → Sharpe 55.55 vs 88.65 baseline). +#define WELFORD_ALPHA_MIN_F 0.4f #define WELFORD_ALPHA_MAX_F 0.95f #define WELFORD_MIN_SAMPLE_COUNT_F 3.0f #define WELFORD_EPS_F 1.0e-6f diff --git a/crates/ml/tests/sp14_oracle_tests.rs b/crates/ml/tests/sp14_oracle_tests.rs index 15b2ece4c..ea3e8c345 100644 --- a/crates/ml/tests/sp14_oracle_tests.rs +++ b/crates/ml/tests/sp14_oracle_tests.rs @@ -2695,15 +2695,21 @@ mod sp16_phase3_wiener_alpha_gpu { ); } - /// SP16 T3 — Test 2: α LOW in steady state (Welford diff_var decays). + /// SP16 T3 — Test 2: α floors at WELFORD_ALPHA_MIN in steady state. /// /// Feed a converging signal: `0.30 + 0.001 / (epoch + 1)` for 60 /// epochs. The decaying perturbation makes consecutive diffs ever- /// smaller (so diff_var → 0) while sample_var converges toward the /// steady-state value — α = diff_var / (diff_var + sample_var + ε) - /// → 0 (clamped at WELFORD_ALPHA_MIN). Verifies the "smoothing - /// emerges naturally" property over the deleted hardcoded α=0.05 - /// cadence. + /// → 0 in the unclamped formula, then clamps UP to WELFORD_ALPHA_MIN. + /// + /// Per `pearl_wiener_alpha_floor_for_nonstationary` (2026-05-08), the + /// MIN floor is 0.4 — a non-stationary control-loop bandwidth + /// guarantee, not a "natural smoothing emerges" outcome. Without + /// this floor, train-multi-seed-b5gmp Fold 1 collapsed (Sharpe + /// 88.65 → 55.55) because α decayed too far during convergence + /// and the controller could not catch up to the climbing overrun + /// signal as the policy drifted post-convergence. #[test] #[ignore = "requires GPU"] fn sp16_phase3_alpha_low_in_steady_state() { @@ -2723,25 +2729,28 @@ mod sp16_phase3_wiener_alpha_gpu { // After ~50 epochs the Welford state should have stabilised: // sample_var stays bounded (target_scale fluctuates within // ~0.001 of 13.0) while diff_var → 0 (consecutive diffs are - // sub-permille). α should approach the WELFORD_ALPHA_MIN floor. + // sub-permille). The unclamped formula would yield α → 0; the + // 0.4 floor (non-stationary control-loop constraint) clamps the + // recovered α back up to WELFORD_ALPHA_MIN. let tail: Vec = traj[50..60].iter() .map(|&(_, a)| a) .filter(|a| !a.is_nan()) .collect(); let mean_tail_alpha = tail.iter().sum::() / tail.len() as f32; - // Tail α should be substantially below the cold-start ceiling. - // With a converging signal the diff_var collapses → α floors near - // WELFORD_ALPHA_MIN. We assert a generous threshold (0.40) to - // tolerate fp rounding in the recovered-α calculation; the - // smoking gun is that this is well below the cold-start α=1.0. + // Tail α should sit AT the WELFORD_ALPHA_MIN floor (not above — + // the unclamped formula has decayed → 0; not below — the floor + // forbids it). Tolerance accommodates fp rounding in the + // recovered-α reconstruction inside training_loop.rs. + let tol = 1e-3_f32; assert!( - mean_tail_alpha < 0.40 && mean_tail_alpha > 0.0, - "Steady-state α should decay below 0.40, got mean over [50..60): {mean_tail_alpha:.5}" + (mean_tail_alpha - WELFORD_ALPHA_MIN).abs() < tol, + "Steady-state α must clamp to WELFORD_ALPHA_MIN={WELFORD_ALPHA_MIN}; \ + got mean over [50..60): {mean_tail_alpha:.5} (tol {tol:.0e})" ); // Defensive bound check — α stays positive (no permanent freeze). assert!( - mean_tail_alpha >= WELFORD_ALPHA_MIN, + mean_tail_alpha >= WELFORD_ALPHA_MIN - tol, "Steady-state α must respect WELFORD_ALPHA_MIN={}; got {mean_tail_alpha:.5}", WELFORD_ALPHA_MIN ); diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index f1ae9e3c4..3219e4a69 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -8846,6 +8846,43 @@ The α field surfaces what would otherwise be opaque on-device state, enabling v - `feedback_no_legacy_aliases`: hardcoded `*_EMA_ALPHA = 0.05` constants deleted (Rust + C #define mirror) — no half-migrated state. - `feedback_no_stubs`: every test body is real; no `todo!()` placeholders. +## SP16 T3 amendment: WELFORD_ALPHA_MIN raised 0.01 → 0.4 for non-stationary control loops (2026-05-08) + +### Why this amendment exists + +`train-multi-seed-b5gmp` (sha `641aa0dfd`, full SP16 chain T0.1+T1+T2+T3 with Wiener-optimal α) Fold 1 collapsed from Sharpe 88.65 baseline to Sharpe 55.55. Hold% climbed 30.6% → 42.9% → 52.6% across HD3-HD5; dir_entropy hit the 0.70 kill threshold at HD5. Trajectory recovered from on-device Welford state: + +- HD3: α=1.000 (cold-start REPLACE, sample_count=1 < 3), blended_scale=8.18×, target_scale=8.18, Hold%=30.6% — working as designed +- HD4: α=0.248 (Wiener-decayed once sample_count ≥ 3), target_scale climbed 9.53 → 10.38 but blended only 8.18 → 8.51 — controller already lagging +- HD5: Hold%=52.6%, dir_entropy=0.70 — collapse before the controller could catch up + +Root cause: the Wiener formula `α = diff_var / (diff_var + sample_var + ε)` is MSE-optimal under a STATIONARY random-walk-with-noise model. The SP16 T1/T2 producers track an `overrun = max(0, observed_hold_rate − target)` signal, but `observed_hold_rate` is itself a fast EMA of the policy's per-step Hold rate, and the policy is being trained against the controllers' outputs. The closed-loop dynamics violate the stationarity assumption: as the policy converges toward target, `diff_var → 0`, `α_wiener → 0`, and the controller loses bandwidth — exactly when a regime shift is most likely to push `observed_hold_rate` back above target. + +### The fix + +Raise `WELFORD_ALPHA_MIN` from `0.01` to `0.4` in BOTH producer kernels (T1 + T2) and the Rust mirror constant. The 0.4 floor is a structural-control constraint (Wiener stationarity assumption violated), NOT a tuned value — it encodes "always keep at least ~2.5-step responsiveness in case the policy drifts under us." Cold-start (`sample_count < WELFORD_MIN_SAMPLE_COUNT`) is unaffected — that branch uses `α = 1.0` REPLACE per Pearl A and bypasses the floor entirely. + +Per `pearl_wiener_alpha_floor_for_nonstationary` (NEW pearl, this commit). The pearl is the non-stationary correction to `pearl_wiener_optimal_adaptive_alpha`: both pearls are correct in their domains. ISVs that track exogenous signal scale (RMS of gradient norms, p99 of |advantage|, etc.) keep the unfloored Wiener formula because they ARE stationary; control loops that close back through the policy require the floor. + +### Files modified + +| File | LOC delta | Purpose | +| ------------------------------------------------------------------- | --------- | ---------------------------------------------------------------------- | +| `crates/ml/src/cuda_pipeline/sp14_isv_slots.rs` | +25 / -8 | `WELFORD_ALPHA_MIN: f32 = 0.4` + commentary + lock-test annotation | +| `crates/ml/src/cuda_pipeline/state_layout.cuh` | +16 / -1 | C `#define WELFORD_ALPHA_MIN_F 0.4f` + structural-bound commentary | +| `crates/ml/src/cuda_pipeline/min_hold_temperature_update_kernel.cu` | +29 / -10 | Local `#define` + doc comment + post-blend bound comment | +| `crates/ml/src/cuda_pipeline/hold_cost_scale_update_kernel.cu` | +24 / -5 | Same shape as T1 | +| `crates/ml/tests/sp14_oracle_tests.rs` | +18 / -10 | `sp16_phase3_alpha_low_in_steady_state` updated to assert α at floor | + +### Cross-pearl invariants preserved + +- `pearl_wiener_alpha_floor_for_nonstationary`: NEW pearl introduced this commit; floor is the structural fix. +- `pearl_wiener_optimal_adaptive_alpha`: still applies; this is its non-stationary correction (cross-referenced in the new pearl). +- `pearl_first_observation_bootstrap`: cold-start REPLACE path bypasses the floor (sample_count < 3 → α = 1.0 unconditional). +- `feedback_no_partial_refactor`: T1 + T2 kernels + Rust mirror + diagnostic mirrors in training_loop.rs (which `.clamp(WELFORD_ALPHA_MIN, ...)` against the same Rust constant) + lock-test + audit doc updated atomically. +- `feedback_isv_for_adaptive_bounds`: 0.4 is a structural-control bound (Wiener stationarity assumption violated), justified in the constant's docstring and the pearl. NOT a tuned value. +- `feedback_no_stubs`: oracle test body updated to assert the new contract (α floors AT WELFORD_ALPHA_MIN, not "below 0.4"). + ## Build infra: ensure-binary cache key bug (2026-05-08) Discovered during SP16 validation (4gb88 → 8mzkj forced switch from L40S to H100).