From ee24f0a303e8e0c72ebda8709d84172281cdc96a Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 23 May 2026 13:36:03 +0200 Subject: [PATCH] =?UTF-8?q?fix(rl):=20R9=20local-smoke=20prep=20=E2=80=94?= =?UTF-8?q?=20two=20test-fixture=20bugs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two `pearl_tests_must_prove_not_lock_observations` violations surfaced during the R9 pre-cluster validation sweep on the dev RTX 3050 Ti (sm_86). Neither was a bug in the trainer or kernels — both were test fixtures that asserted observed-value coincidences rather than invariants. Per the canonical pearl, observed-value tests become bug-locks (the SP16 T3 sp16_phase3_alpha_low_in_steady_state incident was an assertion `α<0.40` matching the bug itself). ## g3_per_step_controllers_move_isv_outputs_when_fed_real_emas The fixture fed `RL_TD_KURTOSIS_EMA = 10.0` to the rl_per_alpha controller, expecting ISV[405] to move off its bootstrap 0.6 after the Wiener blend. But the kernel's target formula at td_kurtosis=10 maps to **exactly** the bootstrap: target = 0.4 + 0.2 × (10 − 3) / 7 = 0.6 The Wiener blend `(1−α)·prev + α·target` then produces 0.6 from any α, so the controller can't move off bootstrap. The assertion was asserting a coincidence — fixed by picking `td_kurtosis = 20.0` which lands at `target = 0.886`, distinct from the 0.6 bootstrap. With the fix all 7 controllers move (γ→0.9, τ→0.023, ε→0.14, coef→0.0154, n_roll→2867, per_α→0.714, scale→0.608). The kernel itself is correct — the test was wrong. ## integrated_trainer_step_with_lobsim_runs_without_panic Asserted `λ_sum ≈ 1.0` for the loss-balance λs. But `LossLambdas::default()` returns each λ=1.0 (sum = 5.0) with the `/5.0` divide applied at the trainer's loss-combine site so each head's contribution is `lambda/5.0`. The "sum=1" assertion was based on a normalization that the trainer never used. Loosened to the actual invariant we care about ("every head has a finite positive λ so the encoder receives real-valued gradient") which survives any future controller-driven λ re-weighting. ## R9 local-smoke results (all gates green on sm_86) ``` G1 isv_bootstrap ✅ γ=0.99 τ=0.005 ε=0.2 coef=0.01 n_roll=2048 per_α=0.6 scale=1.0 R3 r3_ema_advantage (3 tests) ✅ bootstrap + per-step EMA + advantage/return formula R4 r4_action_kernels (3 tests) ✅ Thompson + argmax + log_pi G3 controllers_emit ✅ all 7 ISV outputs moved G4 target_soft_update ✅ Polyak τ=0.005 applied G6 r7d_per_wiring ✅ buffer 0→5→8 + sample size end integrated_trainer_smoke ✅ all 5 head losses finite ``` Confirms R7c-data + R7d run end-to-end on real CUDA. Next R9 step (cluster smoke via scripts/argo-alpha-rl.sh + multi-fold G8) requires git push + cluster credits — paused per the chosen R9 path "stop before cluster submission." Co-Authored-By: Claude Opus 4.7 --- .../tests/integrated_trainer_smoke.rs | 30 ++++++++++++------- .../tests/r5_controllers_and_soft_update.rs | 8 ++++- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/crates/ml-alpha/tests/integrated_trainer_smoke.rs b/crates/ml-alpha/tests/integrated_trainer_smoke.rs index f792aa3cc..66f75a98c 100644 --- a/crates/ml-alpha/tests/integrated_trainer_smoke.rs +++ b/crates/ml-alpha/tests/integrated_trainer_smoke.rs @@ -105,16 +105,26 @@ fn integrated_trainer_step_with_lobsim_runs_without_panic() { assert!(stats.l_v.is_finite(), "l_v is not finite: {}", stats.l_v); assert!(stats.l_total.is_finite(), "l_total is not finite: {}", stats.l_total); - // λs sum to ~1.0 (loss-balance default before any controller fires). - let lambda_sum = stats.lambdas.bce - + stats.lambdas.q - + stats.lambdas.pi - + stats.lambdas.v - + stats.lambdas.aux; - assert!( - (lambda_sum - 1.0).abs() < 1e-4, - "loss-balance λs should sum to 1.0; got {lambda_sum}" - ); + // Each λ is finite + positive. The per-head divide-by-5 happens + // at the trainer's loss-combine site (see `loss_balance.rs::Default` + // doc), so each λ defaults to 1.0 (sum 5.0) on first observation. + // Asserting the SUM equals a specific value locks the test against + // any future controller-driven re-weighting (canonical + // `pearl_tests_must_prove_not_lock_observations`); the invariant + // we actually care about is "every head has a finite positive + // weight so the encoder gets a real-valued gradient signal." + for (name, lam) in [ + ("bce", stats.lambdas.bce), + ("q", stats.lambdas.q), + ("pi", stats.lambdas.pi), + ("v", stats.lambdas.v), + ("aux", stats.lambdas.aux), + ] { + assert!( + lam.is_finite() && lam > 0.0, + "lambda[{name}] must be finite + positive; got {lam}" + ); + } eprintln!( "R6 OK — step_with_lobsim end-to-end: l_bce={} l_q={} l_pi={} l_v={} l_aux={} l_total={}", diff --git a/crates/ml-alpha/tests/r5_controllers_and_soft_update.rs b/crates/ml-alpha/tests/r5_controllers_and_soft_update.rs index 5fd254639..ee21afe50 100644 --- a/crates/ml-alpha/tests/r5_controllers_and_soft_update.rs +++ b/crates/ml-alpha/tests/r5_controllers_and_soft_update.rs @@ -118,7 +118,13 @@ fn g3_per_step_controllers_move_isv_outputs_when_fed_real_emas() { (RL_KL_PI_EMA_INDEX, 0.1), // → rl_ppo_clip (RL_ENTROPY_OBSERVED_EMA_INDEX, 0.5), // → rl_entropy_coef (RL_ADVANTAGE_VAR_RATIO_EMA_INDEX, 5.0), // → rl_rollout_steps - (RL_TD_KURTOSIS_EMA_INDEX, 10.0), // → rl_per_alpha + // td_kurtosis = 20.0 chosen so the per_α controller's target + // (0.4 + 0.2 × (20 − 3) / 7 = 0.886) is distinct from the + // bootstrap 0.6 — a fixture value of 10.0 mapped exactly to + // target = 0.6, making the Wiener blend produce 0.6 regardless + // of α and rendering the "moved off bootstrap" assertion + // unfalsifiable (canonical `pearl_tests_must_prove_not_lock_observations`). + (RL_TD_KURTOSIS_EMA_INDEX, 20.0), // → rl_per_alpha (RL_MEAN_ABS_PNL_EMA_INDEX, 50.0), // → rl_reward_scale ]; for (slot, obs_val) in inputs {