diff --git a/crates/ml/src/trainers/dqn/fused_training.rs b/crates/ml/src/trainers/dqn/fused_training.rs index e5b047d9f..c7f5258b4 100644 --- a/crates/ml/src/trainers/dqn/fused_training.rs +++ b/crates/ml/src/trainers/dqn/fused_training.rs @@ -2039,6 +2039,82 @@ impl FusedTrainingCtx { self.trainer.launch_sp14_alpha_grad_compute(alpha_var) .map_err(|e| anyhow::anyhow!("SP14 B.11 alpha_grad_compute: {e}"))?; + // 2026-05-07 cadence-fix continuation (after SP14 B.11 commit + // 200f05fce): four MORE producers whose docstrings explicitly + // claim "per-step" cadence but whose launches lived in + // `process_epoch_boundary` (fires once per epoch). Each has at + // least one captured-graph or per-step-host consumer that read + // (steps_per_epoch − 1)-step-stale ISV values per the same + // failure mode SP14 B.11 documented for ALPHA_GRAD_SMOOTHED. + // + // Atomic migration per `feedback_no_partial_refactor`. Producers + // are ordered to preserve the existing epoch-block dependency + // chain (moe_lambda_eff_update reads MOE_GATE_ENTROPY_EMA which + // moe_expert_util_ema produces — same-step ordering preserved). + // + // * launch_h_s2_rms_ema → ISV[H_S2_RMS_EMA_INDEX=96] + // Consumer: mag_concat_kernel (per-step in forward graph, + // reads ISV[96] for adaptive scale of save_h_s2 → magnitude + // branch concat). Also `dqn_clamp_finite_f32_kernel` cuBLAS + // backward sanitiser uses 1e6 × ISV[96] for the IQN trunk + // path. Both per-step. Producer kernel reads `save_h_s2` + // populated by THIS step's online forward. + // + // * launch_fold_warmup_factor → ISV[FOLD_WARMUP_FACTOR_INDEX=130] + // Consumer: training_loop.rs:2662 `read_isv_signal_at(130)` + // in per-step `update_adaptive_clip` block, derives + // lr_eff and clip_eff. Producer kernel reads fast/slow + // grad-norm EMAs filled by `update_grad_norm_emas_kernel` + // per step. Note: `WARMUP_FACTOR_MIN_STEPS` and + // `steps_observed_i32` kernel args are FROZEN at capture + // time — the gate is monotone (once passed, stays passed), + // and the host-side step counter increments outside graph + // capture, so freezing the captured value at >200 keeps + // the gate permanently open as designed. + // + // * launch_moe_expert_util_ema → ISV[MOE_EXPERT_UTIL_EMA_BASE + // ..MOE_EXPERT_UTIL_EMA_BASE+8) = [118..126) + + // ISV[MOE_GATE_ENTROPY_EMA_INDEX=126] + // Consumer: launch_moe_lambda_eff_update reads ISV[126] + // (next entry below — same-step). Also HEALTH_DIAG. Producer + // reads `moe_gate_softmax_buf` populated by + // `submit_forward_ops_main`'s `launch_moe_forward` THIS step. + // + // * launch_moe_lambda_eff_update → ISV[MOE_LAMBDA_EFF_INDEX=128] + // Consumer: `moe_load_balance_loss` kernel (in MoE backward + // forward — captured per step) reads ISV[128] at runtime + // to scale the load-balance loss. Producer reads ISV[126] + // written by `launch_moe_expert_util_ema` immediately + // above on the same stream. + // + // All four launchers use pre-loaded `CudaFunction` fields + // (no per-call `load_cubin` / `load_function`) — graph-capture + // safe per `pearl_no_host_branches_in_captured_graph` (verified + // gpu_dqn_trainer.rs:12646, 30983, 16901, 16969). + // + // Cold-start ordering: h_s2_rms_ema reads `save_h_s2` from the + // forward this step (forward_child runs BEFORE aux_child per + // `capture_training_graph`). MoE producer reads + // `moe_gate_softmax_buf` populated by `launch_moe_forward` + // inside the same forward_child. Both inputs are valid by + // the time submit_aux_ops runs. + // + // The per-epoch slots written here are STILL read by HEALTH_DIAG + // emits in `process_epoch_boundary` further down — those reads + // continue to see the latest per-step value (per-epoch read of + // per-step-updated slot is a strict improvement). + self.trainer.launch_h_s2_rms_ema() + .map_err(|e| anyhow::anyhow!("Plan 4 2c.3c.5 h_s2_rms_ema: {e}"))?; + self.trainer.launch_fold_warmup_factor() + .map_err(|e| anyhow::anyhow!("Plan C K fold_warmup_factor: {e}"))?; + // MoE producer + λ-controller chain — order load-bearing per + // gpu_dqn_trainer.rs:16962 ("MUST be called AFTER + // launch_moe_expert_util_ema"). + self.trainer.launch_moe_expert_util_ema() + .map_err(|e| anyhow::anyhow!("Phase 3 T3.5 moe_expert_util_ema: {e}"))?; + self.trainer.launch_moe_lambda_eff_update() + .map_err(|e| anyhow::anyhow!("moe_lambda_eff_update: {e}"))?; + // 2. Gather Q(s, a_taken) for both IQL trainers. let q_out = self.trainer.q_out_buf(); let actions = self.trainer.actions_buf(); diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 35de394cb..4799f61c8 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -4137,32 +4137,23 @@ impl DQNTrainer { } } - // Plan 4 Task 2c.3c.5: per-step H_S2 RMS EMA into ISV[96]. - // Single-block 256-thread shmem-reduction kernel reads the - // online trunk's `save_h_s2 [B, SH2]` (populated by the most - // recent forward) and EMAs sqrt(sum_sq / N) into ISV[96] at - // α=0.05. Producer-only — 2c.3c.6 wires the consumer in - // `mag_concat_qdir`'s adaptive-scale path. Same launch - // cadence as the Plan 3 producers above. - if let Some(ref fused) = self.fused_ctx { - if let Err(e) = fused.trainer().launch_h_s2_rms_ema() { - tracing::warn!("Plan 4 2c.3c.5 h_s2_rms_ema launch failed: {e}"); - } - } + // 2026-05-07 cadence-fix continuation (after SP14 B.11 + // commit 200f05fce): launch_h_s2_rms_ema MOVED to + // submit_aux_ops (per-step, captured into aux_child graph). + // Consumer (mag_concat_kernel + dqn_clamp_finite_f32_kernel) + // reads ISV[96] every training step — per-epoch producer + // here meant (steps_per_epoch − 1)-step-stale values for + // every captured-graph replay. See submit_aux_ops in + // fused_training.rs for the new hook. - // Plan C Phase 2 follow-up K (2026-04-29): per-step adaptive - // fold-boundary warmup factor producer. Reads the fast/slow - // grad-norm EMA mapped-pinned scalars (filled by - // `update_adaptive_clip` in the per-step guard block above) - // and writes ISV[FOLD_WARMUP_FACTOR_INDEX=130] = clamp(fast/slow, 0, 1). - // Two consumers below derive lr_eff (via set_lr) and - // clip_eff (via update_adaptive_clip's pinned slot). Same - // launch cadence as h_s2_rms_ema — once per step. - if let Some(ref fused) = self.fused_ctx { - if let Err(e) = fused.trainer().launch_fold_warmup_factor() { - tracing::warn!("Plan C K fold_warmup_factor launch failed: {e}"); - } - } + // 2026-05-07 cadence-fix continuation (after SP14 B.11 + // commit 200f05fce): launch_fold_warmup_factor MOVED to + // submit_aux_ops (per-step, captured into aux_child graph). + // Consumer at training_loop.rs:2662 reads + // ISV[FOLD_WARMUP_FACTOR_INDEX=130] every training step to + // derive lr_eff and clip_eff — per-epoch producer here + // meant the warmup factor lagged by an entire epoch. See + // submit_aux_ops in fused_training.rs for the new hook. // Plan 4 Task 3 (E.3): per-step IQN multi-quantile diagnostic // EMAs into ISV[99..103). 4-block kernel reads the IQN online @@ -4239,27 +4230,17 @@ impl DQNTrainer { // breaker stays in `process_epoch_boundary` below (its // lockout decrement IS one-per-epoch by design). - // Phase 3 T3.5: MoE expert-utilisation EMA + gate entropy EMA. - // Reads `moe_gate_softmax_buf [B, K]` (valid after the captured - // forward graph ran) and EMA-updates ISV[118..127). - if let Some(ref fused) = self.fused_ctx { - if let Err(e) = fused.trainer().launch_moe_expert_util_ema() { - tracing::warn!("Phase 3 T3.5 launch_moe_expert_util_ema failed: {e}"); - } - } - - // Adaptive MoE load-balance λ controller (2026-04-27). - // MUST run AFTER `launch_moe_expert_util_ema` so the gate- - // entropy EMA fed into the controller is fresh for the NEXT - // step's `moe_load_balance_loss` consumer. Reads ISV[126] - // (entropy EMA), writes ISV[MOE_LAMBDA_EFF_INDEX=128]. - // Per `pearl_blend_formulas_must_have_permanent_floor.md`, - // λ_floor is a permanent minimum. - if let Some(ref fused) = self.fused_ctx { - if let Err(e) = fused.trainer().launch_moe_lambda_eff_update() { - tracing::warn!("launch_moe_lambda_eff_update failed: {e}"); - } - } + // 2026-05-07 cadence-fix continuation (after SP14 B.11 + // commit 200f05fce): launch_moe_expert_util_ema + + // launch_moe_lambda_eff_update MOVED to submit_aux_ops + // (per-step, captured into aux_child graph). Consumer + // `moe_load_balance_loss` reads ISV[MOE_LAMBDA_EFF_INDEX=128] + // every training step (and the controller reads + // ISV[MOE_GATE_ENTROPY_EMA_INDEX=126] from the producer); + // per-epoch producers here meant the load-balance λ lagged + // by an entire epoch. Order preserved (expert_util BEFORE + // lambda_eff_update). See submit_aux_ops in + // fused_training.rs for the new hook. // SP4 Layer A Tasks A10 + A11: per-step ISV-bound producer // launches for the 40 new SP4 slots (TARGET_Q_BOUND, ATOM_POS diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index d1a271c59..3d1e5bb26 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -2,6 +2,8 @@ **Status:** Populated during Plan 1 Task 6 (A.5 orphan audit). Updated on every commit per Invariant 7. +Producer-cadence sweep — 4 more per-step producers MOVED out of `process_epoch_boundary` (2026-05-07): continuation of SP14 B.11 (commit `200f05fce`). Audit of `training_loop.rs:4100-4262` per the `feedback_no_partial_refactor` rule found 4 ADDITIONAL producers whose docstrings explicitly claim "per-step" cadence but whose launches lived in `process_epoch_boundary` (single call site at `training_loop.rs:780`, fires once per epoch — not per training step inside `run_training_steps_slices`). Each has a verified per-step consumer reading the ISV slot every training step inside the parent captured graph: `launch_h_s2_rms_ema` (ISV[H_S2_RMS_EMA_INDEX=96] → `mag_concat_kernel` per-step in forward graph for adaptive `save_h_s2` magnitude-branch concat scale, AND `dqn_clamp_finite_f32_kernel`'s `1e6 × ISV[96]` cuBLAS-backward IQN-trunk sanitiser bound), `launch_fold_warmup_factor` (ISV[FOLD_WARMUP_FACTOR_INDEX=130] → `training_loop.rs:2662` per-step host read deriving `lr_eff = lr_base × max(0.05, factor)` and `clip_eff = clip_base × (0.1 + 0.9 × factor)`), `launch_moe_expert_util_ema` (ISV[MOE_EXPERT_UTIL_EMA_BASE..+8) + ISV[MOE_GATE_ENTROPY_EMA_INDEX=126] → consumed by `launch_moe_lambda_eff_update` immediately below), and `launch_moe_lambda_eff_update` (ISV[MOE_LAMBDA_EFF_INDEX=128] → `moe_load_balance_loss` kernel reads `λ` per-step at `gpu_dqn_trainer.rs:16864` in the captured-graph backward path). All 4 launchers use pre-loaded `CudaFunction` fields with no per-call `load_cubin` (`gpu_dqn_trainer.rs:12646, 30983, 16901, 16969`) — graph-capture safe per `pearl_no_host_branches_in_captured_graph`. Atomic migration per `feedback_no_partial_refactor`. MoE producer + λ-controller order preserved (`launch_moe_lambda_eff_update` MUST run AFTER `launch_moe_expert_util_ema` per the kernel docstring at `gpu_dqn_trainer.rs:16962`). Cold-start ordering: `h_s2_rms_ema` reads `save_h_s2` populated by this step's online forward, MoE producers read `moe_gate_softmax_buf` populated by `launch_moe_forward` (in `submit_forward_ops_main`); both run BEFORE `submit_aux_ops` per `capture_training_graph` ordering (forward_child → ddqn_child → aux_child). State-reset registry coverage already in place (`isv_h_s2_rms_ema`, `isv_fold_warmup_factor`, `isv_moe_expert_util_ema`, `isv_moe_gate_entropy_ema`, `isv_moe_lambda_eff` at `state_reset_registry.rs:304/427/380/387/398` plus companion `isv_grad_norm_fast_ema` at line 442 for the warmup factor's numerator). Producers DELIBERATELY KEPT per-epoch (audit trail): `launch_trade_attempt_rate_ema_inplace` / `launch_plan_threshold_update_inplace` / `launch_seed_step_counter_update_inplace` / `launch_cql_alpha_seed_update_inplace` (collector EMAs reading per-sample buffers populated by `experience_env_step` during `collect_experiences_gpu` — input data only updates per-epoch, so per-step launches would compute the same EMA repeatedly off the same data); `launch_iqn_quantile_ema` / `launch_vsn_mask_ema` / `launch_aux_heads_loss_ema` (HEALTH_DIAG diag-only consumers — slots 99..103, 105..111, 113, 114 are read by per-epoch HEALTH_DIAG emit lines and the GPU `health_diag_kernel`'s per-epoch snapshot path; no per-step consumer). HEALTH_DIAG read points unchanged — per-epoch reads of per-step-updated slots get the latest value (strict improvement over per-epoch reads of per-epoch-stale slots). Verification: `cargo check -p ml --tests --all-targets` clean (dev profile, only pre-existing unused-var warnings unrelated to this change). Files modified: `crates/ml/src/trainers/dqn/fused_training.rs` (+72 lines: 4 launches + comment block in `submit_aux_ops` immediately after the SP14 EGF chain); `crates/ml/src/trainers/dqn/trainer/training_loop.rs` (−45 / +21 lines: deletion of stale per-epoch launches + replacement audit comments). DEFERRED follow-up — additional SP4 Layer A / SP5 Pearl producers in `process_epoch_boundary` lines 4264-4406 (`launch_sp4_target_q_p99`, `launch_sp4_atom_pos_p99_all_branches`, `launch_sp4_grad_norm_p99`, `launch_sp4_h_s2_p99`, `launch_sp4_param_group_oracles_all_groups`, `launch_sp5_pearl_1_atom`, `launch_sp5_pearl_3_sigma`, `launch_sp5_pearl_2_budget`, `launch_max_budget_compute`, `launch_loss_balance_controller`, `launch_sp5_pearl_4_adam_hparams`, `launch_sp5_pearl_5_iqn_tau`, `launch_sp5_pearl_8_trail`, `launch_sp5_pearl_1_ext_num_atoms`) write ISV bounds with VERIFIED per-step consumers (Mech 1 target_q clip, ISV-driven `read_group_adam_bounds` at `submit_aux_ops` lines 1648/2062/2074, `apply_c51_budget_scale` at line 1941, IQN τ schedule, atoms_update via `pearl_1_ext_num_atoms`'s ATOM_NUM_ATOMS_BASE and trail_dist consumers in `experience_env_step`); their docstrings DON'T explicitly claim per-step cadence (silence on cadence) so they do not match the explicit migration heuristic from this commit's audit. Per the user-specified strict heuristic ("docstring per-step + per-step consumer ⇒ migrate"), this sweep deliberately scoped only to docstring-claimant producers; the SP4/SP5 epoch-block producers staleness is a known follow-up tracked here but NOT changed in this commit to keep the scope tight and the validation surface manageable. The atom_pos/SP4 bound producers AND the SP7 controller chain were intentionally LEFT in `process_epoch_boundary` for that reason. + SP14 B.11 cadence fix — EGF producer chain MOVED to per-step in `submit_aux_ops` (2026-05-07): root-caused the L40S `train-v8ztm` 10-epoch run's `α_smoothed=0.0002` + `gate1=closed` + `var_aux:var_q = 290:1` symptoms (HEALTH_DIAG `pearl_egf_diag` at epoch 10) to the original B.11 wire-up (commit `857722e77`) placing the per-step launches in `process_epoch_boundary` instead of the per-step training body. `process_epoch_boundary` runs ONCE per epoch (single call site at `training_loop.rs:780`, called from the per-epoch `for epoch in 0..n_epochs` loop, NOT from the per-step `for _step in 0..num_training_steps` loop in `run_training_steps_slices`). The captured backward consumer `launch_sp14_scale_wire_col` (inside `launch_cublas_backward_to`, called via the parent graph's replay every training step) reads `ISV[ALPHA_GRAD_SMOOTHED=393]` per step but the PRODUCER chain was firing only at epoch boundary — every step inside the epoch observed (steps_per_epoch − 1)-step-stale `α_grad_smoothed`, with the EMA chain barely accumulating past sentinel between rare per-epoch updates. Fix (atomic): MOVED `launch_sp13_aux_dir_metrics`, `launch_sp14_q_disagreement_update`, `launch_sp14_alpha_grad_compute` from `training_loop.rs:process_epoch_boundary` (lines 4245-4319) into `fused_training.rs:submit_aux_ops` immediately after `populate_q_out` (line ~1972). `submit_aux_ops` is captured into the `aux_child` sub-graph (see `capture_training_graph`), so each parent-graph replay re-fires the full producer chain — restoring the per-step cadence the SP14 plan §2550 specifies. `launch_sp13_aux_dir_metrics` had to migrate alongside the SP14 launches because `alpha_grad_compute_kernel` consumes its outputs (`ISV[373/374]`); leaving sp13 per-epoch while SP14 went per-step would have re-introduced the same staleness bug for the aux-dir-acc reads (atomic dependency migration per `feedback_no_partial_refactor`). The per-epoch `launch_sp14_gradient_hack_detect` circuit breaker stays in `process_epoch_boundary` — its lockout decrement is one-per-epoch by design. Forward consumers (the 6 `launch_sp14_dir_concat_qaux` sites at lines 12120, 25666, 25735, 27788, 27978, 29076 in `gpu_dqn_trainer.rs`) and backward consumers (the 2 `launch_sp14_scale_wire_col` sites at lines 12171, 29126) are unchanged — they continue reading the same ISV slot 393, but now see a value that updates every step instead of every epoch. Build clean (`cargo check -p ml --tests --all-targets`); all 6 SP14 oracle GPU tests pass (`alpha_grad_adaptive_beta`, `alpha_grad_schmitt_hysteresis`, `dir_concat_qaux_correct`, `gradient_hack_circuit_breaker_fires`, `q_disagreement_all_hold_no_contribution`, `q_disagreement_k4_k2_mapping`). HEALTH_DIAG validation pending L40S re-dispatch — expect `α_smoothed` to track real EGF-driven values (~0.5 in steady state) instead of the bootstrap-near-zero observed in `train-v8ztm`. Invariants honored: `pearl_no_host_branches_in_captured_graph` (kernels are pure GPU state machines, capture-safe — verified all use `launch_builder` with primitive args, no per-call `load_cubin`); `feedback_no_partial_refactor` (sp13 + 2 SP14 launches migrated atomically); `feedback_wire_everything_up` (all 3 producers fire on every parent graph replay; the previously-stale-cadence EGF wire is now production hot-path); `feedback_isv_for_adaptive_bounds` (no `warmup_gate` parameter — variance-driven k_aux/k_q in alpha_grad_compute_kernel handles cold-start adaptively, per c0fc28e45). Files modified: `crates/ml/src/trainers/dqn/fused_training.rs` (+74 lines: 3 launches + comment block in `submit_aux_ops`); `crates/ml/src/trainers/dqn/trainer/training_loop.rs` (−109 / +24 lines: deletion of stale per-epoch launches + replacement audit comment). SP15 Wave 5 follow-up — pre-load sp15_baseline + cost_net cubins (2026-05-07): root-caused L40S workflow `train-xggfc` (commit `5d63762ab`) hyperopt-trial CUDA failure to the same per-call `load_cubin` + `load_function` anti-pattern that Wave 4.1b's bn_tanh_concat_dd-fix (commit `5d63762ab`, see entry below) addressed for the trainer's bottleneck-concat path. The 5 SP15 evaluation launchers in `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs` (`launch_sp15_cost_net_sharpe` at ~787, `launch_sp15_baseline_buyhold` at ~1171, `launch_sp15_baseline_hold_only` at ~1218, `launch_sp15_baseline_naive_momentum` at ~1263, `launch_sp15_baseline_naive_reversion` at ~1308) were resolving `cost_net_sharpe_kernel` / `baseline_*_kernel` symbols via `stream.context().load_cubin(...)` + `module.load_function(...)` ON EVERY CALL inside `GpuBacktestEvaluator`'s per-window eval hot loop (`gpu_backtest_evaluator.rs:2877..2922`, `for w in 0..self.n_windows`). Pattern works in single-pass train-best context (smoke `train-9bcwm` at `5d63762ab` ran the SAME launcher chain successfully through 2 folds × 5 epochs of train-best evaluation), but fails when re-entered from a hyperopt trial's child stream context: `train-xggfc` ran 8 hyperopt search epochs successfully, then started 20 hyperopt trial trainings; trial 1 failed at `load sp15_baseline_kernels cubin: DriverError(CUDA_ERROR_ILLEGAL_ADDRESS, "an illegal memory access was encountered")`; after trial 1's CUDA context was poisoned, trials 2-20 cascade-failed at `Fork CUDA stream for trial: ILLEGAL_ADDRESS`. Fix (atomic, mirrors `5d63762ab` precedent): added 5 `CudaFunction` fields on `GpuBacktestEvaluator` (`sp15_cost_net_sharpe_kernel`, `sp15_baseline_buyhold_kernel`, `sp15_baseline_hold_only_kernel`, `sp15_baseline_naive_momentum_kernel`, `sp15_baseline_naive_reversion_kernel`); pre-loaded both `SP15_COST_NET_SHARPE_CUBIN` (1 function) and `SP15_BASELINE_KERNELS_CUBIN` (4 functions) once in `GpuBacktestEvaluator::new()` alongside the existing `env_module` / `metrics_module` / `gather_module` loads; changed all 5 launcher signatures in `gpu_dqn_trainer.rs` to take `&CudaFunction` as a trailing parameter (no more per-call resolution); updated all 5 call sites in `gpu_backtest_evaluator.rs:2877..2922` to pass `&self.sp15_*_kernel`; updated 4 oracle test call sites in `crates/ml/tests/sp15_phase1_oracle_tests.rs` (cost_net + 3 baseline tests covering all 4 baseline kernels) to pre-load the cubin and pass the kernel handle. Both `SP15_BASELINE_KERNELS_CUBIN` and `SP15_COST_NET_SHARPE_CUBIN` were already `pub static` so no visibility promotion needed. Verification: `SQLX_OFFLINE=true CUDA_COMPUTE_CAP=86 cargo check -p ml --tests --all-targets` clean (dev profile, warnings only — no errors). Per `pearl_no_host_branches_in_captured_graph` (this is the eval-context analogue: even outside graph capture, host-side `cuModuleLoadData` is fragile across CUDA context lifetimes when the trial-level context fork hasn't fully promoted the parent's loaded modules), `feedback_no_partial_refactor`, `feedback_wire_everything_up`.