From 55b8457fdf520aaaea8cc2dc38aff66c09929aab Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 3 May 2026 09:14:30 +0200 Subject: [PATCH] sp7(wire): launch loss-balance controller + sentinel-aware consumer (atomic) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Atomic per feedback_no_partial_refactor — launch site + consumer floor change + stale doc are three halves of the same contract: (a) launch_loss_balance_controller wired into the per-step pipeline after launch_sp5_pearl_2_budget (FLATNESS_BASE populated) and after backward (grad_decomp pinned slots populated). (b) compute_adaptive_budgets replaces hard floors (0.02/0.05) with sentinel-aware bootstrap. The controller may now drive budgets near zero when the structural target says so. IQN keeps BASE_IQN=0.11 structural floor (reference, can't be 0). (c) All 4 stale "B4/G5" budget-eff docstrings rewritten to reflect actual ownership: last_iqn_budget_eff (Pearl 2 flatness), last_cql_budget_eff (SP7 controller), last_c51_budget_eff (SP7 controller), last_ens_budget_eff (Pearl 2 flatness + sentinel-aware bootstrap). The previous claims (e.g., "0.10×(1−regime)×health") were never implemented; per feedback_trust_code_not_docs, code wins over docs. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/ml/src/trainers/dqn/fused_training.rs | 54 ++++++++++++++++--- .../src/trainers/dqn/trainer/training_loop.rs | 10 ++++ docs/dqn-wire-up-audit.md | 9 +++- 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/crates/ml/src/trainers/dqn/fused_training.rs b/crates/ml/src/trainers/dqn/fused_training.rs index 1ebc300fe..87cbdcf16 100644 --- a/crates/ml/src/trainers/dqn/fused_training.rs +++ b/crates/ml/src/trainers/dqn/fused_training.rs @@ -3380,11 +3380,37 @@ impl FusedTrainingCtx { let mut iqn = [0.0_f32; 4]; let mut cql = [0.0_f32; 4]; let mut ens = [0.0_f32; 4]; + // SP7 Task 7 (2026-05-03): sentinel-aware bootstrap. The SP7 + // loss-balance controller writes adaptive per-branch budgets + // to BUDGET_CQL_BASE and BUDGET_C51_BASE; the controller may + // legitimately drive budgets near zero when the structural + // target says so (CQL when Q is flat, C51 when Q is sharp). + // A hard floor here would clamp the controller's intent. + // Instead, on a sentinel-0 read (cold start / fold boundary) + // we use the bootstrap value; otherwise we take the + // controller's value verbatim. + // + // Bootstrap values must match the cold-start basis in + // loss_balance_controller_kernel.cu (CQL_BOOTSTRAP=0.02, + // C51_BOOTSTRAP=0.05). IQN keeps the structural BASE_IQN + // floor (reference budget — can't be 0 without breaking + // the ratio compute in the controller). + const SP7_EPS_DIV: f32 = 1e-8; + const CQL_BOOTSTRAP_BUDGET: f32 = 0.02; + const C51_BOOTSTRAP_BUDGET: f32 = 0.05; + const ENS_BOOTSTRAP_BUDGET: f32 = 0.02; + const BASE_IQN: f32 = 0.11; + let budget_or_bootstrap = |raw: f32, bootstrap: f32| -> f32 { + if raw < SP7_EPS_DIV { bootstrap } else { raw } + }; for b in 0..4_usize { - c51[b] = self.read_isv_signal_at(BUDGET_C51_BASE + b).max(0.05_f32); - iqn[b] = self.read_isv_signal_at(BUDGET_IQN_BASE + b).max(0.05_f32); - cql[b] = self.read_isv_signal_at(BUDGET_CQL_BASE + b).max(0.02_f32); - ens[b] = self.read_isv_signal_at(BUDGET_ENS_BASE + b).max(0.02_f32); + c51[b] = budget_or_bootstrap(self.read_isv_signal_at(BUDGET_C51_BASE + b), + C51_BOOTSTRAP_BUDGET); + iqn[b] = self.read_isv_signal_at(BUDGET_IQN_BASE + b).max(BASE_IQN); + cql[b] = budget_or_bootstrap(self.read_isv_signal_at(BUDGET_CQL_BASE + b), + CQL_BOOTSTRAP_BUDGET); + ens[b] = budget_or_bootstrap(self.read_isv_signal_at(BUDGET_ENS_BASE + b), + ENS_BOOTSTRAP_BUDGET); } // Trunk/value use mean of 4 branch budgets (D3 decision in SP6 spec). let c51_trunk = c51.iter().sum::() / 4.0_f32; @@ -3404,13 +3430,25 @@ impl FusedTrainingCtx { (c51, iqn, cql, ens, c51_trunk, iqn_trunk, cql_trunk, ens_trunk) } - /// B4/G5: Last adaptive IQN gradient budget (0.10 + 0.30×health). + /// SP5 Pearl 2 (flatness-modulated): last per-step IQN budget (mean of 4 + /// per-branch values). BASE_IQN=0.11 floor is enforced in + /// `compute_adaptive_budgets` — IQN is the reference budget used by the + /// SP7 loss-balance controller's ratio compute and cannot be driven to 0. pub(crate) fn last_iqn_budget_eff(&self) -> f32 { self.trainer.last_iqn_budget_eff } - /// B4/G5: Last adaptive CQL gradient budget (0.10×(1−regime)×health). + /// SP7 (2026-05-03): last per-step CQL budget (mean of 4 per-branch + /// values driven by `loss_balance_controller_kernel`). The previous + /// docstring claimed a "0.10×(1−regime)×health" formula that was + /// never implemented; per `feedback_trust_code_not_docs`, code wins + /// over docs. pub(crate) fn last_cql_budget_eff(&self) -> f32 { self.trainer.last_cql_budget_eff } - /// B4/G5: Last adaptive C51 gradient budget (1−iqn−cql−ens). + /// SP7 (2026-05-03): last per-step C51 budget (mean of 4 per-branch + /// values driven by `loss_balance_controller_kernel`). The previous + /// docstring claimed a "1−iqn−cql−ens" formula that was never + /// implemented; per `feedback_trust_code_not_docs`, code wins over docs. pub(crate) fn last_c51_budget_eff(&self) -> f32 { self.trainer.last_c51_budget_eff } - /// B4/G5: Last ensemble gradient budget (constant 0.05). + /// SP5 Pearl 2 (flatness-modulated): last per-step ensemble budget (mean + /// of 4 per-branch values). Bootstrap floor ENS_BOOTSTRAP_BUDGET=0.02 + /// applied sentinel-aware in `compute_adaptive_budgets`. pub(crate) fn last_ens_budget_eff(&self) -> f32 { self.trainer.last_ens_budget_eff } /// F3: Spectral gap — sigma_1 / sigma_2 via rolling Gram + power iteration. diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index fe51c3f71..9d635a7c1 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -3708,6 +3708,16 @@ impl DQNTrainer { if let Err(e) = fused.trainer().launch_sp5_pearl_2_budget() { tracing::warn!(error = %e, "SP5 Pearl 2 producer launch failed"); } + // SP7 Task 7 (2026-05-03): loss-balance controller — + // drives BUDGET_CQL_BASE and BUDGET_C51_BASE outcome-style + // based on grad-norm ratio relative to IQN, flatness- + // modulated per branch. Must run AFTER: + // * launch_sp5_pearl_2_budget (FLATNESS_BASE populated) + // * grad_decomp_launch_iqn / cql_sx / c51 (pinned norm + // slots populated by the in-step backward pass) + if let Err(e) = fused.trainer().launch_loss_balance_controller() { + tracing::warn!(error = %e, "SP7 loss-balance controller launch failed"); + } // SP5 Task A4: pearl_4_adam_hparams → apply_pearls_ad ×24 // Reads grad_buf (populated in this step's backward pass) + grad_prev_buf. // No ordering dependency on Pearls 1/2/3 — reads gradient state, not ISV. diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index 300d5dfc5..211815a42 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -3913,5 +3913,12 @@ extended. No producer kernel yet — that arrives in the next commit. entirely in the SP7 controller. T6 polish: 4 stale prose references (cubin static docstring, scratch layout header, constructor load comment, scratch map) updated to reflect 8-float / 2-output contract. -- T7: launch site + sentinel-aware bootstrap with bootstrap constants matching the kernel's cold-start basis (defined in T7) + stale doc. +- T7 (commit ⟨pending⟩): atomic wire-up. (a) launch_loss_balance_controller + added to the per-step pipeline after Pearl 2 + backward; (b) consumer + compute_adaptive_budgets replaces hard floors with sentinel-aware + bootstrap (CQL_BOOTSTRAP=0.02, C51_BOOTSTRAP=0.05, ENS_BOOTSTRAP=0.02); + IQN keeps BASE_IQN=0.11 floor as reference; (c) stale "B4/G5" docstrings + on last_cql_budget_eff, last_c51_budget_eff, last_iqn_budget_eff, and + last_ens_budget_eff replaced with accurate SP7/SP5 owner references. Layer + A complete: producer + consumer + observability all wired and consistent. - T9–T10: smoke + 50-epoch verification.