From ed4b30b493d73e459429ff597d6fcdacc7dc0578 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 22 Apr 2026 01:16:45 +0200 Subject: [PATCH] =?UTF-8?q?fix(smoke):=20controller=5Factivity=20=E2=80=94?= =?UTF-8?q?=20V7=20intervention-based=20fire=20detection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Redefine the "fire" semantic for adaptive controllers per the V7 audit (policy-quality-design spec §5.3): a controller fires iff it made an ADAPTIVE INTERVENTION this epoch, not merely because its observable output value changed. Previously fire detection was absolute-delta on the output: - grad_clip fired in 98.3% of epochs because the adaptive clip threshold is an EMA recomputed every training step. The EMA drifts by > 1e-3 every epoch regardless of whether the clip actually clamped a gradient. - cost_anneal fired in 98.3% of epochs because it is a deterministic sigmoid of current_epoch (1/(1+exp(-(epoch-10)/3))) with no adaptive or reactive component. Every epoch moves it by > 1e-4 by design. Neither was "load-bearing" in the V7 sense — one was a per-step EMA tracker, the other a pure curriculum schedule. The prior test output "controller 'grad_clip' fires in 98.3%" was a false positive from measuring the wrong signal. New semantics: - anti_lr / tau / gamma / cql_alpha: unchanged — absolute delta vs prior epoch on the effective output value (real adaptive controllers). - grad_clip: intervention-based latch `grad_clip_kicked_this_epoch`, set in run_training_steps_slices iff raw_grad_norm > active clip at any training step this epoch. Reset in reset_epoch_state. - cost_anneal: pure deterministic schedule → never load-bearing → always fires=false. The value is still tracked in prev_controller_values and the HEALTH_DIAG line still emits it for observability, but it cannot trip the 50% load-bearing gate. After fix, controller_activity smoke reports: anti_lr=0.000 tau=0.033 gamma=0.017 clip=0.233 cql=0.033 cost=0.000 All 6 rates ≤ 0.5. Test passes. Touched: - crates/ml/src/trainers/dqn/trainer/mod.rs (add grad_clip_kicked_this_epoch) - crates/ml/src/trainers/dqn/trainer/constructor.rs (init new field) - crates/ml/src/trainers/dqn/trainer/training_loop.rs (latch kick per step, redefine fire_clip + fire_cost in HEALTH_DIAG block) Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/trainers/dqn/trainer/constructor.rs | 1 + crates/ml/src/trainers/dqn/trainer/mod.rs | 8 +++ .../src/trainers/dqn/trainer/training_loop.rs | 57 ++++++++++++++++--- 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/crates/ml/src/trainers/dqn/trainer/constructor.rs b/crates/ml/src/trainers/dqn/trainer/constructor.rs index 918267971..492f14409 100644 --- a/crates/ml/src/trainers/dqn/trainer/constructor.rs +++ b/crates/ml/src/trainers/dqn/trainer/constructor.rs @@ -581,6 +581,7 @@ impl DQNTrainer { prev_controller_values: super::ControllerPrevValues::default(), controller_fire_counts: super::ControllerFireCounts::default(), controller_total_epochs: 0, + grad_clip_kicked_this_epoch: false, last_eval_magnitude_dist: [0.0_f32; 3], prev_reward_contrib_popart_var: 0.0, last_reward_contrib: [0.0_f32; 5], diff --git a/crates/ml/src/trainers/dqn/trainer/mod.rs b/crates/ml/src/trainers/dqn/trainer/mod.rs index d00741899..a76abf27f 100644 --- a/crates/ml/src/trainers/dqn/trainer/mod.rs +++ b/crates/ml/src/trainers/dqn/trainer/mod.rs @@ -305,6 +305,14 @@ pub struct DQNTrainer { pub(crate) controller_fire_counts: ControllerFireCounts, pub(crate) controller_total_epochs: u32, + /// Per-epoch latch: did the adaptive grad-clip controller actually CLAMP a + /// gradient this epoch (i.e. raw_grad_norm > adaptive_clip_value at some + /// training step)? This is the V7-audit definition of a "fire": adaptive + /// grad clip fires iff it made an intervention, not merely because its + /// EMA-smoothed output value ticked. Set true in `run_training_steps_slices` + /// when the guard observation shows a clamp, reset in `reset_epoch_state`. + pub(crate) grad_clip_kicked_this_epoch: bool, + /// Task 0.8 — previous-epoch PopArt variance reading used to compute the /// HEALTH_DIAG reward_contrib[popart] drift signal. Kept separate from /// FusedTrainingCtx::prev_popart_var (which the tau controller mutates on diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 78642d612..0a8dda023 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -912,6 +912,13 @@ impl DQNTrainer { self.epoch_atom_entropy = 0.0; self.epoch_atom_utilization = 0.0; + // Task 0.9 — adaptive grad-clip "kicked in" latch. Fire detection + // for the grad_clip controller is INTERVENTION-based: it fires iff + // at least one step this epoch had raw_grad_norm > adaptive clip + // value (i.e. the clip actually clamped a gradient). EMA ticks + // alone do NOT count as fires. See controller_activity smoke test. + self.grad_clip_kicked_this_epoch = false; + // GPU-persistent epoch state: set reset flags instead of CPU state mutation. if let Some(ref mut collector) = self.gpu_experience_collector { let flags: u32 = 1 | 2; // reset portfolio + DSR normalizer (always enabled) @@ -1646,7 +1653,21 @@ impl DQNTrainer { } // Feed observed grad norm into EMA-based adaptive clip. // The next Adam graph replay will read the updated clip from the device buffer. + // + // Task 0.9: set `grad_clip_kicked_this_epoch` iff this step's + // raw_grad_norm actually exceeded the clip threshold that was + // active FOR THIS step (i.e. before we feed the new observation + // into the EMA below). The clip value is only load-bearing if it + // actually clamped a gradient — EMA ticks of the threshold are + // schedule drift, not interventions. if let Some(ref mut fused) = self.fused_ctx { + let active_clip = fused.adaptive_clip_value(); + if gr.raw_grad_norm.is_finite() + && active_clip.is_finite() + && gr.raw_grad_norm > active_clip + { + self.grad_clip_kicked_this_epoch = true; + } fused.update_adaptive_clip(gr.raw_grad_norm); } } @@ -2394,11 +2415,31 @@ impl DQNTrainer { self.explore_entropy_mag_history.push((epoch as u32, ent_mag)); // Task 0.9: adaptive-controller fire detection. - // A controller "fired" this epoch iff its observable output value - // changed from the prior epoch. NaN prev (first epoch) → no fire - // (no baseline to compare against). Uses absolute delta thresholds - // chosen per controller's natural scale — below threshold is - // floating-point noise, above is real adjustment. + // + // A controller "fires" iff it made an ADAPTIVE INTERVENTION this + // epoch — not merely because its observable output changed. The + // V7 audit asks "is this controller load-bearing (keeps training + // on the rails) or diagnostic (catches edge cases)?". Pure + // schedules (curricula) and per-step EMA trackers would tick + // every epoch by construction, which would incorrectly flag + // them as load-bearing under a delta-only semantic. + // + // Per-controller semantics: + // - anti_lr, tau, gamma, cql_alpha: genuine adaptive controllers + // whose observable output changes iff they made a decision to + // intervene. Delta-vs-prior-epoch is the right signal. + // Sentinel-NaN prev (first epoch) → no fire (no baseline). + // - grad_clip: EMA-smoothed threshold that recomputes every + // training step. The threshold is ONLY load-bearing if it + // actually clamped a gradient (raw_grad_norm > clip). That + // latch is set in run_training_steps_slices; here we just + // read it. + // - cost_anneal: pure deterministic sigmoid of current_epoch + // (crate::trainers::dqn::trainer::training_loop line ~441). + // No adaptive/reactive component, so it CANNOT be + // load-bearing in the V7 sense. Always reported as 0 fires. + // Still logged so the [CTRL_FIRE] line shows the full 6-tuple. + // // LR used for THIS epoch's training. anti-LR runs later in the loop // (computing LR for next epoch) so scheduler value here is the // one that was actually in effect. @@ -2413,9 +2454,11 @@ impl DQNTrainer { let fire_lr = prev.lr.is_finite() && (cur_lr - prev.lr).abs() > 1e-10; let fire_tau = prev.tau.is_finite() && (cur_tau - prev.tau).abs() > 1e-6; let fire_gamma = prev.gamma.is_finite() && (cur_gamma - prev.gamma).abs() > 1e-4; - let fire_clip = prev.grad_clip.is_finite() && (cur_clip - prev.grad_clip).abs() > 1e-3; + // grad_clip: intervention-based (clip actually clamped ≥ 1 step). + let fire_clip = self.grad_clip_kicked_this_epoch; let fire_cql = prev.cql_alpha.is_finite() && (cur_cql - prev.cql_alpha).abs() > 1e-5; - let fire_cost = prev.cost_anneal.is_finite() && (cur_cost - prev.cost_anneal).abs() > 1e-4; + // cost_anneal: pure curriculum schedule → never load-bearing. + let fire_cost = false; if fire_lr { self.controller_fire_counts.anti_lr += 1; } if fire_tau { self.controller_fire_counts.tau += 1; }