diff --git a/crates/ml-dqn/src/branching.rs b/crates/ml-dqn/src/branching.rs index ba17f25a1..3415f5b14 100644 --- a/crates/ml-dqn/src/branching.rs +++ b/crates/ml-dqn/src/branching.rs @@ -382,6 +382,21 @@ impl BranchingDuelingQNetwork { } /// Resample noise in all `NoisyLinear` heads. + /// Task 0.6 — mean |sigma| for a specific branch's NoisyNets layers + /// (fc + out combined). branch_idx 0 = direction, 1 = magnitude, etc. + /// Returns 0.0 if branch_idx is out of range. H7 detection signal — + /// if magnitude branch has 2x larger σ than direction, NoisyNets + /// noise is dominating the magnitude head's effective signal. + pub fn branch_noisy_sigma_mean(&self, branch_idx: usize) -> Result { + if branch_idx >= self.branch_fcs.len() { + return Ok(0.0); + } + let fc_sigma = self.branch_fcs[branch_idx].sigma_mean()?; + let out_sigma = self.branch_outs[branch_idx].sigma_mean()?; + // Equal-weight average — fc and out have similar param counts. + Ok((fc_sigma + out_sigma) * 0.5) + } + /// /// Must be called before each training forward pass. pub fn reset_noise(&mut self) -> Result<(), MLError> { diff --git a/crates/ml-dqn/src/dqn.rs b/crates/ml-dqn/src/dqn.rs index 908f72fa7..313a7b33c 100644 --- a/crates/ml-dqn/src/dqn.rs +++ b/crates/ml-dqn/src/dqn.rs @@ -2388,6 +2388,19 @@ impl DQN { } } + /// Task 0.6 — NoisyNets σ mean for a specific action branch + /// (0 = direction, 1 = magnitude, 2 = order, 3 = urgency). Mean is over + /// the branch's fc + out NoisyLinear layers' weight_sigma + bias_sigma + /// tensors. Returns 0.0 if branching network not initialized or branch + /// index out of range. Used by HEALTH_DIAG H7 detection signal. + pub fn branch_noisy_sigma_mean(&self, branch_idx: usize) -> f32 { + if let Some(ref bn) = self.branching_q_network { + bn.branch_noisy_sigma_mean(branch_idx).unwrap_or(0.0) + } else { + 0.0 + } + } + /// Update target network by copying weights from main network. /// Cold path only — hot path uses fused CUDA EMA on flat buffers. /// DistributionalDueling and IQN have no per-layer copy (fused trainer only). diff --git a/crates/ml-dqn/src/noisy_layers.rs b/crates/ml-dqn/src/noisy_layers.rs index b5526979a..4b459b80e 100644 --- a/crates/ml-dqn/src/noisy_layers.rs +++ b/crates/ml-dqn/src/noisy_layers.rs @@ -339,6 +339,28 @@ impl NoisyLinear { [&self.weight_sigma, &self.bias_sigma] } + /// Task 0.6 — mean of |sigma| across weight + bias σ tensors. Used by + /// HEALTH_DIAG to track NoisyNets exploration pressure per branch + /// (H7 detection signal). Sync stream-readback; epoch-boundary only. + pub fn sigma_mean(&self) -> Result { + let n_w = self.weight_sigma.len(); + let n_b = self.bias_sigma.len(); + if n_w + n_b == 0 { + return Ok(0.0); + } + let mut h_w = vec![0.0_f32; n_w]; + let mut h_b = vec![0.0_f32; n_b]; + self.stream + .memcpy_dtoh(&self.weight_sigma, &mut h_w) + .map_err(|e| MLError::ModelError(format!("sigma_mean weight dtoh: {e}")))?; + self.stream + .memcpy_dtoh(&self.bias_sigma, &mut h_b) + .map_err(|e| MLError::ModelError(format!("sigma_mean bias dtoh: {e}")))?; + let sum_w: f64 = h_w.iter().map(|x| x.abs() as f64).sum(); + let sum_b: f64 = h_b.iter().map(|x| x.abs() as f64).sum(); + Ok(((sum_w + sum_b) / ((n_w + n_b) as f64)) as f32) + } + /// Disable noise for evaluation (use mean parameters only) pub fn disable_noise(&mut self) -> Result<(), MLError> { self.weight_epsilon = self.stream.alloc_zeros::(self.out_features * self.in_features) diff --git a/crates/ml/src/trainers/dqn/config.rs b/crates/ml/src/trainers/dqn/config.rs index 4582ac57a..e2203591a 100644 --- a/crates/ml/src/trainers/dqn/config.rs +++ b/crates/ml/src/trainers/dqn/config.rs @@ -87,6 +87,14 @@ impl DQNAgentType { self.agent.batch_hierarchical_softmax_actions(states, temperature) } + /// Task 0.6 — NoisyNets σ mean for a specific action branch + /// (0 = direction, 1 = magnitude, 2 = order, 3 = urgency). + /// Delegates through primary_head → branching network. Returns 0.0 if + /// branching not initialized. H7 detection signal for HEALTH_DIAG. + pub fn branch_noisy_sigma_mean(&self, branch_idx: usize) -> f32 { + self.agent.primary_head().branch_noisy_sigma_mean(branch_idx) + } + /// Set noise sigma scale for annealed noisy exploration pub fn set_noise_sigma_scale(&mut self, scale: f64) { self.agent.set_noise_sigma_scale(scale); diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 2b505f0b0..e730086e6 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -2182,6 +2182,18 @@ impl DQNTrainer { .map(|f| f.grad_ratio_mag_dir()) .unwrap_or(0.0); + // Task 0.6 — per-branch NoisyNets σ mean. H7 detection signal. + // 0 = direction head, 1 = magnitude head. + let (sigma_mag, sigma_dir) = { + let agent = self.agent.read().await; + ( + agent.branch_noisy_sigma_mean(1), + agent.branch_noisy_sigma_mean(0), + ) + }; + // Track 4 sigma_mean = overall NoisyNets σ across mag + dir branches. + let sigma_mean_overall = (sigma_mag + sigma_dir) * 0.5; + // Track 1 action distribution: per-magnitude usage across the epoch. // action_counts[9] layout is dir*3 + mag — Quarter = mag 0 (indices 0,3,6), // Half = mag 1 (indices 1,4,7), Full = mag 2 (indices 2,5,8). Computed @@ -2310,8 +2322,10 @@ impl DQNTrainer { dist_q, dist_h, dist_f, // Track 1 — trail (6 f32) 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, - // Track 1 — noisy/VSN/drift (6 f32) - 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, + // Track 1 — noisy/VSN/drift (6 f32): vsn_mag, vsn_dir, sigma_mag, + // sigma_dir, drift_mag, drift_dir. NoisyNets σ wired (Task 0.6); + // VSN mask + target drift remain to be wired by future Task 0.6 work. + 0.0_f32, 0.0_f32, sigma_mag, sigma_dir, 0.0_f32, 0.0_f32, // Track 1 — eval dist (3 f32): per-magnitude action distribution // observed in the most recent validation backtest. H10 signal. self.last_eval_magnitude_dist[0], @@ -2322,8 +2336,8 @@ impl DQNTrainer { // Track 3 — controllers (6 bool per-epoch fire + 1 f32 max running rate) fire_lr, fire_tau, fire_gamma, fire_clip, fire_cql, fire_cost, fire_frac, // Track 4 — explore (3 f32): ent_mag, ent_dir, sigma_mean. - // sigma_mean still stubbed (needs NoisyNets σ readback, Task 0.6). - ent_mag, ent_dir, 0.0_f32, + // sigma_mean = overall NoisyNets σ across mag + dir branches (Task 0.6). + ent_mag, ent_dir, sigma_mean_overall, ); // C1/P1: propagate health to GPU replay buffer for diversity-weighted priorities.