diag(policy-quality): NoisyNets σ per branch — partial Task 0.6, completes 0.10
Wires Track 1 noisy_mag/noisy_dir + Track 4 sigma_mean fields with the
mean |sigma| across each action branch's NoisyLinear fc + out layers.
Branch 0 = direction, Branch 1 = magnitude. H7 detection signal — if
magnitude branch has 2x larger σ than direction, NoisyNets noise is
dominating the magnitude head's effective signal.
Cross-crate API chain (no shortcuts):
* ml-dqn::noisy_layers::NoisyLinear::sigma_mean() -> mean |weight_σ| + |bias_σ|
* ml-dqn::branching::BranchingDuelingQNetwork::branch_noisy_sigma_mean(idx)
averages fc + out σ for the named branch
* ml-dqn::dqn::DQN::branch_noisy_sigma_mean(idx) — None-tolerant proxy
* ml::trainers::dqn::config::DQNAgentType::branch_noisy_sigma_mean(idx)
delegates through primary_head
* HEALTH_DIAG reads via self.agent.read().await at epoch boundary
Pinned-readback pattern not used here — NoisyLinear weights live in
candle-managed CudaSlices, not flat trainer params buffer. Per-call
dtoh of ~256 + 768 floats × 2 layers × 4 branches = ~8KB total per
epoch. Negligible.
Track 0.10 (exploration entropy + sigma_mean) is now COMPLETE — the
sigma_mean field that was previously stubbed is now real.
Task 0.6 remains partial: VSN mask (vsn_mag, vsn_dir) and target drift
(drift_mag, drift_dir) still stubbed — they require separate accessors
on different layer types (VSN module + target_params_buf reductions).
This commit is contained in:
@@ -382,6 +382,21 @@ impl BranchingDuelingQNetwork {
|
||||
}
|
||||
|
||||
/// Resample noise in all `NoisyLinear` heads.
|
||||
/// Task 0.6 — mean |sigma| for a specific branch's NoisyNets layers
|
||||
/// (fc + out combined). branch_idx 0 = direction, 1 = magnitude, etc.
|
||||
/// Returns 0.0 if branch_idx is out of range. H7 detection signal —
|
||||
/// if magnitude branch has 2x larger σ than direction, NoisyNets
|
||||
/// noise is dominating the magnitude head's effective signal.
|
||||
pub fn branch_noisy_sigma_mean(&self, branch_idx: usize) -> Result<f32, MLError> {
|
||||
if branch_idx >= self.branch_fcs.len() {
|
||||
return Ok(0.0);
|
||||
}
|
||||
let fc_sigma = self.branch_fcs[branch_idx].sigma_mean()?;
|
||||
let out_sigma = self.branch_outs[branch_idx].sigma_mean()?;
|
||||
// Equal-weight average — fc and out have similar param counts.
|
||||
Ok((fc_sigma + out_sigma) * 0.5)
|
||||
}
|
||||
|
||||
///
|
||||
/// Must be called before each training forward pass.
|
||||
pub fn reset_noise(&mut self) -> Result<(), MLError> {
|
||||
|
||||
@@ -2388,6 +2388,19 @@ impl DQN {
|
||||
}
|
||||
}
|
||||
|
||||
/// Task 0.6 — NoisyNets σ mean for a specific action branch
|
||||
/// (0 = direction, 1 = magnitude, 2 = order, 3 = urgency). Mean is over
|
||||
/// the branch's fc + out NoisyLinear layers' weight_sigma + bias_sigma
|
||||
/// tensors. Returns 0.0 if branching network not initialized or branch
|
||||
/// index out of range. Used by HEALTH_DIAG H7 detection signal.
|
||||
pub fn branch_noisy_sigma_mean(&self, branch_idx: usize) -> f32 {
|
||||
if let Some(ref bn) = self.branching_q_network {
|
||||
bn.branch_noisy_sigma_mean(branch_idx).unwrap_or(0.0)
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
}
|
||||
|
||||
/// Update target network by copying weights from main network.
|
||||
/// Cold path only — hot path uses fused CUDA EMA on flat buffers.
|
||||
/// DistributionalDueling and IQN have no per-layer copy (fused trainer only).
|
||||
|
||||
@@ -339,6 +339,28 @@ impl NoisyLinear {
|
||||
[&self.weight_sigma, &self.bias_sigma]
|
||||
}
|
||||
|
||||
/// Task 0.6 — mean of |sigma| across weight + bias σ tensors. Used by
|
||||
/// HEALTH_DIAG to track NoisyNets exploration pressure per branch
|
||||
/// (H7 detection signal). Sync stream-readback; epoch-boundary only.
|
||||
pub fn sigma_mean(&self) -> Result<f32, MLError> {
|
||||
let n_w = self.weight_sigma.len();
|
||||
let n_b = self.bias_sigma.len();
|
||||
if n_w + n_b == 0 {
|
||||
return Ok(0.0);
|
||||
}
|
||||
let mut h_w = vec![0.0_f32; n_w];
|
||||
let mut h_b = vec![0.0_f32; n_b];
|
||||
self.stream
|
||||
.memcpy_dtoh(&self.weight_sigma, &mut h_w)
|
||||
.map_err(|e| MLError::ModelError(format!("sigma_mean weight dtoh: {e}")))?;
|
||||
self.stream
|
||||
.memcpy_dtoh(&self.bias_sigma, &mut h_b)
|
||||
.map_err(|e| MLError::ModelError(format!("sigma_mean bias dtoh: {e}")))?;
|
||||
let sum_w: f64 = h_w.iter().map(|x| x.abs() as f64).sum();
|
||||
let sum_b: f64 = h_b.iter().map(|x| x.abs() as f64).sum();
|
||||
Ok(((sum_w + sum_b) / ((n_w + n_b) as f64)) as f32)
|
||||
}
|
||||
|
||||
/// Disable noise for evaluation (use mean parameters only)
|
||||
pub fn disable_noise(&mut self) -> Result<(), MLError> {
|
||||
self.weight_epsilon = self.stream.alloc_zeros::<f32>(self.out_features * self.in_features)
|
||||
|
||||
@@ -87,6 +87,14 @@ impl DQNAgentType {
|
||||
self.agent.batch_hierarchical_softmax_actions(states, temperature)
|
||||
}
|
||||
|
||||
/// Task 0.6 — NoisyNets σ mean for a specific action branch
|
||||
/// (0 = direction, 1 = magnitude, 2 = order, 3 = urgency).
|
||||
/// Delegates through primary_head → branching network. Returns 0.0 if
|
||||
/// branching not initialized. H7 detection signal for HEALTH_DIAG.
|
||||
pub fn branch_noisy_sigma_mean(&self, branch_idx: usize) -> f32 {
|
||||
self.agent.primary_head().branch_noisy_sigma_mean(branch_idx)
|
||||
}
|
||||
|
||||
/// Set noise sigma scale for annealed noisy exploration
|
||||
pub fn set_noise_sigma_scale(&mut self, scale: f64) {
|
||||
self.agent.set_noise_sigma_scale(scale);
|
||||
|
||||
@@ -2182,6 +2182,18 @@ impl DQNTrainer {
|
||||
.map(|f| f.grad_ratio_mag_dir())
|
||||
.unwrap_or(0.0);
|
||||
|
||||
// Task 0.6 — per-branch NoisyNets σ mean. H7 detection signal.
|
||||
// 0 = direction head, 1 = magnitude head.
|
||||
let (sigma_mag, sigma_dir) = {
|
||||
let agent = self.agent.read().await;
|
||||
(
|
||||
agent.branch_noisy_sigma_mean(1),
|
||||
agent.branch_noisy_sigma_mean(0),
|
||||
)
|
||||
};
|
||||
// Track 4 sigma_mean = overall NoisyNets σ across mag + dir branches.
|
||||
let sigma_mean_overall = (sigma_mag + sigma_dir) * 0.5;
|
||||
|
||||
// Track 1 action distribution: per-magnitude usage across the epoch.
|
||||
// action_counts[9] layout is dir*3 + mag — Quarter = mag 0 (indices 0,3,6),
|
||||
// Half = mag 1 (indices 1,4,7), Full = mag 2 (indices 2,5,8). Computed
|
||||
@@ -2310,8 +2322,10 @@ impl DQNTrainer {
|
||||
dist_q, dist_h, dist_f,
|
||||
// Track 1 — trail (6 f32)
|
||||
0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32,
|
||||
// Track 1 — noisy/VSN/drift (6 f32)
|
||||
0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32,
|
||||
// Track 1 — noisy/VSN/drift (6 f32): vsn_mag, vsn_dir, sigma_mag,
|
||||
// sigma_dir, drift_mag, drift_dir. NoisyNets σ wired (Task 0.6);
|
||||
// VSN mask + target drift remain to be wired by future Task 0.6 work.
|
||||
0.0_f32, 0.0_f32, sigma_mag, sigma_dir, 0.0_f32, 0.0_f32,
|
||||
// Track 1 — eval dist (3 f32): per-magnitude action distribution
|
||||
// observed in the most recent validation backtest. H10 signal.
|
||||
self.last_eval_magnitude_dist[0],
|
||||
@@ -2322,8 +2336,8 @@ impl DQNTrainer {
|
||||
// Track 3 — controllers (6 bool per-epoch fire + 1 f32 max running rate)
|
||||
fire_lr, fire_tau, fire_gamma, fire_clip, fire_cql, fire_cost, fire_frac,
|
||||
// Track 4 — explore (3 f32): ent_mag, ent_dir, sigma_mean.
|
||||
// sigma_mean still stubbed (needs NoisyNets σ readback, Task 0.6).
|
||||
ent_mag, ent_dir, 0.0_f32,
|
||||
// sigma_mean = overall NoisyNets σ across mag + dir branches (Task 0.6).
|
||||
ent_mag, ent_dir, sigma_mean_overall,
|
||||
);
|
||||
|
||||
// C1/P1: propagate health to GPU replay buffer for diversity-weighted priorities.
|
||||
|
||||
Reference in New Issue
Block a user