diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 2d3962dff..a19e0ac39 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -30,6 +30,27 @@ use super::super::financials::compute_epoch_financials; use super::super::monitoring::TrainingMonitor; use super::DQNTrainer; +/// Normalized Shannon entropy of a discrete probability distribution. +/// +/// Returns the entropy divided by log(N) where N is the number of bins, so +/// the result is in [0.0, 1.0]: 0.0 = policy fully collapsed to one bin, +/// 1.0 = uniform. Inputs need not sum to exactly 1 — the function +/// re-normalizes defensively. +fn shannon_entropy_normalized(dist: &[f32]) -> f32 { + let sum: f32 = dist.iter().sum(); + if sum <= 0.0 || dist.len() < 2 { + return 0.0; + } + let mut h = 0.0_f32; + for &p in dist { + let q = p / sum; + if q > 1e-9 { + h -= q * q.ln(); + } + } + h / (dist.len() as f32).ln() +} + /// Metrics returned from `process_epoch_boundary` for downstream logging. pub(crate) struct EpochBoundaryMetrics { pub avg_loss: f32, @@ -2174,6 +2195,22 @@ impl DQNTrainer { // Cache for smoke-test accessors (magnitude_distribution test). self.last_magnitude_dist = [dist_q, dist_h, dist_f]; + // Track 4 exploration entropy — per-branch action entropy normalized + // to [0, 1]. Magnitude branch uses the [dist_q, dist_h, dist_f] above; + // direction is summed across magnitudes per direction bin. + // Layout (dir*3 + mag): dir 0 = indices 0..3, dir 1 (Flat) = 3..6, dir 2 = 6..9. + let ent_mag = shannon_entropy_normalized(&[dist_q, dist_h, dist_f]); + let (dir0, dir1, dir2) = if action_total_epoch > 0 { + let t = action_total_epoch as f32; + let d0 = (monitor.action_counts[0] + monitor.action_counts[1] + monitor.action_counts[2]) as f32 / t; + let d1 = (monitor.action_counts[3] + monitor.action_counts[4] + monitor.action_counts[5]) as f32 / t; + let d2 = (monitor.action_counts[6] + monitor.action_counts[7] + monitor.action_counts[8]) as f32 / t; + (d0, d1, d2) + } else { + (0.0_f32, 0.0_f32, 0.0_f32) + }; + let ent_dir = shannon_entropy_normalized(&[dir0, dir1, dir2]); + tracing::info!( "HEALTH_DIAG[{}]: health={:.2} components [q_gap={:.2} q_var={:.2} atoms={:.2} grad_stable={:.2} ens_agree={:.2} grad_cos={:.2} spectral={:.2}] effective [cql_alpha={:.4} iqn_budget={:.2} cql_budget={:.2} c51_budget={:.2} tau={:.5} sarsa_tau={:.2} gamma={:.3} cf_ratio={:.2}] novels [distill={} barrier={:.3} plasticity={} ib={:.3} ensemble_collapse={:.2} contrarian={} meta_q_pred={:.2}] diag [sharpe_ema={:.3} action_entropy={:.2}] gems [g12_predictive={:.4}] mag [q_full={:.3} q_half={:.3} q_quarter={:.3} var_scale={:.3} kelly_f={:.3} avg_win_ratio={:.3} grad_ratio_mag_dir={:.4} dist_q={:.3} dist_h={:.3} dist_f={:.3}] trail [fire_q={:.3} fire_h={:.3} fire_f={:.3} hold_q={:.2} hold_h={:.2} hold_f={:.2}] noisy [vsn_mag={:.3} vsn_dir={:.3} sigma_mag={:.4} sigma_dir={:.4} drift_mag={:.3} drift_dir={:.3}] eval_dist [eq={:.3} eh={:.3} ef={:.3}] reward_contrib [popart={:.3} cf={:.3} trail_r={:.3} micro={:.3} la={:.3} seg={:.3}] controller [anti_lr={} tau={} gamma={} clip={} cql={} cost={} fire_frac={:.2}] explore [ent_mag={:.2} ent_dir={:.2} sigma_mean={:.4}]", epoch, @@ -2219,8 +2256,9 @@ impl DQNTrainer { 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, 0.0_f32, // Track 3 — controllers (6 bool, 1 f32) false, false, false, false, false, false, 0.0_f32, - // Track 4 — explore (3 f32) - 0.0_f32, 0.0_f32, 0.0_f32, + // Track 4 — explore (3 f32): ent_mag, ent_dir, sigma_mean. + // sigma_mean still stubbed (needs NoisyNets σ readback, Task 0.6). + ent_mag, ent_dir, 0.0_f32, ); // C1/P1: propagate health to GPU replay buffer for diversity-weighted priorities.