From a5f23b28f0cce2fee1bf657b5bfb0ea4ac313e79 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 22 Apr 2026 01:03:17 +0200 Subject: [PATCH] fix(diag): move GPU summary download before HEALTH_DIAG emit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third root cause of the apparent magnitude collapse in smoke-test data: HEALTH_DIAG reads monitor.action_counts to compute dist_q/h/f and ent_mag/ent_dir, but the block that populated action_counts from the GPU summary ran AFTER HEALTH_DIAG. Every epoch saw all-zero counts so dist_q=dist_h=dist_f=ent_mag=ent_dir=0 in every log line, and the last_magnitude_dist cache the smoke test reads was always zeros. Moved the GPU-summary download + monitor population block up above the HEALTH_DIAG preparation block. Signal is now real per-epoch: dist_q=0.60 dist_h=0.15 dist_f=0.25 (was 0/0/0) ent_mag=0.83 ent_dir=0.89 (was 0/0) magnitude_distribution smoke test now PASSES on local RTX 3050 Ti at 20 epochs. Final: Quarter=0.598 Half=0.154 Full=0.249 — all above 5% smoke threshold. Combined with 2fb30f098 (9→12 bin cascade + magnitude-preference denominator restricted to tradable directions), this resolves the Track-1 "magnitude collapse" pathology. The collapse was never a policy-learning failure — it was a monitoring visibility + ordering bug that made every reading look like mag=0 100%. --- .../src/trainers/dqn/trainer/training_loop.rs | 101 +++++++++--------- 1 file changed, 52 insertions(+), 49 deletions(-) diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index c3c3a1f3c..78642d612 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -2294,6 +2294,58 @@ impl DQNTrainer { (0.0_f32, 0.0_f32) }; + // Download GPU monitoring summary and populate the local per-epoch + // monitor BEFORE the HEALTH_DIAG block reads `monitor.action_counts`. + // Previously this block ran after HEALTH_DIAG, so every epoch's HEALTH_DIAG + // saw zero action_counts and dist_q/h/f + ent_dir were always 0. + if let Some(ref mon) = self.gpu_monitoring { + if let Ok(summary) = mon.download_summary() { + if summary.total_experiences > 0 { + info!( + "GPU epoch summary: mean_reward={:.6}, std={:.6}, sharpe={:.3}, trades={}, actions={:?}", + summary.mean_reward, summary.reward_std, summary.sharpe_estimate, + summary.total_trades, summary.action_counts + ); + // Diagnostic: Shannon entropy of action distribution, normalized to [0, 1]. + // Replaces the deleted position_entropy_weight reward — narrowing policy + // is a signal to monitor and (optionally) trigger plasticity injection, + // not to reward directly. Populated now so THIS epoch's HEALTH_DIAG emits it. + let total_actions: usize = summary.action_counts.iter().sum(); + let entropy_norm = if total_actions > 0 { + let n_bins = summary.action_counts.len() as f32; + let ln_n = n_bins.max(2.0).ln(); + let total_f = total_actions as f32; + let mut h = 0.0_f32; + for &count in summary.action_counts.iter() { + if count > 0 { + let p = count as f32 / total_f; + h -= p * p.ln(); + } + } + (h / ln_n).clamp(0.0, 1.0) + } else { + 0.0 + }; + self.last_action_entropy = Some(entropy_norm); + monitor.track_reward(summary.mean_reward); + self.observed_reward_std = summary.reward_std; + self.observed_reward_mean = summary.mean_reward; + // Feed all 3 branch distributions from GPU into the monitor. + // These are the actual model-selected actions, not + // deterministic OrderRouter re-derivations. + for (idx, &count) in summary.action_counts.iter().enumerate() { + monitor.action_counts[idx] += count; + } + for (idx, &count) in summary.order_counts.iter().enumerate() { + monitor.order_type_counts[idx] += count; + } + for (idx, &count) in summary.urgency_counts.iter().enumerate() { + monitor.urgency_counts[idx] += count; + } + } + } + } + // Track 1 action distribution: per-magnitude usage across the epoch. // Layout: action_counts[12], exp_idx = dir*3 + mag with // dir ∈ {0=Short, 1=Hold, 2=Long, 3=Flat}. @@ -2616,55 +2668,6 @@ impl DQNTrainer { (0.0, 0.0, 0.0) }; - // Download GPU monitoring summary BEFORE epoch log so mean_reward is current. - if let Some(ref mon) = self.gpu_monitoring { - if let Ok(summary) = mon.download_summary() { - if summary.total_experiences > 0 { - info!( - "GPU epoch summary: mean_reward={:.6}, std={:.6}, sharpe={:.3}, trades={}, actions={:?}", - summary.mean_reward, summary.reward_std, summary.sharpe_estimate, - summary.total_trades, summary.action_counts - ); - // Diagnostic: Shannon entropy of action distribution, normalized to [0, 1]. - // Replaces the deleted position_entropy_weight reward — narrowing policy - // is a signal to monitor and (optionally) trigger plasticity injection, - // not to reward directly. Surfaced in next epoch's HEALTH_DIAG. - let total_actions: usize = summary.action_counts.iter().sum(); - let entropy_norm = if total_actions > 0 { - let n_bins = summary.action_counts.len() as f32; - let ln_n = n_bins.max(2.0).ln(); - let total_f = total_actions as f32; - let mut h = 0.0_f32; - for &count in summary.action_counts.iter() { - if count > 0 { - let p = count as f32 / total_f; - h -= p * p.ln(); - } - } - (h / ln_n).clamp(0.0, 1.0) - } else { - 0.0 - }; - self.last_action_entropy = Some(entropy_norm); - monitor.track_reward(summary.mean_reward); - self.observed_reward_std = summary.reward_std; - self.observed_reward_mean = summary.mean_reward; - // Feed all 3 branch distributions from GPU into the monitor. - // These are the actual model-selected actions, not - // deterministic OrderRouter re-derivations. - for (idx, &count) in summary.action_counts.iter().enumerate() { - monitor.action_counts[idx] += count; - } - for (idx, &count) in summary.order_counts.iter().enumerate() { - monitor.order_type_counts[idx] += count; - } - for (idx, &count) in summary.urgency_counts.iter().enumerate() { - monitor.urgency_counts[idx] += count; - } - } - } - } - // Log epoch end — now uses current-epoch per-trade reward from GPU monitoring. use crate::dqn::logging::AggregatedMetrics; let epoch_avg_reward = if !monitor.reward_history.is_empty() {