gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(diag): move GPU summary download before HEALTH_DIAG emit

Browse Source 
Third root cause of the apparent magnitude collapse in smoke-test data:
HEALTH_DIAG reads monitor.action_counts to compute dist_q/h/f and
ent_mag/ent_dir, but the block that populated action_counts from the
GPU summary ran AFTER HEALTH_DIAG. Every epoch saw all-zero counts so
dist_q=dist_h=dist_f=ent_mag=ent_dir=0 in every log line, and the
last_magnitude_dist cache the smoke test reads was always zeros.

Moved the GPU-summary download + monitor population block up above the
HEALTH_DIAG preparation block. Signal is now real per-epoch:

  dist_q=0.60 dist_h=0.15 dist_f=0.25  (was 0/0/0)
  ent_mag=0.83  ent_dir=0.89           (was 0/0)

magnitude_distribution smoke test now PASSES on local RTX 3050 Ti at
20 epochs. Final: Quarter=0.598 Half=0.154 Full=0.249 — all above 5%
smoke threshold.

Combined with 2fb30f098 (9→12 bin cascade + magnitude-preference
denominator restricted to tradable directions), this resolves the
Track-1 "magnitude collapse" pathology. The collapse was never a
policy-learning failure — it was a monitoring visibility + ordering
bug that made every reading look like mag=0 100%.
This commit is contained in:
jgrusewski
2026-04-22 01:03:17 +02:00
parent 2fb30f098e
commit a5f23b28f0
1 changed files with 52 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
crates/ml/src/trainers/dqn/trainer/training_loop.rs
View File

@@ -2294,6 +2294,58 @@ impl DQNTrainer {
(0.0_f32, 0.0_f32)
};
// Download GPU monitoring summary and populate the local per-epoch
// monitor BEFORE the HEALTH_DIAG block reads `monitor.action_counts`.
// Previously this block ran after HEALTH_DIAG, so every epoch's HEALTH_DIAG
// saw zero action_counts and dist_q/h/f + ent_dir were always 0.
if let Some(ref mon) = self.gpu_monitoring {
if let Ok(summary) = mon.download_summary() {
if summary.total_experiences > 0 {
info!(
"GPU epoch summary: mean_reward={:.6}, std={:.6}, sharpe={:.3}, trades={}, actions={:?}",
summary.mean_reward, summary.reward_std, summary.sharpe_estimate,
summary.total_trades, summary.action_counts
);
// Diagnostic: Shannon entropy of action distribution, normalized to [0, 1].
// Replaces the deleted position_entropy_weight reward — narrowing policy
// is a signal to monitor and (optionally) trigger plasticity injection,
// not to reward directly. Populated now so THIS epoch's HEALTH_DIAG emits it.
let total_actions: usize = summary.action_counts.iter().sum();
let entropy_norm = if total_actions > 0 {
let n_bins = summary.action_counts.len() as f32;
let ln_n = n_bins.max(2.0).ln();
let total_f = total_actions as f32;
let mut h = 0.0_f32;
for &count in summary.action_counts.iter() {
if count > 0 {
let p = count as f32 / total_f;
h -= p * p.ln();
}
}
(h / ln_n).clamp(0.0, 1.0)
} else {
0.0
};
self.last_action_entropy = Some(entropy_norm);
monitor.track_reward(summary.mean_reward);
self.observed_reward_std = summary.reward_std;
self.observed_reward_mean = summary.mean_reward;
// Feed all 3 branch distributions from GPU into the monitor.
// These are the actual model-selected actions, not
// deterministic OrderRouter re-derivations.
for (idx, &count) in summary.action_counts.iter().enumerate() {
monitor.action_counts[idx] += count;
}
for (idx, &count) in summary.order_counts.iter().enumerate() {
monitor.order_type_counts[idx] += count;
}
for (idx, &count) in summary.urgency_counts.iter().enumerate() {
monitor.urgency_counts[idx] += count;
}
}
}
}
// Track 1 action distribution: per-magnitude usage across the epoch.
// Layout: action_counts[12], exp_idx = dir*3 + mag with
// dir ∈ {0=Short, 1=Hold, 2=Long, 3=Flat}.
@@ -2616,55 +2668,6 @@ impl DQNTrainer {
(0.0, 0.0, 0.0)
};
// Download GPU monitoring summary BEFORE epoch log so mean_reward is current.
if let Some(ref mon) = self.gpu_monitoring {
if let Ok(summary) = mon.download_summary() {
if summary.total_experiences > 0 {
info!(
"GPU epoch summary: mean_reward={:.6}, std={:.6}, sharpe={:.3}, trades={}, actions={:?}",
summary.mean_reward, summary.reward_std, summary.sharpe_estimate,
summary.total_trades, summary.action_counts
);
// Diagnostic: Shannon entropy of action distribution, normalized to [0, 1].
// Replaces the deleted position_entropy_weight reward — narrowing policy
// is a signal to monitor and (optionally) trigger plasticity injection,
// not to reward directly. Surfaced in next epoch's HEALTH_DIAG.
let total_actions: usize = summary.action_counts.iter().sum();
let entropy_norm = if total_actions > 0 {
let n_bins = summary.action_counts.len() as f32;
let ln_n = n_bins.max(2.0).ln();
let total_f = total_actions as f32;
let mut h = 0.0_f32;
for &count in summary.action_counts.iter() {
if count > 0 {
let p = count as f32 / total_f;
h -= p * p.ln();
}
}
(h / ln_n).clamp(0.0, 1.0)
} else {
0.0
};
self.last_action_entropy = Some(entropy_norm);
monitor.track_reward(summary.mean_reward);
self.observed_reward_std = summary.reward_std;
self.observed_reward_mean = summary.mean_reward;
// Feed all 3 branch distributions from GPU into the monitor.
// These are the actual model-selected actions, not
// deterministic OrderRouter re-derivations.
for (idx, &count) in summary.action_counts.iter().enumerate() {
monitor.action_counts[idx] += count;
}
for (idx, &count) in summary.order_counts.iter().enumerate() {
monitor.order_type_counts[idx] += count;
}
for (idx, &count) in summary.urgency_counts.iter().enumerate() {
monitor.urgency_counts[idx] += count;
}
}
}
}
// Log epoch end — now uses current-epoch per-trade reward from GPU monitoring.
use crate::dqn::logging::AggregatedMetrics;
let epoch_avg_reward = if !monitor.reward_history.is_empty() {