diff --git a/config/training/dqn-production.toml b/config/training/dqn-production.toml index 5350f94e4..35f90372c 100644 --- a/config/training/dqn-production.toml +++ b/config/training/dqn-production.toml @@ -54,7 +54,7 @@ noise_sigma = 0.1 [replay_buffer] buffer_size = 500000 min_replay_size = 1000 -per_alpha = 0.6 +per_alpha = 0.3 # was 0.6. Lower = more uniform. Dense micro-rewards have tiny TD-errors, high alpha ignores them. per_beta_start = 0.4 regime_replay_decay = 0.3 diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 40a47253b..6acd28821 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -1303,7 +1303,7 @@ impl DQNTrainer { ..Default::default() }; - let mut gpu_batch = collector.collect_experiences_gpu( + let gpu_batch = collector.collect_experiences_gpu( features_buf, targets_buf, &episode_starts, &config, ).map_err(|e| anyhow::anyhow!( "GPU zero-roundtrip collection FAILED (no CPU fallback): {e}" @@ -1312,14 +1312,11 @@ impl DQNTrainer { let count = gpu_batch.n_episodes * gpu_batch.timesteps * 2; // counterfactual doubles experiences // G13: Sharpe-aware rank-normalize rewards. - // Computes sharpe[i] = (r - mean) / std, then ranks by |sharpe|. - // Amplifies reward SNR from 0.001 to ~0.5 — counting-sort rank is outlier-resistant. - // Skip epoch 0: observed_reward_std is 0.0 until GPU monitoring computes it at epoch end. - // Raw rewards flow through epoch 0; rank normalization kicks in from epoch 1+. - if self.observed_reward_std > 1e-8 { - collector.shape_rewards(&mut gpu_batch.rewards, count, self.observed_reward_std, self.observed_reward_mean) - .map_err(|e| anyhow::anyhow!("reward rank normalize: {e}"))?; - } + // Rank normalization DISABLED — PopArt handles reward normalization in the training loop. + // Rank normalization destroys magnitude info: micro-rewards (±0.1) and trade exits (±5.0) + // get compressed to similar ranks. PopArt preserves relative magnitude, which C51 needs + // to distinguish between weak holding signals and strong trade outcomes. + // Raw rewards flow directly into the replay buffer. if let Some(ref stream) = self.cuda_stream { self.experience_done_event = Some(stream.record_event(None) @@ -1887,6 +1884,28 @@ impl DQNTrainer { ); } + // OFI feature diagnostic (every 10 epochs) + if (epoch + 1) % 10 == 1 { + if let Some(ref fused) = self.fused_ctx { + match fused.read_state_sample(0) { + Ok(sample) if sample.len() > 83 => { + let ofi_raw: &[f32] = &sample[66..74]; + let ofi_delta: &[f32] = &sample[74..82]; + let book_agg = sample[82]; + let log_dur = sample[83]; + info!( + "OFI_DIAG: raw_mean={:.4}, delta_mean={:.4}, book_agg={:.4}, log_dur={:.4}", + ofi_raw.iter().sum::() / 8.0, + ofi_delta.iter().sum::() / 8.0, + book_agg, log_dur + ); + } + Ok(_) => {} // state too short, skip + Err(_) => {} // GPU read failed, non-fatal + } + } + } + // Safety plateau detection if self.safety_loss_history.len() >= 10 { let recent_losses: Vec = self.safety_loss_history