diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index 045c2db25..aa62d9638 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -458,8 +458,10 @@ impl ParameterSpace for DQNParams { (0.2, 0.6), // 9: per_beta_start (linear) // Rainbow DQN extensions (6D) - (5.0, 30.0), // 10: v_range (symmetric: v_min=-v_range, v_max=+v_range) - (5.0, 30.0), // 11: v_range_mirror (UNUSED — kept for 28D compat, mirrors index 10) + // v_range: with EMA-normalized rewards in [-0.45, +0.32] and gamma~0.93/n_steps~4, + // max discounted return ≈ 0.32 * (1-0.93^4)/(1-0.93) ≈ 1.2. Range [2, 10] covers this. + (2.0, 10.0), // 10: v_range (symmetric: v_min=-v_range, v_max=+v_range) + (2.0, 10.0), // 11: v_range_mirror (UNUSED — kept for 28D compat, mirrors index 10) (0.1_f64.ln(), 1.0_f64.ln()), // 12: noisy_sigma_init (log scale) (128.0, 512.0), // 13: dueling_hidden_dim (linear, step=128) (3.0, 5.0), // 14: n_steps (raised min: n=3 is Rainbow standard) @@ -517,7 +519,7 @@ impl ParameterSpace for DQNParams { // Rainbow DQN extensions — SYMMETRIC support (prevents init bias) // Asymmetric v_min/v_max causes Q-values to initialize at midpoint ≠ 0, // creating a self-reinforcing equilibrium that freezes learning. - let v_range = x[10].clamp(5.0, 30.0); + let v_range = x[10].clamp(2.0, 10.0); let v_min = -v_range; let v_max = v_range; let noisy_sigma_init = x[12].exp().clamp(0.1, 1.0); @@ -3595,8 +3597,8 @@ mod tests { assert_eq!(bounds[7], (0.5, 2.0)); // transaction_cost_multiplier assert_eq!(bounds[8], (0.4, 0.8)); // per_alpha assert_eq!(bounds[9], (0.2, 0.6)); // per_beta_start - assert_eq!(bounds[10], (5.0, 30.0)); // v_range (symmetric support) - assert_eq!(bounds[11], (5.0, 30.0)); // v_range_mirror (unused, same as 10) + assert_eq!(bounds[10], (2.0, 10.0)); // v_range (symmetric support) + assert_eq!(bounds[11], (2.0, 10.0)); // v_range_mirror (unused, same as 10) assert_eq!(bounds[13], (128.0, 512.0)); // dueling_hidden_dim assert_eq!(bounds[14], (3.0, 5.0)); // n_steps (raised min: Rainbow standard) assert_eq!(bounds[15], (51.0, 201.0)); // num_atoms @@ -3652,7 +3654,7 @@ mod tests { let continuous = vec![ 3e-5_f64.ln(), 92.0, 0.92, 97_273_f64.ln(), 4.0, 24.77_f64.ln(), 0.03, 1.0, 0.6, 0.4, // 8-9: per_alpha, per_beta_start - 15.0, 15.0, 0.5_f64.ln(), // 10-12: v_range, v_range_mirror, noisy_sigma_init + 7.0, 7.0, 0.5_f64.ln(), // 10-12: v_range, v_range_mirror, noisy_sigma_init 256.0, 3.0, 101.0, // 13-15: dueling_hidden_dim, n_steps, num_atoms 1e-4_f64.ln(), // 16: weight_decay 0.5, 0.25, // 17-18: kelly_fractional, kelly_max_fraction @@ -3678,14 +3680,14 @@ mod tests { assert!((params.curiosity_weight - 0.0).abs() < 1e-6); // C2: fixed assert!((params.noisy_epsilon_floor - 0.0).abs() < 1e-6); // C2: fixed // Verify symmetric support - assert!((params.v_min - (-15.0)).abs() < 1e-6, "v_min should be -v_range"); - assert!((params.v_max - 15.0).abs() < 1e-6, "v_max should be +v_range"); + assert!((params.v_min - (-7.0)).abs() < 1e-6, "v_min should be -v_range"); + assert!((params.v_max - 7.0).abs() < 1e-6, "v_max should be +v_range"); // Test PER parameter bounds (min values) -- 28D let continuous_min = vec![ 2e-5_f64.ln(), 64.0, 0.88, 50_000_f64.ln(), 1.0, 10.0_f64.ln(), 0.05, 0.5, 0.4, 0.2, // per_alpha min, per_beta_start min - 5.0, 5.0, 0.1_f64.ln(), // v_range min, v_range_mirror, noisy_sigma_init + 2.0, 2.0, 0.1_f64.ln(), // v_range min, v_range_mirror, noisy_sigma_init 128.0, 3.0, 51.0, // dueling_hidden_dim, n_steps (min=3), num_atoms 1e-4_f64.ln(), // weight_decay min 0.25, 0.1, // kelly_fractional, kelly_max_fraction @@ -3705,14 +3707,14 @@ mod tests { assert!(params_min.use_dueling); assert!(params_min.use_distributional); assert!(params_min.use_noisy_nets); - assert!((params_min.v_min - (-5.0)).abs() < 1e-6, "v_min should be -v_range at min"); - assert!((params_min.v_max - 5.0).abs() < 1e-6, "v_max should be +v_range at min"); + assert!((params_min.v_min - (-2.0)).abs() < 1e-6, "v_min should be -v_range at min"); + assert!((params_min.v_max - 2.0).abs() < 1e-6, "v_max should be +v_range at min"); // Test PER parameter bounds (max values) -- 28D let continuous_max = vec![ 8e-5_f64.ln(), 512.0, 0.99, 100_000_f64.ln(), 4.0, 40.0_f64.ln(), 0.5, 2.0, 0.8, 0.6, // per_alpha max, per_beta_start max - 30.0, 30.0, 1.0_f64.ln(), // v_range max, v_range_mirror, noisy_sigma_init + 10.0, 10.0, 1.0_f64.ln(), // v_range max, v_range_mirror, noisy_sigma_init 512.0, 5.0, 201.0, // dueling_hidden_dim, n_steps, num_atoms 1e-2_f64.ln(), // weight_decay max 0.75, 0.5, // kelly_fractional, kelly_max_fraction @@ -3732,8 +3734,8 @@ mod tests { assert!(params_max.use_dueling); assert!(params_max.use_distributional); assert!(params_max.use_noisy_nets); - assert!((params_max.v_min - (-30.0)).abs() < 1e-6, "v_min should be -v_range at max"); - assert!((params_max.v_max - 30.0).abs() < 1e-6, "v_max should be +v_range at max"); + assert!((params_max.v_min - (-10.0)).abs() < 1e-6, "v_min should be -v_range at max"); + assert!((params_max.v_max - 10.0).abs() < 1e-6, "v_max should be +v_range at max"); } #[test] diff --git a/crates/ml/src/trainers/dqn/trainer.rs b/crates/ml/src/trainers/dqn/trainer.rs index d373f5025..62876240f 100644 --- a/crates/ml/src/trainers/dqn/trainer.rs +++ b/crates/ml/src/trainers/dqn/trainer.rs @@ -1216,13 +1216,6 @@ impl DQNTrainer { .to_vec1::() .map_err(|e| anyhow::anyhow!("Failed to transfer validation argmax to CPU: {}", e))?; - // Extract max Q-values using vectorized operations - let max_q_values = batch_q_values - .max(1) - .map_err(|e| anyhow::anyhow!("Failed to compute max Q-values: {}", e))? - .to_vec1::() - .map_err(|e| anyhow::anyhow!("Failed to transfer max Q-values to CPU: {}", e))?; - // Convert action indices to FactoredAction for reward calculation // DQN uses 5-exposure actions: index → ExposureLevel → OrderRouter → FactoredAction for &idx in &greedy_action_indices { @@ -1231,8 +1224,8 @@ impl DQNTrainer { actions_for_rewards.push(OrderRouter::route_default(exposure)); } - // Calculate rewards and losses (this part still needs to be sequential due to RewardFunction) - let mut total_loss = 0.0; + // Compute greedy rewards on validation data for Sharpe-based val metric + let mut val_rewards = Vec::with_capacity(sample_size); let recent_actions_vec: Vec = self.recent_actions.iter().copied().collect(); @@ -1243,22 +1236,26 @@ impl DQNTrainer { &next_states[i], &recent_actions_vec, )?; - let reward = reward_decimal.to_f32().unwrap_or(0.0); - let max_q = max_q_values[i] as f64; - - // Reward prediction error: (max_Q - reward)^2 - // NOTE: This is NOT the Bellman residual (which would be (r + gamma*Q_next - Q_pred)^2). - // It measures how well the Q-network's greedy Q-value predicts immediate rewards, - // serving as a proxy for overfitting detection — rising values indicate the - // Q-function is drifting from grounded reward signals. - let loss = (max_q - reward as f64).powi(2); - total_loss += loss; + val_rewards.push(reward_decimal.to_f32().unwrap_or(0.0) as f64); } // Restore original epsilon after evaluation self.set_epsilon(original_epsilon).await?; - Ok(total_loss / sample_size as f64) + // Compute validation Sharpe ratio (annualized, 252 trading days) + // Returns negative Sharpe as "loss" so lower = better (compatible with best-model selection) + let n = val_rewards.len() as f64; + let mean = val_rewards.iter().sum::() / n; + let variance = val_rewards.iter().map(|r| (r - mean).powi(2)).sum::() / n; + let std = variance.sqrt(); + let val_sharpe = if std > 1e-10 { + (mean / std) * (252.0_f64).sqrt() + } else { + 0.0 + }; + + // Return negative Sharpe as the "loss" (lower = better Sharpe) + Ok(-val_sharpe) } /// Get Q-values for a given state @@ -2965,6 +2962,22 @@ impl DQNTrainer { gap_mean, gap_min, gap_max ); } + + // Per-action Q-value averages (sampled from replay buffer) + // Reveals whether the agent differentiates between exposure actions + if let Some(per_action_avgs) = self.compute_per_action_q_values().await { + let names = ["S100", "S50", "Flat", "L50", "L100"]; + let parts: Vec = names + .iter() + .zip(per_action_avgs.iter()) + .map(|(n, q)| format!("{}={:.4}", n, q)) + .collect(); + info!( + "Epoch {}/{}: Per-action Q: {}", + epoch + 1, self.hyperparams.epochs, + parts.join(", ") + ); + } } // Compute validation loss @@ -4235,6 +4248,72 @@ impl DQNTrainer { Some((mean, min, max)) } + /// Compute per-action average Q-values from a sample of replay buffer states. + /// Returns `[f64; 5]` averages (one per exposure action) or `None` if insufficient data. + async fn compute_per_action_q_values(&self) -> Option<[f64; 5]> { + let agent = self.agent.read().await; + let buffer = agent.memory(); + + if buffer.len() < 10 { + return None; + } + + let sample_size = buffer.len().min(200); + let batch_sample = match buffer.sample(sample_size) { + Ok(s) => s, + Err(_) => return None, + }; + + let state_dim = agent.get_state_dim(); + let batched_states: Vec = batch_sample + .experiences + .iter() + .flat_map(|exp| { + let mut s = exp.state.clone(); + s.resize(state_dim, 0.0); + s + }) + .collect(); + + let batch_tensor = + match Tensor::from_vec(batched_states, (sample_size, state_dim), agent.device()) { + Ok(t) => match t.to_dtype(training_dtype(agent.device())) { + Ok(t) => t, + Err(_) => return None, + }, + Err(_) => return None, + }; + + let batch_q_values = match agent.forward(&batch_tensor) { + Ok(q) => q, + Err(_) => return None, + }; + + let q_2d: Vec> = match batch_q_values.to_vec2::() { + Ok(v) => v, + Err(_) => return None, + }; + + let mut sums = [0.0_f64; 5]; + let mut counts = [0_usize; 5]; + for row in &q_2d { + for (i, &v) in row.iter().enumerate().take(5) { + if v.is_finite() { + sums[i] += v as f64; + counts[i] += 1; + } + } + } + + let mut avgs = [0.0_f64; 5]; + for i in 0..5 { + if counts[i] > 0 { + avgs[i] = sums[i] / counts[i] as f64; + } + } + Some(avgs) + } + /// Get current epsilon value async fn get_epsilon(&self) -> Result { let agent = self.agent.read().await;