fix(dqn): tighten v_range, replace val_loss proxy, add per-action Q diagnostics
- v_range search bounds [5,30] → [2,10] to match EMA-normalized reward range [-0.45,+0.32] (max discounted return ≈1.2) - Replace val_loss proxy (max_Q - reward)^2 with Sharpe-based validation metric: returns -Sharpe so lower = better, economically meaningful - Add compute_per_action_q_values(): samples 200 states from replay buffer, logs per-action Q averages (S100/S50/Flat/L50/L100) at each epoch — exposes whether agent differentiates exposure actions - Remove dead max_q_values computation (leftover from old proxy) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
@@ -458,8 +458,10 @@ impl ParameterSpace for DQNParams {
|
||||
(0.2, 0.6), // 9: per_beta_start (linear)
|
||||
|
||||
// Rainbow DQN extensions (6D)
|
||||
(5.0, 30.0), // 10: v_range (symmetric: v_min=-v_range, v_max=+v_range)
|
||||
(5.0, 30.0), // 11: v_range_mirror (UNUSED — kept for 28D compat, mirrors index 10)
|
||||
// v_range: with EMA-normalized rewards in [-0.45, +0.32] and gamma~0.93/n_steps~4,
|
||||
// max discounted return ≈ 0.32 * (1-0.93^4)/(1-0.93) ≈ 1.2. Range [2, 10] covers this.
|
||||
(2.0, 10.0), // 10: v_range (symmetric: v_min=-v_range, v_max=+v_range)
|
||||
(2.0, 10.0), // 11: v_range_mirror (UNUSED — kept for 28D compat, mirrors index 10)
|
||||
(0.1_f64.ln(), 1.0_f64.ln()), // 12: noisy_sigma_init (log scale)
|
||||
(128.0, 512.0), // 13: dueling_hidden_dim (linear, step=128)
|
||||
(3.0, 5.0), // 14: n_steps (raised min: n=3 is Rainbow standard)
|
||||
@@ -517,7 +519,7 @@ impl ParameterSpace for DQNParams {
|
||||
// Rainbow DQN extensions — SYMMETRIC support (prevents init bias)
|
||||
// Asymmetric v_min/v_max causes Q-values to initialize at midpoint ≠ 0,
|
||||
// creating a self-reinforcing equilibrium that freezes learning.
|
||||
let v_range = x[10].clamp(5.0, 30.0);
|
||||
let v_range = x[10].clamp(2.0, 10.0);
|
||||
let v_min = -v_range;
|
||||
let v_max = v_range;
|
||||
let noisy_sigma_init = x[12].exp().clamp(0.1, 1.0);
|
||||
@@ -3595,8 +3597,8 @@ mod tests {
|
||||
assert_eq!(bounds[7], (0.5, 2.0)); // transaction_cost_multiplier
|
||||
assert_eq!(bounds[8], (0.4, 0.8)); // per_alpha
|
||||
assert_eq!(bounds[9], (0.2, 0.6)); // per_beta_start
|
||||
assert_eq!(bounds[10], (5.0, 30.0)); // v_range (symmetric support)
|
||||
assert_eq!(bounds[11], (5.0, 30.0)); // v_range_mirror (unused, same as 10)
|
||||
assert_eq!(bounds[10], (2.0, 10.0)); // v_range (symmetric support)
|
||||
assert_eq!(bounds[11], (2.0, 10.0)); // v_range_mirror (unused, same as 10)
|
||||
assert_eq!(bounds[13], (128.0, 512.0)); // dueling_hidden_dim
|
||||
assert_eq!(bounds[14], (3.0, 5.0)); // n_steps (raised min: Rainbow standard)
|
||||
assert_eq!(bounds[15], (51.0, 201.0)); // num_atoms
|
||||
@@ -3652,7 +3654,7 @@ mod tests {
|
||||
let continuous = vec![
|
||||
3e-5_f64.ln(), 92.0, 0.92, 97_273_f64.ln(), 4.0, 24.77_f64.ln(), 0.03, 1.0,
|
||||
0.6, 0.4, // 8-9: per_alpha, per_beta_start
|
||||
15.0, 15.0, 0.5_f64.ln(), // 10-12: v_range, v_range_mirror, noisy_sigma_init
|
||||
7.0, 7.0, 0.5_f64.ln(), // 10-12: v_range, v_range_mirror, noisy_sigma_init
|
||||
256.0, 3.0, 101.0, // 13-15: dueling_hidden_dim, n_steps, num_atoms
|
||||
1e-4_f64.ln(), // 16: weight_decay
|
||||
0.5, 0.25, // 17-18: kelly_fractional, kelly_max_fraction
|
||||
@@ -3678,14 +3680,14 @@ mod tests {
|
||||
assert!((params.curiosity_weight - 0.0).abs() < 1e-6); // C2: fixed
|
||||
assert!((params.noisy_epsilon_floor - 0.0).abs() < 1e-6); // C2: fixed
|
||||
// Verify symmetric support
|
||||
assert!((params.v_min - (-15.0)).abs() < 1e-6, "v_min should be -v_range");
|
||||
assert!((params.v_max - 15.0).abs() < 1e-6, "v_max should be +v_range");
|
||||
assert!((params.v_min - (-7.0)).abs() < 1e-6, "v_min should be -v_range");
|
||||
assert!((params.v_max - 7.0).abs() < 1e-6, "v_max should be +v_range");
|
||||
|
||||
// Test PER parameter bounds (min values) -- 28D
|
||||
let continuous_min = vec![
|
||||
2e-5_f64.ln(), 64.0, 0.88, 50_000_f64.ln(), 1.0, 10.0_f64.ln(), 0.05, 0.5,
|
||||
0.4, 0.2, // per_alpha min, per_beta_start min
|
||||
5.0, 5.0, 0.1_f64.ln(), // v_range min, v_range_mirror, noisy_sigma_init
|
||||
2.0, 2.0, 0.1_f64.ln(), // v_range min, v_range_mirror, noisy_sigma_init
|
||||
128.0, 3.0, 51.0, // dueling_hidden_dim, n_steps (min=3), num_atoms
|
||||
1e-4_f64.ln(), // weight_decay min
|
||||
0.25, 0.1, // kelly_fractional, kelly_max_fraction
|
||||
@@ -3705,14 +3707,14 @@ mod tests {
|
||||
assert!(params_min.use_dueling);
|
||||
assert!(params_min.use_distributional);
|
||||
assert!(params_min.use_noisy_nets);
|
||||
assert!((params_min.v_min - (-5.0)).abs() < 1e-6, "v_min should be -v_range at min");
|
||||
assert!((params_min.v_max - 5.0).abs() < 1e-6, "v_max should be +v_range at min");
|
||||
assert!((params_min.v_min - (-2.0)).abs() < 1e-6, "v_min should be -v_range at min");
|
||||
assert!((params_min.v_max - 2.0).abs() < 1e-6, "v_max should be +v_range at min");
|
||||
|
||||
// Test PER parameter bounds (max values) -- 28D
|
||||
let continuous_max = vec![
|
||||
8e-5_f64.ln(), 512.0, 0.99, 100_000_f64.ln(), 4.0, 40.0_f64.ln(), 0.5, 2.0,
|
||||
0.8, 0.6, // per_alpha max, per_beta_start max
|
||||
30.0, 30.0, 1.0_f64.ln(), // v_range max, v_range_mirror, noisy_sigma_init
|
||||
10.0, 10.0, 1.0_f64.ln(), // v_range max, v_range_mirror, noisy_sigma_init
|
||||
512.0, 5.0, 201.0, // dueling_hidden_dim, n_steps, num_atoms
|
||||
1e-2_f64.ln(), // weight_decay max
|
||||
0.75, 0.5, // kelly_fractional, kelly_max_fraction
|
||||
@@ -3732,8 +3734,8 @@ mod tests {
|
||||
assert!(params_max.use_dueling);
|
||||
assert!(params_max.use_distributional);
|
||||
assert!(params_max.use_noisy_nets);
|
||||
assert!((params_max.v_min - (-30.0)).abs() < 1e-6, "v_min should be -v_range at max");
|
||||
assert!((params_max.v_max - 30.0).abs() < 1e-6, "v_max should be +v_range at max");
|
||||
assert!((params_max.v_min - (-10.0)).abs() < 1e-6, "v_min should be -v_range at max");
|
||||
assert!((params_max.v_max - 10.0).abs() < 1e-6, "v_max should be +v_range at max");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1216,13 +1216,6 @@ impl DQNTrainer {
|
||||
.to_vec1::<u32>()
|
||||
.map_err(|e| anyhow::anyhow!("Failed to transfer validation argmax to CPU: {}", e))?;
|
||||
|
||||
// Extract max Q-values using vectorized operations
|
||||
let max_q_values = batch_q_values
|
||||
.max(1)
|
||||
.map_err(|e| anyhow::anyhow!("Failed to compute max Q-values: {}", e))?
|
||||
.to_vec1::<f32>()
|
||||
.map_err(|e| anyhow::anyhow!("Failed to transfer max Q-values to CPU: {}", e))?;
|
||||
|
||||
// Convert action indices to FactoredAction for reward calculation
|
||||
// DQN uses 5-exposure actions: index → ExposureLevel → OrderRouter → FactoredAction
|
||||
for &idx in &greedy_action_indices {
|
||||
@@ -1231,8 +1224,8 @@ impl DQNTrainer {
|
||||
actions_for_rewards.push(OrderRouter::route_default(exposure));
|
||||
}
|
||||
|
||||
// Calculate rewards and losses (this part still needs to be sequential due to RewardFunction)
|
||||
let mut total_loss = 0.0;
|
||||
// Compute greedy rewards on validation data for Sharpe-based val metric
|
||||
let mut val_rewards = Vec::with_capacity(sample_size);
|
||||
let recent_actions_vec: Vec<FactoredAction> =
|
||||
self.recent_actions.iter().copied().collect();
|
||||
|
||||
@@ -1243,22 +1236,26 @@ impl DQNTrainer {
|
||||
&next_states[i],
|
||||
&recent_actions_vec,
|
||||
)?;
|
||||
let reward = reward_decimal.to_f32().unwrap_or(0.0);
|
||||
let max_q = max_q_values[i] as f64;
|
||||
|
||||
// Reward prediction error: (max_Q - reward)^2
|
||||
// NOTE: This is NOT the Bellman residual (which would be (r + gamma*Q_next - Q_pred)^2).
|
||||
// It measures how well the Q-network's greedy Q-value predicts immediate rewards,
|
||||
// serving as a proxy for overfitting detection — rising values indicate the
|
||||
// Q-function is drifting from grounded reward signals.
|
||||
let loss = (max_q - reward as f64).powi(2);
|
||||
total_loss += loss;
|
||||
val_rewards.push(reward_decimal.to_f32().unwrap_or(0.0) as f64);
|
||||
}
|
||||
|
||||
// Restore original epsilon after evaluation
|
||||
self.set_epsilon(original_epsilon).await?;
|
||||
|
||||
Ok(total_loss / sample_size as f64)
|
||||
// Compute validation Sharpe ratio (annualized, 252 trading days)
|
||||
// Returns negative Sharpe as "loss" so lower = better (compatible with best-model selection)
|
||||
let n = val_rewards.len() as f64;
|
||||
let mean = val_rewards.iter().sum::<f64>() / n;
|
||||
let variance = val_rewards.iter().map(|r| (r - mean).powi(2)).sum::<f64>() / n;
|
||||
let std = variance.sqrt();
|
||||
let val_sharpe = if std > 1e-10 {
|
||||
(mean / std) * (252.0_f64).sqrt()
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
// Return negative Sharpe as the "loss" (lower = better Sharpe)
|
||||
Ok(-val_sharpe)
|
||||
}
|
||||
|
||||
/// Get Q-values for a given state
|
||||
@@ -2965,6 +2962,22 @@ impl DQNTrainer {
|
||||
gap_mean, gap_min, gap_max
|
||||
);
|
||||
}
|
||||
|
||||
// Per-action Q-value averages (sampled from replay buffer)
|
||||
// Reveals whether the agent differentiates between exposure actions
|
||||
if let Some(per_action_avgs) = self.compute_per_action_q_values().await {
|
||||
let names = ["S100", "S50", "Flat", "L50", "L100"];
|
||||
let parts: Vec<String> = names
|
||||
.iter()
|
||||
.zip(per_action_avgs.iter())
|
||||
.map(|(n, q)| format!("{}={:.4}", n, q))
|
||||
.collect();
|
||||
info!(
|
||||
"Epoch {}/{}: Per-action Q: {}",
|
||||
epoch + 1, self.hyperparams.epochs,
|
||||
parts.join(", ")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Compute validation loss
|
||||
@@ -4235,6 +4248,72 @@ impl DQNTrainer {
|
||||
Some((mean, min, max))
|
||||
}
|
||||
|
||||
/// Compute per-action average Q-values from a sample of replay buffer states.
|
||||
/// Returns `[f64; 5]` averages (one per exposure action) or `None` if insufficient data.
|
||||
async fn compute_per_action_q_values(&self) -> Option<[f64; 5]> {
|
||||
let agent = self.agent.read().await;
|
||||
let buffer = agent.memory();
|
||||
|
||||
if buffer.len() < 10 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let sample_size = buffer.len().min(200);
|
||||
let batch_sample = match buffer.sample(sample_size) {
|
||||
Ok(s) => s,
|
||||
Err(_) => return None,
|
||||
};
|
||||
|
||||
let state_dim = agent.get_state_dim();
|
||||
let batched_states: Vec<f32> = batch_sample
|
||||
.experiences
|
||||
.iter()
|
||||
.flat_map(|exp| {
|
||||
let mut s = exp.state.clone();
|
||||
s.resize(state_dim, 0.0);
|
||||
s
|
||||
})
|
||||
.collect();
|
||||
|
||||
let batch_tensor =
|
||||
match Tensor::from_vec(batched_states, (sample_size, state_dim), agent.device()) {
|
||||
Ok(t) => match t.to_dtype(training_dtype(agent.device())) {
|
||||
Ok(t) => t,
|
||||
Err(_) => return None,
|
||||
},
|
||||
Err(_) => return None,
|
||||
};
|
||||
|
||||
let batch_q_values = match agent.forward(&batch_tensor) {
|
||||
Ok(q) => q,
|
||||
Err(_) => return None,
|
||||
};
|
||||
|
||||
let q_2d: Vec<Vec<f32>> = match batch_q_values.to_vec2::<f32>() {
|
||||
Ok(v) => v,
|
||||
Err(_) => return None,
|
||||
};
|
||||
|
||||
let mut sums = [0.0_f64; 5];
|
||||
let mut counts = [0_usize; 5];
|
||||
for row in &q_2d {
|
||||
for (i, &v) in row.iter().enumerate().take(5) {
|
||||
if v.is_finite() {
|
||||
sums[i] += v as f64;
|
||||
counts[i] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut avgs = [0.0_f64; 5];
|
||||
for i in 0..5 {
|
||||
if counts[i] > 0 {
|
||||
avgs[i] = sums[i] / counts[i] as f64;
|
||||
}
|
||||
}
|
||||
Some(avgs)
|
||||
}
|
||||
|
||||
/// Get current epsilon value
|
||||
async fn get_epsilon(&self) -> Result<f64> {
|
||||
let agent = self.agent.read().await;
|
||||
|
||||
Reference in New Issue
Block a user