gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ml): DQN hyperopt overhaul — C2 triple exploration + eval 5-action compat

Browse Source 
Critical fixes:
- hyperopt backtest: ExposureLevel::from_index() replaces FactoredAction::from_index()
  which mapped all DQN indices 0-4 to Short100 (every trial ran all-short)
- evaluate_baseline: same fix + DQN num_actions default 5, PPO hardcoded 45
- simulate_chunk_trades: is_dqn dispatch for DQN vs PPO action decoding

C2 triple exploration stacking:
- Removed count bonus UCB from Q-value computation in both batch paths
  (noisy nets are sole exploration mechanism)
- Narrowed noisy_epsilon_floor from [0.02, 0.10] to [0.0, 0.05], default 0.0
- Removed count_bonus_coefficient from search space (30D → 29D)
- Count bonus module kept for diversity metrics tracking only

Smoke tests: 6 new tests verifying 5-action space, 29D search space,
epsilon floor defaults, count bonus coefficient fixed at 0.1

2735 tests passed, 0 clippy warnings.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-06 15:29:02 +01:00
parent e3c3cf0fa3
commit 5ace5dd24f
3 changed files with 162 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
crates/ml/examples/evaluate_baseline.rs
View File

@@ -42,7 +42,7 @@ use tracing::{error, info, warn};
use common::metrics::{server as metrics_server, training_metrics as tm};
use ml::common::action::{ExposureLevel, FactoredAction, OrderType as ActionOrderType, Urgency};
use ml::dqn::{DQNConfig, DQN};
use ml::dqn::{DQNConfig, OrderRouter, DQN};
#[allow(unreachable_pub)]
mod baseline_common;
@@ -92,8 +92,8 @@ struct Args {
#[arg(long, default_value_t = 54)]
feature_dim: usize,
/// Number of actions for the DQN/PPO action space (45 = 5 exposure x 3 order x 3 urgency)
#[arg(long, default_value_t = 45)]
/// Number of actions (5 exposure levels for DQN, pass --num-actions 45 for PPO)
#[arg(long, default_value_t = 5)]
num_actions: usize,
/// Walk-forward: initial training window in months
@@ -459,15 +459,26 @@ fn simulate_chunk_trades(
returns: &mut Vec<f64>,
action_counts: &mut [usize; 3],
model_name: &str,
is_dqn: bool,
) {
for (i, &action_idx) in action_indices.iter().enumerate() {
let bar_idx = chunk_start + i;
let action = match FactoredAction::from_index(action_idx) {
Ok(fa) => fa,
Err(e) => {
warn!(" [{}] from_index({}) error at bar {}: {}", model_name, action_idx, bar_idx, e);
FactoredAction::new(ExposureLevel::Flat, ActionOrderType::Market, Urgency::Normal)
let action = if is_dqn {
match ExposureLevel::from_index(action_idx) {
Ok(exposure) => OrderRouter::route_default(exposure),
Err(e) => {
warn!(" [{}] exposure_from_index({}) error at bar {}: {}", model_name, action_idx, bar_idx, e);
FactoredAction::new(ExposureLevel::Flat, ActionOrderType::Market, Urgency::Normal)
}
}
} else {
match FactoredAction::from_index(action_idx) {
Ok(fa) => fa,
Err(e) => {
warn!(" [{}] from_index({}) error at bar {}: {}", model_name, action_idx, bar_idx, e);
FactoredAction::new(ExposureLevel::Flat, ActionOrderType::Market, Urgency::Normal)
}
}
};
@@ -673,7 +684,7 @@ fn evaluate_dqn_fold(
// Simulate trade and update portfolio state (B3: state synced per bar)
simulate_chunk_trades(
&action_indices, bar_idx, test_bars, args,
&mut portfolio, &mut returns, &mut action_counts, "DQN",
&mut portfolio, &mut returns, &mut action_counts, "DQN", true,
);
}
@@ -716,11 +727,11 @@ fn evaluate_ppo_fold(
);
}
// Create PPO config matching training
// Create PPO config matching training — PPO always uses 45 factored actions
#[allow(clippy::integer_division)]
let config = PPOConfig {
state_dim: args.feature_dim,
num_actions: args.num_actions,
num_actions: 45,
policy_hidden_dims: {
let base = hp_usize(hp, "hidden_dim_base").unwrap_or(256);
let align = |x: usize| x.div_ceil(8) * 8; // tensor-core alignment (must match PpoTrainer)
@@ -823,7 +834,7 @@ fn evaluate_ppo_fold(
// 3. Sequential trade simulation on CPU — updates portfolio state
simulate_chunk_trades(
&action_indices, chunk_start, test_bars, args,
&mut portfolio, &mut returns, &mut action_counts, "PPO",
&mut portfolio, &mut returns, &mut action_counts, "PPO", false,
);
}

105
crates/ml/src/hyperopt/adapters/dqn.rs
View File

@@ -3,7 +3,7 @@
//! This module provides a production-ready adapter for optimizing DQN
//! hyperparameters using the generic optimization framework. It implements:
//!
//! - 30D continuous parameter space with log-scale handling
//! - 29D continuous parameter space with log-scale handling
//! - Training wrapper that integrates with existing DQN pipeline
//! - Backtest-based objective using Sharpe ratio (production metric)
//! - Prioritized Experience Replay (PER) tuning for Rainbow DQN performance (+25-40% convergence speed)
@@ -153,7 +153,7 @@ const TRIAL_VRAM_MB: f64 = 7000.0;
/// Corrected: was 0.02 (20KB) — 100x too high
const MB_PER_SAMPLE: f64 = 0.0005;
/// DQN hyperparameter space (31D continuous - WAVE 11 with Rainbow booleans hardcoded to TRUE)
/// DQN hyperparameter space (29D continuous - C2c: count_bonus_coefficient removed from search)
///
/// Defines the hyperparameters to optimize for DQN training:
/// **Base Parameters (11D from Wave 1-2)**:
@@ -462,7 +462,7 @@ impl Default for DQNParams {
num_quantiles: 32,
qr_kappa: 1.0,
hidden_dim_base: 256, // Conservative default (backward compatible)
noisy_epsilon_floor: 0.05, // Minimum random exploration with noisy nets
noisy_epsilon_floor: 0.0, // C2: Noisy nets provide exploration; no epsilon floor during training
count_bonus_coefficient: 0.1, // UCB exploration bonus
cql_alpha: 0.1, // CQL regularization: mild penalty on OOD Q-values
eval_softmax_temp: 0.1, // Default: nearly greedy (hyperopt will tune)
@@ -473,7 +473,7 @@ impl Default for DQNParams {
impl ParameterSpace for DQNParams {
fn continuous_bounds() -> Vec<(f64, f64)> {
// 30D search space (C3: hold_penalty_weight removed — hold reward is neutral 0.0)
// 29D search space (C3: hold_penalty_weight removed, C2c: count_bonus_coefficient removed)
// 16 exotic parameters fixed to validated defaults in from_continuous()
// This makes PSO (20 particles) and TPE dramatically more effective
vec![
@@ -515,7 +515,7 @@ impl ParameterSpace for DQNParams {
(256.0, 4096.0), // 22: hidden_dim_base (linear, step=256)
// Exploration anti-collapse (1D)
(0.02, 0.10), // 23: noisy_epsilon_floor (linear)
(0.0, 0.05), // 23: noisy_epsilon_floor (linear, 0.0=noisy-nets-only)
// CQL regularization (1D)
(0.0, 0.5), // 24: cql_alpha (linear)
@@ -527,21 +527,20 @@ impl ParameterSpace for DQNParams {
(0.0, 2.0), // 26: lr_decay_type (discrete: 0=constant, 1=linear, 2=cosine)
(2.0, 6.0), // 27: min_epochs_before_stopping (discrete)
(1.1, 2.0), // 28: minimum_profit_factor (linear)
// Count-based exploration bonus (1D — kept for diversity metrics)
(0.05, 1.0), // 29: count_bonus_coefficient (linear)
// C2c: count_bonus_coefficient removed — count bonus no longer modifies Q-values
]
}
fn from_continuous(x: &[f64]) -> Result<Self, MLError> {
if x.len() != 30 {
if x.len() != 29 {
return Err(MLError::ConfigError {
reason: format!("Expected 30 continuous parameters, got {}", x.len()),
reason: format!("Expected 29 continuous parameters, got {}", x.len()),
});
}
// === 30 TUNED parameters (from search space) ===
// === 29 TUNED parameters (from search space) ===
// C3: hold_penalty_weight removed — hold reward is neutral 0.0
// C2c: count_bonus_coefficient removed — count bonus no longer modifies Q-values
let learning_rate = x[0].exp();
let mut batch_size = x[1].round().max(64.0) as usize;
let buffer_size = x[3].exp().round().max(50_000.0) as usize;
@@ -580,7 +579,7 @@ impl ParameterSpace for DQNParams {
let hidden_dim_base = ((x[22].round() / 256.0).round() * 256.0).clamp(256.0, 4096.0) as usize;
// Exploration anti-collapse
let noisy_epsilon_floor = x[23].clamp(0.02, 0.10);
let noisy_epsilon_floor = x[23].clamp(0.0, 0.05);
// CQL regularization
let cql_alpha = x[24].clamp(0.0, 0.5);
@@ -593,9 +592,6 @@ impl ParameterSpace for DQNParams {
let min_epochs_before_stopping = x[27].round().clamp(2.0, 6.0) as usize;
let minimum_profit_factor = x[28].clamp(1.1, 2.0);
// Count-based exploration bonus
let count_bonus_coefficient = x[29].clamp(0.05, 1.0);
// === 15 FIXED parameters (validated defaults, removed from search) ===
let kelly_min_trades: usize = 20;
let ensemble_size = 5.0;
@@ -678,7 +674,7 @@ impl ParameterSpace for DQNParams {
qr_kappa,
hidden_dim_base,
noisy_epsilon_floor,
count_bonus_coefficient,
count_bonus_coefficient: 0.1, // C2c: fixed default (not in search space)
cql_alpha,
eval_softmax_temp,
min_epochs_before_stopping,
@@ -688,7 +684,7 @@ impl ParameterSpace for DQNParams {
}
fn to_continuous(&self) -> Vec<f64> {
// 30D search space — C3: hold_penalty_weight removed
// 29D search space — C3: hold_penalty_weight removed, C2c: count_bonus_coefficient removed
vec![
self.learning_rate.ln(), // 0
self.batch_size as f64, // 1
@@ -719,12 +715,11 @@ impl ParameterSpace for DQNParams {
self.lr_decay_type, // 26
self.min_epochs_before_stopping as f64, // 27
self.minimum_profit_factor, // 28
self.count_bonus_coefficient, // 29
]
}
fn param_names() -> Vec<&'static str> {
// 30 tuned parameters — C3: hold_penalty_weight removed
// 29 tuned parameters — C3: hold_penalty_weight removed, C2c: count_bonus_coefficient removed
vec![
"learning_rate", // 0
"batch_size", // 1
@@ -755,7 +750,6 @@ impl ParameterSpace for DQNParams {
"lr_decay_type", // 26
"min_epochs_before_stopping", // 27
"minimum_profit_factor", // 28
"count_bonus_coefficient", // 29
]
}
@@ -767,7 +761,7 @@ impl ParameterSpace for DQNParams {
batch_bound.1 = max_batch;
}
}
// Cap hidden_dim_base by VRAM (index 22 in 30D space)
// Cap hidden_dim_base by VRAM (index 22 in 29D space)
let max_base = budget.max_hidden_dim_base(4, 256, 54, 45);
if let Some(dim_bound) = bounds.get_mut(22) {
dim_bound.1 = max_base as f64;
@@ -2533,8 +2527,8 @@ impl HyperparameterOptimizable for DQNTrainer {
use_cql: true,
cql_alpha: params.cql_alpha,
// BUG #7: minimum_profit_factor — now tunable via hyperopt (idx 30, [1.1, 2.0])
// Previously hardcoded to 1.5, meaning dimension 30 of the search space was wasted.
// BUG #7: minimum_profit_factor — now tunable via hyperopt (idx 28, [1.1, 2.0])
// Previously hardcoded to 1.5, meaning this dimension of the search space was wasted.
minimum_profit_factor: params.minimum_profit_factor,
// Phase 3: GPU experience collection (defaults, not in search space)
@@ -2937,7 +2931,8 @@ impl HyperparameterOptimizable for DQNTrainer {
let close = win_prices[bar_idx] as f32;
unique_actions.insert(action_idx);
let factored = crate::dqn::FactoredAction::from_index(action_idx)?;
let exposure = crate::dqn::action_space::ExposureLevel::from_index(action_idx)?;
let factored = crate::dqn::OrderRouter::route_default(exposure);
let bar = OHLCVBarF32 {
timestamp: (abs_start + i) as i64,
@@ -2982,7 +2977,7 @@ impl HyperparameterOptimizable for DQNTrainer {
MaxDD {:.2}%, Return {:.2}%, unique_actions={}/{}, conv_fail={}",
win_idx + 1, WINDOW_COUNT, ohlcv_bars.len(), wm.total_trades,
wm.sharpe_ratio, wm.win_rate, wm.max_drawdown_pct,
wm.total_return_pct, unique_actions.len(), 45, conversion_failures,
wm.total_return_pct, unique_actions.len(), 5, conversion_failures,
);
all_unique_actions.extend(&unique_actions);
@@ -3473,8 +3468,8 @@ mod tests {
#[test]
fn test_dqn_params_roundtrip() {
// Roundtrip test for the 30D search space
// Only the 30 tuned parameters roundtrip; the 17 fixed params get defaults from from_continuous
// Roundtrip test for the 29D search space
// Only the 29 tuned parameters roundtrip; the 17 fixed params get defaults from from_continuous
let params = DQNParams {
learning_rate: 3.37e-05,
batch_size: 92,
@@ -3535,10 +3530,10 @@ mod tests {
};
let continuous = params.to_continuous();
assert_eq!(continuous.len(), 30, "to_continuous must return 30D vector");
assert_eq!(continuous.len(), 29, "to_continuous must return 29D vector");
let recovered = DQNParams::from_continuous(&continuous).unwrap();
// Tuned parameters must roundtrip exactly (30D — C3: hold_penalty_weight removed)
// Tuned parameters must roundtrip exactly (29D — C3: hold_penalty_weight removed, C2c: count_bonus_coefficient removed)
assert!((recovered.learning_rate - params.learning_rate).abs() < 1e-6);
assert_eq!(recovered.batch_size, params.batch_size);
assert!((recovered.gamma - params.gamma).abs() < 1e-6);
@@ -3572,13 +3567,13 @@ mod tests {
assert!((recovered.activation_type - 1.0).abs() < 1e-6); // LeakyReLU
assert_eq!(recovered.num_quantiles, 64);
assert!((recovered.qr_kappa - 1.0).abs() < 1e-6);
assert!((recovered.count_bonus_coefficient - params.count_bonus_coefficient).abs() < 1e-6);
assert!((recovered.count_bonus_coefficient - 0.1).abs() < 1e-6); // C2c: fixed default
}
#[test]
fn test_dqn_params_bounds() {
let bounds = DQNParams::continuous_bounds();
assert_eq!(bounds.len(), 30); // 30D search space (C3: hold_penalty_weight removed)
assert_eq!(bounds.len(), 29); // 29D search space (C3: hold_penalty_weight removed, C2c: count_bonus_coefficient removed)
// Check log-scale bounds are reasonable
assert!(bounds[0].0 < bounds[0].1); // learning_rate
@@ -3617,7 +3612,7 @@ mod tests {
// Curiosity + exploration
assert_eq!(bounds[20], (0.01, 0.5)); // curiosity_weight
assert_eq!(bounds[22], (256.0, 4096.0)); // hidden_dim_base
assert_eq!(bounds[23], (0.02, 0.10)); // noisy_epsilon_floor
assert_eq!(bounds[23], (0.0, 0.05)); // noisy_epsilon_floor
// CQL regularization
assert_eq!(bounds[24], (0.0, 0.5)); // cql_alpha
@@ -3626,7 +3621,7 @@ mod tests {
#[test]
fn test_param_names() {
let names = DQNParams::param_names();
assert_eq!(names.len(), 30); // 30D search space (C3: hold_penalty_weight removed)
assert_eq!(names.len(), 29); // 29D search space (C3: hold_penalty_weight removed, C2c: count_bonus_coefficient removed)
assert_eq!(names[0], "learning_rate");
assert_eq!(names[1], "batch_size");
assert_eq!(names[2], "gamma");
@@ -3656,13 +3651,12 @@ mod tests {
assert_eq!(names[26], "lr_decay_type");
assert_eq!(names[27], "min_epochs_before_stopping");
assert_eq!(names[28], "minimum_profit_factor");
assert_eq!(names[29], "count_bonus_coefficient");
}
#[test]
fn test_per_params_always_enabled() {
// Test that PER is always enabled with tunable alpha/beta parameters
// 30D search space (C3: hold_penalty_weight removed)
// 29D search space (C3: hold_penalty_weight removed, C2c: count_bonus_coefficient removed)
let continuous = vec![
3e-5_f64.ln(), 92.0, 0.9588, 97_273_f64.ln(), 4.0, 24.77_f64.ln(), 0.15, 1.0,
0.6, 0.4, // 8-9: per_alpha, per_beta_start
@@ -3680,7 +3674,6 @@ mod tests {
2.0, // 26: lr_decay_type (cosine)
4.0, // 27: min_epochs_before_stopping
1.5, // 28: minimum_profit_factor
0.2, // 29: count_bonus_coefficient
];
let params = DQNParams::from_continuous(&continuous).unwrap();
@@ -3692,7 +3685,7 @@ mod tests {
assert!(params.use_noisy_nets);
assert!((params.warmup_ratio - 0.0).abs() < 1e-6); // Always fixed to 0.0
// Test PER parameter bounds (min values) — 30D
// Test PER parameter bounds (min values) — 29D
let continuous_min = vec![
2e-5_f64.ln(), 64.0, 0.95, 50_000_f64.ln(), 1.0, 10.0_f64.ln(), 0.05, 0.5,
0.4, 0.2, // per_alpha min, per_beta_start min
@@ -3704,13 +3697,12 @@ mod tests {
0.01, // curiosity_weight min
0.0001_f64.ln(), // tau min
256.0, // hidden_dim_base min
0.02, // noisy_epsilon_floor min
0.0, // noisy_epsilon_floor min
0.0, // cql_alpha min
0.5_f64.ln(), // eval_softmax_temp min
0.0, // 26: lr_decay_type (constant)
2.0, // 27: min_epochs_before_stopping min
1.1, // 28: minimum_profit_factor min
0.05, // 29: count_bonus_coefficient min
];
let params_min = DQNParams::from_continuous(&continuous_min).unwrap();
assert!((params_min.per_alpha - 0.4).abs() < 1e-6);
@@ -3719,7 +3711,7 @@ mod tests {
assert!(params_min.use_distributional);
assert!(params_min.use_noisy_nets);
// Test PER parameter bounds (max values) — 30D
// Test PER parameter bounds (max values) — 29D
let continuous_max = vec![
8e-5_f64.ln(), 4096.0, 0.99, 100_000_f64.ln(), 4.0, 40.0_f64.ln(), 0.5, 2.0,
0.8, 0.6, // per_alpha max, per_beta_start max
@@ -3731,13 +3723,12 @@ mod tests {
0.5, // curiosity_weight max
0.01_f64.ln(), // tau max
4096.0, // hidden_dim_base max
0.10, // noisy_epsilon_floor max
0.05, // noisy_epsilon_floor max
0.5, // cql_alpha max
2.0_f64.ln(), // eval_softmax_temp max
2.0, // 26: lr_decay_type (cosine)
6.0, // 27: min_epochs_before_stopping max
2.0, // 28: minimum_profit_factor max
1.0, // 29: count_bonus_coefficient max
];
let params_max = DQNParams::from_continuous(&continuous_max).unwrap();
assert!((params_max.per_alpha - 0.8).abs() < 1e-6);
@@ -4037,7 +4028,7 @@ mod tests {
fn test_qr_dqn_roundtrip_continuous() {
let params = DQNParams::default();
let continuous = params.to_continuous();
assert_eq!(continuous.len(), 30, "Should have 30 continuous dimensions");
assert_eq!(continuous.len(), 29, "Should have 29 continuous dimensions");
let roundtrip = DQNParams::from_continuous(&continuous).unwrap();
// num_quantiles and qr_kappa are now fixed defaults (not in search space)
assert_eq!(roundtrip.num_quantiles, 64); // Fixed default
@@ -4062,7 +4053,7 @@ mod tests {
fn test_batch_size_respects_wide_bounds() {
// Simulate PSO choosing batch_size=2048 (within VRAM-aware bounds)
let bounds = DQNParams::continuous_bounds();
let mut params = vec![0.0_f64; 30]; // 30D (C3: hold_penalty_weight removed)
let mut params = vec![0.0_f64; 29]; // 29D (C3: hold_penalty_weight removed, C2c: count_bonus_coefficient removed)
params[1] = 2048.0; // batch_size (index 1)
// Fill other required params with valid defaults
params[0] = (1e-4_f64).ln(); // learning_rate
@@ -4074,8 +4065,8 @@ mod tests {
params[7] = 0.5; // transaction_cost_multiplier
params[8] = 0.6; // per_alpha
params[9] = 0.4; // per_beta_start
// Remaining params (indices 10-29) -- use midpoint of bounds
for i in 10..30 {
// Remaining params (indices 10-28) -- use midpoint of bounds
for i in 10..29 {
params[i] = (bounds[i].0 + bounds[i].1) / 2.0;
}
let result = DQNParams::from_continuous(&params).unwrap();
@@ -4199,9 +4190,9 @@ mod tests {
);
// from_continuous should clamp to 0.5 even if value is below
let mut low_temp_vec = vec![0.0_f64; 30]; // 30D (C3: hold_penalty_weight removed)
let mut low_temp_vec = vec![0.0_f64; 29]; // 29D (C3: hold_penalty_weight removed, C2c: count_bonus_coefficient removed)
// Set all to midpoint of bounds
for i in 0..30 {
for i in 0..29 {
low_temp_vec[i] = (bounds[i].0 + bounds[i].1) / 2.0;
}
low_temp_vec[25] = 0.01_f64.ln(); // Way below floor (eval_softmax_temp at index 25)
@@ -4261,4 +4252,22 @@ mod tests {
objective
);
}
#[test]
fn test_dqn_exposure_indices_produce_distinct_actions() {
use crate::dqn::action_space::ExposureLevel;
use crate::dqn::order_router::OrderRouter;
// DQN outputs 0-4. Each must map to a distinct exposure level.
let actions: Vec<_> = (0..5)
.map(|idx| {
let exposure = ExposureLevel::from_index(idx).expect("valid index 0-4");
OrderRouter::route_default(exposure)
})
.collect();
// All 5 exposures must be distinct
let exposures: std::collections::HashSet<_> = actions.iter().map(|a| a.exposure).collect();
assert_eq!(exposures.len(), 5, "DQN 5-action space must produce 5 distinct exposure levels");
}
}

113
crates/ml/src/trainers/dqn/trainer.rs
View File

@@ -3289,22 +3289,9 @@ impl DQNTrainer {
.forward(&batch_tensor)
.map_err(|e| anyhow::anyhow!("Batched forward pass failed: {}", e))?;
// Apply count bonus (UCB exploration) to Q-values before argmax.
// Previously only in DQN::select_action() (single-sample path),
// making the batch training path have no UCB exploration.
let use_cb = agent.use_count_bonus();
let n_actions = agent.num_actions();
let batch_q_values = if use_cb {
let bonuses = agent.get_count_bonuses();
let bonus_tensor = Tensor::from_vec(bonuses, (1, n_actions), batch_q_values.device())
.map_err(|e| anyhow::anyhow!("Failed to create bonus tensor: {}", e))?
.to_dtype(batch_q_values.dtype())
.map_err(|e| anyhow::anyhow!("Failed to cast bonus tensor: {}", e))?;
batch_q_values.broadcast_add(&bonus_tensor)
.map_err(|e| anyhow::anyhow!("Failed to add count bonus: {}", e))?
} else {
batch_q_values
};
// C2 FIX: Count bonus removed from Q-value computation.
// Noisy nets are the sole exploration mechanism during training.
// Count bonus kept for diversity metrics only (record_action tracking).
drop(agent); // Release lock early
@@ -3364,21 +3351,9 @@ impl DQNTrainer {
.forward(batch_tensor)
.map_err(|e| anyhow::anyhow!("GPU batched forward pass failed: {}", e))?;
// Apply count bonus (UCB exploration) to Q-values before argmax.
// Mirrors the same fix in select_actions_batch() above.
let use_cb = agent.use_count_bonus();
let n_actions = agent.num_actions();
let batch_q_values = if use_cb {
let bonuses = agent.get_count_bonuses();
let bonus_tensor = Tensor::from_vec(bonuses, (1, n_actions), batch_q_values.device())
.map_err(|e| anyhow::anyhow!("Failed to create bonus tensor: {}", e))?
.to_dtype(batch_q_values.dtype())
.map_err(|e| anyhow::anyhow!("Failed to cast bonus tensor: {}", e))?;
batch_q_values.broadcast_add(&bonus_tensor)
.map_err(|e| anyhow::anyhow!("Failed to add count bonus: {}", e))?
} else {
batch_q_values
};
// C2 FIX: Count bonus removed from Q-value computation.
// Noisy nets are the sole exploration mechanism during training.
// Count bonus kept for diversity metrics only (record_action tracking).
drop(agent);
@@ -4151,6 +4126,7 @@ fn gpu_batch_to_experiences(
#[cfg(test)]
mod tests {
use super::*;
use crate::hyperopt::ParameterSpace;
// Helper function to create test hyperparameters
// Uses conservative defaults suitable for testing
@@ -4624,4 +4600,79 @@ mod tests {
let batch = budget.max_batch_size(50.0, 0.0005, 64.0, 8192.0);
assert!((batch.unwrap_or(0.0) - 8192.0).abs() < 1.0, "H100 should hit 8192 ceiling, got {:?}", batch);
}
// ── C2 Overhaul Smoke Tests ─────────────────────────────────────────
/// Verify DQN action space is 5 exposure levels (not 45 factored actions).
#[test]
fn test_c2_dqn_default_num_actions_is_5() {
let config = crate::dqn::DQNConfig::default();
assert_eq!(config.num_actions, 5, "DQN default must be 5 exposure-level actions");
}
/// Verify 5 exposure indices produce 5 distinct exposure levels.
#[test]
fn test_c2_five_actions_produce_distinct_exposures() {
use crate::dqn::action_space::ExposureLevel;
use crate::dqn::order_router::OrderRouter;
let actions: Vec<_> = (0..5)
.filter_map(|idx| ExposureLevel::from_index(idx).ok())
.map(|e| OrderRouter::route_default(e))
.collect();
assert_eq!(actions.len(), 5);
let unique: std::collections::HashSet<_> = actions.iter().map(|a| a.exposure).collect();
assert_eq!(unique.len(), 5, "All 5 exposure levels must be distinct");
}
/// Verify hyperopt search space is 29D (count_bonus_coefficient removed).
#[test]
fn test_c2_search_space_is_29d() {
let bounds = crate::hyperopt::adapters::dqn::DQNParams::continuous_bounds();
assert_eq!(bounds.len(), 29, "Search space must be 29D after C2c removal");
let names = crate::hyperopt::adapters::dqn::DQNParams::param_names();
assert_eq!(names.len(), 29);
assert!(!names.contains(&"count_bonus_coefficient"), "count_bonus_coefficient must not be in search space");
}
/// Verify noisy_epsilon_floor defaults to 0.0 (noisy nets only).
#[test]
fn test_c2_noisy_epsilon_floor_defaults_to_zero() {
let params = crate::hyperopt::adapters::dqn::DQNParams::default();
assert!(
(params.noisy_epsilon_floor - 0.0).abs() < f64::EPSILON,
"noisy_epsilon_floor must default to 0.0 (noisy nets only)"
);
}
/// Verify noisy_epsilon_floor search bounds are [0.0, 0.05].
#[test]
fn test_c2_noisy_epsilon_floor_bounds() {
let bounds = crate::hyperopt::adapters::dqn::DQNParams::continuous_bounds();
// Index 23 = noisy_epsilon_floor
let (lo, hi) = bounds[23];
assert!((lo - 0.0).abs() < f64::EPSILON, "noisy_epsilon_floor lower bound must be 0.0");
assert!((hi - 0.05).abs() < f64::EPSILON, "noisy_epsilon_floor upper bound must be 0.05");
}
/// Verify count_bonus_coefficient is fixed at 0.1 (not tunable).
#[test]
fn test_c2_count_bonus_coefficient_fixed() {
let params = crate::hyperopt::adapters::dqn::DQNParams::default();
assert!(
(params.count_bonus_coefficient - 0.1).abs() < f64::EPSILON,
"count_bonus_coefficient must be fixed at 0.1"
);
// Roundtrip through from_continuous should preserve fixed value
let continuous = params.to_continuous();
let recovered = crate::hyperopt::adapters::dqn::DQNParams::from_continuous(&continuous).unwrap();
assert!(
(recovered.count_bonus_coefficient - 0.1).abs() < f64::EPSILON,
"count_bonus_coefficient must remain 0.1 after roundtrip"
);
}
}