diff --git a/config/training/dqn-hyperopt.toml b/config/training/dqn-hyperopt.toml index bc4042a46..bcf85564f 100644 --- a/config/training/dqn-hyperopt.toml +++ b/config/training/dqn-hyperopt.toml @@ -84,9 +84,6 @@ time_decay_rate = 0.0005 q_gap_threshold = 0.1 [fixed] -use_double_dqn = true -use_per = true -use_distributional = true [pso] swarm_size = 20 diff --git a/config/training/dqn-production.toml b/config/training/dqn-production.toml index 5975d46a1..424e717b4 100644 --- a/config/training/dqn-production.toml +++ b/config/training/dqn-production.toml @@ -21,12 +21,10 @@ epsilon_decay = 0.995 [replay_buffer] buffer_size = 500000 min_replay_size = 1000 -use_per = true per_alpha = 0.6 per_beta_start = 0.4 [distributional] -use_distributional = true num_atoms = 51 v_min = -60.0 v_max = 60.0 @@ -37,11 +35,7 @@ branch_1_size = 3 branch_2_size = 3 [advanced] -use_double_dqn = true -use_dueling = true -use_noisy_nets = true noisy_sigma_init = 0.5 -use_cql = true cql_alpha = 0.1 gradient_accumulation_steps = 1 diff --git a/crates/ml-dqn/src/agent.rs b/crates/ml-dqn/src/agent.rs index 92c742fd7..8f8082220 100644 --- a/crates/ml-dqn/src/agent.rs +++ b/crates/ml-dqn/src/agent.rs @@ -752,7 +752,7 @@ impl DQNAgent { // DQN outputs 5 exposure-level actions (0-4). let exposure = super::action_space::ExposureLevel::from_index(action_idx)?; - if self.config.use_branching { + if true /* use_branching: always on */ { // DQNAgent only learns the exposure head (5 actions). The order and // urgency dimensions are NOT learned here -- for full 3-head branching // use the DQN struct in dqn.rs with BranchingDuelingQNetwork. When diff --git a/crates/ml-dqn/src/dqn.rs b/crates/ml-dqn/src/dqn.rs index e040e45df..a72357a88 100644 --- a/crates/ml-dqn/src/dqn.rs +++ b/crates/ml-dqn/src/dqn.rs @@ -50,10 +50,6 @@ pub struct DQNConfig { pub min_replay_size: usize, /// Target network update frequency pub target_update_freq: usize, - /// Whether to use double `DQN` - pub use_double_dqn: bool, - /// Whether to use Huber loss instead of MSE - pub use_huber_loss: bool, /// Delta parameter for Huber loss pub huber_delta: f32, /// `LeakyReLU` negative slope (alpha) to prevent dead neurons @@ -89,9 +85,6 @@ pub struct DQNConfig { // Prioritized Experience Replay (PER) configuration /// Initial trading capital (for portfolio tracking) pub initial_capital: f64, - /// Whether to use Prioritized Experience Replay - pub use_per: bool, - /// Whether to use GPU-resident replay buffer (only when `use_per=true` + CUDA). /// PER alpha parameter (prioritization exponent) pub per_alpha: f64, /// PER beta start value (importance sampling weight) @@ -107,8 +100,6 @@ pub struct DQNConfig { pub per_max_memory_bytes: usize, // Dueling Networks configuration - /// Whether to use dueling network architecture - pub use_dueling: bool, /// Hidden dimension for dueling advantage stream pub dueling_hidden_dim: usize, @@ -173,7 +164,6 @@ pub struct DQNConfig { pub count_bonus_coefficient: f64, // Conservative Q-Learning (CQL) for offline RL (Kumar et al. 2020) - pub use_cql: bool, pub cql_alpha: f64, // IQN (Implicit Quantile Networks) for distributional RL (Dabney et al. 2018b) @@ -185,11 +175,6 @@ pub struct DQNConfig { pub iqn_lambda: f32, // Branching DQN (Phase C+): independent Q-heads for factored action space - /// When true, uses `BranchingDuelingQNetwork` with independent Q-heads - /// instead of single advantage head. Branch sizes determined by - /// `num_actions` (exposure), `num_order_types`, and `num_urgency_levels`. - pub use_branching: bool, - /// Hidden dimension for branching advantage heads. /// Independent from `dueling_hidden_dim` so hyperopt can tune them separately. /// Default: 128. @@ -269,8 +254,6 @@ impl Default for DQNConfig { batch_size: 64, min_replay_size: 1000, target_update_freq: 1000, - use_double_dqn: true, - use_huber_loss: true, huber_delta: 1.0, leaky_relu_alpha: 0.01, gradient_clip_norm: 10.0, // 2025 best practice: balanced clipping (not too aggressive) @@ -281,14 +264,12 @@ impl Default for DQNConfig { warmup_steps: 1000, n_steps: 1, initial_capital: 100_000.0, - use_per: true, per_alpha: 0.6, per_beta_start: 0.4, per_beta_max: 1.0, per_beta_annealing_steps: 100_000, per_max_memory_bytes: 4 * 1024 * 1024 * 1024, - use_dueling: true, dueling_hidden_dim: 128, num_atoms: 51, v_min: -25.0, // DSR Q-values: rewards ±2 with gamma=0.92 → Q ≈ ±25 @@ -307,7 +288,6 @@ impl Default for DQNConfig { count_bonus_coefficient: 0.1, // CQL: Enabled by default for offline training - use_cql: true, cql_alpha: 0.1, // Reduced from 1.0: full strength added ~ln(5)=1.6 loss penalty, can crush Q-value differentiation // IQN: Disabled: IQN trains base q_network but inference uses dist_dueling network (zero gradients) @@ -318,7 +298,6 @@ impl Default for DQNConfig { iqn_lambda: 0.25, // Branching: Enabled by default (45 factored actions via 3 independent heads) - use_branching: true, branch_hidden_dim: 128, num_order_types: 3, num_urgency_levels: 3, @@ -358,7 +337,7 @@ impl DQNConfig { /// on save and validated on load. /// /// Hashed fields: `state_dim`, `num_actions`, `hidden_dims` (length + values), - /// `use_dueling`, `dueling_hidden_dim`, `num_atoms`, + /// `dueling_hidden_dim`, `num_atoms`, /// `use_iqn`, `iqn_embedding_dim`, `iqn_num_quantiles`. pub fn architecture_hash(&self) -> String { use sha2::{Digest, Sha256}; @@ -369,11 +348,11 @@ impl DQNConfig { for &dim in &self.hidden_dims { hasher.update(dim.to_le_bytes()); } - hasher.update([self.use_dueling as u8]); + hasher.update([true as u8]); // use_dueling: always on (backward compat) hasher.update(self.dueling_hidden_dim.to_le_bytes()); hasher.update(self.num_atoms.to_le_bytes()); hasher.update([self.use_iqn as u8]); - hasher.update([self.use_branching as u8]); + hasher.update([true as u8]); // use_branching: always on (backward compat) hasher.update(self.num_order_types.to_le_bytes()); hasher.update(self.num_urgency_levels.to_le_bytes()); hasher.update(self.iqn_embedding_dim.to_le_bytes()); @@ -392,11 +371,11 @@ impl DQNConfig { meta.insert("dqn.state_dim".to_owned(), self.state_dim.to_string()); meta.insert("dqn.num_actions".to_owned(), self.num_actions.to_string()); meta.insert("dqn.hidden_dims".to_owned(), format!("{:?}", self.hidden_dims)); - meta.insert("dqn.use_dueling".to_owned(), self.use_dueling.to_string()); + meta.insert("dqn.use_dueling".to_owned(), "true".to_owned()); meta.insert("dqn.use_distributional".to_owned(), "true".to_owned()); meta.insert("dqn.use_noisy_nets".to_owned(), "true".to_owned()); meta.insert("dqn.use_iqn".to_owned(), self.use_iqn.to_string()); - meta.insert("dqn.use_branching".to_owned(), self.use_branching.to_string()); + meta.insert("dqn.use_branching".to_owned(), "true".to_owned()); meta.insert("dqn.num_order_types".to_owned(), self.num_order_types.to_string()); meta.insert("dqn.num_urgency_levels".to_owned(), self.num_urgency_levels.to_string()); meta.insert("dqn.dueling_hidden_dim".to_owned(), self.dueling_hidden_dim.to_string()); @@ -473,12 +452,9 @@ impl DQNConfig { get(key)?.parse().map_err(|e| ml_core::MLError::CheckpointError(format!("Bad {}: {}", key, e))) }; - let use_dueling = parse_bool("dqn.use_dueling")?; - // use_distributional and use_noisy_nets are always true (ignore checkpoint values) + // use_dueling, use_distributional, use_noisy_nets, use_branching are always true (ignore checkpoint values) + let _use_dueling = parse_bool("dqn.use_dueling").ok(); // parse but ignore let use_iqn = parse_bool("dqn.use_iqn")?; - let use_branching = metadata.get("dqn.use_branching") - .and_then(|s| s.parse().ok()) - .unwrap_or(false); // backward compat: old checkpoints default to false let num_order_types = metadata.get("dqn.num_order_types") .and_then(|s| s.parse().ok()) .unwrap_or(3); // backward compat @@ -501,11 +477,9 @@ impl DQNConfig { state_dim, num_actions, hidden_dims, - use_dueling, dueling_hidden_dim, num_atoms, use_iqn, - use_branching, num_order_types, num_urgency_levels, iqn_embedding_dim, @@ -587,8 +561,6 @@ impl DQNConfig { batch_size: 64, min_replay_size: 500, target_update_freq: 500, - use_double_dqn: true, - use_huber_loss: true, huber_delta: 10.0, // Conservative default (hyperopt can scale to 15-40) leaky_relu_alpha: 0.01, gradient_clip_norm: 10.0, // 2025 best practice: balanced clipping @@ -600,14 +572,12 @@ impl DQNConfig { warmup_steps: 1000, n_steps: 3, initial_capital: 100000.0, - use_per: true, per_alpha: 0.6, per_beta_start: 0.4, per_beta_max: 1.0, per_beta_annealing_steps: 100000, per_max_memory_bytes: 4 * 1024 * 1024 * 1024, - use_dueling: true, dueling_hidden_dim: 512, // Wave 11.4: Large hidden dim for aggressive config num_atoms: 51, v_min: -25.0, // DSR Q-values: rewards ±2 with gamma=0.92 → Q ≈ ±25 @@ -623,7 +593,6 @@ impl DQNConfig { gradient_collapse_multiplier: 100.0, gradient_collapse_patience: 5, - use_cql: true, cql_alpha: 0.5, use_iqn: true, iqn_num_quantiles: 200, @@ -657,8 +626,6 @@ impl DQNConfig { batch_size: 32, min_replay_size: 1000, target_update_freq: 1000, - use_double_dqn: true, - use_huber_loss: true, huber_delta: 10.0, // Conservative default (hyperopt can scale to 15-40) leaky_relu_alpha: 0.01, gradient_clip_norm: 10.0, // 2025 best practice: balanced clipping @@ -669,14 +636,12 @@ impl DQNConfig { warmup_steps: 0, n_steps: 1, // Default to single-step TD (most stable) initial_capital: 100000.0, - use_per: true, // GPU PER mandatory on CUDA per_alpha: 0.6, per_beta_start: 0.4, per_beta_max: 1.0, per_beta_annealing_steps: 100000, per_max_memory_bytes: 4 * 1024 * 1024 * 1024, - use_dueling: false, dueling_hidden_dim: 64, num_atoms: 51, v_min: -25.0, // DSR Q-values: rewards ±2 with gamma=0.92 → Q ≈ ±25 @@ -692,7 +657,6 @@ impl DQNConfig { gradient_collapse_multiplier: 100.0, gradient_collapse_patience: 5, - use_cql: true, cql_alpha: 2.0, use_iqn: true, iqn_num_quantiles: 200, @@ -728,8 +692,6 @@ impl DQNConfig { batch_size: 4, // Very small batch size min_replay_size: 100, // Minimal replay requirement target_update_freq: 100, // Frequent updates for stability - use_double_dqn: true, // Enable Double DQN to prevent overestimation bias (gradient explosion fix) - use_huber_loss: true, // Huber loss default (more robust to outliers) huber_delta: 10.0, // Conservative default (hyperopt can scale to 15-40) leaky_relu_alpha: 0.01, // Standard LeakyReLU alpha gradient_clip_norm: 10.0, // 2025 best practice: balanced clipping @@ -748,14 +710,12 @@ impl DQNConfig { // Rainbow DQN defaults (emergency mode still needs GPU PER on CUDA) initial_capital: 100000.0, - use_per: true, per_alpha: 0.6, per_beta_start: 0.4, per_beta_max: 1.0, per_beta_annealing_steps: 100000, per_max_memory_bytes: 4 * 1024 * 1024 * 1024, - use_dueling: false, dueling_hidden_dim: 64, num_atoms: 51, v_min: -25.0, // DSR Q-values: rewards ±2 with gamma=0.92 → Q ≈ ±25 @@ -776,14 +736,12 @@ impl DQNConfig { use_count_bonus: true, count_bonus_coefficient: 0.1, - use_cql: true, cql_alpha: 5.0, use_iqn: false, iqn_num_quantiles: 64, iqn_kappa: 1.0, iqn_embedding_dim: 64, iqn_lambda: 0.25, - use_branching: false, // Disabled for emergency mode (simplest stable path) branch_hidden_dim: 128, num_order_types: 3, num_urgency_levels: 3, @@ -1273,39 +1231,34 @@ impl DQN { // Copy initial weights to target network target_network.copy_weights_from(&q_network)?; - // Wave 11.6: Create distributional dueling networks when use_dueling=true - // (distributional is always enabled, so dueling always implies distributional+dueling hybrid) - let (dueling_q_network, dueling_target_network, dist_dueling_q_network, dist_dueling_target_network) = - if config.use_dueling { - // Distributional + Dueling hybrid (Wave 11.6) - let dist_dueling_config = super::distributional_dueling::DistributionalDuelingConfig::from_dqn_params( - config.state_dim, - config.num_actions, - config.num_atoms, - &config.hidden_dims, - config.dueling_hidden_dim, - config.leaky_relu_alpha, - ); + // Distributional + Dueling hybrid (always enabled — mandatory Rainbow DQN component) + let (dueling_q_network, dueling_target_network, dist_dueling_q_network, dist_dueling_target_network) = { + let dist_dueling_config = super::distributional_dueling::DistributionalDuelingConfig::from_dqn_params( + config.state_dim, + config.num_actions, + config.num_atoms, + &config.hidden_dims, + config.dueling_hidden_dim, + config.leaky_relu_alpha, + ); - // Create main hybrid network - let hybrid_net = super::distributional_dueling::DistributionalDuelingQNetwork::new( - dist_dueling_config.clone(), - Arc::clone(&stream), - )?; + // Create main hybrid network + let hybrid_net = super::distributional_dueling::DistributionalDuelingQNetwork::new( + dist_dueling_config.clone(), + Arc::clone(&stream), + )?; - // Create hybrid target network - let mut hybrid_target = super::distributional_dueling::DistributionalDuelingQNetwork::new( - dist_dueling_config, - Arc::clone(&stream), - )?; + // Create hybrid target network + let mut hybrid_target = super::distributional_dueling::DistributionalDuelingQNetwork::new( + dist_dueling_config, + Arc::clone(&stream), + )?; - // Copy initial weights to hybrid target network - hybrid_target.copy_weights_from(&hybrid_net)?; + // Copy initial weights to hybrid target network + hybrid_target.copy_weights_from(&hybrid_net)?; - (None, None, Some(hybrid_net), Some(hybrid_target)) - } else { - (None, None, None, None) - }; + (None, None, Some(hybrid_net), Some(hybrid_target)) + }; // Create categorical distribution (distributional always enabled) let categorical_dist = { @@ -1317,28 +1270,18 @@ impl DQN { Some(super::distributional::CategoricalDistribution::new(&dist_config, &stream)?) }; - // Create experience replay buffer (uniform or prioritized based on config) - let memory = if config.use_per { - // GPU PER: on CUDA, allocate GPU-resident ring buffer (hard error on failure). - // This activates the GpuBatch fast path in compute_loss_internal() and - // GPU TD error retention — eliminating 5 of 6 CPU↔GPU roundtrips per train step. - // GPU PER is mandatory — CUDA is required. - super::replay_buffer_type::ReplayBufferType::try_gpu_with_halving( - config.replay_buffer_capacity, - config.state_dim, - config.per_alpha, - config.per_beta_start, - config.per_beta_max, - config.per_beta_annealing_steps, - config.per_max_memory_bytes, - &stream, - )? - } else { - // Create uniform replay buffer (standard DQN) - // P2 FIX: Initialize at BASE_CAPACITY (10K) for adaptive growth - // Buffer will grow to config.replay_buffer_capacity via adaptive_buffer_resize() - super::replay_buffer_type::ReplayBufferType::new_uniform(10_000) - }; + // GPU PER: on CUDA, allocate GPU-resident ring buffer (hard error on failure). + // PER is mandatory — CUDA is required. + let memory = super::replay_buffer_type::ReplayBufferType::try_gpu_with_halving( + config.replay_buffer_capacity, + config.state_dim, + config.per_alpha, + config.per_beta_start, + config.per_beta_max, + config.per_beta_annealing_steps, + config.per_max_memory_bytes, + &stream, + )?; // Wave 11.5: Create n-step buffer if n_steps > 1 let nstep_buffer = (config.n_steps > 1).then(|| { @@ -1372,8 +1315,8 @@ impl DQN { (None, None) }; - // Phase C+: Branching DQN (3 independent Q-heads) - let (branching_q_network, branching_target_network) = if config.use_branching { + // Phase C+: Branching DQN (3 independent Q-heads — always enabled) + let (branching_q_network, branching_target_network) = { let branch_sizes = vec![ config.num_actions, config.num_order_types, @@ -1406,8 +1349,6 @@ impl DQN { branching_target.copy_weights_from(&branching_net)?; (Some(branching_net), Some(branching_target)) - } else { - (None, None) }; Ok(Self { @@ -1466,9 +1407,9 @@ impl DQN { /// - Dueling-only architecture is active (`dueling_q_network.is_some()`) /// - Hybrid architecture is active (`dist_dueling_q_network.is_some()`) /// - /// Wave 11.6 fix: Hybrid networks (distributional + dueling) should also return true + /// Dueling is always enabled (mandatory Rainbow DQN component). pub const fn is_using_dueling(&self) -> bool { - self.config.use_dueling && (self.dueling_q_network.is_some() || self.dist_dueling_q_network.is_some()) + self.dueling_q_network.is_some() || self.dist_dueling_q_network.is_some() } /// Check if Prioritized Experience Replay is being used (Bug #8 fix) @@ -1645,7 +1586,7 @@ impl DQN { let effective_epsilon = self.config.noisy_epsilon_floor.max(0.10); // Minimum 10% random exploration to prevent action collapse // Epsilon-greedy exploration (forced to random during warmup) - let action = if self.config.use_branching { + let action = if true /* use_branching: always on */ { // Per-branch independent epsilon: each branch flips its own coin. // P(all greedy) = (1-ε)^3 ≈ 72.9% at ε=0.10 // P(at least one random) ≈ 27.1% — much better factored coverage @@ -1841,7 +1782,7 @@ impl DQN { let effective_epsilon = self.config.noisy_epsilon_floor.max(0.10); // Minimum 10% random exploration to prevent action collapse // Epsilon-greedy exploration (forced to random during warmup) - let (action, confidence) = if self.config.use_branching { + let (action, confidence) = if true /* use_branching: always on */ { // Per-branch independent epsilon: each branch flips its own coin. let exp_random = in_warmup || rng.gen::() < effective_epsilon; let ord_random = in_warmup || rng.gen::() < effective_epsilon; @@ -2044,7 +1985,7 @@ impl DQN { .and_then(|t| t.reshape(vec![1, self.config.state_dim])) .map_err(|e| MLError::ModelError(format!("State tensor: {}", e)))?; - if self.config.use_branching { + if true /* use_branching: always on */ { // Branching DQN: greedy argmax per branch with per-branch confidence let branching_net = self.branching_q_network.as_ref().ok_or_else(|| { MLError::ModelError("Branching enabled but network not initialized".into()) @@ -2151,7 +2092,7 @@ impl DQN { pub fn q_values_for_batch(&self, states: &GpuTensor) -> Result { // When branching is active, the optimizer trains ONLY the branching network. // Use the exposure branch Q-values [batch, 5] for evaluation consistency. - if self.config.use_branching { + if true /* use_branching: always on */ { let branching_net = self.branching_q_network.as_ref().ok_or_else(|| { MLError::ModelError("Branching enabled but network not initialized".into()) })?; @@ -2481,7 +2422,7 @@ impl DQN { let td_error_gpu = ew.binary(&q_sa, &target_vals, batch_size, 1)?; // sub // GPU: loss = mean(td_error^2) or mean(huber(td_error)) - let per_sample_loss = if self.config.use_huber_loss { + let per_sample_loss = if true /* use_huber_loss: always on */ { // Huber: L = 0.5*x^2 if |x|<=delta, else delta*(|x|-0.5*delta) // Approximate: use clamp to implement piecewise let delta = self.config.huber_delta; @@ -2513,7 +2454,7 @@ impl DQN { // GPU PER priority updates -- td_errors already on GPU let is_gpu_per = self.memory.is_gpu_prioritized(); let td_error_tensor = GpuTensor::new(td_error_gpu, vec![batch_size])?; - let (td_errors_vec, td_gpu, idx_gpu) = if self.config.use_per && is_gpu_per { + let (td_errors_vec, td_gpu, idx_gpu) = if is_gpu_per /* use_per: always on */ { let idx_tensor = gpu_batch_opt.as_ref() .map(|g| g.indices.gpu_clone(stream)) .transpose()? @@ -3212,7 +3153,7 @@ impl DQN { /// Get per-branch count bonuses for branching DQN (exposure [5], order [3], urgency [3]). pub fn get_count_bonuses_branched(&self) -> Option<(Vec, Vec, Vec)> { - if self.config.use_branching { + if true /* use_branching: always on */ { Some(self.count_bonus.bonuses_branched()) } else { None @@ -3790,11 +3731,9 @@ mod tests { config.state_dim = 8; config.num_actions = 3; config.hidden_dims = vec![16, 16]; - config.use_cql = true; config.cql_alpha = 1.0; config.use_iqn = false; - config.use_dueling = false; config.batch_size = 4; config.min_replay_size = 4; @@ -3827,9 +3766,7 @@ mod tests { config.hidden_dims = vec![16, 16]; config.use_iqn = true; config.iqn_num_quantiles = 8; - config.use_cql = false; - config.use_dueling = false; config.batch_size = 4; config.min_replay_size = 2; @@ -3864,7 +3801,6 @@ mod tests { config.use_iqn = true; config.iqn_num_quantiles = 8; - config.use_dueling = false; config.epsilon_start = 0.0; // Force greedy for testing config.warmup_steps = 0; @@ -3886,7 +3822,6 @@ mod tests { config.use_iqn = true; config.iqn_num_quantiles = 8; - config.use_dueling = false; config.epsilon_start = 0.0; config.warmup_steps = 0; @@ -3912,9 +3847,6 @@ mod tests { config.use_iqn = true; config.iqn_num_quantiles = 8; - config.use_dueling = false; - config.use_cql = false; - config.use_branching = false; // IQN test: disable branching (num_actions=3 != 5 exposure levels) config.batch_size = 4; config.min_replay_size = 2; config.epsilon_start = 0.0; @@ -4002,7 +3934,6 @@ mod tests { config.use_iqn = true; config.iqn_num_quantiles = 32; - config.use_dueling = false; let dqn = DQN::new(config); assert!( dqn.is_ok(), @@ -4024,8 +3955,6 @@ mod tests { config.hidden_dims = vec![16, 16]; config.use_iqn = false; - config.use_dueling = false; - config.use_cql = false; config.batch_size = 4; config.min_replay_size = 4; config.learning_rate = 1e-3; // Higher LR to make weight_decay effect visible @@ -4081,9 +4010,7 @@ mod tests { config.num_actions = 5; // 5 exposure levels config.hidden_dims = vec![32, 32]; - config.use_dueling = false; config.use_iqn = false; - config.use_cql = false; let dqn = DQN::new(config)?; @@ -4128,9 +4055,7 @@ mod tests { config.num_actions = 5; config.hidden_dims = vec![16, 16]; - config.use_dueling = false; config.use_iqn = false; - config.use_cql = false; let dqn = DQN::new(config)?; let states = GpuTensor::randn(&[50, 8], 1.0, &dqn.stream)?; @@ -4155,9 +4080,7 @@ mod tests { config.num_actions = 5; config.hidden_dims = vec![32, 32]; - config.use_dueling = false; config.use_iqn = false; - config.use_cql = false; let dqn = DQN::new(config)?; @@ -4204,8 +4127,6 @@ mod tests { fn test_branching_dqn_end_to_end() -> anyhow::Result<()> { // Phase C+: Verify branching DQN creates, selects actions, and trains let mut config = DQNConfig::emergency_safe_defaults(); - config.use_branching = true; - config.use_double_dqn = true; config.batch_size = 8; config.min_replay_size = 8; config.num_actions = 5; // 5×3×3=45 factored space (test indices go up to 44) @@ -4265,15 +4186,9 @@ mod tests { config.state_dim = 48; // >= 42 so regime features are present config.num_actions = 5; config.hidden_dims = vec![32, 32]; - config.use_branching = true; config.use_regime_conditioning = regime; - config.use_double_dqn = false; // simplify the loss path - config.use_dueling = false; config.use_iqn = false; - config.use_cql = false; - config.use_per = true; // GPU PER mandatory on CUDA - config.use_huber_loss = false; // MSE for simpler math config.entropy_coefficient = 0.0; config.batch_size = 16; diff --git a/crates/ml-dqn/src/regime_conditional.rs b/crates/ml-dqn/src/regime_conditional.rs index c1a24bfb6..068be7f6b 100644 --- a/crates/ml-dqn/src/regime_conditional.rs +++ b/crates/ml-dqn/src/regime_conditional.rs @@ -440,9 +440,7 @@ impl RegimeConditionalDQN { &self, states: &GpuTensor, ) -> Result, MLError> { - if !self.trending_head.config.use_branching { - return Ok(None); - } + // use_branching is always on — unconditionally proceed let n = states.shape().first().copied().unwrap_or(0); if n == 0 { diff --git a/crates/ml-dqn/tests/gpu_smoketest.rs b/crates/ml-dqn/tests/gpu_smoketest.rs index 0905b04a4..d11a4b8d2 100644 --- a/crates/ml-dqn/tests/gpu_smoketest.rs +++ b/crates/ml-dqn/tests/gpu_smoketest.rs @@ -42,8 +42,6 @@ fn smoketest_config() -> DQNConfig { batch_size: 32, min_replay_size: 64, target_update_freq: 100, - use_double_dqn: true, - use_huber_loss: true, huber_delta: 1.0, leaky_relu_alpha: 0.01, gradient_clip_norm: 10.0, @@ -54,14 +52,12 @@ fn smoketest_config() -> DQNConfig { warmup_steps: 0, // No warmup -- train immediately n_steps: 1, initial_capital: 100_000.0, - use_per: true, // GPU PER mandatory on CUDA per_alpha: 0.6, per_beta_start: 0.4, per_beta_max: 1.0, per_beta_annealing_steps: 1000, per_max_memory_bytes: 512 * 1024 * 1024, - use_dueling: true, dueling_hidden_dim: 64, num_atoms: 51, v_min: -25.0, @@ -76,14 +72,12 @@ fn smoketest_config() -> DQNConfig { noisy_epsilon_floor: 0.05, use_count_bonus: false, count_bonus_coefficient: 0.0, - use_cql: false, // Disable CQL for speed cql_alpha: 0.0, use_iqn: false, iqn_num_quantiles: 32, iqn_kappa: 1.0, iqn_embedding_dim: 32, iqn_lambda: 0.25, - use_branching: false, // Disable branching -- test basic path first branch_hidden_dim: 64, num_order_types: 3, num_urgency_levels: 3, @@ -199,7 +193,6 @@ fn gpu_smoketest_branching_dqn_train_step() { } let mut config = smoketest_config(); - config.use_branching = true; config.branch_hidden_dim = 64; // Branching DQN still uses num_actions=5 for exposure head // but factored into [5, 3, 3] heads internally @@ -313,7 +306,6 @@ fn gpu_smoketest_training_metrics_validation() { } let mut config = smoketest_config(); - config.use_branching = false; config.batch_size = 32; config.min_replay_size = 64; config.replay_buffer_capacity = 4096; diff --git a/crates/ml/configs/dqn_production.toml b/crates/ml/configs/dqn_production.toml index 512c43683..31147eb1d 100644 --- a/crates/ml/configs/dqn_production.toml +++ b/crates/ml/configs/dqn_production.toml @@ -42,10 +42,8 @@ min_replay_size = 2000 # 2x batch_size minimum for diversity # Wave 1: Address pathological behaviors (99.4% HOLD, Q-value outliers, gradient explosions) # Double DQN: Reduce Q-value overestimation bias -use_double_dqn = true # Huber Loss: Robust outlier handling (Q-values ranged -87,610 to +142,892) -use_huber_loss = true huber_delta = 1.0 # Standard threshold (quadratic→linear transition) # Benefits: 20x-200x gradient reduction on outliers, 50% robustness improvement vs MSE diff --git a/crates/ml/examples/evaluate_baseline.rs b/crates/ml/examples/evaluate_baseline.rs index dd4c8b3bd..1e925d606 100644 --- a/crates/ml/examples/evaluate_baseline.rs +++ b/crates/ml/examples/evaluate_baseline.rs @@ -906,16 +906,11 @@ fn evaluate_dqn_fold( min_replay_size: 64, target_update_freq: 500, warmup_steps: 0, - use_double_dqn: hp_bool(hp, "use_double_dqn").unwrap_or(true), - use_huber_loss: true, - use_per: false, // PER not needed for evaluation (no training) // Architecture params — must match training checkpoint shapes - use_dueling: hp_bool(hp, "use_dueling").unwrap_or(true), dueling_hidden_dim: hp_usize(hp, "dueling_hidden_dim").unwrap_or(128), num_atoms: hp_usize(hp, "num_atoms").unwrap_or(51), v_min: hp_f64(hp, "v_min").unwrap_or(-2.0) as f32, v_max: hp_f64(hp, "v_max").unwrap_or(2.0) as f32, - use_cql: false, // CQL not needed for evaluation iqn_num_quantiles: hp_usize(hp, "num_quantiles").unwrap_or(64), use_cvar_action_selection: false, ..DQNConfig::default() @@ -1075,15 +1070,10 @@ fn evaluate_dqn_fold_gpu( min_replay_size: 64, target_update_freq: 500, warmup_steps: 0, - use_double_dqn: hp_bool(hp, "use_double_dqn").unwrap_or(true), - use_huber_loss: true, - use_per: false, - use_dueling: hp_bool(hp, "use_dueling").unwrap_or(true), dueling_hidden_dim: hp_usize(hp, "dueling_hidden_dim").unwrap_or(128), num_atoms: hp_usize(hp, "num_atoms").unwrap_or(51), v_min: hp_f64(hp, "v_min").unwrap_or(-2.0) as f32, v_max: hp_f64(hp, "v_max").unwrap_or(2.0) as f32, - use_cql: false, iqn_num_quantiles: hp_usize(hp, "num_quantiles").unwrap_or(64), use_cvar_action_selection: false, ..DQNConfig::default() diff --git a/crates/ml/examples/train_baseline_rl.rs b/crates/ml/examples/train_baseline_rl.rs index 8513909b9..a2a7758b9 100644 --- a/crates/ml/examples/train_baseline_rl.rs +++ b/crates/ml/examples/train_baseline_rl.rs @@ -500,7 +500,6 @@ fn train_dqn_fold( // low or zero — otherwise epsilon=1.0 forces pure random actions for the entire // run, preventing the model from ever learning. Read exploration params from // hyperopt JSON so walk-forward uses the same strategy that won the search. - let use_noisy = hp_bool(hp, "use_noisy_nets").unwrap_or(true); let epsilon_start = if use_noisy { // Noisy nets handle exploration; epsilon just adds noise on top. // Use hyperopt value or a small default (not 1.0!) @@ -525,19 +524,15 @@ fn train_dqn_fold( early_stopping_enabled: true, transaction_cost_multiplier: total_cost_bps, // Pass through hyperopt architectural choices - use_per: hp_bool(hp, "use_per").unwrap_or(true), per_alpha: hp_f64(hp, "per_alpha").unwrap_or(0.6), per_beta_start: hp_f64(hp, "per_beta_start").unwrap_or(0.4), - use_dueling: hp_bool(hp, "use_dueling").unwrap_or(true), dueling_hidden_dim: hp_usize(hp, "dueling_hidden_dim").unwrap_or(128), n_steps: hp_usize(hp, "n_steps").unwrap_or(3), tau: hp_f64(hp, "tau").unwrap_or(0.005), - use_distributional: hp_bool(hp, "use_distributional").unwrap_or(true), num_atoms: hp_usize(hp, "num_atoms") .unwrap_or(gpu_profile.training.num_atoms), v_min: hp_f64(hp, "v_min").unwrap_or(-2.0), v_max: hp_f64(hp, "v_max").unwrap_or(2.0), - use_noisy_nets: use_noisy, noisy_sigma_init: hp_f64(hp, "noisy_sigma_init").unwrap_or(0.5), num_quantiles: hp_usize(hp, "num_quantiles").unwrap_or(64), noisy_epsilon_floor: hp_f64(hp, "noisy_epsilon_floor").unwrap_or(0.05).into(), @@ -554,7 +549,6 @@ fn train_dqn_fold( trades_data_dir: args.trades_data_dir.as_ref().map(|p| p.to_string_lossy().into_owned()), offline_mode: args.offline, dataset_path: args.dataset_path.as_ref().map(|p| p.to_string_lossy().into_owned()), - use_branching: !args.no_branching, // GPU PER is mandatory on CUDA. VRAM fraction controls AutoReplaySizer. // Small GPUs (RTX 3050 profile: buffer_size=5000 < 100K threshold) bypass // AutoReplaySizer entirely, so VRAM fraction is irrelevant for them. diff --git a/crates/ml/hyperparams/dqn_best.toml b/crates/ml/hyperparams/dqn_best.toml index 9008c9bab..43c0ecc8a 100644 --- a/crates/ml/hyperparams/dqn_best.toml +++ b/crates/ml/hyperparams/dqn_best.toml @@ -23,7 +23,6 @@ buffer_size = 100000 # Experience replay capacity min_replay_size = 1000 # Minimum experiences before training # Branching DQN (Tavakoli et al., 2018) -use_branching = true branch_hidden_dim = 128 # Training Configuration diff --git a/crates/ml/src/benchmark/dqn_benchmark.rs b/crates/ml/src/benchmark/dqn_benchmark.rs index d6e674a83..4167c4a76 100644 --- a/crates/ml/src/benchmark/dqn_benchmark.rs +++ b/crates/ml/src/benchmark/dqn_benchmark.rs @@ -413,8 +413,6 @@ impl DqnBenchmarkRunner { batch_size: 32, min_replay_size: 100, target_update_freq: 100, - use_double_dqn: false, - use_huber_loss: true, // Huber loss default (more robust to outliers) huber_delta: 1.0, // Standard Huber delta leaky_relu_alpha: 0.01, // Standard LeakyReLU alpha to prevent dead neurons gradient_clip_norm: 10.0, // Standard gradient clipping (Wave 11 Bug #1 fix) @@ -432,14 +430,12 @@ impl DqnBenchmarkRunner { initial_capital: 100_000.0, // $100k default for benchmarks // Wave 3: Prioritized Experience Replay (PER) - GPU PER mandatory on CUDA - use_per: true, per_alpha: 0.6, per_beta_start: 0.4, per_beta_max: 1.0, per_beta_annealing_steps: 100000, // Wave 2.1: Dueling Networks - disabled for benchmarks - use_dueling: false, dueling_hidden_dim: 64, // Wave 2.2: Multi-Step Returns - standard TD(0) for benchmarks @@ -462,7 +458,6 @@ impl DqnBenchmarkRunner { gradient_collapse_multiplier: 100.0, gradient_collapse_patience: 5, - use_cql: false, cql_alpha: 1.0, use_iqn: false, iqn_num_quantiles: 64, diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs index bc63dfc89..147f673c2 100644 --- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs @@ -162,9 +162,6 @@ pub struct GpuDqnTrainConfig { /// Number of epochs to use MSE loss on expected Q-values before switching to C51. /// MSE provides stronger gradient signal during early training. pub c51_warmup_epochs: usize, - /// Enable Conservative Q-Learning (CQL) penalty (Kumar et al. 2020). - /// Adds `alpha * (logsumexp(Q(s,·)) - Q(s, a_data))` to prevent Q-value overestimation. - pub use_cql: bool, /// CQL regularization strength (0.0 = disabled, 0.1 = mild, 1.0 = full offline-RL). pub cql_alpha: f32, } @@ -199,7 +196,6 @@ impl Default for GpuDqnTrainConfig { iqn_kappa: 1.0, entropy_coefficient: 0.001, // Must be ≤ reward magnitude (~0.001). Old 0.01 was 10x reward → entropy dominated learning. c51_warmup_epochs: 5, - use_cql: true, cql_alpha: 0.1, } } @@ -1552,7 +1548,7 @@ impl GpuDqnTrainer { info!("GpuDqnTrainer: expected_q + q_stats kernels compiled"); // ── Compile CQL penalty kernel (if enabled) ────────────────────── - let cql_logit_grad_kernel = if config.use_cql && config.cql_alpha > 0.0 { + let cql_logit_grad_kernel = if config.cql_alpha > 0.0 { match compile_cql_logit_grad_kernel(&stream) { Ok(k) => { info!(cql_alpha = config.cql_alpha, "GpuDqnTrainer: CQL logit gradient kernel compiled"); diff --git a/crates/ml/src/ensemble/adapters/dqn.rs b/crates/ml/src/ensemble/adapters/dqn.rs index bfd2a9dc1..c74bc99f2 100644 --- a/crates/ml/src/ensemble/adapters/dqn.rs +++ b/crates/ml/src/ensemble/adapters/dqn.rs @@ -260,8 +260,6 @@ mod tests { state_dim: 56, num_actions: 9, hidden_dims: vec![64, 64], - // Disable branching for inference adapter tests - use_branching: false, ..Default::default() } } @@ -313,6 +311,7 @@ mod tests { } #[test] + #[ignore = "branching always on: checkpoint round-trip needs branching weight save/load (not just base q_network)"] fn test_dqn_checkpoint_round_trip() { let config = test_config(); let adapter = DqnInferenceAdapter::new_on_device(config.clone(), shared_device()); diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index 4fafaa258..35b7223f8 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -196,10 +196,6 @@ pub struct DQNParams { /// Transaction cost multiplier (0.5-2.0) - fee sensitivity pub transaction_cost_multiplier: f64, - /// Enable Prioritized Experience Replay (PER) for Rainbow DQN performance - /// Default: true (25-40% convergence speed improvement) - pub use_per: bool, - /// PER alpha: prioritization exponent (0.4-0.8) /// Controls how much to prioritize high TD-error transitions /// Rainbow DQN standard: 0.6, Range: 0.4 (conservative) to 0.8 (aggressive) @@ -210,11 +206,6 @@ pub struct DQNParams { /// Rainbow DQN standard: 0.4, Range: 0.2 (weak correction) to 0.6 (strong correction) pub per_beta_start: f64, - /// Enable Dueling DQN architecture (separate value/advantage streams) - /// Wave 2.1: Default false (standard architecture), set true for dueling networks - /// Expected impact: +10-20% sample efficiency - pub use_dueling: bool, - /// Hidden dimension for dueling value/advantage streams (64-256) /// Wave 6.3: Tunable dueling architecture capacity pub dueling_hidden_dim: usize, @@ -229,11 +220,6 @@ pub struct DQNParams { /// Default: 0.001 (Rainbow standard) pub tau: f64, - /// Enable Distributional RL (C51) - /// Wave 2.3: Model return distribution instead of expected value - /// Expected impact: +15-25% performance, better risk modeling - pub use_distributional: bool, - /// Number of atoms for distributional RL (21, 51, 101) /// Wave 2.3: Distribution resolution (more atoms = finer granularity) /// Rainbow DQN standard: 51 @@ -247,11 +233,6 @@ pub struct DQNParams { /// Wave 2.3 / Q-freeze fix: v_range = v_max, v_min = -v_max pub v_max: f64, - /// Enable Noisy Networks for exploration - /// Wave 2.4: Learned exploration via noisy linear layers - /// Expected impact: +10-15% sample efficiency, better than epsilon-greedy - pub use_noisy_nets: bool, - /// Initial noise parameter for NoisyNets (0.1-1.0) /// Wave 2.4: Controls exploration magnitude /// Rainbow DQN standard: 0.5 @@ -374,13 +355,8 @@ pub struct DQNParams { /// Hidden dimension for each branching advantage head (64-256, step=64). /// Controls capacity of the per-dimension advantage streams in Branching DQN. - /// Only used when `use_branching` is true. pub branch_hidden_dim: usize, - /// Whether to use Branching DQN (3 independent heads) vs standard Dueling DQN. - /// Hyperopt-tunable via index 11 (repurposed from dead v_range_mirror slot). - pub use_branching: bool, - /// Gradient accumulation steps (1=disabled, 2/4/8 for larger effective batch). /// Only in search space for large GPUs (≥40GB): effective_batch = batch_size × accum_steps. /// H100 with batch=1024, accum=4 → effective batch 4096 (more stable gradients). @@ -443,10 +419,8 @@ impl Default for DQNParams { huber_delta: 10.0, // Conservative default (hyperopt can scale to 15-40) entropy_coefficient: 0.02, // C2: SAC-style regularization only (narrowed from 0.1) transaction_cost_multiplier: 1.0, - use_per: true, // P0: Default enabled for Rainbow DQN performance per_alpha: 0.6, // Rainbow DQN standard per_beta_start: 0.4, // Rainbow DQN standard - use_dueling: true, // Wave 8: Default ENABLED for full Rainbow DQN dueling_hidden_dim: 128, // Wave 6.3: Default 128 hidden units n_steps: 3, // Rainbow standard: 3-step TD (raised from 1) tau: 0.005, // Wave 2.2: Polyak soft update (raised lower bound for short hyperopt runs) @@ -459,11 +433,9 @@ impl Default for DQNParams { // Status: BUG #36 FIXED — scatter_add gradient flow verified (test_scatter_add_gradient_flow) // Re-enabled: Candle's scatter_add preserves BackpropOp on GPU // ============================================================================= - use_distributional: true, // ENABLED: BUG #36 fixed, full Rainbow DQN C51 num_atoms: 51, // Wave 2.3: Rainbow DQN standard v_min: -25.0, // DSR Q-values: rewards ±2 with gamma=0.92 → Q ≈ ±25 v_max: 25.0, // DSR Q-values: rewards ±2 with gamma=0.92 → Q ≈ ±25 - use_noisy_nets: true, // Wave 8: Default ENABLED for full Rainbow DQN noisy_sigma_init: 0.5, // Wave 2.4: Rainbow DQN standard minimum_profit_factor: 1.5, // BUG #7: Default 50% margin above breakeven weight_decay: 1e-4, // Default: 0.0001 (standard L2 regularization strength) @@ -507,7 +479,6 @@ impl Default for DQNParams { eval_softmax_temp: 0.1, // Default: nearly greedy (hyperopt will tune) dsr_eta: 0.01, // Default: ~100-step DSR lookback window branch_hidden_dim: 128, // Default: 128 hidden units per branching head - use_branching: true, // Default: branching enabled (3 heads) gradient_accumulation_steps: 1, // Default: no accumulation // Composite reward weights (defaults match dqn-production.toml) w_dsr: 1.0, @@ -795,7 +766,7 @@ impl ParameterSpace for DQNParams { let v_range = (max_abs_reward / (1.0 - gamma_val) * 1.2).clamp(20.0, 300.0); let v_min = -v_range; let v_max = v_range; - let use_branching = x[11].round() >= 1.0; + // x[11] was use_branching — always true, slot kept for index compatibility let noisy_sigma_init = x[12].exp().clamp(0.1, 1.0); let dueling_hidden_dim = (x[13].round() / 128.0).round() * 128.0; // Upper clamp matches large_gpu expansion in continuous_bounds_for (768); @@ -899,17 +870,13 @@ impl ParameterSpace for DQNParams { huber_delta, entropy_coefficient, transaction_cost_multiplier, - use_per: true, per_alpha, per_beta_start, - use_dueling: true, dueling_hidden_dim, n_steps, - use_distributional: true, num_atoms, v_min, v_max, - use_noisy_nets: true, noisy_sigma_init, minimum_profit_factor, weight_decay, @@ -949,7 +916,6 @@ impl ParameterSpace for DQNParams { eval_softmax_temp, dsr_eta, branch_hidden_dim, - use_branching, gradient_accumulation_steps, // C8: Composite reward weights (indices 31-38) w_dsr, @@ -986,7 +952,7 @@ impl ParameterSpace for DQNParams { self.per_alpha, // 8 self.per_beta_start, // 9 self.v_max, // 10: v_range (symmetric: v_min=-v_range) - if self.use_branching { 1.0 } else { 0.0 }, // 11: use_branching + 1.0, // 11: use_branching (always on — fixed at 1.0) self.noisy_sigma_init.ln(), // 12 self.dueling_hidden_dim as f64, // 13 self.n_steps as f64, // 14 @@ -1039,7 +1005,7 @@ impl ParameterSpace for DQNParams { "per_alpha", // 8 "per_beta_start", // 9 "v_range", // 10: symmetric support ±v_range - "use_branching", // 11: branching vs standard dueling + "use_branching_fixed", // 11: always true (slot kept for index compat) "noisy_sigma_init", // 12 "dueling_hidden_dim", // 13 "n_steps", // 14 @@ -1602,7 +1568,6 @@ impl DQNTrainer { let mut preload_hyperparams = DQNHyperparameters::default(); preload_hyperparams.mbp10_data_dir = self.mbp10_data_dir.clone(); preload_hyperparams.trades_data_dir = self.trades_data_dir.clone(); - preload_hyperparams.use_per = false; preload_hyperparams.enable_regime_qnetwork = false; preload_hyperparams.buffer_size = 1; let mut loader = InternalDQNTrainer::new_with_device(preload_hyperparams, self.device.clone()) @@ -2868,9 +2833,7 @@ impl HyperparameterOptimizable for DQNTrainer { min_epochs_before_stopping: self.epochs, // Disabled: early stopping off in hyperopt hold_penalty: -0.001, // Aligned with production (train_dqn.rs:285) // WAVE 1 AGENT 3: Huber loss configuration (matches production) - use_huber_loss: true, // CRITICAL: Must match production (--use-huber-loss) huber_delta: params.huber_delta, // WAVE 17: Use hyperopt value instead of fixed 1.0 - use_double_dqn: true, // Production feature: --use-double-dqn gradient_clip_norm: Some(gradient_clip_norm), // WAVE 64: Dynamic clipping (5.0 for high LR, 10.0 for low LR) hold_penalty_weight: params.w_idle, // Deprecated: mapped from w_idle for backward compat movement_threshold: 0.02, // Aligned with production (fixed at 2%, train_dqn.rs:296) @@ -2929,12 +2892,10 @@ impl HyperparameterOptimizable for DQNTrainer { triple_barrier_max_holding_seconds: 3600, // Default: 1 hour max holding // P0 CRITICAL: Prioritized Experience Replay (now tunable in hyperopt) - use_per: params.use_per, // Tunable boolean (default: true) per_alpha: params.per_alpha, // Tunable 0.4-0.8 (default: 0.6) per_beta_start: params.per_beta_start, // Tunable 0.2-0.6 (default: 0.4) // Wave 2.1: Dueling Networks - use_dueling: params.use_dueling, // Tunable boolean (default: false) dueling_hidden_dim: params.dueling_hidden_dim, // Wave 6.3: Use hyperopt value // Wave 2.2: Multi-Step Returns + Soft Updates @@ -2942,13 +2903,11 @@ impl HyperparameterOptimizable for DQNTrainer { // Note: tau is already set above in preprocessing section (line 1590) // Wave 2.3: Distributional RL (C51) - use_distributional: params.use_distributional, // Wave 6.3: Tunable boolean (default: false) num_atoms: params.num_atoms, // Wave 6.3: Tunable categorical (default: 51) v_min: params.v_min, // Wave 6.3: Tunable continuous (default: -1000.0) v_max: params.v_max, // Wave 6.3: Tunable continuous (default: 1000.0) // Wave 2.4: Noisy Networks - use_noisy_nets: params.use_noisy_nets, // Wave 6.3: Tunable boolean (default: false) noisy_sigma_init: params.noisy_sigma_init, // Wave 6.3: Tunable continuous (default: 0.5) // Two-Phase Feature Normalization Configuration @@ -3041,7 +3000,6 @@ impl HyperparameterOptimizable for DQNTrainer { spectral_norm_sigma_max: params.spectral_norm_sigma_max, // Conservative Q-Learning (CQL) — wired from hyperopt search space - use_cql: params.cql_alpha > 1e-6, cql_alpha: params.cql_alpha, // BUG #7: minimum_profit_factor — tunable via hyperopt (idx 25 in 26D, [1.1, 2.0]) @@ -3078,7 +3036,6 @@ impl HyperparameterOptimizable for DQNTrainer { iql_expectile_tau: 0.7, iql_advantage_temperature: 3.0, use_iql: false, - use_branching: params.use_branching, // Phase C: hyperopt-tunable (index 11) branch_hidden_dim: params.branch_hidden_dim, // AutoReplaySizer VRAM fraction: scale linearly from 0% at ≤8GB to 80% at ≥80GB. // Below 8GB the static buffer_size is used (regime heads + C51 need the headroom). @@ -3876,18 +3833,14 @@ mod tests { huber_delta: 24.77, entropy_coefficient: 0.03, // C2: narrowed range [0.005, 0.05] transaction_cost_multiplier: 1.0, - use_per: true, per_alpha: 0.6, per_beta_start: 0.4, - use_dueling: true, dueling_hidden_dim: 128, n_steps: 3, tau: 0.007, - use_distributional: true, num_atoms: 51, v_min: -25.0, v_max: 25.0, - use_noisy_nets: true, noisy_sigma_init: 0.5, minimum_profit_factor: 1.5, warmup_ratio: 0.0, @@ -3923,7 +3876,6 @@ mod tests { cql_alpha: 0.05, eval_softmax_temp: 0.8, branch_hidden_dim: 128, - use_branching: true, gradient_accumulation_steps: 4, // C8: Composite reward weights w_dsr: 1.5, @@ -3983,7 +3935,6 @@ mod tests { assert!((recovered.qr_kappa - 1.0).abs() < 1e-6); // C6: use_branching roundtrips via index 11 - assert_eq!(recovered.use_branching, params.use_branching); // C6: gradient_accumulation_steps roundtrips via index 29 assert_eq!(recovered.gradient_accumulation_steps, params.gradient_accumulation_steps); @@ -4078,7 +4029,7 @@ mod tests { assert_eq!(names[8], "per_alpha"); assert_eq!(names[9], "per_beta_start"); assert_eq!(names[10], "v_range"); - assert_eq!(names[11], "use_branching"); + assert_eq!(names[11], "use_branching_fixed"); assert_eq!(names[12], "noisy_sigma_init"); assert_eq!(names[13], "dueling_hidden_dim"); assert_eq!(names[14], "n_steps"); @@ -4141,12 +4092,8 @@ mod tests { ]; let params = DQNParams::from_continuous(&continuous).unwrap(); - assert!(params.use_per); assert!((params.per_alpha - 0.6).abs() < 1e-6); assert!((params.per_beta_start - 0.4).abs() < 1e-6); - assert!(params.use_dueling); - assert!(params.use_distributional); - assert!(params.use_noisy_nets); assert!((params.warmup_ratio - 0.0).abs() < 1e-6); assert!((params.curiosity_weight - 0.0).abs() < 1e-6); // C2: fixed assert!((params.noisy_epsilon_floor - 0.10).abs() < 1e-6); // Fixed: 10% floor @@ -4189,9 +4136,6 @@ mod tests { let params_min = DQNParams::from_continuous(&continuous_min).unwrap(); assert!((params_min.per_alpha - 0.4).abs() < 1e-6); assert!((params_min.per_beta_start - 0.2).abs() < 1e-6); - assert!(params_min.use_dueling); - assert!(params_min.use_distributional); - assert!(params_min.use_noisy_nets); // v_range now dynamic from gamma — just verify symmetric and positive assert!(params_min.v_min < 0.0, "v_min should be negative at min"); assert!(params_min.v_max > 0.0, "v_max should be positive at min"); @@ -4230,9 +4174,6 @@ mod tests { let params_max = DQNParams::from_continuous(&continuous_max).unwrap(); assert!((params_max.per_alpha - 0.8).abs() < 1e-6); assert!((params_max.per_beta_start - 0.6).abs() < 1e-6); - assert!(params_max.use_dueling); - assert!(params_max.use_distributional); - assert!(params_max.use_noisy_nets); // v_range dynamic from gamma — verify symmetric assert!(params_max.v_min < 0.0, "v_min should be negative at max"); assert!(params_max.v_max > 0.0, "v_max should be positive at max"); diff --git a/crates/ml/src/hyperopt/campaign.rs b/crates/ml/src/hyperopt/campaign.rs index b520c8391..665a985dc 100644 --- a/crates/ml/src/hyperopt/campaign.rs +++ b/crates/ml/src/hyperopt/campaign.rs @@ -311,11 +311,11 @@ mod tests { let config = CampaignConfig { model_type: ModelType::DQN, - num_trials: 5, // Quick local: 5 trials + num_trials: 3, // Quick validation: 3 trials data_dir: effective_dir, max_batch_size: 64, // RTX 3050 4GB early_stopping_eta: 3, - max_epochs_per_trial: 50, + max_epochs_per_trial: 20, // 20 epochs enough to validate metrics results_base_dir: PathBuf::from("ml/hyperopt_results"), mode: CampaignMode::Full, }; diff --git a/crates/ml/src/trainers/dqn/config.rs b/crates/ml/src/trainers/dqn/config.rs index 9428d0576..1510c93ad 100644 --- a/crates/ml/src/trainers/dqn/config.rs +++ b/crates/ml/src/trainers/dqn/config.rs @@ -102,7 +102,7 @@ impl DQNAgentType { ) -> Result, MLError> { match self { Self::Standard(agent) => { - if !agent.config.use_branching { + if !true { return Ok(None); } let branching_net = agent.branching_q_network.as_ref().ok_or_else(|| { @@ -754,10 +754,10 @@ impl DQNAgentType { /// - `false` → `extract_dueling_weights()` (reads `advantage_fc.*` keys) pub fn is_using_branching(&self) -> bool { match self { - Self::Standard(agent) => agent.config.use_branching && agent.branching_q_network.is_some(), + Self::Standard(agent) => true && agent.branching_q_network.is_some(), Self::RegimeConditional(agent) => { agent.get_trending_head() - .is_some_and(|h| h.config.use_branching && h.branching_q_network.is_some()) + .is_some_and(|h| h.branching_q_network.is_some()) } } } @@ -773,7 +773,7 @@ impl DQNAgentType { let shared_h1 = h.first().copied().unwrap_or(256); let shared_h2 = h.get(1).copied().unwrap_or(shared_h1); let value_h = agent.config.dueling_hidden_dim; - let adv_h = if agent.config.use_branching { + let adv_h = if true { agent.config.branch_hidden_dim } else { agent.config.dueling_hidden_dim @@ -788,7 +788,7 @@ impl DQNAgentType { let shared_h1 = h.first().copied().unwrap_or(256); let shared_h2 = h.get(1).copied().unwrap_or(shared_h1); let value_h = head.config.dueling_hidden_dim; - let adv_h = if head.config.use_branching { + let adv_h = if true { head.config.branch_hidden_dim } else { head.config.dueling_hidden_dim @@ -861,12 +861,8 @@ pub struct DQNHyperparameters { /// When greedy Q(best_exposure) - Q(flat) < threshold, default to flat. /// 0.0 = disabled. Typical range [0.0, 0.5]. pub q_gap_threshold: f64, - /// Use Huber loss instead of MSE (more robust to outliers) - pub use_huber_loss: bool, /// Huber loss delta threshold (default: 1.0) pub huber_delta: f64, - /// Use Double DQN to reduce overestimation bias - pub use_double_dqn: bool, /// Gradient clipping max norm (None = disabled) pub gradient_clip_norm: Option, /// Deprecated: was HOLD action penalty weight, replaced by `w_idle` in GPU composite reward. @@ -965,15 +961,11 @@ pub struct DQNHyperparameters { pub triple_barrier_stop_loss_bps: u32, pub triple_barrier_max_holding_seconds: u64, - // P0: Prioritized Experience Replay - /// Enable Prioritized Experience Replay (PER) - pub use_per: bool, + // P0: Prioritized Experience Replay (always enabled) pub per_alpha: f64, pub per_beta_start: f64, - // Wave 2.1: Dueling Networks - /// Enable Dueling DQN architecture (separate value/advantage streams) - pub use_dueling: bool, + // Wave 2.1: Dueling Networks (always enabled) /// Hidden dimension for dueling value/advantage streams pub dueling_hidden_dim: usize, @@ -982,9 +974,7 @@ pub struct DQNHyperparameters { /// Recommended: 3-5 for balance between bias and variance pub n_steps: usize, - // Wave 2.3: Distributional RL (C51) - /// Enable distributional RL (C51 algorithm) - pub use_distributional: bool, + // Wave 2.3: Distributional RL (C51) — always enabled /// Number of atoms for value distribution (Rainbow DQN standard: 51) pub num_atoms: usize, /// Minimum value for distribution support (must be -v_max for symmetric support) @@ -992,10 +982,7 @@ pub struct DQNHyperparameters { /// Maximum value for distribution support (v_min = -v_max enforced by hyperopt) pub v_max: f64, - // Wave 2.4: Noisy Networks for Exploration - /// Enable Noisy Networks (replaces epsilon-greedy exploration) - /// CRITICAL: Mutually exclusive with epsilon decay (set epsilon to 0 when enabled) - pub use_noisy_nets: bool, + // Wave 2.4: Noisy Networks for Exploration (always enabled) /// Initial noise std dev (Rainbow DQN standard: 0.5, scaled by 1/√in_features) pub noisy_sigma_init: f64, @@ -1110,9 +1097,7 @@ pub struct DQNHyperparameters { /// Recommended: 1e-4 for standard training, 1e-3 for aggressive regularization. pub weight_decay: f64, - // Conservative Q-Learning (CQL) — prevents Q-value overestimation on offline/replay data - /// Enable Conservative Q-Learning (default: true, alpha controls strength) - pub use_cql: bool, + // Conservative Q-Learning (CQL) — always enabled, alpha controls strength /// CQL regularization strength (default: 0.1, range 0.0-1.0) /// 0.0 = disabled, 0.1 = mild conservatism, 1.0 = full offline-RL strength pub cql_alpha: f64, @@ -1200,11 +1185,7 @@ pub struct DQNHyperparameters { /// and extracts the policy via advantage-weighted regression. Complementary to CQL. pub use_iql: bool, - // Phase C+: Branching DQN (Tavakoli et al., 2018) - /// Enable Branching DQN with independent advantage heads per action dimension. - /// When enabled, 3 independent heads (exposure=5, order=3, urgency=3) replace - /// the single Q-network, and actions are stored as factored indices (0-44). - pub use_branching: bool, + // Phase C+: Branching DQN (Tavakoli et al., 2018) — always enabled /// Hidden dimension for each branching advantage head (64-256, step=64). /// Controls capacity of the per-dimension advantage streams in Branching DQN. /// Only used when `use_branching` is true. @@ -1330,9 +1311,7 @@ impl DQNHyperparameters { loss_aversion: 1.5, time_decay_rate: 0.0005, q_gap_threshold: 0.05, // Tier 2 default: mild conviction gating (hyperopt searches [0.0, 0.5]) - use_huber_loss: true, // Default: Huber loss enabled (more robust) huber_delta: 100.0, // BUG #12 FIX: Scale delta 100x for gradient explosion fix (was 1.0) - use_double_dqn: true, // Default: Double DQN enabled (prevents overestimation bias) gradient_clip_norm: Some(10.0), // C51 101-atoms × 3 branches → gradient naturally 300x larger than DQN hold_penalty_weight: 0.01, // Default: 1% penalty weight movement_threshold: 0.02, // Default: 2% price movement threshold @@ -1392,26 +1371,22 @@ impl DQNHyperparameters { triple_barrier_max_holding_seconds: 3600, // P0: Prioritized Experience Replay (WAVE 6.4: ENABLED BY DEFAULT) - use_per: true, per_alpha: 0.6, per_beta_start: 0.6, // Higher start → faster IS weight correction (was 0.4) // Wave 2.1: Dueling Networks (WAVE 6.4: ENABLED BY DEFAULT) - use_dueling: true, // Default: enabled (Rainbow DQN standard) dueling_hidden_dim: 128, // Default: 128 hidden units // Wave 2.2: Multi-Step Returns (WAVE 6.4: ENABLED BY DEFAULT) n_steps: 3, // Default: 3 (Rainbow DQN standard) // Wave 2.3: Distributional RL (BUG #36 FIXED — scatter_add gradient flow verified) - use_distributional: true, // Default: ENABLED (C51 Rainbow DQN standard, BUG #36 fixed) num_atoms: 51, // Rainbow DQN standard: 51 atoms v_min: -50.0, // Reward v6: SCALE=10, gamma=0.95 → Q*≈200. Default ±50 for smoke tests. v_max: 50.0, // Hyperopt computes dynamically from gamma: ±(10/(1-γ)*1.2). // Wave 2.4: Noisy Networks (WAVE 6.4: ENABLED BY DEFAULT) - use_noisy_nets: true, // Default: enabled (replaces epsilon-greedy) noisy_sigma_init: 0.5, // Rainbow DQN standard: 0.5 // Two-Phase Feature Normalization Configuration @@ -1473,7 +1448,6 @@ impl DQNHyperparameters { weight_decay: 1e-4, // Default: 0.0001 (standard regularization strength) // Conservative Q-Learning (CQL) - use_cql: true, // Default: enabled (prevents Q-value overestimation) cql_alpha: 0.1, // Default: mild conservatism (0.1 of 0.0-1.0 range) // QR-DQN (complementary to C51 — IQN for quantile estimation) @@ -1511,7 +1485,6 @@ impl DQNHyperparameters { iql_expectile_tau: 0.7, iql_advantage_temperature: 3.0, use_iql: false, - use_branching: true, branch_hidden_dim: 128, // GPU walk-forward: disabled by default (single-pass training) @@ -1613,12 +1586,10 @@ pub(crate) fn dqn_default_config() -> DQNConfig { gamma: 0.95, gradient_clip_norm: 100.0, huber_delta: 10.0, - use_huber_loss: true, // Replay Buffer replay_buffer_capacity: 500_000, min_replay_size: 10_000, - use_per: true, per_alpha: 0.6, per_beta_start: 0.4, per_beta_max: 1.0, @@ -1639,8 +1610,6 @@ pub(crate) fn dqn_default_config() -> DQNConfig { tau_anneal_steps: 100_000, // Rainbow DQN Components - use_double_dqn: true, - use_dueling: true, dueling_hidden_dim: 256, num_atoms: 51, v_min: -2.0, // Reward v4: tight C51 atoms for per-bar percentage returns @@ -1658,7 +1627,6 @@ pub(crate) fn dqn_default_config() -> DQNConfig { initial_capital: 100_000.0, // CQL + IQN (2026 modernization) - use_cql: true, cql_alpha: 1.0, use_iqn: true, iqn_num_quantiles: 64, diff --git a/crates/ml/src/trainers/dqn/fused_training.rs b/crates/ml/src/trainers/dqn/fused_training.rs index fd7730090..bf504bead 100644 --- a/crates/ml/src/trainers/dqn/fused_training.rs +++ b/crates/ml/src/trainers/dqn/fused_training.rs @@ -191,7 +191,6 @@ impl FusedTrainingCtx { iqn_kappa: dqn.config.iqn_kappa, entropy_coefficient: dqn.config.entropy_coefficient as f32, c51_warmup_epochs: hyperparams.c51_warmup_epochs, - use_cql: hyperparams.use_cql, cql_alpha: hyperparams.cql_alpha as f32, }; diff --git a/crates/ml/src/trainers/dqn/smoke_tests/helpers.rs b/crates/ml/src/trainers/dqn/smoke_tests/helpers.rs index ac7cc9ff6..89630d389 100644 --- a/crates/ml/src/trainers/dqn/smoke_tests/helpers.rs +++ b/crates/ml/src/trainers/dqn/smoke_tests/helpers.rs @@ -34,9 +34,7 @@ pub(super) fn smoke_params() -> DQNHyperparameters { profile.apply_to(&mut p); // ── Production features (always on — not in the TOML profile) ── - p.use_branching = true; p.curiosity_weight = 0.1; // Curiosity — always on - p.use_cql = true; // Conservative Q-Learning — always on p.cql_alpha = 0.1; p.enable_kelly_sizing = true; // Kelly criterion — always on p.kelly_fractional = 0.5; @@ -44,7 +42,6 @@ pub(super) fn smoke_params() -> DQNHyperparameters { p.enable_action_masking = true; // Action masking — always on p.max_position_absolute = 2.0; p.enable_circuit_breaker = true; // Circuit breaker — always on - p.use_double_dqn = true; // Double DQN — always on // ── Testing flags (not training config) ── p.enable_stress_testing = false; diff --git a/crates/ml/src/trainers/dqn/trainer/action.rs b/crates/ml/src/trainers/dqn/trainer/action.rs index f6c02c39d..23c73c268 100644 --- a/crates/ml/src/trainers/dqn/trainer/action.rs +++ b/crates/ml/src/trainers/dqn/trainer/action.rs @@ -72,7 +72,7 @@ impl DQNTrainer { let epsilon = adjusted_epsilon as f32; debug!("Epsilon: base={:.4}, volatility-adjusted={:.4}", base_epsilon, adjusted_epsilon); let batch_q_values = agent.forward(&batch_tensor).map_err(|e| anyhow::anyhow!("Batched forward pass failed: {}", e))?; - let branching_q_tensors: Option<(GpuTensor, GpuTensor, GpuTensor)> = if self.hyperparams.use_branching { + let branching_q_tensors: Option<(GpuTensor, GpuTensor, GpuTensor)> = if true { agent.batch_branching_q_values(&batch_tensor).map_err(|e| anyhow::anyhow!("Branching Q-values failed: {}", e))? } else { None }; drop(agent); @@ -90,7 +90,7 @@ impl DQNTrainer { // Upload UCB count bonuses for branching exploration. // CountBonus tracks per-branch action counts and computes UCB bonuses. // Bonuses encourage trying under-explored actions (exploration pressure). - if self.hyperparams.use_branching { + if true { let agent_r = self.agent.read().await; if let Some(bonuses) = agent_r.get_count_bonuses_branched() { let coeff = self.hyperparams.count_bonus_coefficient.unwrap_or(0.0) as f32; diff --git a/crates/ml/src/trainers/dqn/trainer/constructor.rs b/crates/ml/src/trainers/dqn/trainer/constructor.rs index d85f00b78..bc4a37537 100644 --- a/crates/ml/src/trainers/dqn/trainer/constructor.rs +++ b/crates/ml/src/trainers/dqn/trainer/constructor.rs @@ -252,8 +252,6 @@ impl DQNTrainer { batch_size: hyperparams.batch_size, min_replay_size: hyperparams.min_replay_size.min(hyperparams.buffer_size), // Cap at buffer_size to prevent deadlock target_update_freq: hyperparams.target_update_frequency, // Use hyperparameter instead of hardcoded 1000 - use_double_dqn: true, - use_huber_loss: hyperparams.use_huber_loss, huber_delta: hyperparams.huber_delta as f32, leaky_relu_alpha: 0.01, // Standard LeakyReLU alpha (prevents dead neurons) gradient_clip_norm: hyperparams.gradient_clip_norm.unwrap_or(10.0), // Wave 11 Bug #1 fix: Dynamic clipping @@ -269,7 +267,6 @@ impl DQNTrainer { // PER configuration initial_capital: hyperparams.initial_capital as f64, - use_per: hyperparams.use_per, per_alpha: hyperparams.per_alpha, per_beta_start: hyperparams.per_beta_start, @@ -278,7 +275,6 @@ impl DQNTrainer { per_max_memory_bytes, // Wave 2.1: Dueling Networks (ENABLED BY DEFAULT - Wave 6.4) - use_dueling: hyperparams.use_dueling, dueling_hidden_dim: hyperparams.dueling_hidden_dim, // Wave 2.2: Multi-Step Returns (N-step TD) (ENABLED BY DEFAULT - Wave 6.4) @@ -301,13 +297,11 @@ impl DQNTrainer { gradient_collapse_multiplier: hyperparams.gradient_collapse_multiplier, gradient_collapse_patience: hyperparams.gradient_collapse_patience, - use_cql: hyperparams.use_cql, cql_alpha: hyperparams.cql_alpha, iqn_num_quantiles: hyperparams.num_quantiles, // Controlled by hyperopt iqn_kappa: hyperparams.qr_kappa as f32, // Controlled by hyperopt (f64→f32) iqn_embedding_dim: 64, // Fixed (not in search space) iqn_lambda: hyperparams.iqn_lambda as f32, // IQN dual-head loss weight (f64→f32) - use_branching: hyperparams.use_branching, branch_hidden_dim: hyperparams.branch_hidden_dim, use_regime_conditioning: true, // Always enable per-regime IS weights for branching loss use_cvar_action_selection: hyperparams.use_cvar_action_selection, diff --git a/crates/ml/src/trainers/dqn/trainer/metrics.rs b/crates/ml/src/trainers/dqn/trainer/metrics.rs index dc3c3fd40..fc01c6d8a 100644 --- a/crates/ml/src/trainers/dqn/trainer/metrics.rs +++ b/crates/ml/src/trainers/dqn/trainer/metrics.rs @@ -526,7 +526,7 @@ impl DQNTrainer { // Direction LUT maps exposure head index → position size direction. // 0=Short100(-1.0), 1=Short50(-0.5), 2=Flat(0.0), 3=Long50(0.5), 4=Long100(1.0) const DIRECTION_LUT: [f32; 9] = [-1.0, -0.75, -0.5, -0.25, 0.0, 0.25, 0.5, 0.75, 1.0]; - let use_branching = self.hyperparams.use_branching && total_actions >= 15; + let use_branching = true && total_actions >= 15; let rewards: Vec = (0..sample_size).map(|i| { let row_start = i * total_actions; diff --git a/crates/ml/src/trainers/dqn/trainer/mod.rs b/crates/ml/src/trainers/dqn/trainer/mod.rs index 7013aba79..e81acba84 100644 --- a/crates/ml/src/trainers/dqn/trainer/mod.rs +++ b/crates/ml/src/trainers/dqn/trainer/mod.rs @@ -358,7 +358,6 @@ impl DQNTrainer { let mut inner_hp = self.hyperparams.clone(); inner_hp.enable_stress_testing = false; inner_hp.enable_regime_qnetwork = false; - inner_hp.use_per = false; inner_hp.buffer_size = 1; let inner_trainer = Self::new_with_device(inner_hp, self.device.clone()) diff --git a/crates/ml/src/trainers/dqn/trainer/tests.rs b/crates/ml/src/trainers/dqn/trainer/tests.rs index 9c7f0711c..1933b9ba8 100644 --- a/crates/ml/src/trainers/dqn/trainer/tests.rs +++ b/crates/ml/src/trainers/dqn/trainer/tests.rs @@ -37,7 +37,6 @@ fn create_test_params() -> DQNHyperparameters { let mut params = DQNHyperparameters::conservative(); // Production default: branching DQN (3 heads: exposure, order, urgency). // Always enabled — the warp-cooperative kernel on H100 requires it. - params.use_branching = true; params.hidden_dim_base = Some(32); // Small for fast test iterations params.buffer_size = 10_000; params.replay_buffer_vram_fraction = 0.0; // Disable AutoReplaySizer in tests diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index fa745a8a2..509ef5150 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -484,13 +484,13 @@ impl DQNTrainer { info!("Rainbow DQN Components:"); info!(" Double DQN (always enabled)"); - if self.hyperparams.use_dueling { + if true { info!(" Dueling Networks (value/advantage streams, hidden_dim={})", self.hyperparams.dueling_hidden_dim); } else { info!(" Dueling Networks: disabled"); } - if self.hyperparams.use_per { + if true { info!(" Prioritized Experience Replay (a={}, b={}->1.0)", self.hyperparams.per_alpha, self.hyperparams.per_beta_start); } else { @@ -503,20 +503,20 @@ impl DQNTrainer { info!(" N-Step Returns (n=1, standard TD)"); } - if self.hyperparams.use_distributional { + if true { info!(" Categorical DQN (atoms={}, V=[{}, {}])", self.hyperparams.num_atoms, self.hyperparams.v_min, self.hyperparams.v_max); } else { info!(" Categorical DQN / C51: disabled"); } - if self.hyperparams.use_noisy_nets { + if true { info!(" Noisy Networks (sigma_init={})", self.hyperparams.noisy_sigma_init); } else { info!(" Noisy Networks: disabled"); } - if self.hyperparams.use_noisy_nets { + if true { let floor = self.hyperparams.noisy_epsilon_floor.unwrap_or(0.05); let mut agent = self.agent.write().await; agent.set_epsilon(floor); @@ -1019,14 +1019,14 @@ impl DQNTrainer { .unwrap_or(0.0) as f32, q_clip_min: -500.0, q_clip_max: 500.0, - huber_kappa: if self.hyperparams.use_huber_loss { + huber_kappa: if true { self.hyperparams.huber_delta as f32 } else { 0.0 }, - use_noisy_nets: self.hyperparams.use_noisy_nets, + use_noisy_nets: true, noisy_sigma_init: self.hyperparams.noisy_sigma_init as f32, - use_distributional: self.hyperparams.use_distributional, + use_distributional: true, num_atoms: self.hyperparams.num_atoms as i32, v_min: self.hyperparams.v_min as f32, v_max: self.hyperparams.v_max as f32, @@ -1605,7 +1605,7 @@ impl DQNTrainer { /// Samples stale experiences, runs cuBLAS forward via fused_ctx to compute /// max Q-values (TD error proxy), then updates priorities via GPU PER. pub(crate) async fn refresh_stale_per_priorities(&mut self, epoch: usize) -> Result<()> { - if !self.hyperparams.use_per || self.hyperparams.n_steps <= 1 { + if !true || self.hyperparams.n_steps <= 1 { return Ok(()); } @@ -1878,7 +1878,7 @@ impl DQNTrainer { ); // Epsilon decay (skip when noisy nets) - if !self.hyperparams.use_noisy_nets { + if !true { let mut agent = self.agent.write().await; agent.update_epsilon(); } @@ -2066,7 +2066,7 @@ impl DQNTrainer { " Exploration: entropy={:.3} (1.0=uniform), epsilon={:.4}, noisy_nets={}, count_bonus={}", epoch_entropy, self.get_epsilon().await.unwrap_or(0.0), - self.hyperparams.use_noisy_nets, + true, self.hyperparams.count_bonus_coefficient.unwrap_or(0.0) > 0.0, ); diff --git a/crates/ml/src/training_profile.rs b/crates/ml/src/training_profile.rs index 3746d173d..42f1e1715 100644 --- a/crates/ml/src/training_profile.rs +++ b/crates/ml/src/training_profile.rs @@ -78,7 +78,6 @@ pub struct ExplorationSection { pub struct ReplayBufferSection { pub buffer_size: Option, pub min_replay_size: Option, - pub use_per: Option, pub per_alpha: Option, pub per_beta_start: Option, } @@ -86,7 +85,6 @@ pub struct ReplayBufferSection { /// C51 distributional RL parameters. #[derive(Debug, Clone, Deserialize, Default)] pub struct DistributionalSection { - pub use_distributional: Option, pub num_atoms: Option, pub v_min: Option, pub v_max: Option, @@ -95,7 +93,6 @@ pub struct DistributionalSection { /// Branching DQN per-head action-space sizes. #[derive(Debug, Clone, Deserialize, Default)] pub struct BranchingSection { - pub use_branching: Option, /// Branch 0: exposure head size (default 5). pub branch_0_size: Option, /// Branch 1: order-type head size (default 3). @@ -107,11 +104,7 @@ pub struct BranchingSection { /// Advanced Rainbow DQN features. #[derive(Debug, Clone, Deserialize, Default)] pub struct AdvancedSection { - pub use_double_dqn: Option, - pub use_dueling: Option, - pub use_noisy_nets: Option, pub noisy_sigma_init: Option, - pub use_cql: Option, pub cql_alpha: Option, pub gradient_accumulation_steps: Option, } @@ -598,9 +591,6 @@ impl DqnTrainingProfile { if let Some(v) = rb.min_replay_size { hp.min_replay_size = v; } - if let Some(v) = rb.use_per { - hp.use_per = v; - } if let Some(v) = rb.per_alpha { hp.per_alpha = v; } @@ -611,9 +601,6 @@ impl DqnTrainingProfile { // [distributional] if let Some(ref d) = self.distributional { - if let Some(v) = d.use_distributional { - hp.use_distributional = v; - } if let Some(v) = d.num_atoms { hp.num_atoms = v; } @@ -625,33 +612,15 @@ impl DqnTrainingProfile { } } - // [branching] - if let Some(ref b) = self.branching { - if let Some(v) = b.use_branching { - hp.use_branching = v; - } - // branch_0_size / branch_1_size / branch_2_size are informational only - // (action space is fixed by DQN architecture constants in CUDA header). - // We store them for documentation but do not map to hp fields. - } + // [branching] — informational only (use_branching always enabled). + // branch_0_size / branch_1_size / branch_2_size are documentation fields; + // action space is fixed by DQN architecture constants in CUDA header. // [advanced] if let Some(ref a) = self.advanced { - if let Some(v) = a.use_double_dqn { - hp.use_double_dqn = v; - } - if let Some(v) = a.use_dueling { - hp.use_dueling = v; - } - if let Some(v) = a.use_noisy_nets { - hp.use_noisy_nets = v; - } if let Some(v) = a.noisy_sigma_init { hp.noisy_sigma_init = v; } - if let Some(v) = a.use_cql { - hp.use_cql = v; - } if let Some(v) = a.cql_alpha { hp.cql_alpha = v; } diff --git a/crates/ml/src/validation/adapters.rs b/crates/ml/src/validation/adapters.rs index 03b67017d..5d7bb1941 100644 --- a/crates/ml/src/validation/adapters.rs +++ b/crates/ml/src/validation/adapters.rs @@ -223,9 +223,6 @@ mod tests { config.min_replay_size = 4; config.warmup_steps = 0; config.use_iqn = false; - config.use_dueling = false; - config.use_per = true; // GPU PER mandatory on CUDA - config.use_branching = false; config.epsilon_start = 0.5; config } diff --git a/crates/ml/tests/dqn_action_collapse_fix_test.rs b/crates/ml/tests/dqn_action_collapse_fix_test.rs index e44a53ab1..d11a6268b 100644 --- a/crates/ml/tests/dqn_action_collapse_fix_test.rs +++ b/crates/ml/tests/dqn_action_collapse_fix_test.rs @@ -142,7 +142,6 @@ fn test_cql_configurable_via_hyperparameters() { let hp = DQNHyperparameters::default(); // use_cql and cql_alpha should exist and have correct defaults - assert!(hp.use_cql, "use_cql should default to true"); let alpha_diff = (hp.cql_alpha - 0.1).abs(); assert!( alpha_diff < 1e-6, @@ -152,9 +151,7 @@ fn test_cql_configurable_via_hyperparameters() { // Should be overridable let mut hp2 = DQNHyperparameters::default(); - hp2.use_cql = false; hp2.cql_alpha = 0.0; - assert!(!hp2.use_cql, "use_cql should be overridable to false"); assert!( hp2.cql_alpha.abs() < 1e-10, "cql_alpha should be overridable to 0.0" @@ -167,7 +164,6 @@ async fn test_cql_alpha_wired_through_trainer() -> Result<()> { // Create trainer with custom CQL alpha let mut hp = DQNHyperparameters::default(); - hp.use_cql = true; hp.cql_alpha = 0.05; // Custom value hp.epochs = 1; @@ -316,10 +312,7 @@ fn test_count_bonus_accessible() { config.state_dim = 8; config.num_actions = 45; // Factored action space (non-branching) config.hidden_dims = vec![32, 16]; - config.use_dueling = false; - config.use_per = true; // GPU PER mandatory on CUDA config.use_iqn = false; - config.use_branching = false; // Non-branching: all 45 actions flat config.batch_size = 8; config.min_replay_size = 8; config.warmup_steps = 0; diff --git a/crates/ml/tests/dqn_feature_defaults_test.rs b/crates/ml/tests/dqn_feature_defaults_test.rs index 509f8fef2..10b317ac3 100644 --- a/crates/ml/tests/dqn_feature_defaults_test.rs +++ b/crates/ml/tests/dqn_feature_defaults_test.rs @@ -90,10 +90,6 @@ fn test_all_rainbow_features_enabled_by_default() { let hyperparams = DQNHyperparameters::conservative(); // All Rainbow components should be enabled by default - assert!(hyperparams.use_dueling, "Dueling should be enabled by default"); - assert!(hyperparams.use_distributional, "Distributional should be enabled by default"); - assert!(hyperparams.use_noisy_nets, "Noisy nets should be enabled by default"); - assert!(hyperparams.use_per, "PER should be enabled by default"); assert_eq!(hyperparams.n_steps, 3, "Multi-step (n=3) should be enabled by default"); } @@ -137,7 +133,6 @@ fn test_dueling_parameters_valid() { let hyperparams = DQNHyperparameters::conservative(); // Validate dueling network parameters - assert!(hyperparams.use_dueling, "Dueling should be enabled"); assert_eq!(hyperparams.dueling_hidden_dim, 128, "Hidden dim should be 128"); assert!(hyperparams.dueling_hidden_dim >= 32, "Hidden dim should be at least 32"); assert!(hyperparams.dueling_hidden_dim <= 512, "Hidden dim should be reasonable (<= 512)"); @@ -148,7 +143,6 @@ fn test_noisy_nets_parameters_valid() { let hyperparams = DQNHyperparameters::conservative(); // Validate noisy networks parameters - assert!(hyperparams.use_noisy_nets, "Noisy nets should be enabled"); assert_eq!(hyperparams.noisy_sigma_init, 0.5, "Sigma init should be 0.5 (Rainbow standard)"); assert!(hyperparams.noisy_sigma_init > 0.0, "Sigma must be positive"); assert!(hyperparams.noisy_sigma_init <= 1.0, "Sigma should be reasonable (<= 1.0)"); @@ -159,7 +153,6 @@ fn test_per_parameters_valid() { let hyperparams = DQNHyperparameters::conservative(); // Validate PER parameters - assert!(hyperparams.use_per, "PER should be enabled"); assert_eq!(hyperparams.per_alpha, 0.6, "PER alpha should be 0.6"); assert_eq!(hyperparams.per_beta_start, 0.4, "PER beta start should be 0.4"); assert!(hyperparams.per_alpha >= 0.0 && hyperparams.per_alpha <= 1.0, "Alpha must be in [0, 1]"); @@ -170,19 +163,11 @@ fn test_per_parameters_valid() { fn test_can_create_vanilla_dqn_config() { // Test that we can create a vanilla DQN config with all Rainbow features disabled let vanilla_config = DQNHyperparameters { - use_dueling: false, - use_distributional: false, - use_noisy_nets: false, - use_per: false, n_steps: 1, ..DQNHyperparameters::conservative() }; // Verify vanilla DQN configuration - assert!(!vanilla_config.use_dueling, "Dueling should be disabled"); - assert!(!vanilla_config.use_distributional, "Distributional should be disabled"); - assert!(!vanilla_config.use_noisy_nets, "Noisy nets should be disabled"); - assert!(!vanilla_config.use_per, "PER should be disabled"); assert_eq!(vanilla_config.n_steps, 1, "n_steps should be 1 for vanilla DQN"); } @@ -190,31 +175,16 @@ fn test_can_create_vanilla_dqn_config() { fn test_can_disable_individual_rainbow_features() { // Test that we can selectively disable Rainbow features let config_no_dueling = DQNHyperparameters { - use_dueling: false, ..DQNHyperparameters::conservative() }; - assert!(!config_no_dueling.use_dueling); - assert!(config_no_dueling.use_distributional); // Others still enabled - assert!(config_no_dueling.use_noisy_nets); - assert!(config_no_dueling.use_per); let config_no_distributional = DQNHyperparameters { - use_distributional: false, ..DQNHyperparameters::conservative() }; - assert!(config_no_distributional.use_dueling); // Others still enabled - assert!(!config_no_distributional.use_distributional); - assert!(config_no_distributional.use_noisy_nets); - assert!(config_no_distributional.use_per); let config_no_noisy = DQNHyperparameters { - use_noisy_nets: false, ..DQNHyperparameters::conservative() }; - assert!(config_no_noisy.use_dueling); // Others still enabled - assert!(config_no_noisy.use_distributional); - assert!(!config_no_noisy.use_noisy_nets); - assert!(config_no_noisy.use_per); } #[test] @@ -223,37 +193,19 @@ fn test_rainbow_feature_combinations() { // Rainbow without noisy nets (use epsilon-greedy instead) let rainbow_epsilon_greedy = DQNHyperparameters { - use_noisy_nets: false, ..DQNHyperparameters::conservative() }; - assert!(rainbow_epsilon_greedy.use_dueling); - assert!(rainbow_epsilon_greedy.use_distributional); - assert!(!rainbow_epsilon_greedy.use_noisy_nets); - assert!(rainbow_epsilon_greedy.use_per); // Rainbow without PER (uniform replay) let rainbow_uniform_replay = DQNHyperparameters { - use_per: false, ..DQNHyperparameters::conservative() }; - assert!(rainbow_uniform_replay.use_dueling); - assert!(rainbow_uniform_replay.use_distributional); - assert!(rainbow_uniform_replay.use_noisy_nets); - assert!(!rainbow_uniform_replay.use_per); // Double DQN + Dueling only (no distributional, no noisy, no PER) let double_dueling_only = DQNHyperparameters { - use_dueling: true, - use_distributional: false, - use_noisy_nets: false, - use_per: false, n_steps: 1, ..DQNHyperparameters::conservative() }; - assert!(double_dueling_only.use_dueling); - assert!(!double_dueling_only.use_distributional); - assert!(!double_dueling_only.use_noisy_nets); - assert!(!double_dueling_only.use_per); } #[test] @@ -316,9 +268,5 @@ fn test_backward_compatibility() { assert!(config.epochs > 0); // Rainbow features should all be enabled - assert!(config.use_dueling); - assert!(config.use_distributional); - assert!(config.use_noisy_nets); - assert!(config.use_per); assert_eq!(config.n_steps, 3); } diff --git a/crates/ml/tests/dqn_hyperopt_json_export_test.rs b/crates/ml/tests/dqn_hyperopt_json_export_test.rs index abe26ab31..17bde390d 100644 --- a/crates/ml/tests/dqn_hyperopt_json_export_test.rs +++ b/crates/ml/tests/dqn_hyperopt_json_export_test.rs @@ -290,18 +290,14 @@ mod hyperopt_json_export_tests { assert_eq!(saved_params.huber_delta, roundtrip_params.huber_delta); assert_eq!(saved_params.entropy_coefficient, roundtrip_params.entropy_coefficient); assert_eq!(saved_params.transaction_cost_multiplier, roundtrip_params.transaction_cost_multiplier); - assert_eq!(saved_params.use_per, roundtrip_params.use_per); assert_eq!(saved_params.per_alpha, roundtrip_params.per_alpha); assert_eq!(saved_params.per_beta_start, roundtrip_params.per_beta_start); - assert_eq!(saved_params.use_dueling, roundtrip_params.use_dueling); assert_eq!(saved_params.dueling_hidden_dim, roundtrip_params.dueling_hidden_dim); assert_eq!(saved_params.n_steps, roundtrip_params.n_steps); assert_eq!(saved_params.tau, roundtrip_params.tau); - assert_eq!(saved_params.use_distributional, roundtrip_params.use_distributional); assert_eq!(saved_params.num_atoms, roundtrip_params.num_atoms); assert_eq!(saved_params.v_min, roundtrip_params.v_min); assert_eq!(saved_params.v_max, roundtrip_params.v_max); - assert_eq!(saved_params.use_noisy_nets, roundtrip_params.use_noisy_nets); assert_eq!(saved_params.noisy_sigma_init, roundtrip_params.noisy_sigma_init); assert_eq!(saved_params.minimum_profit_factor, roundtrip_params.minimum_profit_factor); diff --git a/crates/ml/tests/dqn_inference_test.rs b/crates/ml/tests/dqn_inference_test.rs index 0355008ff..69a0b9e69 100644 --- a/crates/ml/tests/dqn_inference_test.rs +++ b/crates/ml/tests/dqn_inference_test.rs @@ -134,7 +134,6 @@ fn build_matching_config(checkpoint_tensors: &safetensors::SafeTensors<'_>) -> D config.noisy_sigma_init = 0.5; // Match trainer defaults: IQN disabled, CQL enabled with alpha=0.1 config.use_iqn = false; - config.use_cql = true; config.cql_alpha = 0.1; config } diff --git a/crates/ml/tests/dqn_iqn_integration_test.rs b/crates/ml/tests/dqn_iqn_integration_test.rs index e8f5923b6..7d612dcc0 100644 --- a/crates/ml/tests/dqn_iqn_integration_test.rs +++ b/crates/ml/tests/dqn_iqn_integration_test.rs @@ -90,10 +90,7 @@ fn test_full_iqn_cql_training_loop() { config.hidden_dims = vec![32, 16]; config.use_iqn = true; config.iqn_num_quantiles = 16; - config.use_cql = true; config.cql_alpha = 1.0; - config.use_dueling = false; - config.use_per = true; // GPU PER mandatory on CUDA config.batch_size = 8; config.min_replay_size = 8; config.warmup_steps = 0; @@ -144,8 +141,6 @@ fn test_iqn_only_no_cql() { config.hidden_dims = vec![16, 16]; config.use_iqn = true; config.iqn_num_quantiles = 8; - config.use_cql = false; - config.use_dueling = false; config.batch_size = 4; config.min_replay_size = 4; config.warmup_steps = 0; @@ -175,7 +170,6 @@ fn test_cvar_action_selection_integration() { config.hidden_dims = vec![16, 16]; config.use_iqn = true; config.iqn_num_quantiles = 8; - config.use_dueling = false; config.epsilon_start = 0.0; config.warmup_steps = 0; config.use_cvar_action_selection = true; diff --git a/crates/ml/tests/dqn_trainer_integration_tests.rs b/crates/ml/tests/dqn_trainer_integration_tests.rs index 9afd512ab..f70b2a1a9 100644 --- a/crates/ml/tests/dqn_trainer_integration_tests.rs +++ b/crates/ml/tests/dqn_trainer_integration_tests.rs @@ -336,7 +336,6 @@ async fn test_p0_batch_diversity_cooldown() -> Result<()> { async fn test_p0_all_features_together() -> Result<()> { // Create hyperparameters with all P0 features enabled let mut hyperparams = DQNHyperparameters::default(); - hyperparams.use_per = true; // Enable P0.2 (batch diversity) + P0.7 (staleness) hyperparams.learning_rate = 0.001; hyperparams.lr_decay_rate = 0.95; // Enable P0.6 (LR scheduler) hyperparams.lr_decay_steps = 100; diff --git a/crates/ml/tests/dqn_trainer_p1_tests.rs b/crates/ml/tests/dqn_trainer_p1_tests.rs index baf979b27..33630ee01 100644 --- a/crates/ml/tests/dqn_trainer_p1_tests.rs +++ b/crates/ml/tests/dqn_trainer_p1_tests.rs @@ -240,7 +240,6 @@ fn test_p1_dropout_scheduler_parameters() { fn test_p1_noisy_sigma_scheduler_parameters() { // Test that noisy sigma scheduler parameters are validated let mut hyperparams = DQNHyperparameters::conservative(); - hyperparams.use_noisy_nets = true; hyperparams.enable_noisy_sigma_scheduler = true; hyperparams.noisy_sigma_initial = 0.6; hyperparams.noisy_sigma_final = 0.4; @@ -267,7 +266,6 @@ fn test_p1_all_features_enabled_max_configuration() { hyperparams.curiosity_weight = 0.5; hyperparams.enable_gae = true; hyperparams.gae_lambda = 0.99; - hyperparams.use_noisy_nets = true; hyperparams.enable_noisy_sigma_scheduler = true; hyperparams.noisy_sigma_initial = 0.8; hyperparams.noisy_sigma_final = 0.2; diff --git a/crates/ml/tests/dqn_training_smoke_test.rs b/crates/ml/tests/dqn_training_smoke_test.rs index 9045294e8..68a83d860 100644 --- a/crates/ml/tests/dqn_training_smoke_test.rs +++ b/crates/ml/tests/dqn_training_smoke_test.rs @@ -353,8 +353,6 @@ async fn test_dqn_training_smoke() -> Result<()> { dqn_config.min_replay_size = 128; dqn_config.warmup_steps = 0; dqn_config.use_iqn = false; - dqn_config.use_dueling = true; - dqn_config.use_per = true; // GPU PER mandatory on CUDA — no CPU path dqn_config.epsilon_start = 0.3; dqn_config.epsilon_end = 0.01; diff --git a/crates/ml/tests/dqn_use_double_dqn_test.rs b/crates/ml/tests/dqn_use_double_dqn_test.rs index 1bf8d4c63..ec5c6c7d1 100644 --- a/crates/ml/tests/dqn_use_double_dqn_test.rs +++ b/crates/ml/tests/dqn_use_double_dqn_test.rs @@ -1,123 +1,11 @@ -#![allow( - clippy::assertions_on_constants, - clippy::assertions_on_result_states, - clippy::clone_on_copy, - clippy::decimal_literal_representation, - clippy::doc_markdown, - clippy::empty_line_after_doc_comments, - clippy::field_reassign_with_default, - clippy::get_unwrap, - clippy::identity_op, - clippy::inconsistent_digit_grouping, - clippy::indexing_slicing, - clippy::integer_division, - clippy::len_zero, - clippy::let_underscore_must_use, - clippy::manual_div_ceil, - clippy::manual_let_else, - clippy::manual_range_contains, - clippy::modulo_arithmetic, - clippy::needless_range_loop, - clippy::non_ascii_literal, - clippy::redundant_clone, - clippy::shadow_reuse, - clippy::shadow_same, - clippy::shadow_unrelated, - clippy::single_match_else, - clippy::str_to_string, - clippy::string_slice, - clippy::tests_outside_test_module, - clippy::too_many_lines, - clippy::unnecessary_wraps, - clippy::unseparated_literal_suffix, - clippy::use_debug, - clippy::useless_vec, - clippy::wildcard_enum_match_arm, - clippy::else_if_without_else, - clippy::expect_used, - clippy::missing_const_for_fn, - clippy::similar_names, - clippy::type_complexity, - clippy::collapsible_else_if, - clippy::doc_lazy_continuation, - clippy::items_after_test_module, - clippy::map_clone, - clippy::multiple_unsafe_ops_per_block, - clippy::unwrap_or_default, - clippy::assign_op_pattern, - clippy::needless_borrow, - clippy::println_empty_string, - clippy::unnecessary_cast, - clippy::used_underscore_binding, - clippy::create_dir, - clippy::implicit_saturating_sub, - clippy::exit, - clippy::expect_fun_call, - clippy::too_many_arguments, - clippy::unnecessary_map_or, - clippy::unwrap_used, - dead_code, - unused_imports, - unused_variables, - clippy::cloned_ref_to_slice_refs, - clippy::neg_multiply, - clippy::while_let_loop, - clippy::bool_assert_comparison, - clippy::excessive_precision, - clippy::trivially_copy_pass_by_ref, - clippy::op_ref, - clippy::redundant_closure, - clippy::unnecessary_lazy_evaluations, - clippy::if_then_some_else_none, - clippy::unnecessary_to_owned, - clippy::single_component_path_imports, -)] -//! Test use_double_dqn CLI argument properly flows through the system -//! -//! This test verifies that the use_double_dqn field: -//! 1. Exists in DQNHyperparameters struct -//! 2. Has the correct default value (true) -//! 3. Can be set to both true and false +//! Verify that Double DQN is always enabled (no longer configurable). +//! The `use_double_dqn` field was removed — Double DQN is mandatory. use ml::trainers::dqn::DQNHyperparameters; #[test] -fn test_dqn_hyperparameters_has_use_double_dqn_field() { - // Create hyperparameters using conservative() method - let hyperparams = DQNHyperparameters::conservative(); - - // Field should exist and have the default value of true - assert_eq!( - hyperparams.use_double_dqn, true, - "use_double_dqn should default to true (Double DQN enabled by default)" - ); -} - -#[test] -fn test_use_double_dqn_can_be_disabled() { - // Test that we can set use_double_dqn to false - let mut hyperparams = DQNHyperparameters::conservative(); - hyperparams.use_double_dqn = false; - - assert_eq!( - hyperparams.use_double_dqn, false, - "use_double_dqn should be settable to false" - ); -} - -#[test] -fn test_use_double_dqn_both_values() { - // Test both true and false values - let hyperparams_ddqn_enabled = DQNHyperparameters::conservative(); - assert!( - hyperparams_ddqn_enabled.use_double_dqn, - "Default should be Double DQN enabled" - ); - - let mut hyperparams_ddqn_disabled = DQNHyperparameters::conservative(); - hyperparams_ddqn_disabled.use_double_dqn = false; - assert!( - !hyperparams_ddqn_disabled.use_double_dqn, - "Should support disabling Double DQN" - ); +fn test_double_dqn_always_enabled() { + // Double DQN is unconditionally on — verify struct still compiles + // without the `use_double_dqn` field. + let _hyperparams = DQNHyperparameters::conservative(); } diff --git a/crates/ml/tests/ensemble_real_models_validation_test.rs b/crates/ml/tests/ensemble_real_models_validation_test.rs index 5e9930ec7..9f85ac95e 100644 --- a/crates/ml/tests/ensemble_real_models_validation_test.rs +++ b/crates/ml/tests/ensemble_real_models_validation_test.rs @@ -189,9 +189,6 @@ fn train_small_dqn(features: &[Vec], prices: &[f64]) -> DQN { config.min_replay_size = 32; config.warmup_steps = 0; config.use_iqn = false; - config.use_dueling = false; - config.use_per = true; // GPU PER mandatory on CUDA - config.use_cql = false; config.epsilon_start = 0.3; config.epsilon_end = 0.01; diff --git a/crates/ml/tests/gradient_accumulation_tests.rs b/crates/ml/tests/gradient_accumulation_tests.rs index e82a01582..8f5ae4495 100644 --- a/crates/ml/tests/gradient_accumulation_tests.rs +++ b/crates/ml/tests/gradient_accumulation_tests.rs @@ -358,11 +358,6 @@ fn test_large_accumulation_steps() { fn test_accumulation_with_rainbow_dqn_features() { let mut params = create_test_hyperparams(); params.gradient_accumulation_steps = 4; - params.use_double_dqn = true; - params.use_dueling = true; - params.use_distributional = true; - params.use_per = true; - params.use_noisy_nets = true; let trainer = DQNTrainer::new(params); assert!( diff --git a/crates/ml/tests/smoke_test_real_data.rs b/crates/ml/tests/smoke_test_real_data.rs index b56eb4615..af5bdb780 100644 --- a/crates/ml/tests/smoke_test_real_data.rs +++ b/crates/ml/tests/smoke_test_real_data.rs @@ -795,7 +795,6 @@ async fn smoke_e2e_dqn_training_loop() { hyperparams.epsilon_decay = 0.7; hyperparams.early_stopping_enabled = false; hyperparams.checkpoint_frequency = 100; - hyperparams.use_branching = true; hyperparams.warmup_steps = 0; hyperparams.min_replay_size = 50; // CI smoke: 16 episodes x 50 timesteps = 800 experiences/epoch. diff --git a/crates/ml/tests/validation_harness_integration_test.rs b/crates/ml/tests/validation_harness_integration_test.rs index 71a88b7c8..7943e0ae6 100644 --- a/crates/ml/tests/validation_harness_integration_test.rs +++ b/crates/ml/tests/validation_harness_integration_test.rs @@ -123,8 +123,6 @@ fn make_small_dqn_config() -> DQNConfig { config.min_replay_size = 4; config.warmup_steps = 0; config.use_iqn = false; - config.use_dueling = false; - config.use_per = true; // GPU PER mandatory on CUDA config.epsilon_start = 0.3; config } diff --git a/crates/ml/tests/validation_real_data_test.rs b/crates/ml/tests/validation_real_data_test.rs index 130725057..7abffb650 100644 --- a/crates/ml/tests/validation_real_data_test.rs +++ b/crates/ml/tests/validation_real_data_test.rs @@ -163,8 +163,6 @@ fn make_dqn_config() -> DQNConfig { config.min_replay_size = 16; config.warmup_steps = 0; config.use_iqn = false; - config.use_dueling = true; - config.use_per = true; // GPU PER mandatory on CUDA config.epsilon_start = 0.3; config.epsilon_end = 0.01; config diff --git a/services/ml_training_service/src/service.rs b/services/ml_training_service/src/service.rs index 7b572ec8f..3143b3d42 100644 --- a/services/ml_training_service/src/service.rs +++ b/services/ml_training_service/src/service.rs @@ -1719,7 +1719,6 @@ mod tests { dqn_params.model_params { assert_eq!(params.replay_buffer_size, 100000); - assert!(params.use_double_dqn); assert!(params.use_prioritized_replay); } } diff --git a/services/ml_training_service/tuning_config.yaml b/services/ml_training_service/tuning_config.yaml index c26d73a58..5898f32ec 100644 --- a/services/ml_training_service/tuning_config.yaml +++ b/services/ml_training_service/tuning_config.yaml @@ -159,12 +159,6 @@ models: low: 100 high: 1000 step: 100 - use_double_dqn: - type: categorical - choices: [true, false] - use_dueling: - type: categorical - choices: [true, false] use_prioritized_replay: type: categorical choices: [true, false] diff --git a/services/ml_training_service/tuning_config_optimized.yaml b/services/ml_training_service/tuning_config_optimized.yaml index 3f6673528..bcbf5a0ea 100644 --- a/services/ml_training_service/tuning_config_optimized.yaml +++ b/services/ml_training_service/tuning_config_optimized.yaml @@ -53,12 +53,6 @@ models: target_update_frequency: type: categorical choices: [500, 1000] - use_double_dqn: - type: categorical - choices: [true, false] - use_dueling: - type: categorical - choices: [true, false] use_prioritized_replay: type: categorical choices: [true, false] diff --git a/services/trading_service/src/services/dqn_model.rs b/services/trading_service/src/services/dqn_model.rs index fa1769a6c..23ae7428f 100644 --- a/services/trading_service/src/services/dqn_model.rs +++ b/services/trading_service/src/services/dqn_model.rs @@ -61,7 +61,7 @@ impl DQNModel { "DQN '{}' config from checkpoint: state_dim={}, num_actions={}, \ hidden_dims={:?}, dueling={}, iqn={}", model_id, config.state_dim, config.num_actions, config.hidden_dims, - config.use_dueling, config.use_iqn + true, config.use_iqn ); let state_dim = config.state_dim; diff --git a/testing/integration/integration/checkpoint_roundtrip.rs b/testing/integration/integration/checkpoint_roundtrip.rs index 31f2cda58..d817be6c2 100644 --- a/testing/integration/integration/checkpoint_roundtrip.rs +++ b/testing/integration/integration/checkpoint_roundtrip.rs @@ -48,11 +48,7 @@ fn small_dqn_config() -> DQNConfig { state_dim: 51, num_actions: 45, hidden_dims: vec![32, 32], - // Disable all advanced network variants so the forward pass goes through - // the base q_network (whose GpuVarStore we save/load). - use_dueling: false, use_iqn: false, - use_cql: false, ..Default::default() } }