From 0acb89b638acf07b747c57030712d7096b92fbcc Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 4 Mar 2026 21:02:05 +0100 Subject: [PATCH] feat(ml): make CQL configurable via DQNHyperparameters (default alpha=0.1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add use_cql and cql_alpha fields to DQNHyperparameters - Wire through trainer (was hardcoded use_cql: true, cql_alpha: 1.0) - Add DQNAgentWrapper delegation for count bonus methods - Zero hold_reward (was +0.001, 20x trade PnL — biased toward holding) - Populate pnl_history from GPU experience collector for financial metrics - Wire count bonus (UCB) into select_actions_batch for exploration Co-Authored-By: Claude Opus 4.6 --- crates/ml/src/trainers/dqn/config.rs | 36 ++++++++++++++++++++ crates/ml/src/trainers/dqn/trainer.rs | 49 +++++++++++++++++++++++++-- 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/crates/ml/src/trainers/dqn/config.rs b/crates/ml/src/trainers/dqn/config.rs index e43cbbc96..e82cf6fd0 100644 --- a/crates/ml/src/trainers/dqn/config.rs +++ b/crates/ml/src/trainers/dqn/config.rs @@ -79,6 +79,31 @@ impl DQNAgentType { } } + /// Get count bonuses for all actions (UCB exploration). + /// Returns a Vec of length num_actions with exploration bonuses. + pub fn get_count_bonuses(&self) -> Vec { + match self { + Self::Standard(agent) => agent.get_count_bonuses(), + Self::RegimeConditional(agent) => agent.get_count_bonuses(), + } + } + + /// Whether count-based exploration bonus is enabled. + pub fn use_count_bonus(&self) -> bool { + match self { + Self::Standard(agent) => agent.config.use_count_bonus, + Self::RegimeConditional(agent) => agent.config().use_count_bonus, + } + } + + /// Number of discrete actions in the action space. + pub fn num_actions(&self) -> usize { + match self { + Self::Standard(agent) => agent.config.num_actions, + Self::RegimeConditional(agent) => agent.config().num_actions, + } + } + /// Update epsilon for exploration decay pub fn update_epsilon(&mut self) { match self { @@ -691,6 +716,13 @@ pub struct DQNHyperparameters { /// Recommended: 1e-4 for standard training, 1e-3 for aggressive regularization. pub weight_decay: f64, + // Conservative Q-Learning (CQL) — prevents Q-value overestimation on offline/replay data + /// Enable Conservative Q-Learning (default: true, alpha controls strength) + pub use_cql: bool, + /// CQL regularization strength (default: 0.1, range 0.0-1.0) + /// 0.0 = disabled, 0.1 = mild conservatism, 1.0 = full offline-RL strength + pub cql_alpha: f64, + // QR-DQN (Quantile Regression DQN) -- replaces disabled C51 /// Enable QR-DQN (IQN) for distributional RL pub use_qr_dqn: bool, @@ -887,6 +919,10 @@ impl DQNHyperparameters { // WAVE 30: L2 Weight Decay weight_decay: 1e-4, // Default: 0.0001 (standard regularization strength) + // Conservative Q-Learning (CQL) + use_cql: true, // Default: enabled (prevents Q-value overestimation) + cql_alpha: 0.1, // Default: mild conservatism (0.1 of 0.0-1.0 range) + // QR-DQN (complementary to C51 — IQN for quantile estimation) use_qr_dqn: true, // Default: enabled (IQN distributional RL) num_quantiles: 32, // Default: 32 quantiles diff --git a/crates/ml/src/trainers/dqn/trainer.rs b/crates/ml/src/trainers/dqn/trainer.rs index dbec51359..1d48672e8 100644 --- a/crates/ml/src/trainers/dqn/trainer.rs +++ b/crates/ml/src/trainers/dqn/trainer.rs @@ -440,8 +440,8 @@ impl DQNTrainer { gradient_collapse_multiplier: hyperparams.gradient_collapse_multiplier, gradient_collapse_patience: hyperparams.gradient_collapse_patience, - use_cql: true, - cql_alpha: 1.0, + use_cql: hyperparams.use_cql, + cql_alpha: hyperparams.cql_alpha, use_iqn: hyperparams.use_qr_dqn, // Controlled by hyperopt iqn_num_quantiles: hyperparams.num_quantiles, // Controlled by hyperopt iqn_kappa: hyperparams.qr_kappa as f32, // Controlled by hyperopt (f64→f32) @@ -489,7 +489,7 @@ impl DQNTrainer { pnl_weight: Decimal::ONE, risk_weight: Decimal::try_from(0.1).unwrap_or(Decimal::ZERO), cost_weight: Decimal::ONE, // Bug #2 fix: 100% transaction cost weight (was 0.05, 20x too low) - hold_reward: Decimal::try_from(0.001).unwrap_or(Decimal::ZERO), + hold_reward: Decimal::ZERO, // Flat position = no edge = zero reward (was +0.001, 20x trade PnL) movement_threshold: Decimal::try_from(hyperparams.movement_threshold) .unwrap_or(Decimal::ZERO), hold_penalty_weight: Decimal::try_from(hyperparams.hold_penalty_weight) @@ -1769,6 +1769,16 @@ impl DQNTrainer { monitor.action_counts[clamped] += 1; } + // Populate pnl_history from GPU-collected rewards for financial metrics. + // Without this, compute_epoch_financials() sees an empty deque and + // reports Trades=0, Sharpe=0.00 even when the agent is trading. + for &reward in &batch.rewards { + self.pnl_history.push_back(reward as f64); + if self.pnl_history.len() > 1000 { + self.pnl_history.pop_front(); + } + } + let experiences = gpu_batch_to_experiences(&batch); let count = experiences.len(); info!("GPU collected {} experiences ({} episodes × {} timesteps)", @@ -3240,6 +3250,23 @@ impl DQNTrainer { .forward(&batch_tensor) .map_err(|e| anyhow::anyhow!("Batched forward pass failed: {}", e))?; + // Apply count bonus (UCB exploration) to Q-values before argmax. + // Previously only in DQN::select_action() (single-sample path), + // making the batch training path have no UCB exploration. + let use_cb = agent.use_count_bonus(); + let n_actions = agent.num_actions(); + let batch_q_values = if use_cb { + let bonuses = agent.get_count_bonuses(); + let bonus_tensor = Tensor::from_vec(bonuses, (1, n_actions), batch_q_values.device()) + .map_err(|e| anyhow::anyhow!("Failed to create bonus tensor: {}", e))? + .to_dtype(batch_q_values.dtype()) + .map_err(|e| anyhow::anyhow!("Failed to cast bonus tensor: {}", e))?; + batch_q_values.broadcast_add(&bonus_tensor) + .map_err(|e| anyhow::anyhow!("Failed to add count bonus: {}", e))? + } else { + batch_q_values + }; + drop(agent); // Release lock early // Extract Q-values and select actions (epsilon-greedy) @@ -3295,6 +3322,22 @@ impl DQNTrainer { .forward(batch_tensor) .map_err(|e| anyhow::anyhow!("GPU batched forward pass failed: {}", e))?; + // Apply count bonus (UCB exploration) to Q-values before argmax. + // Mirrors the same fix in select_actions_batch() above. + let use_cb = agent.use_count_bonus(); + let n_actions = agent.num_actions(); + let batch_q_values = if use_cb { + let bonuses = agent.get_count_bonuses(); + let bonus_tensor = Tensor::from_vec(bonuses, (1, n_actions), batch_q_values.device()) + .map_err(|e| anyhow::anyhow!("Failed to create bonus tensor: {}", e))? + .to_dtype(batch_q_values.dtype()) + .map_err(|e| anyhow::anyhow!("Failed to cast bonus tensor: {}", e))?; + batch_q_values.broadcast_add(&bonus_tensor) + .map_err(|e| anyhow::anyhow!("Failed to add count bonus: {}", e))? + } else { + batch_q_values + }; + drop(agent); // GPU argmax + epsilon-greedy (same as CPU path)