feat(ml): make CQL configurable via DQNHyperparameters (default alpha=0.1)

- Add use_cql and cql_alpha fields to DQNHyperparameters
- Wire through trainer (was hardcoded use_cql: true, cql_alpha: 1.0)
- Add DQNAgentWrapper delegation for count bonus methods
- Zero hold_reward (was +0.001, 20x trade PnL — biased toward holding)
- Populate pnl_history from GPU experience collector for financial metrics
- Wire count bonus (UCB) into select_actions_batch for exploration

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

crates/ml/src/trainers/dqn/config.rs

@@ -79,6 +79,31 @@ impl DQNAgentType {
         }
     }
 
+    /// Get count bonuses for all actions (UCB exploration).
+    /// Returns a Vec<f32> of length num_actions with exploration bonuses.
+    pub fn get_count_bonuses(&self) -> Vec<f32> {
+        match self {
+            Self::Standard(agent) => agent.get_count_bonuses(),
+            Self::RegimeConditional(agent) => agent.get_count_bonuses(),
+        }
+    }
+
+    /// Whether count-based exploration bonus is enabled.
+    pub fn use_count_bonus(&self) -> bool {
+        match self {
+            Self::Standard(agent) => agent.config.use_count_bonus,
+            Self::RegimeConditional(agent) => agent.config().use_count_bonus,
+        }
+    }
+
+    /// Number of discrete actions in the action space.
+    pub fn num_actions(&self) -> usize {
+        match self {
+            Self::Standard(agent) => agent.config.num_actions,
+            Self::RegimeConditional(agent) => agent.config().num_actions,
+        }
+    }
+
     /// Update epsilon for exploration decay
     pub fn update_epsilon(&mut self) {
         match self {
@@ -691,6 +716,13 @@ pub struct DQNHyperparameters {
     /// Recommended: 1e-4 for standard training, 1e-3 for aggressive regularization.
     pub weight_decay: f64,
 
+    // Conservative Q-Learning (CQL) — prevents Q-value overestimation on offline/replay data
+    /// Enable Conservative Q-Learning (default: true, alpha controls strength)
+    pub use_cql: bool,
+    /// CQL regularization strength (default: 0.1, range 0.0-1.0)
+    /// 0.0 = disabled, 0.1 = mild conservatism, 1.0 = full offline-RL strength
+    pub cql_alpha: f64,
+
     // QR-DQN (Quantile Regression DQN) -- replaces disabled C51
     /// Enable QR-DQN (IQN) for distributional RL
     pub use_qr_dqn: bool,
@@ -887,6 +919,10 @@ impl DQNHyperparameters {
             // WAVE 30: L2 Weight Decay
             weight_decay: 1e-4,                   // Default: 0.0001 (standard regularization strength)
 
+            // Conservative Q-Learning (CQL)
+            use_cql: true,                        // Default: enabled (prevents Q-value overestimation)
+            cql_alpha: 0.1,                       // Default: mild conservatism (0.1 of 0.0-1.0 range)
+
             // QR-DQN (complementary to C51 — IQN for quantile estimation)
             use_qr_dqn: true,                     // Default: enabled (IQN distributional RL)
             num_quantiles: 32,                    // Default: 32 quantiles

crates/ml/src/trainers/dqn/trainer.rs

@@ -440,8 +440,8 @@ impl DQNTrainer {
             gradient_collapse_multiplier: hyperparams.gradient_collapse_multiplier,
             gradient_collapse_patience: hyperparams.gradient_collapse_patience,
 
-            use_cql: true,
-            cql_alpha: 1.0,
+            use_cql: hyperparams.use_cql,
+            cql_alpha: hyperparams.cql_alpha,
             use_iqn: hyperparams.use_qr_dqn,                  // Controlled by hyperopt
             iqn_num_quantiles: hyperparams.num_quantiles,       // Controlled by hyperopt
             iqn_kappa: hyperparams.qr_kappa as f32,            // Controlled by hyperopt (f64→f32)
@@ -489,7 +489,7 @@ impl DQNTrainer {
             pnl_weight: Decimal::ONE,
             risk_weight: Decimal::try_from(0.1).unwrap_or(Decimal::ZERO),
             cost_weight: Decimal::ONE, // Bug #2 fix: 100% transaction cost weight (was 0.05, 20x too low)
-            hold_reward: Decimal::try_from(0.001).unwrap_or(Decimal::ZERO),
+            hold_reward: Decimal::ZERO,  // Flat position = no edge = zero reward (was +0.001, 20x trade PnL)
             movement_threshold: Decimal::try_from(hyperparams.movement_threshold)
                 .unwrap_or(Decimal::ZERO),
             hold_penalty_weight: Decimal::try_from(hyperparams.hold_penalty_weight)
@@ -1769,6 +1769,16 @@ impl DQNTrainer {
                                 monitor.action_counts[clamped] += 1;
                             }
 
+                            // Populate pnl_history from GPU-collected rewards for financial metrics.
+                            // Without this, compute_epoch_financials() sees an empty deque and
+                            // reports Trades=0, Sharpe=0.00 even when the agent is trading.
+                            for &reward in &batch.rewards {
+                                self.pnl_history.push_back(reward as f64);
+                                if self.pnl_history.len() > 1000 {
+                                    self.pnl_history.pop_front();
+                                }
+                            }
+
                             let experiences = gpu_batch_to_experiences(&batch);
                             let count = experiences.len();
                             info!("GPU collected {} experiences ({} episodes × {} timesteps)",
@@ -3240,6 +3250,23 @@ impl DQNTrainer {
             .forward(&batch_tensor)
             .map_err(|e| anyhow::anyhow!("Batched forward pass failed: {}", e))?;
 
+        // Apply count bonus (UCB exploration) to Q-values before argmax.
+        // Previously only in DQN::select_action() (single-sample path),
+        // making the batch training path have no UCB exploration.
+        let use_cb = agent.use_count_bonus();
+        let n_actions = agent.num_actions();
+        let batch_q_values = if use_cb {
+            let bonuses = agent.get_count_bonuses();
+            let bonus_tensor = Tensor::from_vec(bonuses, (1, n_actions), batch_q_values.device())
+                .map_err(|e| anyhow::anyhow!("Failed to create bonus tensor: {}", e))?
+                .to_dtype(batch_q_values.dtype())
+                .map_err(|e| anyhow::anyhow!("Failed to cast bonus tensor: {}", e))?;
+            batch_q_values.broadcast_add(&bonus_tensor)
+                .map_err(|e| anyhow::anyhow!("Failed to add count bonus: {}", e))?
+        } else {
+            batch_q_values
+        };
+
         drop(agent); // Release lock early
 
         // Extract Q-values and select actions (epsilon-greedy)
@@ -3295,6 +3322,22 @@ impl DQNTrainer {
             .forward(batch_tensor)
             .map_err(|e| anyhow::anyhow!("GPU batched forward pass failed: {}", e))?;
 
+        // Apply count bonus (UCB exploration) to Q-values before argmax.
+        // Mirrors the same fix in select_actions_batch() above.
+        let use_cb = agent.use_count_bonus();
+        let n_actions = agent.num_actions();
+        let batch_q_values = if use_cb {
+            let bonuses = agent.get_count_bonuses();
+            let bonus_tensor = Tensor::from_vec(bonuses, (1, n_actions), batch_q_values.device())
+                .map_err(|e| anyhow::anyhow!("Failed to create bonus tensor: {}", e))?
+                .to_dtype(batch_q_values.dtype())
+                .map_err(|e| anyhow::anyhow!("Failed to cast bonus tensor: {}", e))?;
+            batch_q_values.broadcast_add(&bonus_tensor)
+                .map_err(|e| anyhow::anyhow!("Failed to add count bonus: {}", e))?
+        } else {
+            batch_q_values
+        };
+
         drop(agent);
 
         // GPU argmax + epsilon-greedy (same as CPU path)