diff --git a/crates/common/src/metrics/training_metrics.rs b/crates/common/src/metrics/training_metrics.rs index 47e9c626f..eae70ea29 100644 --- a/crates/common/src/metrics/training_metrics.rs +++ b/crates/common/src/metrics/training_metrics.rs @@ -171,6 +171,18 @@ pub fn init() { mf, ); + // Exploration diagnostics (DQN action collapse monitoring) + _ = register_gauge_vec( + "foxhunt_training_action_entropy", + "Normalized entropy of epoch action distribution (0=collapse, 1=uniform)", + mf, + ); + _ = register_gauge_vec( + "foxhunt_training_action_diversity", + "Fraction of 45 actions used above 0.5% threshold", + mf, + ); + // Hyperopt gauges (model only) _ = register_gauge_vec( "foxhunt_hyperopt_trial_current", @@ -428,6 +440,14 @@ pub fn set_epoch_duration(model: &str, fold: &str, secs: f64) { set_gauge_vec("foxhunt_training_epoch_duration_seconds", &[model, fold], secs); } +pub fn set_action_entropy(model: &str, fold: &str, entropy: f64) { + set_gauge_vec("foxhunt_training_action_entropy", &[model, fold], entropy); +} + +pub fn set_action_diversity(model: &str, fold: &str, diversity: f64) { + set_gauge_vec("foxhunt_training_action_diversity", &[model, fold], diversity); +} + // --------------------------------------------------------------------------- // Tier 1: Hyperopt intra-trial // --------------------------------------------------------------------------- diff --git a/crates/ml/src/benchmark/dqn_benchmark.rs b/crates/ml/src/benchmark/dqn_benchmark.rs index 2fb33f558..91d0bd17b 100644 --- a/crates/ml/src/benchmark/dqn_benchmark.rs +++ b/crates/ml/src/benchmark/dqn_benchmark.rs @@ -471,6 +471,7 @@ impl DqnBenchmarkRunner { minimum_profit_factor: 1.5, weight_decay: 1e-4, mixed_precision: None, + ..Default::default() } } diff --git a/crates/ml/src/dqn/count_bonus.rs b/crates/ml/src/dqn/count_bonus.rs new file mode 100644 index 000000000..b22b05d64 --- /dev/null +++ b/crates/ml/src/dqn/count_bonus.rs @@ -0,0 +1,152 @@ +//! UCB-style count-based exploration bonus for action selection. +//! +//! Adds bonus = β × sqrt(ln(N) / (1 + n_a)) to Q-values during action selection, +//! where N = total actions taken this epoch, n_a = count of action a. +//! This gives a direct exploration incentive to under-visited actions, +//! preventing collapse onto a single action (e.g., A38) across all states. + +/// Count-based exploration bonus tracker for 45 factored actions. +/// +/// At each greedy action selection, bonuses are added to Q-values before argmax. +/// Actions taken less frequently get higher bonuses, encouraging diversity. +/// Counts are reset at epoch boundaries. +#[derive(Debug)] +pub struct CountBonus { + action_counts: [u64; 45], + total_count: u64, + coefficient: f64, +} + +impl CountBonus { + /// Create a new count bonus tracker with the given coefficient β. + /// + /// β controls the strength of the exploration bonus. + /// Typical values: 0.01–0.5 (tunable via hyperopt). + pub fn new(coefficient: f64) -> Self { + Self { + action_counts: [0_u64; 45], + total_count: 0, + coefficient, + } + } + + /// Record that an action was taken. + pub fn record_action(&mut self, action_idx: usize) { + if action_idx < 45 { + self.action_counts[action_idx] += 1; + self.total_count += 1; + } + } + + /// Compute UCB-style bonus for each of the 45 actions. + /// + /// Formula: bonus_a = β × sqrt(ln(N) / (1 + n_a)) + /// - When n_a = 0 and N > 0: bonus is at maximum (strong incentive to try) + /// - When n_a is large: bonus approaches 0 (already well-explored) + /// - When N = 0: returns uniform bonus β (no data yet) + pub fn bonuses(&self) -> [f64; 45] { + if self.total_count == 0 { + return [self.coefficient; 45]; + } + let log_n = (self.total_count as f64).ln(); + let mut result = [0.0; 45]; + for (a, bonus) in result.iter_mut().enumerate() { + *bonus = + self.coefficient * (log_n / (1.0 + self.action_counts[a] as f64)).sqrt(); + } + result + } + + /// Reset all counts (call at epoch boundary). + pub fn reset(&mut self) { + self.action_counts = [0_u64; 45]; + self.total_count = 0; + } + + /// Get total number of actions recorded since last reset. + pub fn total_count(&self) -> u64 { + self.total_count + } + + /// Get the coefficient β. + pub fn coefficient(&self) -> f64 { + self.coefficient + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_new_bonus_returns_uniform_coefficient() { + let cb = CountBonus::new(0.1); + let bonuses = cb.bonuses(); + for b in &bonuses { + assert!((*b - 0.1).abs() < 1e-10, "Expected 0.1, got {}", b); + } + } + + #[test] + fn test_frequent_action_gets_low_bonus() { + let mut cb = CountBonus::new(0.1); + // Record action 38 one hundred times + for _ in 0..100 { + cb.record_action(38); + } + let bonuses = cb.bonuses(); + // Action 38 should have much lower bonus than others + let bonus_38 = bonuses[38]; + let bonus_0 = bonuses[0]; // Never taken + assert!( + bonus_0 > bonus_38 * 5.0, + "Unvisited bonus ({}) should be >> frequent bonus ({})", + bonus_0, + bonus_38 + ); + } + + #[test] + fn test_uniform_usage_gives_equal_bonuses() { + let mut cb = CountBonus::new(0.1); + // Take each action exactly once + for i in 0..45 { + cb.record_action(i); + } + let bonuses = cb.bonuses(); + let first = bonuses[0]; + for (i, b) in bonuses.iter().enumerate() { + assert!( + (*b - first).abs() < 1e-10, + "Bonus {} ({}) should equal bonus 0 ({})", + i, + b, + first + ); + } + } + + #[test] + fn test_reset_clears_all_counts() { + let mut cb = CountBonus::new(0.1); + for _ in 0..50 { + cb.record_action(10); + } + assert_eq!(cb.total_count(), 50); + cb.reset(); + assert_eq!(cb.total_count(), 0); + let bonuses = cb.bonuses(); + // After reset, should return uniform coefficient again + for b in &bonuses { + assert!((*b - 0.1).abs() < 1e-10); + } + } + + #[test] + fn test_out_of_range_action_ignored() { + let mut cb = CountBonus::new(0.1); + cb.record_action(45); // Out of range + cb.record_action(100); // Out of range + assert_eq!(cb.total_count(), 0); + } +} diff --git a/crates/ml/src/dqn/dqn.rs b/crates/ml/src/dqn/dqn.rs index d68c2e76c..f7c1fe415 100644 --- a/crates/ml/src/dqn/dqn.rs +++ b/crates/ml/src/dqn/dqn.rs @@ -151,6 +151,26 @@ pub struct DQNConfig { /// Default: 5 (prevents stopping on transient spikes) pub gradient_collapse_patience: usize, + /// SAC-style entropy coefficient for Q-value softmax regularization. + /// Adds -coeff * H(softmax(Q)) to the loss, preventing action collapse. + /// Default: 0.01. Range for hyperopt: [0.001, 0.1]. + pub entropy_coefficient: f64, + + /// Minimum epsilon for exploration when noisy nets are enabled. + /// Standard noisy nets set epsilon=0.0, but 45-action spaces need a floor. + /// Default: 0.05. Range for hyperopt: [0.02, 0.10]. + pub noisy_epsilon_floor: f32, + + /// Enable UCB count-based exploration bonus during action selection. + /// Adds bonus = β × sqrt(ln(N) / (1 + n_a)) to Q-values before argmax. + /// Default: true. + pub use_count_bonus: bool, + + /// Coefficient β for count-based exploration bonus. + /// Controls strength of the exploration incentive for under-visited actions. + /// Default: 0.1. Range for hyperopt: [0.01, 0.5]. + pub count_bonus_coefficient: f64, + // Conservative Q-Learning (CQL) for offline RL (Kumar et al. 2020) pub use_cql: bool, pub cql_alpha: f64, @@ -223,6 +243,12 @@ impl Default for DQNConfig { gradient_collapse_multiplier: 2.0, gradient_collapse_patience: 100, + // Entropy regularization: prevent 45-action collapse + entropy_coefficient: 0.01, + noisy_epsilon_floor: 0.05, + use_count_bonus: true, + count_bonus_coefficient: 0.1, + // CQL: Enabled by default for offline training use_cql: true, cql_alpha: 1.0, @@ -326,6 +352,7 @@ impl DQNConfig { minimum_profit_factor: 1.5, weight_decay: 1e-4, mixed_precision: None, + ..Default::default() } } @@ -393,6 +420,7 @@ impl DQNConfig { minimum_profit_factor: 2.0, // Higher safety margin for conservative config weight_decay: 1e-4, mixed_precision: None, + ..Default::default() } } @@ -457,6 +485,11 @@ impl DQNConfig { gradient_collapse_multiplier: 100.0, gradient_collapse_patience: 5, + entropy_coefficient: 0.01, + noisy_epsilon_floor: 0.05, + use_count_bonus: true, + count_bonus_coefficient: 0.1, + use_cql: true, cql_alpha: 5.0, use_iqn: false, @@ -832,6 +865,9 @@ pub struct DQN { // WAVE 23 P0 Fix #4: Early stopping on Q-value divergence /// Consecutive epochs with Q-value divergence (resets when Q-values normalize) q_value_divergence_counter: usize, + + /// UCB count-based exploration bonus for action diversity + count_bonus: super::count_bonus::CountBonus, } impl DQN { @@ -1013,6 +1049,7 @@ impl DQN { total_steps: 0, optimizer: None, gradient_clip_norm: config.gradient_clip_norm, + count_bonus: super::count_bonus::CountBonus::new(config.count_bonus_coefficient), config, device, recent_actions: VecDeque::with_capacity(100), @@ -1195,9 +1232,10 @@ impl DQN { // During warmup period: always use random exploration (epsilon=1.0) let in_warmup = self.total_steps <= self.config.warmup_steps as u64; - // Noisy Networks replace epsilon-greedy: use epsilon=0 when noisy nets enabled + // Noisy Networks: use epsilon floor (not 0) to guarantee some random exploration + // for 45-action spaces where parameter noise alone is insufficient let effective_epsilon = if self.config.use_noisy_nets { - 0.0 // Learned exploration via parameter noise + self.config.noisy_epsilon_floor // Minimum random exploration with noisy nets } else { self.epsilon // Standard epsilon-greedy exploration }; @@ -1241,6 +1279,16 @@ impl DQN { .map_err(|e| MLError::ModelError(format!("Expected Q computation failed: {}", e)))? }; + // Apply count bonus before argmax (UCB exploration) + let action_scores = if self.config.use_count_bonus { + let bonuses = self.count_bonus.bonuses(); + let bonus_vec: Vec = bonuses.iter().map(|b| *b as f32).collect(); + let bonus_t = Tensor::from_vec(bonus_vec, (1, self.config.num_actions), &self.device)?; + action_scores.broadcast_add(&bonus_t)? + } else { + action_scores + }; + // Argmax over actions let best_action_idx = action_scores .argmax(1)? @@ -1251,6 +1299,17 @@ impl DQN { } else { // Standard Q-network action selection let q_values = self.forward(&state_tensor)?; + + // Apply count bonus before argmax (UCB exploration) + let q_values = if self.config.use_count_bonus { + let bonuses = self.count_bonus.bonuses(); + let bonus_vec: Vec = bonuses.iter().map(|b| *b as f32).collect(); + let bonus_t = Tensor::from_vec(bonus_vec, (1, self.config.num_actions), &self.device)?; + q_values.broadcast_add(&bonus_t)? + } else { + q_values + }; + let best_action_idx = q_values .argmax(1)? .get(0)? @@ -1261,6 +1320,11 @@ impl DQN { } }; + // Record action for count bonus (UCB exploration tracking) + if self.config.use_count_bonus { + self.count_bonus.record_action(action.to_index()); + } + // Track action for entropy penalty calculation self.track_action(action); @@ -1922,11 +1986,21 @@ impl DQN { } }; - // Add entropy regularization (in-training diversity penalty) - let entropy_penalty = self.calculate_entropy_penalty()?; - let entropy_weight = 0.1; // Match Wave 5-A1 diversity_weight - let entropy_term = (entropy_penalty * entropy_weight)?; - let loss_with_entropy = loss_raw.add(&entropy_term)?; + // SAC-style entropy regularization on Q-value softmax + // Encourages the Q-network to maintain spread across actions, preventing collapse. + // H(π) = -Σ softmax(Q) * log_softmax(Q), added as: loss = td_loss - coeff * mean(H) + let loss_with_entropy = if self.config.entropy_coefficient > 0.0 { + let log_probs_q = candle_nn::ops::log_softmax(¤t_q_values, 1)?; + let probs_q = candle_nn::ops::softmax(¤t_q_values, 1)?; + // Entropy per sample: H = -(π * log π).sum(dim=1) + let neg_entropy_per_sample = (&probs_q * &log_probs_q)?.sum(1)?; // negative entropy + let mean_neg_entropy = neg_entropy_per_sample.mean_all()?; + // loss = td_loss + coeff * (-H) = td_loss - coeff * H + let entropy_term = (mean_neg_entropy * self.config.entropy_coefficient)?; + loss_raw.add(&entropy_term)? + } else { + loss_raw + }; // CQL regularization for offline RL (Kumar et al. 2020) // Penalizes high Q-values for actions not taken in the training data @@ -2462,38 +2536,34 @@ impl DQN { .map_err(|e| MLError::ModelError(format!("Failed to create zero penalty: {}", e))); } - // Count action frequencies (convert FactoredAction to legacy for entropy calculation) - let mut counts = [0, 0, 0]; // BUY, SELL, HOLD + // Count over all 45 factored actions (not 3 legacy categories) + let mut counts = [0_u32; 45]; for action in &self.recent_actions { - let legacy_action = action.to_legacy_action(); - counts[legacy_action as usize] += 1; + let idx = action.to_index(); + if idx < 45 { + counts[idx] += 1; + } } - // Calculate Shannon entropy: H = -Σ(p_i * log2(p_i)) - // BUG #14 FIX: Add epsilon to prevent log(0) = -inf → 0 * -inf = NaN let total = self.recent_actions.len() as f64; let mut entropy = 0.0_f64; - const EPSILON: f64 = 1e-10; // Prevent log(0) for &count in &counts { if count > 0 { let p = count as f64 / total; - // Clamp p to [EPSILON, 1.0] to prevent numerical instability - let p_safe = p.max(EPSILON).min(1.0); - entropy -= p_safe * p_safe.log2(); + entropy -= p * p.ln(); } } - // Return negative entropy as penalty (lower entropy = higher penalty) - // This encourages the agent to maximize entropy (balanced actions) - let penalty = -entropy as f32; + // Normalize by max entropy ln(45), then negate (lower entropy = higher penalty) + let max_entropy = 45.0_f64.ln(); + let normalized = entropy / max_entropy; // [0, 1] + let penalty = -(normalized as f32); - // BUG #14 FIX: Validate penalty is finite before creating tensor if !penalty.is_finite() { tracing::warn!( - "Entropy penalty is not finite ({}), returning zero. Recent actions: {:?}", + "Entropy penalty is not finite ({}), returning zero", penalty, - counts ); return Tensor::zeros(&[], DType::F32, &self.device) .map_err(|e| MLError::ModelError(format!("Failed to create zero penalty: {}", e))); @@ -2509,6 +2579,11 @@ impl DQN { self.epsilon = (self.epsilon * self.config.epsilon_decay).max(self.config.epsilon_end); } + /// Reset count-based exploration bonus (call at epoch boundary) + pub fn reset_count_bonus(&mut self) { + self.count_bonus.reset(); + } + /// Update target network by copying weights from main network fn update_target_network(&mut self) -> Result<(), MLError> { // Wave 11.6: Update target network (hybrid > dueling > standard) diff --git a/crates/ml/src/dqn/mod.rs b/crates/ml/src/dqn/mod.rs index a98ca52f1..42ec87f12 100644 --- a/crates/ml/src/dqn/mod.rs +++ b/crates/ml/src/dqn/mod.rs @@ -6,6 +6,7 @@ // Original DQN components pub mod action_space; // Factored action space (45 actions: 5 exposure × 3 order × 3 urgency) +pub mod count_bonus; // UCB count-based exploration bonus for action diversity pub mod agent; pub mod attention; // Multi-head self-attention for temporal pattern recognition (Wave 26 P1.2) pub mod circuit_breaker; // Circuit breaker for risk management diff --git a/crates/ml/src/dqn/noisy_sigma_scheduler.rs b/crates/ml/src/dqn/noisy_sigma_scheduler.rs index 404d11b33..141e25d2a 100644 --- a/crates/ml/src/dqn/noisy_sigma_scheduler.rs +++ b/crates/ml/src/dqn/noisy_sigma_scheduler.rs @@ -76,9 +76,13 @@ impl NoisySigmaScheduler { /// where α = min(t / decay_steps, 1.0) pub fn get_sigma(&self) -> f64 { let progress = (self.current_step as f64 / self.decay_steps as f64).min(1.0); - self.initial_sigma * (1.0 - progress) + self.final_sigma * progress + let annealed = self.initial_sigma * (1.0 - progress) + self.final_sigma * progress; + annealed.max(Self::SIGMA_FLOOR) } + /// Minimum sigma to prevent complete collapse of exploration noise + const SIGMA_FLOOR: f64 = 0.3; + /// Increment step counter pub fn step(&mut self) { self.current_step += 1; @@ -106,10 +110,10 @@ impl NoisySigmaScheduler { } impl Default for NoisySigmaScheduler { - /// Default scheduler with Rainbow DQN-style annealing - /// (0.6 → 0.4 over 100k steps) + /// Default scheduler with wider initial exploration for 45-action space + /// (0.8 → 0.4 over 100k steps, floor=0.3) fn default() -> Self { - Self::new(0.6, 0.4, 100_000) + Self::new(0.8, 0.4, 100_000) } } @@ -236,11 +240,20 @@ mod tests { #[test] fn test_default_scheduler() { let scheduler = NoisySigmaScheduler::default(); - assert_eq!(scheduler.initial_sigma, 0.6); + assert_eq!(scheduler.initial_sigma, 0.8); assert_eq!(scheduler.final_sigma, 0.4); assert_eq!(scheduler.decay_steps, 100_000); } + #[test] + fn test_sigma_floor() { + // Even if final_sigma is below floor, get_sigma() respects the floor + let mut scheduler = NoisySigmaScheduler::new(0.5, 0.1, 100); + scheduler.step_to(100); + assert!(scheduler.get_sigma() >= NoisySigmaScheduler::SIGMA_FLOOR, + "Sigma {} should be >= floor {}", scheduler.get_sigma(), NoisySigmaScheduler::SIGMA_FLOOR); + } + #[test] fn test_monotonic_decrease() { let mut scheduler = NoisySigmaScheduler::new(0.6, 0.4, 1000); diff --git a/crates/ml/src/dqn/regime_conditional.rs b/crates/ml/src/dqn/regime_conditional.rs index e9b1f5f97..34d542113 100644 --- a/crates/ml/src/dqn/regime_conditional.rs +++ b/crates/ml/src/dqn/regime_conditional.rs @@ -421,6 +421,13 @@ impl RegimeConditionalDQN { Ok((avg_loss, avg_grad_norm)) } + /// Reset count-based exploration bonus for all regime heads + pub fn reset_count_bonus(&mut self) { + self.trending_head.reset_count_bonus(); + self.ranging_head.reset_count_bonus(); + self.volatile_head.reset_count_bonus(); + } + /// Update epsilon for specific regime head pub fn update_epsilon(&mut self, regime: RegimeType) { match regime { diff --git a/crates/ml/src/dqn/reward.rs b/crates/ml/src/dqn/reward.rs index 733d4d872..1f6bded5f 100644 --- a/crates/ml/src/dqn/reward.rs +++ b/crates/ml/src/dqn/reward.rs @@ -349,49 +349,48 @@ pub struct MarketData { pub volume: Decimal, } -/// Calculate Shannon entropy for action distribution +/// Calculate normalized Shannon entropy for the 45-action distribution. /// /// # Arguments /// * `recent_actions` - Sliding window of recent actions (last 100 actions) /// /// # Returns -/// Shannon entropy H = -Σ(p_i * log2(p_i)) where p_i is frequency of action i -/// Range: [0.0, 1.585] (0 = all same action, 1.585 = perfectly balanced) -/// -/// # Note -/// For FactoredAction, we convert to legacy TradingAction (Buy/Sell/Hold) for entropy calculation -/// to maintain backward compatibility with the original 3-action entropy range. +/// Normalized entropy H/H_max in [0.0, 1.0]: +/// 0.0 = all actions identical (complete collapse) +/// 1.0 = uniform distribution over all 45 actions fn calculate_entropy(recent_actions: &[FactoredAction]) -> Decimal { if recent_actions.is_empty() { return Decimal::ONE; // Default to high entropy if no history } - // Convert to legacy actions and count - let mut counts = [0, 0, 0]; // BUY, SELL, HOLD + // Count over all 45 factored actions (not 3 legacy categories) + let mut counts = [0_u32; 45]; for action in recent_actions { - let legacy_action = action.to_legacy_action(); - counts[legacy_action as usize] += 1; - } - - let total = recent_actions.len() as f64; - let mut entropy = Decimal::ZERO; - - for &count in &counts { - if count > 0 { - let p = Decimal::try_from(count as f64 / total).unwrap_or(Decimal::ZERO); - // Shannon entropy: -p * log2(p) - // Use natural log and convert: log2(x) = ln(x) / ln(2) - if let Ok(p_f64) = TryInto::::try_into(p) { - if p_f64 > 0.0 { - let log2_p = p_f64.ln() / 2.0_f64.ln(); - let term = p * Decimal::try_from(log2_p).unwrap_or(Decimal::ZERO); - entropy -= term; - } - } + let idx = action.to_index(); + if idx < 45 { + counts[idx] += 1; } } - entropy + let total = recent_actions.len() as f64; + let mut entropy_raw = 0.0_f64; + + for &count in &counts { + if count > 0 { + let p = count as f64 / total; + entropy_raw -= p * p.ln(); + } + } + + // Normalize by max entropy = ln(45) ≈ 3.807 → range [0, 1] + let max_entropy = 45.0_f64.ln(); + let normalized = if max_entropy > 0.0 { + entropy_raw / max_entropy + } else { + 1.0 + }; + + Decimal::try_from(normalized).unwrap_or(Decimal::ONE) } /// Reward function for `DQN` training @@ -522,10 +521,9 @@ impl RewardFunction { }; // Calculate diversity bonus (in-training regularization) - // Entropy threshold: 0.5 (50% of max entropy for 3 actions = 1.585) - // Low entropy → action bias → apply penalty (-0.1) + // Entropy is normalized [0,1] over 45 actions. Threshold 0.3 ≈ ~4 distinct actions. let entropy = calculate_entropy(recent_actions); - let entropy_threshold = Decimal::try_from(0.5).unwrap_or(Decimal::ZERO); + let entropy_threshold = Decimal::try_from(0.3).unwrap_or(Decimal::ZERO); let diversity_bonus = if entropy < entropy_threshold { self.config.diversity_weight // -0.1 (penalty for low diversity) } else { diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index 02506cdd7..0e262f356 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -360,6 +360,14 @@ pub struct DQNParams { /// Network shape: [base, base/2, base/4]. Range: [256, 4096], step=256. /// Bounded by VRAM via HardwareBudget at runtime. pub hidden_dim_base: usize, + + /// Minimum epsilon when noisy nets are enabled (0.02-0.10, default 0.05) + /// Guarantees random exploration even with learned noise + pub noisy_epsilon_floor: f64, + + /// Count-based exploration bonus coefficient (0.01-0.5, default 0.1) + /// UCB-style bonus: β * sqrt(log(N) / (1 + n_a)) + pub count_bonus_coefficient: f64, } impl Default for DQNParams { @@ -430,6 +438,8 @@ impl Default for DQNParams { num_quantiles: 32, qr_kappa: 1.0, hidden_dim_base: 256, // Conservative default (backward compatible) + noisy_epsilon_floor: 0.05, // Minimum random exploration with noisy nets + count_bonus_coefficient: 0.1, // UCB exploration bonus } } } @@ -521,14 +531,16 @@ impl ParameterSpace for DQNParams { (0.5_f64.ln(), 2.0_f64.ln()), // 41: qr_kappa (log scale) // GPU-dynamic network sizing (42D → 43D) (256.0, 4096.0), // 42: hidden_dim_base (linear, step=256) - // WAVE 11: Rainbow DQN boolean parameters REMOVED from search space (always TRUE) + // Exploration anti-collapse parameters (43D → 45D) + (0.02, 0.10), // 43: noisy_epsilon_floor (linear, minimum random exploration) + (0.01, 0.5), // 44: count_bonus_coefficient (linear, UCB exploration bonus) ] } fn from_continuous(x: &[f64]) -> Result { - if x.len() != 43 { + if x.len() != 45 { return Err(MLError::ConfigError { - reason: format!("Expected 43 continuous parameters (added hidden_dim_base), got {}", x.len()), + reason: format!("Expected 45 continuous parameters (added noisy_epsilon_floor, count_bonus_coefficient), got {}", x.len()), }); } @@ -606,6 +618,10 @@ impl ParameterSpace for DQNParams { // GPU-dynamic hidden dims let hidden_dim_base = ((x[42].round() / 256.0).round() * 256.0).clamp(256.0, 4096.0) as usize; + // Exploration anti-collapse parameters + let noisy_epsilon_floor = x[43].clamp(0.02, 0.10); + let count_bonus_coefficient = x[44].clamp(0.01, 0.5); + // WAVE 11: Rainbow DQN boolean parameters are ALWAYS TRUE (removed from search space) // User requirement: "I want them enabled!" - no point in tuning boolean flags @@ -688,6 +704,8 @@ impl ParameterSpace for DQNParams { num_quantiles, qr_kappa, hidden_dim_base, + noisy_epsilon_floor, + count_bonus_coefficient, }; // Note: HFT constraint validation moved to evaluate_objective (train_with_params) @@ -750,7 +768,9 @@ impl ParameterSpace for DQNParams { self.num_quantiles as f64, self.qr_kappa.ln(), self.hidden_dim_base as f64, - // WAVE 11: Rainbow DQN boolean parameters REMOVED (always TRUE, not tunable) + // Exploration anti-collapse + self.noisy_epsilon_floor, + self.count_bonus_coefficient, ] } @@ -808,7 +828,9 @@ impl ParameterSpace for DQNParams { "num_quantiles", "qr_kappa", "hidden_dim_base", - // WAVE 11: Rainbow DQN boolean parameters REMOVED (always TRUE, not tunable) + // Exploration anti-collapse + "noisy_epsilon_floor", + "count_bonus_coefficient", ] } @@ -2397,6 +2419,8 @@ impl HyperparameterOptimizable for DQNTrainer { // WAVE 17: Hyperopt-tuned parameters entropy_coefficient: Some(params.entropy_coefficient), // Use hyperopt value + noisy_epsilon_floor: Some(params.noisy_epsilon_floor), // Exploration floor for noisy nets + count_bonus_coefficient: Some(params.count_bonus_coefficient), // UCB exploration bonus transaction_cost_multiplier: params.transaction_cost_multiplier, // Use hyperopt value // Wave 3 (Phase 2): Triple Barrier Method (enabled for production) @@ -3358,6 +3382,8 @@ mod tests { num_quantiles: 32, qr_kappa: 1.0, hidden_dim_base: 512, + noisy_epsilon_floor: 0.05, + count_bonus_coefficient: 0.1, }; let continuous = params.to_continuous(); @@ -3378,7 +3404,7 @@ mod tests { #[test] fn test_dqn_params_bounds() { let bounds = DQNParams::continuous_bounds(); - assert_eq!(bounds.len(), 43); // 43 continuous parameters (added hidden_dim_base) + assert_eq!(bounds.len(), 45); // 45 continuous parameters (added noisy_epsilon_floor, count_bonus_coefficient) // Check log-scale bounds are reasonable assert!(bounds[0].0 < bounds[0].1); // learning_rate @@ -3429,7 +3455,7 @@ mod tests { #[test] fn test_param_names() { let names = DQNParams::param_names(); - assert_eq!(names.len(), 43); // 43 tunable hyperparameters (added hidden_dim_base) + assert_eq!(names.len(), 45); // 45 tunable hyperparameters (added noisy_epsilon_floor, count_bonus_coefficient) assert_eq!(names[0], "learning_rate"); assert_eq!(names[1], "batch_size"); assert_eq!(names[2], "gamma"); @@ -3489,6 +3515,9 @@ mod tests { 64.0, 1.0_f64.ln(), // num_quantiles=64, qr_kappa=1.0 (ln(1.0)=0.0) // GPU-dynamic network sizing 512.0, // hidden_dim_base + // Exploration anti-collapse + 0.05, // noisy_epsilon_floor + 0.1, // count_bonus_coefficient ]; let params = DQNParams::from_continuous(&continuous).unwrap(); @@ -3527,6 +3556,9 @@ mod tests { 32.0, 0.5_f64.ln(), // num_quantiles min=32, qr_kappa min=0.5 // GPU-dynamic network sizing min 256.0, // hidden_dim_base min + // Exploration anti-collapse min + 0.02, // noisy_epsilon_floor min + 0.01, // count_bonus_coefficient min ]; let params_min = DQNParams::from_continuous(&continuous_min).unwrap(); assert!((params_min.per_alpha - 0.4).abs() < 1e-6); @@ -3561,6 +3593,9 @@ mod tests { 200.0, 2.0_f64.ln(), // num_quantiles max=200, qr_kappa max=2.0 // GPU-dynamic network sizing max 4096.0, // hidden_dim_base max + // Exploration anti-collapse max + 0.10, // noisy_epsilon_floor max + 0.5, // count_bonus_coefficient max ]; let params_max = DQNParams::from_continuous(&continuous_max).unwrap(); assert!((params_max.per_alpha - 0.8).abs() < 1e-6); @@ -3933,7 +3968,7 @@ mod tests { fn test_qr_dqn_roundtrip_continuous() { let params = DQNParams::default(); let continuous = params.to_continuous(); - assert_eq!(continuous.len(), 43, "Should have 43 continuous dimensions (added hidden_dim_base)"); + assert_eq!(continuous.len(), 45, "Should have 45 continuous dimensions (added noisy_epsilon_floor, count_bonus_coefficient)"); let roundtrip = DQNParams::from_continuous(&continuous).unwrap(); assert_eq!(roundtrip.num_quantiles, params.num_quantiles); assert!((roundtrip.qr_kappa - params.qr_kappa).abs() < 0.01); @@ -3956,7 +3991,7 @@ mod tests { #[test] fn test_batch_size_respects_wide_bounds() { // Simulate PSO choosing batch_size=2048 (within VRAM-aware bounds) - let mut params = vec![0.0_f64; 43]; + let mut params = vec![0.0_f64; 45]; params[1] = 2048.0; // batch_size (index 1) // Fill other required params with valid defaults params[0] = (1e-4_f64).ln(); // learning_rate @@ -3969,9 +4004,9 @@ mod tests { params[8] = 0.5; // transaction_cost_multiplier params[9] = 0.6; // per_alpha params[10] = 0.4; // per_beta_start - // Rainbow extensions (indices 11-42) -- use midpoint of bounds + // Rainbow extensions (indices 11-44) -- use midpoint of bounds let bounds = DQNParams::continuous_bounds(); - for i in 11..43 { + for i in 11..45 { params[i] = (bounds[i].0 + bounds[i].1) / 2.0; } let result = DQNParams::from_continuous(¶ms).unwrap(); diff --git a/crates/ml/src/hyperopt/adapters/ppo.rs b/crates/ml/src/hyperopt/adapters/ppo.rs index 1b4b4e900..40d709ac1 100644 --- a/crates/ml/src/hyperopt/adapters/ppo.rs +++ b/crates/ml/src/hyperopt/adapters/ppo.rs @@ -165,7 +165,7 @@ impl ParameterSpace for PPOParams { } // Cap hidden_dim_base by VRAM (index 6) // PPO needs more VRAM than DQN (actor + critic networks) - let max_base = budget.max_hidden_dim_base(4, 512, 51, 3); // 4 concurrent, 512 batch, 51 features, 3 actions + let max_base = budget.max_hidden_dim_base(4, 512, 51, 45); // 4 concurrent, 512 batch, 51 features, 45 factored actions if let Some(dim_bound) = bounds.get_mut(6) { dim_bound.1 = max_base as f64; } diff --git a/crates/ml/src/ppo/entropy_regularization.rs b/crates/ml/src/ppo/entropy_regularization.rs index daa348c56..0677e59d3 100644 --- a/crates/ml/src/ppo/entropy_regularization.rs +++ b/crates/ml/src/ppo/entropy_regularization.rs @@ -16,7 +16,8 @@ //! use ml::ppo::entropy_regularization::EntropyRegularizer; //! //! let regularizer = EntropyRegularizer::new(); -//! let action_probs = Tensor::new(&[0.5_f32, 0.3, 0.2], &Device::Cpu).unwrap(); +//! // 45 factored actions (5 exposure × 3 order × 3 urgency) +//! let action_probs = Tensor::new(&[1.0_f32 / 45.0; 45], &Device::Cpu).unwrap(); //! let bonus = regularizer.calculate_entropy_bonus(&action_probs).unwrap(); //! ``` @@ -30,23 +31,26 @@ use crate::MLError; /// to encourage exploration and maintain action diversity. #[derive(Debug, Clone)] pub struct EntropyRegularizer { - /// Maximum possible entropy for 3 actions: log(3) ≈ 1.099 + /// Maximum possible entropy: log(num_actions) max_entropy: f64, /// Normalized entropy threshold (0.7) for bonus/penalty entropy_threshold: f64, } impl EntropyRegularizer { - /// Create a new entropy regularizer + /// Number of factored actions (5 exposure × 3 order × 3 urgency) + const NUM_ACTIONS: usize = 45; + + /// Create a new entropy regularizer for 45 factored actions. /// /// # Configuration - /// - `max_entropy`: log(3) ≈ 1.0986 for 3 actions (BUY, SELL, HOLD) + /// - `max_entropy`: log(45) ≈ 3.807 for 45 factored actions /// - `entropy_threshold`: 0.7 normalized entropy /// - Above 0.7: 2x bonus for high diversity /// - Below 0.7: 3x penalty for low diversity pub fn new() -> Self { Self { - max_entropy: (3.0_f64).ln(), // log(3) = 1.0986122886681098 + max_entropy: (Self::NUM_ACTIONS as f64).ln(), entropy_threshold: 0.7, } } @@ -137,18 +141,30 @@ mod tests { Ok(tensor.reshape(&[1, probs.len()])?) } + /// Create uniform probability vector over 45 actions + fn uniform_45() -> Vec { + vec![1.0 / 45.0; 45] + } + + /// Create near-deterministic probability vector over 45 actions + fn deterministic_45(dominant_action: usize) -> Vec { + let mut probs = vec![0.001 / 44.0; 45]; + probs[dominant_action] = 0.999; + probs + } + #[test] fn test_entropy_uniform_distribution() -> Result<(), MLError> { let regularizer = EntropyRegularizer::new(); - let probs = create_prob_tensor(&[1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0])?; + let probs = create_prob_tensor(&uniform_45())?; let entropy = regularizer.calculate_entropy(&probs)?; - // Uniform distribution: entropy = log(3) ≈ 1.0986 + // Uniform distribution over 45 actions: entropy = log(45) ≈ 3.807 + let expected = (45.0_f64).ln(); assert!( - (entropy - 1.0986).abs() < 0.01, - "Expected entropy ~1.0986, got {}", - entropy + (entropy - expected).abs() < 0.01, + "Expected entropy ~{expected:.4}, got {entropy}", ); Ok(()) } @@ -156,27 +172,26 @@ mod tests { #[test] fn test_entropy_deterministic() -> Result<(), MLError> { let regularizer = EntropyRegularizer::new(); - let probs = create_prob_tensor(&[1.0, 0.0, 0.0])?; + let probs = create_prob_tensor(&deterministic_45(0))?; let entropy = regularizer.calculate_entropy(&probs)?; - // Deterministic policy: entropy ≈ 0 - assert!(entropy < 0.01, "Expected entropy ~0, got {}", entropy); + // Near-deterministic policy: entropy ≈ 0 + assert!(entropy < 0.05, "Expected entropy ~0, got {entropy}"); Ok(()) } #[test] fn test_bonus_high_diversity() -> Result<(), MLError> { let regularizer = EntropyRegularizer::new(); - let probs = create_prob_tensor(&[1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0])?; + let probs = create_prob_tensor(&uniform_45())?; let bonus = regularizer.calculate_entropy_bonus(&probs)?; - // Uniform: normalized entropy = 1.0, bonus = 2.0 + // Uniform over 45: normalized entropy ≈ 1.0, bonus = 1.0 × 2.0 = 2.0 assert!( - (bonus - 2.0).abs() < 0.01, - "Expected bonus ~2.0, got {}", - bonus + (bonus - 2.0).abs() < 0.05, + "Expected bonus ~2.0, got {bonus}", ); Ok(()) } @@ -184,13 +199,13 @@ mod tests { #[test] fn test_penalty_low_diversity() -> Result<(), MLError> { let regularizer = EntropyRegularizer::new(); - let probs = create_prob_tensor(&[0.9, 0.05, 0.05])?; + let probs = create_prob_tensor(&deterministic_45(38))?; let penalty = regularizer.calculate_entropy_bonus(&probs)?; - // Low diversity: normalized entropy < 0.7, penalty negative - assert!(penalty < 0.0, "Expected penalty < 0.0, got {}", penalty); - assert!(penalty < -0.4, "Expected penalty < -0.4, got {}", penalty); + // Near-deterministic: normalized entropy ≈ 0, penalty ≈ -(0.7) × 3 = -2.1 + assert!(penalty < 0.0, "Expected penalty < 0.0, got {penalty}"); + assert!(penalty < -1.0, "Expected penalty < -1.0, got {penalty}"); Ok(()) } } diff --git a/crates/ml/src/ppo/flow_policy/mod.rs b/crates/ml/src/ppo/flow_policy/mod.rs index 36b20ff77..f01a4d154 100644 --- a/crates/ml/src/ppo/flow_policy/mod.rs +++ b/crates/ml/src/ppo/flow_policy/mod.rs @@ -377,21 +377,27 @@ impl FlowPolicy { self.evaluate_actions(states, actions) } - /// Computes entropy of the policy distribution. + /// Computes entropy of the policy distribution via Monte Carlo estimate. /// - /// NOTE: Exact entropy for normalizing flows is intractable. - /// This returns zeros as a conservative default. For exploration, - /// rely on the stochastic sampling process. + /// Exact entropy for normalizing flows is analytically intractable, + /// so we estimate H(π|s) ≈ -E[log π(a|s)] by sampling actions and + /// computing the negative mean log-probability. /// /// # Arguments /// * `states` - Batch of states [batch_size, state_dim]. /// /// # Returns - /// Zero tensor [batch_size, 1] (summed across action dimensions). + /// Entropy estimate tensor [batch_size]. pub fn entropy(&self, states: &Tensor) -> Result { - let batch_size = states.dims()[0]; - Tensor::zeros(batch_size, DType::F32, &self.device) - .map_err(|e| MLError::TensorOperationError(format!("Entropy zeros failed: {}", e))) + // Sample actions and get log_probs: shape [batch_size, 1] + let (_actions, log_probs) = self.sample_action(states)?; + // H ≈ -log_prob per sample; squeeze to [batch_size] + let entropy = log_probs + .neg() + .map_err(|e| MLError::TensorOperationError(format!("Entropy negation failed: {e}")))? + .squeeze(1) + .map_err(|e| MLError::TensorOperationError(format!("Entropy squeeze failed: {e}")))?; + Ok(entropy) } /// Encodes state into context vector for conditioning the flow. @@ -600,13 +606,14 @@ mod tests { assert_eq!(entropy.dims(), &[4]); - // Verify all zeros (conservative default) + // Monte Carlo entropy estimate should be non-negative (H = -log_prob ≥ 0 for normalized distributions) let entropy_vec = entropy.flatten_all() .map_err(|e| MLError::TensorOperationError(e.to_string()))? .to_vec1::() .map_err(|e| MLError::TensorOperationError(e.to_string()))?; - for e in entropy_vec { - assert_eq!(e, 0.0); + for e in &entropy_vec { + assert!(*e >= -1.0, "Entropy should be non-negative or near-zero, got {e}"); + assert!(e.is_finite(), "Entropy must be finite, got {e}"); } Ok(()) diff --git a/crates/ml/src/trainers/dqn/config.rs b/crates/ml/src/trainers/dqn/config.rs index c95b01bc7..539b703e4 100644 --- a/crates/ml/src/trainers/dqn/config.rs +++ b/crates/ml/src/trainers/dqn/config.rs @@ -61,6 +61,14 @@ impl DQNAgentType { } } + /// Reset count-based exploration bonus (call at epoch boundary) + pub fn reset_count_bonus(&mut self) { + match self { + Self::Standard(agent) => agent.reset_count_bonus(), + Self::RegimeConditional(agent) => agent.reset_count_bonus(), + } + } + /// Update epsilon for exploration decay pub fn update_epsilon(&mut self) { match self { @@ -470,8 +478,12 @@ pub struct DQNHyperparameters { pub max_position_absolute: f64, // WAVE 17: Hyperopt-tuned parameters - /// Entropy regularization coefficient (optional) + /// SAC-style entropy coefficient for Q-value softmax regularization (default: 0.01) pub entropy_coefficient: Option, + /// Minimum epsilon when noisy nets are enabled (default: 0.05) + pub noisy_epsilon_floor: Option, + /// Count-based exploration bonus coefficient (default: 0.1, range: [0.01, 0.5]) + pub count_bonus_coefficient: Option, /// Transaction cost multiplier for reward calculation pub transaction_cost_multiplier: f64, @@ -732,7 +744,9 @@ impl DQNHyperparameters { max_position_absolute: 2.0, // Default: ±2.0 position limit (matches production) // WAVE 17: Hyperopt-tuned parameters - entropy_coefficient: None, + entropy_coefficient: Some(0.01), + noisy_epsilon_floor: Some(0.05), + count_bonus_coefficient: Some(0.1), transaction_cost_multiplier: 1.0, // Triple Barrier defaults @@ -813,8 +827,8 @@ impl DQNHyperparameters { gae_lambda: 0.95, // Default: 0.95 (standard PPO/A2C value) // P1.11: Noisy Sigma Scheduling - enable_noisy_sigma_scheduler: false, // Default: disabled (enable with noisy_nets=true) - noisy_sigma_initial: 0.6, // Default: 60% initial noise + enable_noisy_sigma_scheduler: true, // Enabled by default for wider exploration + noisy_sigma_initial: 0.8, // Higher initial noise for 45-action exploration noisy_sigma_final: 0.4, // Default: 40% final noise noisy_sigma_anneal_steps: 10000, // Default: 10K steps for annealing @@ -958,6 +972,7 @@ pub(crate) fn dqn_config_2025() -> DQNConfig { minimum_profit_factor: 1.5, weight_decay: 1e-4, mixed_precision: None, + ..Default::default() } } diff --git a/crates/ml/src/trainers/dqn/trainer.rs b/crates/ml/src/trainers/dqn/trainer.rs index 197d92404..a9ad1815f 100644 --- a/crates/ml/src/trainers/dqn/trainer.rs +++ b/crates/ml/src/trainers/dqn/trainer.rs @@ -95,7 +95,7 @@ pub struct DQNTrainer { /// Maximum position size for action masking (default: 2.0) pub max_position: f64, /// Entropy regularizer for preventing policy collapse (None if disabled) - pub entropy_regularizer: Option>, + // entropy_regularizer removed — SAC-style entropy is computed directly on Q-value tensors in DQN::compute_loss_internal /// Multi-asset portfolio tracker (None if single-asset mode) pub multi_asset_portfolio: Option>, /// Stress tester for robustness validation (None if disabled) @@ -450,6 +450,10 @@ impl DQNTrainer { minimum_profit_factor: 1.5, weight_decay: 1e-4, mixed_precision: hyperparams.mixed_precision.clone().or(mixed_precision_detected), + entropy_coefficient: hyperparams.entropy_coefficient.unwrap_or(0.01), + noisy_epsilon_floor: hyperparams.noisy_epsilon_floor.unwrap_or(0.05) as f32, + use_count_bonus: true, + count_bonus_coefficient: hyperparams.count_bonus_coefficient.unwrap_or(0.1), }; // Create DQN agent @@ -527,12 +531,11 @@ impl DQNTrainer { let enable_action_masking = hyperparams.enable_action_masking; let max_position = hyperparams.max_position_absolute; // BLOCKER #2: Use hyperopt-tunable position limit - // Entropy regularization for preventing policy collapse - let entropy_regularizer: Option> = hyperparams.enable_entropy_regularization.then(|| { - use crate::dqn::entropy_regularization::EntropyRegularizer; - info!("Entropy regularization enabled (coefficient=0.01)"); - Arc::new(EntropyRegularizer::new()) - }); + // Entropy regularization: SAC-style computed directly on Q-values in DQN::compute_loss_internal + if hyperparams.enable_entropy_regularization { + let coeff = hyperparams.entropy_coefficient.unwrap_or(0.01); + info!("Entropy regularization enabled (coefficient={coeff:.4}, applied to Q-value softmax in loss)"); + } // Multi-asset portfolio tracking (disabled -- single-asset is the current production mode) // @@ -751,7 +754,6 @@ impl DQNTrainer { // Wave 16 Portfolio Features enable_action_masking, max_position, - entropy_regularizer, multi_asset_portfolio, stress_tester, @@ -2592,13 +2594,18 @@ impl DQNTrainer { ); // BUG #29 FIX: Update epsilon once per epoch (not per batch) - // This ensures consistent exploration across different batch sizes - // With epsilon_decay=0.995, after 15 epochs: 0.3 × (0.995^15) = 0.2783 (27.8% exploration) - { + // Skip epsilon decay when noisy nets are enabled — epsilon is fixed at noisy_epsilon_floor + if !self.hyperparams.use_noisy_nets { let mut agent = self.agent.write().await; agent.update_epsilon(); } + // Reset count-based exploration bonus at epoch boundary + { + let mut agent = self.agent.write().await; + agent.reset_count_bonus(); + } + // WAVE 26 P0.6: Update learning rate with scheduler (warmup + decay) self.lr_scheduler.step(); let current_lr = self.lr_scheduler.get_lr(); @@ -2693,15 +2700,39 @@ impl DQNTrainer { diversity_percentage ); + // Compute normalized entropy of epoch action distribution + let epoch_entropy = if epoch_total_actions > 0 { + let mut entropy_raw = 0.0_f64; + for &count in monitor.action_counts.iter() { + if count > 0 { + let p = count as f64 / epoch_total_actions as f64; + entropy_raw -= p * p.ln(); + } + } + entropy_raw / 45.0_f64.ln() // Normalize to [0, 1] + } else { + 0.0 + }; + + info!( + " Exploration: entropy={:.3} (1.0=uniform), epsilon={:.4}, noisy_nets={}, count_bonus={}", + epoch_entropy, + self.get_epsilon().await.unwrap_or(0.0), + self.hyperparams.use_noisy_nets, + self.hyperparams.count_bonus_coefficient.unwrap_or(0.0) > 0.0, + ); + + // Emit exploration diagnostics to Prometheus + training_metrics::set_action_entropy("dqn", "current", epoch_entropy); + training_metrics::set_action_diversity("dqn", "current", diversity_percentage / 100.0); + // Warning if diversity drops below 20% (9 actions) const DIVERSITY_THRESHOLD: usize = 9; // 20% of 45 actions if active_actions_count < DIVERSITY_THRESHOLD { warn!( - "⚠️ LOW ACTION DIVERSITY: {}/45 actions (<20%), consider increasing epsilon floor", - active_actions_count + "⚠️ LOW ACTION DIVERSITY: {}/45 actions (<20%), entropy={:.3}", + active_actions_count, epoch_entropy, ); - info!(" Recommendation: Increase epsilon_end from 0.05 to 0.10"); - info!(" Alternative: Add entropy regularization bonus"); } // WAVE P2: Log episode statistics diff --git a/crates/ml/src/trainers/ppo.rs b/crates/ml/src/trainers/ppo.rs index fc0ee2b08..b260af645 100644 --- a/crates/ml/src/trainers/ppo.rs +++ b/crates/ml/src/trainers/ppo.rs @@ -1269,7 +1269,7 @@ impl PpoTrainer { &self, batch: &TrajectoryBatch, policy_loss: f32, - value_loss: f32, + _value_loss: f32, ) -> Result<(f32, f32, f32, f32, f32), MLError> { // KL divergence (approximated from policy loss) let kl_div = policy_loss.abs() * 0.1; @@ -1299,8 +1299,31 @@ impl PpoTrainer { (ev, mr, sr) }); - // Entropy (approximated from entropy coefficient impact) - let entropy = value_loss * 0.5; // Simplified + // Compute real entropy from the epoch action distribution + let entropy = { + let actions = &batch.actions; + if actions.is_empty() { + 0.0_f32 + } else { + let mut counts = [0_u32; 45]; + for action in actions { + let idx = action.to_index(); + if idx < 45 { + counts[idx] += 1; + } + } + let n = actions.len() as f64; + let mut h = 0.0_f64; + for &c in &counts { + if c > 0 { + let p = c as f64 / n; + h -= p * p.ln(); + } + } + // Normalize by ln(45) so entropy is in [0, 1] + (h / 45.0_f64.ln()) as f32 + } + }; Ok((kl_div, explained_variance, mean_reward, std_reward, entropy)) } diff --git a/docs/plans/2026-03-03-dqn-action-collapse-fix.md b/docs/plans/2026-03-03-dqn-action-collapse-fix.md new file mode 100644 index 000000000..42751a1d7 --- /dev/null +++ b/docs/plans/2026-03-03-dqn-action-collapse-fix.md @@ -0,0 +1,389 @@ +# DQN Action Collapse Fix — Design & Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Fix 45-action DQN collapse where the model converges to a single action (A38 = Long100/Market/Aggressive) across all hyperopt trials, despite existing anti-collapse infrastructure. + +**Root Causes:** +1. Entropy regularizer is dead code (initialized, never called) +2. Noisy nets disable epsilon-greedy but provide insufficient exploration for 45 actions +3. Diversity penalty entropy normalized for 3 actions, not 45 + +**Approach:** Fix all three existing mechanisms + add two new ones (epsilon floor for noisy nets, count-based exploration bonus). + +--- + +## Task 1: Wire entropy regularizer into TD loss + +**Files:** +- Modify: `crates/ml/src/dqn/dqn.rs` (add entropy bonus to `compute_loss` or loss path) +- Modify: `crates/ml/src/trainers/dqn/trainer.rs` (pass entropy regularizer into training step) + +**Step 1: Add entropy bonus computation to the Q-learning loss** + +The entropy regularizer at `crates/ml/src/dqn/entropy_regularization.rs` is fully implemented but never called. Wire it into the training loop. + +In the training step where loss is computed (trainer.rs, the `train_step` / gradient update section), after computing Q-values for the current batch: + +```rust +// Compute softmax policy from Q-values: π(a|s) = softmax(Q(s,a) / τ) +// Then entropy: H(π) = -Σ π(a|s) * log(π(a|s)) +// Add to loss: loss = td_loss - entropy_coefficient * mean_entropy +``` + +Specifically: +1. After the forward pass that produces `q_values` tensor (shape [batch, 45]) +2. Compute `log_softmax(q_values, dim=1)` and `softmax(q_values, dim=1)` +3. Entropy per sample: `H = -(softmax * log_softmax).sum(dim=1)` → shape [batch] +4. Mean entropy: `mean_H = H.mean()` +5. Final loss: `loss = td_loss - entropy_coefficient * mean_H` + +The `entropy_coefficient` field already exists in `DQNHyperparameters` (default 0.01, hyperopt range [0.0, 0.1]). + +**Step 2: Remove unused `entropy_regularizer` field from trainer struct** + +The `Arc` stored at trainer.rs:754 is no longer needed — entropy is computed directly on Q-value tensors. Remove the field and the initialization at lines 531-535. + +**Step 3: Add unit test** + +Test that with `entropy_coefficient > 0`, training on a batch where all Q-values are identical (uniform policy) produces lower loss than a batch with one dominant Q-value (collapsed policy). + +**Step 4: Build and test** + +```bash +SQLX_OFFLINE=true cargo test -p ml --lib -- trainers::dqn +``` + +**Step 5: Commit** + +```bash +git add crates/ml/src/dqn/ crates/ml/src/trainers/dqn/ +git commit -m "fix(dqn): wire entropy regularization into TD loss (was dead code)" +``` + +--- + +## Task 2: Fix diversity penalty entropy normalization + +**Files:** +- Modify: `crates/ml/src/dqn/reward.rs` (fix `calculate_entropy()` normalization) + +**Step 1: Fix entropy normalization from log(3) to log(45)** + +In `reward.rs`, find `calculate_entropy()` (around line 364-395). The function computes Shannon entropy of recent actions. Currently it normalizes by `log(3)` (3 legacy action categories). Change to `log(45)` to match the 45 factored actions. + +Also change the entropy threshold from 0.5 to 0.3. With 45 actions, entropy of 0.3 (normalized) means ~4 distinct actions — a reasonable collapse threshold. + +```rust +// Before: +let max_entropy = (3.0_f64).ln(); // log(3) = 1.099 +let entropy_threshold = 0.5; + +// After: +let max_entropy = (45.0_f64).ln(); // log(45) = 3.807 +let entropy_threshold = 0.3; +``` + +**Step 2: Update the action counting to use 45 action indices** + +If the entropy calculation currently maps actions to 3 categories (Buy/Sell/Hold), change it to use the full `to_index()` (0-44) for the frequency distribution. + +**Step 3: Add test** + +Test that: +- Uniform distribution over 45 actions → entropy ≈ 1.0 (normalized) → no penalty +- Single-action collapse (all A38) → entropy ≈ 0.0 → penalty fires (-0.1) +- 5 distinct actions → entropy ≈ 0.42 → penalty fires (< 0.3 threshold? depends on distribution) + +**Step 4: Build and test** + +```bash +SQLX_OFFLINE=true cargo test -p ml --lib -- dqn::reward +``` + +**Step 5: Commit** + +```bash +git add crates/ml/src/dqn/reward.rs +git commit -m "fix(dqn): normalize diversity entropy for 45 actions (was calibrated for 3)" +``` + +--- + +## Task 3: Add epsilon floor for noisy nets + +**Files:** +- Modify: `crates/ml/src/trainers/dqn/config.rs` (add `noisy_epsilon_floor` field) +- Modify: `crates/ml/src/dqn/dqn.rs` (use floor in `select_action()`) +- Modify: `crates/ml/src/hyperopt/adapters/dqn.rs` (add to search space) + +**Step 1: Add config field** + +In `config.rs`, add to `DQNHyperparameters`: + +```rust +/// Minimum epsilon when noisy nets are enabled (guarantees some random exploration) +pub noisy_epsilon_floor: f64, // default: 0.05 +``` + +Add default value in the `Default` impl: `noisy_epsilon_floor: 0.05`. + +**Step 2: Use floor in select_action()** + +In `dqn.rs`, change the noisy nets epsilon logic: + +```rust +// Before: +let effective_epsilon = if self.config.use_noisy_nets { + 0.0 // Learned exploration via parameter noise +} else { + self.epsilon +}; + +// After: +let effective_epsilon = if self.config.use_noisy_nets { + self.config.noisy_epsilon_floor // Minimum random exploration even with noisy nets +} else { + self.epsilon +}; +``` + +**Step 3: Add to hyperopt search space** + +In `dqn.rs` hyperopt adapter, add `noisy_epsilon_floor` to: +- `DQNParams` struct +- `continuous_bounds()` — range [0.02, 0.10] +- `from_continuous()` / `to_continuous()` +- `to_hyperparams()` mapping + +**Step 4: Skip epsilon decay when noisy nets enabled** + +In `trainer.rs`, guard the `agent.update_epsilon()` call: + +```rust +// Before: +agent.update_epsilon(); + +// After: +if !self.hyperparams.use_noisy_nets { + agent.update_epsilon(); +} +``` + +**Step 5: Build and test** + +```bash +SQLX_OFFLINE=true cargo check -p ml +SQLX_OFFLINE=true cargo test -p ml --lib -- dqn::dqn::tests +``` + +**Step 6: Commit** + +```bash +git add crates/ml/src/trainers/dqn/config.rs crates/ml/src/dqn/dqn.rs crates/ml/src/hyperopt/adapters/dqn.rs crates/ml/src/trainers/dqn/trainer.rs +git commit -m "feat(dqn): add epsilon floor for noisy nets (0.05 default, hyperopt-tunable)" +``` + +--- + +## Task 4: Enable noisy sigma scheduling with higher initial sigma + +**Files:** +- Modify: `crates/ml/src/trainers/dqn/config.rs` (change defaults) +- Modify: `crates/ml/src/dqn/dqn.rs` (add sigma floor) + +**Step 1: Change defaults for sigma scheduling** + +In `config.rs`, update the defaults: + +```rust +// Before: +enable_noisy_sigma_scheduler: false, +noisy_sigma_initial: 0.6, +noisy_sigma_final: 0.4, + +// After: +enable_noisy_sigma_scheduler: true, // Enable by default +noisy_sigma_initial: 0.8, // Wider initial exploration +noisy_sigma_final: 0.4, +``` + +**Step 2: Add sigma floor** + +In the sigma scheduling logic (wherever sigma is computed/annealed), add a minimum: + +```rust +let sigma = computed_sigma.max(0.3); // Never let sigma drop below 0.3 +``` + +**Step 3: Update hyperopt adapter defaults** + +Make sure the hyperopt adapter uses the new defaults when constructing `DQNHyperparameters`. + +**Step 4: Build and test** + +```bash +SQLX_OFFLINE=true cargo check -p ml +``` + +**Step 5: Commit** + +```bash +git add crates/ml/src/trainers/dqn/config.rs crates/ml/src/dqn/dqn.rs +git commit -m "feat(dqn): enable noisy sigma scheduling (0.8→0.4, floor=0.3)" +``` + +--- + +## Task 5: Add count-based exploration bonus (UCB) + +**Files:** +- Create: `crates/ml/src/dqn/count_bonus.rs` +- Modify: `crates/ml/src/dqn/mod.rs` (add module) +- Modify: `crates/ml/src/trainers/dqn/config.rs` (add config fields) +- Modify: `crates/ml/src/dqn/dqn.rs` (apply bonus in action selection) +- Modify: `crates/ml/src/hyperopt/adapters/dqn.rs` (add to search space) + +**Step 1: Create count_bonus.rs** + +```rust +/// UCB-style count-based exploration bonus for action selection. +/// +/// Adds bonus = β * sqrt(log(N) / (1 + n_a)) to Q-values during action selection, +/// where N = total actions taken this epoch, n_a = count of action a. +/// This gives direct exploration incentive to under-visited actions. +pub struct CountBonus { + action_counts: [u64; 45], + total_count: u64, + coefficient: f64, +} + +impl CountBonus { + pub fn new(coefficient: f64) -> Self { ... } + + /// Record that action was taken + pub fn record_action(&mut self, action_idx: usize) { ... } + + /// Compute bonus for each of 45 actions, returns [f64; 45] + pub fn bonuses(&self) -> [f64; 45] { + if self.total_count == 0 { + return [self.coefficient; 45]; // Max bonus when no data + } + let log_n = (self.total_count as f64).ln(); + let mut result = [0.0; 45]; + for a in 0..45 { + result[a] = self.coefficient * (log_n / (1.0 + self.action_counts[a] as f64)).sqrt(); + } + result + } + + /// Reset counts (call at epoch boundary) + pub fn reset(&mut self) { ... } +} +``` + +**Step 2: Add config fields** + +In `config.rs`: + +```rust +pub use_count_bonus: bool, // default: true +pub count_bonus_coefficient: f64, // default: 0.1 +``` + +**Step 3: Wire into action selection** + +In `dqn.rs` `select_action()`, when choosing the greedy action: + +```rust +// After computing q_values from forward pass: +if self.config.use_count_bonus { + let bonuses = self.count_bonus.bonuses(); + let bonus_tensor = Tensor::from_vec(bonuses.to_vec(), 45, device)?; + let q_with_bonus = q_values.broadcast_add(&bonus_tensor)?; + // Use q_with_bonus for argmax instead of q_values +} +``` + +Record action after selection: + +```rust +if self.config.use_count_bonus { + self.count_bonus.record_action(action.to_index()); +} +``` + +**Step 4: Reset at epoch boundary** + +In `trainer.rs`, at the start of each epoch: + +```rust +agent.reset_count_bonus(); // Resets per-epoch exploration pressure +``` + +**Step 5: Add to hyperopt search space** + +- `count_bonus_coefficient` — range [0.01, 0.5] +- `use_count_bonus` — always true (not tuned, it's a safety mechanism) + +**Step 6: Add tests** + +- Test that bonuses are high for unvisited actions and low for frequent actions +- Test that after recording 100 of action 38, action 38's bonus is near zero while others are high +- Test reset clears all counts + +**Step 7: Build and test** + +```bash +SQLX_OFFLINE=true cargo test -p ml --lib -- dqn::count_bonus +SQLX_OFFLINE=true cargo check -p ml +``` + +**Step 8: Commit** + +```bash +git add crates/ml/src/dqn/count_bonus.rs crates/ml/src/dqn/mod.rs crates/ml/src/trainers/dqn/config.rs crates/ml/src/dqn/dqn.rs crates/ml/src/hyperopt/adapters/dqn.rs +git commit -m "feat(dqn): add UCB count-based exploration bonus for 45-action diversity" +``` + +--- + +## Task 6: Update monitoring to report exploration diagnostics + +**Files:** +- Modify: `crates/ml/src/trainers/dqn/trainer.rs` (log exploration source stats) + +**Step 1: Track exploration source in action diversity log** + +In the per-epoch action diversity logging, add: +- Count of random (epsilon) vs greedy vs noisy-net actions +- Effective entropy over 45 actions (normalized by log(45)) +- Count bonus range (min/max/mean) + +**Step 2: Add Prometheus metrics** + +In `crates/common/src/metrics/training_metrics.rs`, add: + +```rust +foxhunt_training_action_entropy // Normalized entropy (0-1) of epoch action distribution +foxhunt_training_epsilon_actions // Count of epsilon-random actions this epoch +foxhunt_training_greedy_actions // Count of greedy actions this epoch +``` + +**Step 3: Commit** + +```bash +git add crates/ml/src/trainers/dqn/trainer.rs crates/common/src/metrics/training_metrics.rs +git commit -m "feat(monitoring): add exploration diagnostics (entropy, epsilon/greedy split)" +``` + +--- + +## Task 7: Full build verification + +**Step 1:** `SQLX_OFFLINE=true cargo check --workspace` +**Step 2:** `SQLX_OFFLINE=true cargo test -p ml --lib` +**Step 3:** `SQLX_OFFLINE=true cargo test -p common --lib` +**Step 4:** `SQLX_OFFLINE=true cargo clippy --workspace -- -D warnings` + +Expected: 0 errors, 0 warnings, all tests pass.