gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'worktree-dqn-action-collapse-fix'

Browse Source
This commit is contained in:
jgrusewski
2026-03-03 02:04:57 +01:00
parent eed268c230 864cf8690d
commit 0840f8a971
16 changed files with 909 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
crates/common/src/metrics/training_metrics.rs
View File

@@ -171,6 +171,18 @@ pub fn init() {
mf,
);
// Exploration diagnostics (DQN action collapse monitoring)
_ = register_gauge_vec(
"foxhunt_training_action_entropy",
"Normalized entropy of epoch action distribution (0=collapse, 1=uniform)",
mf,
);
_ = register_gauge_vec(
"foxhunt_training_action_diversity",
"Fraction of 45 actions used above 0.5% threshold",
mf,
);
// Hyperopt gauges (model only)
_ = register_gauge_vec(
"foxhunt_hyperopt_trial_current",
@@ -428,6 +440,14 @@ pub fn set_epoch_duration(model: &str, fold: &str, secs: f64) {
set_gauge_vec("foxhunt_training_epoch_duration_seconds", &[model, fold], secs);
}
pub fn set_action_entropy(model: &str, fold: &str, entropy: f64) {
set_gauge_vec("foxhunt_training_action_entropy", &[model, fold], entropy);
}
pub fn set_action_diversity(model: &str, fold: &str, diversity: f64) {
set_gauge_vec("foxhunt_training_action_diversity", &[model, fold], diversity);
}
// ---------------------------------------------------------------------------
// Tier 1: Hyperopt intra-trial
// ---------------------------------------------------------------------------

1
crates/ml/src/benchmark/dqn_benchmark.rs
View File

@@ -471,6 +471,7 @@ impl DqnBenchmarkRunner {
minimum_profit_factor: 1.5,
weight_decay: 1e-4,
mixed_precision: None,
..Default::default()
}
}

152
crates/ml/src/dqn/count_bonus.rs Normal file
View File

@@ -0,0 +1,152 @@
//! UCB-style count-based exploration bonus for action selection.
//!
//! Adds bonus = β × sqrt(ln(N) / (1 + n_a)) to Q-values during action selection,
//! where N = total actions taken this epoch, n_a = count of action a.
//! This gives a direct exploration incentive to under-visited actions,
//! preventing collapse onto a single action (e.g., A38) across all states.
/// Count-based exploration bonus tracker for 45 factored actions.
///
/// At each greedy action selection, bonuses are added to Q-values before argmax.
/// Actions taken less frequently get higher bonuses, encouraging diversity.
/// Counts are reset at epoch boundaries.
#[derive(Debug)]
pub struct CountBonus {
action_counts: [u64; 45],
total_count: u64,
coefficient: f64,
}
impl CountBonus {
/// Create a new count bonus tracker with the given coefficient β.
///
/// β controls the strength of the exploration bonus.
/// Typical values: 0.01–0.5 (tunable via hyperopt).
pub fn new(coefficient: f64) -> Self {
Self {
action_counts: [0_u64; 45],
total_count: 0,
coefficient,
}
}
/// Record that an action was taken.
pub fn record_action(&mut self, action_idx: usize) {
if action_idx < 45 {
self.action_counts[action_idx] += 1;
self.total_count += 1;
}
}
/// Compute UCB-style bonus for each of the 45 actions.
///
/// Formula: bonus_a = β × sqrt(ln(N) / (1 + n_a))
/// - When n_a = 0 and N > 0: bonus is at maximum (strong incentive to try)
/// - When n_a is large: bonus approaches 0 (already well-explored)
/// - When N = 0: returns uniform bonus β (no data yet)
pub fn bonuses(&self) -> [f64; 45] {
if self.total_count == 0 {
return [self.coefficient; 45];
}
let log_n = (self.total_count as f64).ln();
let mut result = [0.0; 45];
for (a, bonus) in result.iter_mut().enumerate() {
*bonus =
self.coefficient * (log_n / (1.0 + self.action_counts[a] as f64)).sqrt();
}
result
}
/// Reset all counts (call at epoch boundary).
pub fn reset(&mut self) {
self.action_counts = [0_u64; 45];
self.total_count = 0;
}
/// Get total number of actions recorded since last reset.
pub fn total_count(&self) -> u64 {
self.total_count
}
/// Get the coefficient β.
pub fn coefficient(&self) -> f64 {
self.coefficient
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_new_bonus_returns_uniform_coefficient() {
let cb = CountBonus::new(0.1);
let bonuses = cb.bonuses();
for b in &bonuses {
assert!((*b - 0.1).abs() < 1e-10, "Expected 0.1, got {}", b);
}
}
#[test]
fn test_frequent_action_gets_low_bonus() {
let mut cb = CountBonus::new(0.1);
// Record action 38 one hundred times
for _ in 0..100 {
cb.record_action(38);
}
let bonuses = cb.bonuses();
// Action 38 should have much lower bonus than others
let bonus_38 = bonuses[38];
let bonus_0 = bonuses[0]; // Never taken
assert!(
bonus_0 > bonus_38 * 5.0,
"Unvisited bonus ({}) should be >> frequent bonus ({})",
bonus_0,
bonus_38
);
}
#[test]
fn test_uniform_usage_gives_equal_bonuses() {
let mut cb = CountBonus::new(0.1);
// Take each action exactly once
for i in 0..45 {
cb.record_action(i);
}
let bonuses = cb.bonuses();
let first = bonuses[0];
for (i, b) in bonuses.iter().enumerate() {
assert!(
(*b - first).abs() < 1e-10,
"Bonus {} ({}) should equal bonus 0 ({})",
i,
b,
first
);
}
}
#[test]
fn test_reset_clears_all_counts() {
let mut cb = CountBonus::new(0.1);
for _ in 0..50 {
cb.record_action(10);
}
assert_eq!(cb.total_count(), 50);
cb.reset();
assert_eq!(cb.total_count(), 0);
let bonuses = cb.bonuses();
// After reset, should return uniform coefficient again
for b in &bonuses {
assert!((*b - 0.1).abs() < 1e-10);
}
}
#[test]
fn test_out_of_range_action_ignored() {
let mut cb = CountBonus::new(0.1);
cb.record_action(45); // Out of range
cb.record_action(100); // Out of range
assert_eq!(cb.total_count(), 0);
}
}

121
crates/ml/src/dqn/dqn.rs
View File

@@ -151,6 +151,26 @@ pub struct DQNConfig {
/// Default: 5 (prevents stopping on transient spikes)
pub gradient_collapse_patience: usize,
/// SAC-style entropy coefficient for Q-value softmax regularization.
/// Adds -coeff * H(softmax(Q)) to the loss, preventing action collapse.
/// Default: 0.01. Range for hyperopt: [0.001, 0.1].
pub entropy_coefficient: f64,
/// Minimum epsilon for exploration when noisy nets are enabled.
/// Standard noisy nets set epsilon=0.0, but 45-action spaces need a floor.
/// Default: 0.05. Range for hyperopt: [0.02, 0.10].
pub noisy_epsilon_floor: f32,
/// Enable UCB count-based exploration bonus during action selection.
/// Adds bonus = β × sqrt(ln(N) / (1 + n_a)) to Q-values before argmax.
/// Default: true.
pub use_count_bonus: bool,
/// Coefficient β for count-based exploration bonus.
/// Controls strength of the exploration incentive for under-visited actions.
/// Default: 0.1. Range for hyperopt: [0.01, 0.5].
pub count_bonus_coefficient: f64,
// Conservative Q-Learning (CQL) for offline RL (Kumar et al. 2020)
pub use_cql: bool,
pub cql_alpha: f64,
@@ -223,6 +243,12 @@ impl Default for DQNConfig {
gradient_collapse_multiplier: 2.0,
gradient_collapse_patience: 100,
// Entropy regularization: prevent 45-action collapse
entropy_coefficient: 0.01,
noisy_epsilon_floor: 0.05,
use_count_bonus: true,
count_bonus_coefficient: 0.1,
// CQL: Enabled by default for offline training
use_cql: true,
cql_alpha: 1.0,
@@ -326,6 +352,7 @@ impl DQNConfig {
minimum_profit_factor: 1.5,
weight_decay: 1e-4,
mixed_precision: None,
..Default::default()
}
}
@@ -393,6 +420,7 @@ impl DQNConfig {
minimum_profit_factor: 2.0, // Higher safety margin for conservative config
weight_decay: 1e-4,
mixed_precision: None,
..Default::default()
}
}
@@ -457,6 +485,11 @@ impl DQNConfig {
gradient_collapse_multiplier: 100.0,
gradient_collapse_patience: 5,
entropy_coefficient: 0.01,
noisy_epsilon_floor: 0.05,
use_count_bonus: true,
count_bonus_coefficient: 0.1,
use_cql: true,
cql_alpha: 5.0,
use_iqn: false,
@@ -832,6 +865,9 @@ pub struct DQN {
// WAVE 23 P0 Fix #4: Early stopping on Q-value divergence
/// Consecutive epochs with Q-value divergence (resets when Q-values normalize)
q_value_divergence_counter: usize,
/// UCB count-based exploration bonus for action diversity
count_bonus: super::count_bonus::CountBonus,
}
impl DQN {
@@ -1013,6 +1049,7 @@ impl DQN {
total_steps: 0,
optimizer: None,
gradient_clip_norm: config.gradient_clip_norm,
count_bonus: super::count_bonus::CountBonus::new(config.count_bonus_coefficient),
config,
device,
recent_actions: VecDeque::with_capacity(100),
@@ -1195,9 +1232,10 @@ impl DQN {
// During warmup period: always use random exploration (epsilon=1.0)
let in_warmup = self.total_steps <= self.config.warmup_steps as u64;
// Noisy Networks replace epsilon-greedy: use epsilon=0 when noisy nets enabled
// Noisy Networks: use epsilon floor (not 0) to guarantee some random exploration
// for 45-action spaces where parameter noise alone is insufficient
let effective_epsilon = if self.config.use_noisy_nets {
0.0 // Learned exploration via parameter noise
self.config.noisy_epsilon_floor // Minimum random exploration with noisy nets
} else {
self.epsilon // Standard epsilon-greedy exploration
};
@@ -1241,6 +1279,16 @@ impl DQN {
.map_err(|e| MLError::ModelError(format!("Expected Q computation failed: {}", e)))?
};
// Apply count bonus before argmax (UCB exploration)
let action_scores = if self.config.use_count_bonus {
let bonuses = self.count_bonus.bonuses();
let bonus_vec: Vec<f32> = bonuses.iter().map(|b| *b as f32).collect();
let bonus_t = Tensor::from_vec(bonus_vec, (1, self.config.num_actions), &self.device)?;
action_scores.broadcast_add(&bonus_t)?
} else {
action_scores
};
// Argmax over actions
let best_action_idx = action_scores
.argmax(1)?
@@ -1251,6 +1299,17 @@ impl DQN {
} else {
// Standard Q-network action selection
let q_values = self.forward(&state_tensor)?;
// Apply count bonus before argmax (UCB exploration)
let q_values = if self.config.use_count_bonus {
let bonuses = self.count_bonus.bonuses();
let bonus_vec: Vec<f32> = bonuses.iter().map(|b| *b as f32).collect();
let bonus_t = Tensor::from_vec(bonus_vec, (1, self.config.num_actions), &self.device)?;
q_values.broadcast_add(&bonus_t)?
} else {
q_values
};
let best_action_idx = q_values
.argmax(1)?
.get(0)?
@@ -1261,6 +1320,11 @@ impl DQN {
}
};
// Record action for count bonus (UCB exploration tracking)
if self.config.use_count_bonus {
self.count_bonus.record_action(action.to_index());
}
// Track action for entropy penalty calculation
self.track_action(action);
@@ -1922,11 +1986,21 @@ impl DQN {
}
};
// Add entropy regularization (in-training diversity penalty)
let entropy_penalty = self.calculate_entropy_penalty()?;
let entropy_weight = 0.1; // Match Wave 5-A1 diversity_weight
let entropy_term = (entropy_penalty * entropy_weight)?;
let loss_with_entropy = loss_raw.add(&entropy_term)?;
// SAC-style entropy regularization on Q-value softmax
// Encourages the Q-network to maintain spread across actions, preventing collapse.
// H(π) = -Σ softmax(Q) * log_softmax(Q), added as: loss = td_loss - coeff * mean(H)
let loss_with_entropy = if self.config.entropy_coefficient > 0.0 {
let log_probs_q = candle_nn::ops::log_softmax(&current_q_values, 1)?;
let probs_q = candle_nn::ops::softmax(&current_q_values, 1)?;
// Entropy per sample: H = -(π * log π).sum(dim=1)
let neg_entropy_per_sample = (&probs_q * &log_probs_q)?.sum(1)?; // negative entropy
let mean_neg_entropy = neg_entropy_per_sample.mean_all()?;
// loss = td_loss + coeff * (-H) = td_loss - coeff * H
let entropy_term = (mean_neg_entropy * self.config.entropy_coefficient)?;
loss_raw.add(&entropy_term)?
} else {
loss_raw
};
// CQL regularization for offline RL (Kumar et al. 2020)
// Penalizes high Q-values for actions not taken in the training data
@@ -2462,38 +2536,34 @@ impl DQN {
.map_err(|e| MLError::ModelError(format!("Failed to create zero penalty: {}", e)));
}
// Count action frequencies (convert FactoredAction to legacy for entropy calculation)
let mut counts = [0, 0, 0]; // BUY, SELL, HOLD
// Count over all 45 factored actions (not 3 legacy categories)
let mut counts = [0_u32; 45];
for action in &self.recent_actions {
let legacy_action = action.to_legacy_action();
counts[legacy_action as usize] += 1;
let idx = action.to_index();
if idx < 45 {
counts[idx] += 1;
}
}
// Calculate Shannon entropy: H = -Σ(p_i * log2(p_i))
// BUG #14 FIX: Add epsilon to prevent log(0) = -inf → 0 * -inf = NaN
let total = self.recent_actions.len() as f64;
let mut entropy = 0.0_f64;
const EPSILON: f64 = 1e-10; // Prevent log(0)
for &count in &counts {
if count > 0 {
let p = count as f64 / total;
// Clamp p to [EPSILON, 1.0] to prevent numerical instability
let p_safe = p.max(EPSILON).min(1.0);
entropy -= p_safe * p_safe.log2();
entropy -= p * p.ln();
}
}
// Return negative entropy as penalty (lower entropy = higher penalty)
// This encourages the agent to maximize entropy (balanced actions)
let penalty = -entropy as f32;
// Normalize by max entropy ln(45), then negate (lower entropy = higher penalty)
let max_entropy = 45.0_f64.ln();
let normalized = entropy / max_entropy; // [0, 1]
let penalty = -(normalized as f32);
// BUG #14 FIX: Validate penalty is finite before creating tensor
if !penalty.is_finite() {
tracing::warn!(
"Entropy penalty is not finite ({}), returning zero. Recent actions: {:?}",
"Entropy penalty is not finite ({}), returning zero",
penalty,
counts
);
return Tensor::zeros(&[], DType::F32, &self.device)
.map_err(|e| MLError::ModelError(format!("Failed to create zero penalty: {}", e)));
@@ -2509,6 +2579,11 @@ impl DQN {
self.epsilon = (self.epsilon * self.config.epsilon_decay).max(self.config.epsilon_end);
}
/// Reset count-based exploration bonus (call at epoch boundary)
pub fn reset_count_bonus(&mut self) {
self.count_bonus.reset();
}
/// Update target network by copying weights from main network
fn update_target_network(&mut self) -> Result<(), MLError> {
// Wave 11.6: Update target network (hybrid > dueling > standard)

1
crates/ml/src/dqn/mod.rs
View File

@@ -6,6 +6,7 @@
// Original DQN components
pub mod action_space; // Factored action space (45 actions: 5 exposure × 3 order × 3 urgency)
pub mod count_bonus; // UCB count-based exploration bonus for action diversity
pub mod agent;
pub mod attention; // Multi-head self-attention for temporal pattern recognition (Wave 26 P1.2)
pub mod circuit_breaker; // Circuit breaker for risk management

23
crates/ml/src/dqn/noisy_sigma_scheduler.rs
View File

@@ -76,9 +76,13 @@ impl NoisySigmaScheduler {
/// where α = min(t / decay_steps, 1.0)
pub fn get_sigma(&self) -> f64 {
let progress = (self.current_step as f64 / self.decay_steps as f64).min(1.0);
self.initial_sigma * (1.0 - progress) + self.final_sigma * progress
let annealed = self.initial_sigma * (1.0 - progress) + self.final_sigma * progress;
annealed.max(Self::SIGMA_FLOOR)
}
/// Minimum sigma to prevent complete collapse of exploration noise
const SIGMA_FLOOR: f64 = 0.3;
/// Increment step counter
pub fn step(&mut self) {
self.current_step += 1;
@@ -106,10 +110,10 @@ impl NoisySigmaScheduler {
}
impl Default for NoisySigmaScheduler {
/// Default scheduler with Rainbow DQN-style annealing
/// (0.6 → 0.4 over 100k steps)
/// Default scheduler with wider initial exploration for 45-action space
/// (0.8 → 0.4 over 100k steps, floor=0.3)
fn default() -> Self {
Self::new(0.6, 0.4, 100_000)
Self::new(0.8, 0.4, 100_000)
}
}
@@ -236,11 +240,20 @@ mod tests {
#[test]
fn test_default_scheduler() {
let scheduler = NoisySigmaScheduler::default();
assert_eq!(scheduler.initial_sigma, 0.6);
assert_eq!(scheduler.initial_sigma, 0.8);
assert_eq!(scheduler.final_sigma, 0.4);
assert_eq!(scheduler.decay_steps, 100_000);
}
#[test]
fn test_sigma_floor() {
// Even if final_sigma is below floor, get_sigma() respects the floor
let mut scheduler = NoisySigmaScheduler::new(0.5, 0.1, 100);
scheduler.step_to(100);
assert!(scheduler.get_sigma() >= NoisySigmaScheduler::SIGMA_FLOOR,
"Sigma {} should be >= floor {}", scheduler.get_sigma(), NoisySigmaScheduler::SIGMA_FLOOR);
}
#[test]
fn test_monotonic_decrease() {
let mut scheduler = NoisySigmaScheduler::new(0.6, 0.4, 1000);

7
crates/ml/src/dqn/regime_conditional.rs
View File

@@ -421,6 +421,13 @@ impl RegimeConditionalDQN {
Ok((avg_loss, avg_grad_norm))
}
/// Reset count-based exploration bonus for all regime heads
pub fn reset_count_bonus(&mut self) {
self.trending_head.reset_count_bonus();
self.ranging_head.reset_count_bonus();
self.volatile_head.reset_count_bonus();
}
/// Update epsilon for specific regime head
pub fn update_epsilon(&mut self, regime: RegimeType) {
match regime {

62
crates/ml/src/dqn/reward.rs
View File

@@ -349,49 +349,48 @@ pub struct MarketData {
pub volume: Decimal,
}
/// Calculate Shannon entropy for action distribution
/// Calculate normalized Shannon entropy for the 45-action distribution.
///
/// # Arguments
/// * `recent_actions` - Sliding window of recent actions (last 100 actions)
///
/// # Returns
/// Shannon entropy H = -Σ(p_i * log2(p_i)) where p_i is frequency of action i
/// Range: [0.0, 1.585] (0 = all same action, 1.585 = perfectly balanced)
///
/// # Note
/// For FactoredAction, we convert to legacy TradingAction (Buy/Sell/Hold) for entropy calculation
/// to maintain backward compatibility with the original 3-action entropy range.
/// Normalized entropy H/H_max in [0.0, 1.0]:
/// 0.0 = all actions identical (complete collapse)
/// 1.0 = uniform distribution over all 45 actions
fn calculate_entropy(recent_actions: &[FactoredAction]) -> Decimal {
if recent_actions.is_empty() {
return Decimal::ONE; // Default to high entropy if no history
}
// Convert to legacy actions and count
let mut counts = [0, 0, 0]; // BUY, SELL, HOLD
// Count over all 45 factored actions (not 3 legacy categories)
let mut counts = [0_u32; 45];
for action in recent_actions {
let legacy_action = action.to_legacy_action();
counts[legacy_action as usize] += 1;
}
let total = recent_actions.len() as f64;
let mut entropy = Decimal::ZERO;
for &count in &counts {
if count > 0 {
let p = Decimal::try_from(count as f64 / total).unwrap_or(Decimal::ZERO);
// Shannon entropy: -p * log2(p)
// Use natural log and convert: log2(x) = ln(x) / ln(2)
if let Ok(p_f64) = TryInto::<f64>::try_into(p) {
if p_f64 > 0.0 {
let log2_p = p_f64.ln() / 2.0_f64.ln();
let term = p * Decimal::try_from(log2_p).unwrap_or(Decimal::ZERO);
entropy -= term;
}
}
let idx = action.to_index();
if idx < 45 {
counts[idx] += 1;
}
}
entropy
let total = recent_actions.len() as f64;
let mut entropy_raw = 0.0_f64;
for &count in &counts {
if count > 0 {
let p = count as f64 / total;
entropy_raw -= p * p.ln();
}
}
// Normalize by max entropy = ln(45) ≈ 3.807 → range [0, 1]
let max_entropy = 45.0_f64.ln();
let normalized = if max_entropy > 0.0 {
entropy_raw / max_entropy
} else {
1.0
};
Decimal::try_from(normalized).unwrap_or(Decimal::ONE)
}
/// Reward function for `DQN` training
@@ -522,10 +521,9 @@ impl RewardFunction {
};
// Calculate diversity bonus (in-training regularization)
// Entropy threshold: 0.5 (50% of max entropy for 3 actions = 1.585)
// Low entropy → action bias → apply penalty (-0.1)
// Entropy is normalized [0,1] over 45 actions. Threshold 0.3 ≈ ~4 distinct actions.
let entropy = calculate_entropy(recent_actions);
let entropy_threshold = Decimal::try_from(0.5).unwrap_or(Decimal::ZERO);
let entropy_threshold = Decimal::try_from(0.3).unwrap_or(Decimal::ZERO);
let diversity_bonus = if entropy < entropy_threshold {
self.config.diversity_weight // -0.1 (penalty for low diversity)
} else {

57
crates/ml/src/hyperopt/adapters/dqn.rs
View File

@@ -360,6 +360,14 @@ pub struct DQNParams {
/// Network shape: [base, base/2, base/4]. Range: [256, 4096], step=256.
/// Bounded by VRAM via HardwareBudget at runtime.
pub hidden_dim_base: usize,
/// Minimum epsilon when noisy nets are enabled (0.02-0.10, default 0.05)
/// Guarantees random exploration even with learned noise
pub noisy_epsilon_floor: f64,
/// Count-based exploration bonus coefficient (0.01-0.5, default 0.1)
/// UCB-style bonus: β * sqrt(log(N) / (1 + n_a))
pub count_bonus_coefficient: f64,
}
impl Default for DQNParams {
@@ -430,6 +438,8 @@ impl Default for DQNParams {
num_quantiles: 32,
qr_kappa: 1.0,
hidden_dim_base: 256, // Conservative default (backward compatible)
noisy_epsilon_floor: 0.05, // Minimum random exploration with noisy nets
count_bonus_coefficient: 0.1, // UCB exploration bonus
}
}
}
@@ -521,14 +531,16 @@ impl ParameterSpace for DQNParams {
(0.5_f64.ln(), 2.0_f64.ln()), // 41: qr_kappa (log scale)
// GPU-dynamic network sizing (42D → 43D)
(256.0, 4096.0), // 42: hidden_dim_base (linear, step=256)
// WAVE 11: Rainbow DQN boolean parameters REMOVED from search space (always TRUE)
// Exploration anti-collapse parameters (43D → 45D)
(0.02, 0.10), // 43: noisy_epsilon_floor (linear, minimum random exploration)
(0.01, 0.5), // 44: count_bonus_coefficient (linear, UCB exploration bonus)
]
}
fn from_continuous(x: &[f64]) -> Result<Self, MLError> {
if x.len() != 43 {
if x.len() != 45 {
return Err(MLError::ConfigError {
reason: format!("Expected 43 continuous parameters (added hidden_dim_base), got {}", x.len()),
reason: format!("Expected 45 continuous parameters (added noisy_epsilon_floor, count_bonus_coefficient), got {}", x.len()),
});
}
@@ -606,6 +618,10 @@ impl ParameterSpace for DQNParams {
// GPU-dynamic hidden dims
let hidden_dim_base = ((x[42].round() / 256.0).round() * 256.0).clamp(256.0, 4096.0) as usize;
// Exploration anti-collapse parameters
let noisy_epsilon_floor = x[43].clamp(0.02, 0.10);
let count_bonus_coefficient = x[44].clamp(0.01, 0.5);
// WAVE 11: Rainbow DQN boolean parameters are ALWAYS TRUE (removed from search space)
// User requirement: "I want them enabled!" - no point in tuning boolean flags
@@ -688,6 +704,8 @@ impl ParameterSpace for DQNParams {
num_quantiles,
qr_kappa,
hidden_dim_base,
noisy_epsilon_floor,
count_bonus_coefficient,
};
// Note: HFT constraint validation moved to evaluate_objective (train_with_params)
@@ -750,7 +768,9 @@ impl ParameterSpace for DQNParams {
self.num_quantiles as f64,
self.qr_kappa.ln(),
self.hidden_dim_base as f64,
// WAVE 11: Rainbow DQN boolean parameters REMOVED (always TRUE, not tunable)
// Exploration anti-collapse
self.noisy_epsilon_floor,
self.count_bonus_coefficient,
]
}
@@ -808,7 +828,9 @@ impl ParameterSpace for DQNParams {
"num_quantiles",
"qr_kappa",
"hidden_dim_base",
// WAVE 11: Rainbow DQN boolean parameters REMOVED (always TRUE, not tunable)
// Exploration anti-collapse
"noisy_epsilon_floor",
"count_bonus_coefficient",
]
}
@@ -2397,6 +2419,8 @@ impl HyperparameterOptimizable for DQNTrainer {
// WAVE 17: Hyperopt-tuned parameters
entropy_coefficient: Some(params.entropy_coefficient), // Use hyperopt value
noisy_epsilon_floor: Some(params.noisy_epsilon_floor), // Exploration floor for noisy nets
count_bonus_coefficient: Some(params.count_bonus_coefficient), // UCB exploration bonus
transaction_cost_multiplier: params.transaction_cost_multiplier, // Use hyperopt value
// Wave 3 (Phase 2): Triple Barrier Method (enabled for production)
@@ -3358,6 +3382,8 @@ mod tests {
num_quantiles: 32,
qr_kappa: 1.0,
hidden_dim_base: 512,
noisy_epsilon_floor: 0.05,
count_bonus_coefficient: 0.1,
};
let continuous = params.to_continuous();
@@ -3378,7 +3404,7 @@ mod tests {
#[test]
fn test_dqn_params_bounds() {
let bounds = DQNParams::continuous_bounds();
assert_eq!(bounds.len(), 43); // 43 continuous parameters (added hidden_dim_base)
assert_eq!(bounds.len(), 45); // 45 continuous parameters (added noisy_epsilon_floor, count_bonus_coefficient)
// Check log-scale bounds are reasonable
assert!(bounds[0].0 < bounds[0].1); // learning_rate
@@ -3429,7 +3455,7 @@ mod tests {
#[test]
fn test_param_names() {
let names = DQNParams::param_names();
assert_eq!(names.len(), 43); // 43 tunable hyperparameters (added hidden_dim_base)
assert_eq!(names.len(), 45); // 45 tunable hyperparameters (added noisy_epsilon_floor, count_bonus_coefficient)
assert_eq!(names[0], "learning_rate");
assert_eq!(names[1], "batch_size");
assert_eq!(names[2], "gamma");
@@ -3489,6 +3515,9 @@ mod tests {
64.0, 1.0_f64.ln(), // num_quantiles=64, qr_kappa=1.0 (ln(1.0)=0.0)
// GPU-dynamic network sizing
512.0, // hidden_dim_base
// Exploration anti-collapse
0.05, // noisy_epsilon_floor
0.1, // count_bonus_coefficient
];
let params = DQNParams::from_continuous(&continuous).unwrap();
@@ -3527,6 +3556,9 @@ mod tests {
32.0, 0.5_f64.ln(), // num_quantiles min=32, qr_kappa min=0.5
// GPU-dynamic network sizing min
256.0, // hidden_dim_base min
// Exploration anti-collapse min
0.02, // noisy_epsilon_floor min
0.01, // count_bonus_coefficient min
];
let params_min = DQNParams::from_continuous(&continuous_min).unwrap();
assert!((params_min.per_alpha - 0.4).abs() < 1e-6);
@@ -3561,6 +3593,9 @@ mod tests {
200.0, 2.0_f64.ln(), // num_quantiles max=200, qr_kappa max=2.0
// GPU-dynamic network sizing max
4096.0, // hidden_dim_base max
// Exploration anti-collapse max
0.10, // noisy_epsilon_floor max
0.5, // count_bonus_coefficient max
];
let params_max = DQNParams::from_continuous(&continuous_max).unwrap();
assert!((params_max.per_alpha - 0.8).abs() < 1e-6);
@@ -3933,7 +3968,7 @@ mod tests {
fn test_qr_dqn_roundtrip_continuous() {
let params = DQNParams::default();
let continuous = params.to_continuous();
assert_eq!(continuous.len(), 43, "Should have 43 continuous dimensions (added hidden_dim_base)");
assert_eq!(continuous.len(), 45, "Should have 45 continuous dimensions (added noisy_epsilon_floor, count_bonus_coefficient)");
let roundtrip = DQNParams::from_continuous(&continuous).unwrap();
assert_eq!(roundtrip.num_quantiles, params.num_quantiles);
assert!((roundtrip.qr_kappa - params.qr_kappa).abs() < 0.01);
@@ -3956,7 +3991,7 @@ mod tests {
#[test]
fn test_batch_size_respects_wide_bounds() {
// Simulate PSO choosing batch_size=2048 (within VRAM-aware bounds)
let mut params = vec![0.0_f64; 43];
let mut params = vec![0.0_f64; 45];
params[1] = 2048.0; // batch_size (index 1)
// Fill other required params with valid defaults
params[0] = (1e-4_f64).ln(); // learning_rate
@@ -3969,9 +4004,9 @@ mod tests {
params[8] = 0.5; // transaction_cost_multiplier
params[9] = 0.6; // per_alpha
params[10] = 0.4; // per_beta_start
// Rainbow extensions (indices 11-42) -- use midpoint of bounds
// Rainbow extensions (indices 11-44) -- use midpoint of bounds
let bounds = DQNParams::continuous_bounds();
for i in 11..43 {
for i in 11..45 {
params[i] = (bounds[i].0 + bounds[i].1) / 2.0;
}
let result = DQNParams::from_continuous(&params).unwrap();

2
crates/ml/src/hyperopt/adapters/ppo.rs
View File

@@ -165,7 +165,7 @@ impl ParameterSpace for PPOParams {
}
// Cap hidden_dim_base by VRAM (index 6)
// PPO needs more VRAM than DQN (actor + critic networks)
let max_base = budget.max_hidden_dim_base(4, 512, 51, 3); // 4 concurrent, 512 batch, 51 features, 3 actions
let max_base = budget.max_hidden_dim_base(4, 512, 51, 45); // 4 concurrent, 512 batch, 51 features, 45 factored actions
if let Some(dim_bound) = bounds.get_mut(6) {
dim_bound.1 = max_base as f64;
}

59
crates/ml/src/ppo/entropy_regularization.rs
View File

@@ -16,7 +16,8 @@
//! use ml::ppo::entropy_regularization::EntropyRegularizer;
//!
//! let regularizer = EntropyRegularizer::new();
//! let action_probs = Tensor::new(&[0.5_f32, 0.3, 0.2], &Device::Cpu).unwrap();
//! // 45 factored actions (5 exposure × 3 order × 3 urgency)
//! let action_probs = Tensor::new(&[1.0_f32 / 45.0; 45], &Device::Cpu).unwrap();
//! let bonus = regularizer.calculate_entropy_bonus(&action_probs).unwrap();
//! ```
@@ -30,23 +31,26 @@ use crate::MLError;
/// to encourage exploration and maintain action diversity.
#[derive(Debug, Clone)]
pub struct EntropyRegularizer {
/// Maximum possible entropy for 3 actions: log(3) ≈ 1.099
/// Maximum possible entropy: log(num_actions)
max_entropy: f64,
/// Normalized entropy threshold (0.7) for bonus/penalty
entropy_threshold: f64,
}
impl EntropyRegularizer {
/// Create a new entropy regularizer
/// Number of factored actions (5 exposure × 3 order × 3 urgency)
const NUM_ACTIONS: usize = 45;
/// Create a new entropy regularizer for 45 factored actions.
///
/// # Configuration
/// - `max_entropy`: log(3) ≈ 1.0986 for 3 actions (BUY, SELL, HOLD)
/// - `max_entropy`: log(45) ≈ 3.807 for 45 factored actions
/// - `entropy_threshold`: 0.7 normalized entropy
/// - Above 0.7: 2x bonus for high diversity
/// - Below 0.7: 3x penalty for low diversity
pub fn new() -> Self {
Self {
max_entropy: (3.0_f64).ln(), // log(3) = 1.0986122886681098
max_entropy: (Self::NUM_ACTIONS as f64).ln(),
entropy_threshold: 0.7,
}
}
@@ -137,18 +141,30 @@ mod tests {
Ok(tensor.reshape(&[1, probs.len()])?)
}
/// Create uniform probability vector over 45 actions
fn uniform_45() -> Vec<f32> {
vec![1.0 / 45.0; 45]
}
/// Create near-deterministic probability vector over 45 actions
fn deterministic_45(dominant_action: usize) -> Vec<f32> {
let mut probs = vec![0.001 / 44.0; 45];
probs[dominant_action] = 0.999;
probs
}
#[test]
fn test_entropy_uniform_distribution() -> Result<(), MLError> {
let regularizer = EntropyRegularizer::new();
let probs = create_prob_tensor(&[1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0])?;
let probs = create_prob_tensor(&uniform_45())?;
let entropy = regularizer.calculate_entropy(&probs)?;
// Uniform distribution: entropy = log(3) ≈ 1.0986
// Uniform distribution over 45 actions: entropy = log(45) ≈ 3.807
let expected = (45.0_f64).ln();
assert!(
(entropy - 1.0986).abs() < 0.01,
"Expected entropy ~1.0986, got {}",
entropy
(entropy - expected).abs() < 0.01,
"Expected entropy ~{expected:.4}, got {entropy}",
);
Ok(())
}
@@ -156,27 +172,26 @@ mod tests {
#[test]
fn test_entropy_deterministic() -> Result<(), MLError> {
let regularizer = EntropyRegularizer::new();
let probs = create_prob_tensor(&[1.0, 0.0, 0.0])?;
let probs = create_prob_tensor(&deterministic_45(0))?;
let entropy = regularizer.calculate_entropy(&probs)?;
// Deterministic policy: entropy ≈ 0
assert!(entropy < 0.01, "Expected entropy ~0, got {}", entropy);
// Near-deterministic policy: entropy ≈ 0
assert!(entropy < 0.05, "Expected entropy ~0, got {entropy}");
Ok(())
}
#[test]
fn test_bonus_high_diversity() -> Result<(), MLError> {
let regularizer = EntropyRegularizer::new();
let probs = create_prob_tensor(&[1.0 / 3.0, 1.0 / 3.0, 1.0 / 3.0])?;
let probs = create_prob_tensor(&uniform_45())?;
let bonus = regularizer.calculate_entropy_bonus(&probs)?;
// Uniform: normalized entropy = 1.0, bonus = 2.0
// Uniform over 45: normalized entropy ≈ 1.0, bonus = 1.0 × 2.0 = 2.0
assert!(
(bonus - 2.0).abs() < 0.01,
"Expected bonus ~2.0, got {}",
bonus
(bonus - 2.0).abs() < 0.05,
"Expected bonus ~2.0, got {bonus}",
);
Ok(())
}
@@ -184,13 +199,13 @@ mod tests {
#[test]
fn test_penalty_low_diversity() -> Result<(), MLError> {
let regularizer = EntropyRegularizer::new();
let probs = create_prob_tensor(&[0.9, 0.05, 0.05])?;
let probs = create_prob_tensor(&deterministic_45(38))?;
let penalty = regularizer.calculate_entropy_bonus(&probs)?;
// Low diversity: normalized entropy < 0.7, penalty negative
assert!(penalty < 0.0, "Expected penalty < 0.0, got {}", penalty);
assert!(penalty < -0.4, "Expected penalty < -0.4, got {}", penalty);
// Near-deterministic: normalized entropy ≈ 0, penalty ≈ -(0.7) × 3 = -2.1
assert!(penalty < 0.0, "Expected penalty < 0.0, got {penalty}");
assert!(penalty < -1.0, "Expected penalty < -1.0, got {penalty}");
Ok(())
}
}

29
crates/ml/src/ppo/flow_policy/mod.rs
View File

@@ -377,21 +377,27 @@ impl FlowPolicy {
self.evaluate_actions(states, actions)
}
/// Computes entropy of the policy distribution.
/// Computes entropy of the policy distribution via Monte Carlo estimate.
///
/// NOTE: Exact entropy for normalizing flows is intractable.
/// This returns zeros as a conservative default. For exploration,
/// rely on the stochastic sampling process.
/// Exact entropy for normalizing flows is analytically intractable,
/// so we estimate H(π|s) ≈ -E[log π(a|s)] by sampling actions and
/// computing the negative mean log-probability.
///
/// # Arguments
/// * `states` - Batch of states [batch_size, state_dim].
///
/// # Returns
/// Zero tensor [batch_size, 1] (summed across action dimensions).
/// Entropy estimate tensor [batch_size].
pub fn entropy(&self, states: &Tensor) -> Result<Tensor, MLError> {
let batch_size = states.dims()[0];
Tensor::zeros(batch_size, DType::F32, &self.device)
.map_err(|e| MLError::TensorOperationError(format!("Entropy zeros failed: {}", e)))
// Sample actions and get log_probs: shape [batch_size, 1]
let (_actions, log_probs) = self.sample_action(states)?;
// H ≈ -log_prob per sample; squeeze to [batch_size]
let entropy = log_probs
.neg()
.map_err(|e| MLError::TensorOperationError(format!("Entropy negation failed: {e}")))?
.squeeze(1)
.map_err(|e| MLError::TensorOperationError(format!("Entropy squeeze failed: {e}")))?;
Ok(entropy)
}
/// Encodes state into context vector for conditioning the flow.
@@ -600,13 +606,14 @@ mod tests {
assert_eq!(entropy.dims(), &[4]);
// Verify all zeros (conservative default)
// Monte Carlo entropy estimate should be non-negative (H = -log_prob ≥ 0 for normalized distributions)
let entropy_vec = entropy.flatten_all()
.map_err(|e| MLError::TensorOperationError(e.to_string()))?
.to_vec1::<f32>()
.map_err(|e| MLError::TensorOperationError(e.to_string()))?;
for e in entropy_vec {
assert_eq!(e, 0.0);
for e in &entropy_vec {
assert!(*e >= -1.0, "Entropy should be non-negative or near-zero, got {e}");
assert!(e.is_finite(), "Entropy must be finite, got {e}");
}
Ok(())

23
crates/ml/src/trainers/dqn/config.rs
View File

@@ -61,6 +61,14 @@ impl DQNAgentType {
}
}
/// Reset count-based exploration bonus (call at epoch boundary)
pub fn reset_count_bonus(&mut self) {
match self {
Self::Standard(agent) => agent.reset_count_bonus(),
Self::RegimeConditional(agent) => agent.reset_count_bonus(),
}
}
/// Update epsilon for exploration decay
pub fn update_epsilon(&mut self) {
match self {
@@ -470,8 +478,12 @@ pub struct DQNHyperparameters {
pub max_position_absolute: f64,
// WAVE 17: Hyperopt-tuned parameters
/// Entropy regularization coefficient (optional)
/// SAC-style entropy coefficient for Q-value softmax regularization (default: 0.01)
pub entropy_coefficient: Option<f64>,
/// Minimum epsilon when noisy nets are enabled (default: 0.05)
pub noisy_epsilon_floor: Option<f64>,
/// Count-based exploration bonus coefficient (default: 0.1, range: [0.01, 0.5])
pub count_bonus_coefficient: Option<f64>,
/// Transaction cost multiplier for reward calculation
pub transaction_cost_multiplier: f64,
@@ -732,7 +744,9 @@ impl DQNHyperparameters {
max_position_absolute: 2.0, // Default: ±2.0 position limit (matches production)
// WAVE 17: Hyperopt-tuned parameters
entropy_coefficient: None,
entropy_coefficient: Some(0.01),
noisy_epsilon_floor: Some(0.05),
count_bonus_coefficient: Some(0.1),
transaction_cost_multiplier: 1.0,
// Triple Barrier defaults
@@ -813,8 +827,8 @@ impl DQNHyperparameters {
gae_lambda: 0.95, // Default: 0.95 (standard PPO/A2C value)
// P1.11: Noisy Sigma Scheduling
enable_noisy_sigma_scheduler: false, // Default: disabled (enable with noisy_nets=true)
noisy_sigma_initial: 0.6, // Default: 60% initial noise
enable_noisy_sigma_scheduler: true, // Enabled by default for wider exploration
noisy_sigma_initial: 0.8, // Higher initial noise for 45-action exploration
noisy_sigma_final: 0.4, // Default: 40% final noise
noisy_sigma_anneal_steps: 10000, // Default: 10K steps for annealing
@@ -958,6 +972,7 @@ pub(crate) fn dqn_config_2025() -> DQNConfig {
minimum_profit_factor: 1.5,
weight_decay: 1e-4,
mixed_precision: None,
..Default::default()
}
}

61
crates/ml/src/trainers/dqn/trainer.rs
View File

@@ -95,7 +95,7 @@ pub struct DQNTrainer {
/// Maximum position size for action masking (default: 2.0)
pub max_position: f64,
/// Entropy regularizer for preventing policy collapse (None if disabled)
pub entropy_regularizer: Option<Arc<crate::dqn::entropy_regularization::EntropyRegularizer>>,
// entropy_regularizer removed — SAC-style entropy is computed directly on Q-value tensors in DQN::compute_loss_internal
/// Multi-asset portfolio tracker (None if single-asset mode)
pub multi_asset_portfolio: Option<Arc<crate::dqn::multi_asset::MultiAssetPortfolioTracker>>,
/// Stress tester for robustness validation (None if disabled)
@@ -450,6 +450,10 @@ impl DQNTrainer {
minimum_profit_factor: 1.5,
weight_decay: 1e-4,
mixed_precision: hyperparams.mixed_precision.clone().or(mixed_precision_detected),
entropy_coefficient: hyperparams.entropy_coefficient.unwrap_or(0.01),
noisy_epsilon_floor: hyperparams.noisy_epsilon_floor.unwrap_or(0.05) as f32,
use_count_bonus: true,
count_bonus_coefficient: hyperparams.count_bonus_coefficient.unwrap_or(0.1),
};
// Create DQN agent
@@ -527,12 +531,11 @@ impl DQNTrainer {
let enable_action_masking = hyperparams.enable_action_masking;
let max_position = hyperparams.max_position_absolute; // BLOCKER #2: Use hyperopt-tunable position limit
// Entropy regularization for preventing policy collapse
let entropy_regularizer: Option<Arc<crate::dqn::entropy_regularization::EntropyRegularizer>> = hyperparams.enable_entropy_regularization.then(|| {
use crate::dqn::entropy_regularization::EntropyRegularizer;
info!("Entropy regularization enabled (coefficient=0.01)");
Arc::new(EntropyRegularizer::new())
});
// Entropy regularization: SAC-style computed directly on Q-values in DQN::compute_loss_internal
if hyperparams.enable_entropy_regularization {
let coeff = hyperparams.entropy_coefficient.unwrap_or(0.01);
info!("Entropy regularization enabled (coefficient={coeff:.4}, applied to Q-value softmax in loss)");
}
// Multi-asset portfolio tracking (disabled -- single-asset is the current production mode)
//
@@ -751,7 +754,6 @@ impl DQNTrainer {
// Wave 16 Portfolio Features
enable_action_masking,
max_position,
entropy_regularizer,
multi_asset_portfolio,
stress_tester,
@@ -2592,13 +2594,18 @@ impl DQNTrainer {
);
// BUG #29 FIX: Update epsilon once per epoch (not per batch)
// This ensures consistent exploration across different batch sizes
// With epsilon_decay=0.995, after 15 epochs: 0.3 × (0.995^15) = 0.2783 (27.8% exploration)
{
// Skip epsilon decay when noisy nets are enabled — epsilon is fixed at noisy_epsilon_floor
if !self.hyperparams.use_noisy_nets {
let mut agent = self.agent.write().await;
agent.update_epsilon();
}
// Reset count-based exploration bonus at epoch boundary
{
let mut agent = self.agent.write().await;
agent.reset_count_bonus();
}
// WAVE 26 P0.6: Update learning rate with scheduler (warmup + decay)
self.lr_scheduler.step();
let current_lr = self.lr_scheduler.get_lr();
@@ -2693,15 +2700,39 @@ impl DQNTrainer {
diversity_percentage
);
// Compute normalized entropy of epoch action distribution
let epoch_entropy = if epoch_total_actions > 0 {
let mut entropy_raw = 0.0_f64;
for &count in monitor.action_counts.iter() {
if count > 0 {
let p = count as f64 / epoch_total_actions as f64;
entropy_raw -= p * p.ln();
}
}
entropy_raw / 45.0_f64.ln() // Normalize to [0, 1]
} else {
0.0
};
info!(
" Exploration: entropy={:.3} (1.0=uniform), epsilon={:.4}, noisy_nets={}, count_bonus={}",
epoch_entropy,
self.get_epsilon().await.unwrap_or(0.0),
self.hyperparams.use_noisy_nets,
self.hyperparams.count_bonus_coefficient.unwrap_or(0.0) > 0.0,
);
// Emit exploration diagnostics to Prometheus
training_metrics::set_action_entropy("dqn", "current", epoch_entropy);
training_metrics::set_action_diversity("dqn", "current", diversity_percentage / 100.0);
// Warning if diversity drops below 20% (9 actions)
const DIVERSITY_THRESHOLD: usize = 9; // 20% of 45 actions
if active_actions_count < DIVERSITY_THRESHOLD {
warn!(
"⚠️ LOW ACTION DIVERSITY: {}/45 actions (<20%), consider increasing epsilon floor",
active_actions_count
"⚠️ LOW ACTION DIVERSITY: {}/45 actions (<20%), entropy={:.3}",
active_actions_count, epoch_entropy,
);
info!(" Recommendation: Increase epsilon_end from 0.05 to 0.10");
info!(" Alternative: Add entropy regularization bonus");
}
// WAVE P2: Log episode statistics

29
crates/ml/src/trainers/ppo.rs
View File

@@ -1269,7 +1269,7 @@ impl PpoTrainer {
&self,
batch: &TrajectoryBatch,
policy_loss: f32,
value_loss: f32,
_value_loss: f32,
) -> Result<(f32, f32, f32, f32, f32), MLError> {
// KL divergence (approximated from policy loss)
let kl_div = policy_loss.abs() * 0.1;
@@ -1299,8 +1299,31 @@ impl PpoTrainer {
(ev, mr, sr)
});
// Entropy (approximated from entropy coefficient impact)
let entropy = value_loss * 0.5; // Simplified
// Compute real entropy from the epoch action distribution
let entropy = {
let actions = &batch.actions;
if actions.is_empty() {
0.0_f32
} else {
let mut counts = [0_u32; 45];
for action in actions {
let idx = action.to_index();
if idx < 45 {
counts[idx] += 1;
}
}
let n = actions.len() as f64;
let mut h = 0.0_f64;
for &c in &counts {
if c > 0 {
let p = c as f64 / n;
h -= p * p.ln();
}
}
// Normalize by ln(45) so entropy is in [0, 1]
(h / 45.0_f64.ln()) as f32
}
};
Ok((kl_div, explained_variance, mean_reward, std_reward, entropy))
}

389
docs/plans/2026-03-03-dqn-action-collapse-fix.md Normal file
View File

@@ -0,0 +1,389 @@
# DQN Action Collapse Fix — Design & Implementation Plan
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
**Goal:** Fix 45-action DQN collapse where the model converges to a single action (A38 = Long100/Market/Aggressive) across all hyperopt trials, despite existing anti-collapse infrastructure.
**Root Causes:**
1. Entropy regularizer is dead code (initialized, never called)
2. Noisy nets disable epsilon-greedy but provide insufficient exploration for 45 actions
3. Diversity penalty entropy normalized for 3 actions, not 45
**Approach:** Fix all three existing mechanisms + add two new ones (epsilon floor for noisy nets, count-based exploration bonus).
---
## Task 1: Wire entropy regularizer into TD loss
**Files:**
- Modify: `crates/ml/src/dqn/dqn.rs` (add entropy bonus to `compute_loss` or loss path)
- Modify: `crates/ml/src/trainers/dqn/trainer.rs` (pass entropy regularizer into training step)
**Step 1: Add entropy bonus computation to the Q-learning loss**
The entropy regularizer at `crates/ml/src/dqn/entropy_regularization.rs` is fully implemented but never called. Wire it into the training loop.
In the training step where loss is computed (trainer.rs, the `train_step` / gradient update section), after computing Q-values for the current batch:
```rust
// Compute softmax policy from Q-values: π(a|s) = softmax(Q(s,a) / τ)
// Then entropy: H(π) = -Σ π(a|s) * log(π(a|s))
// Add to loss: loss = td_loss - entropy_coefficient * mean_entropy
```
Specifically:
1. After the forward pass that produces `q_values` tensor (shape [batch, 45])
2. Compute `log_softmax(q_values, dim=1)` and `softmax(q_values, dim=1)`
3. Entropy per sample: `H = -(softmax * log_softmax).sum(dim=1)` → shape [batch]
4. Mean entropy: `mean_H = H.mean()`
5. Final loss: `loss = td_loss - entropy_coefficient * mean_H`
The `entropy_coefficient` field already exists in `DQNHyperparameters` (default 0.01, hyperopt range [0.0, 0.1]).
**Step 2: Remove unused `entropy_regularizer` field from trainer struct**
The `Arc<EntropyRegularizer>` stored at trainer.rs:754 is no longer needed — entropy is computed directly on Q-value tensors. Remove the field and the initialization at lines 531-535.
**Step 3: Add unit test**
Test that with `entropy_coefficient > 0`, training on a batch where all Q-values are identical (uniform policy) produces lower loss than a batch with one dominant Q-value (collapsed policy).
**Step 4: Build and test**
```bash
SQLX_OFFLINE=true cargo test -p ml --lib -- trainers::dqn
```
**Step 5: Commit**
```bash
git add crates/ml/src/dqn/ crates/ml/src/trainers/dqn/
git commit -m "fix(dqn): wire entropy regularization into TD loss (was dead code)"
```
---
## Task 2: Fix diversity penalty entropy normalization
**Files:**
- Modify: `crates/ml/src/dqn/reward.rs` (fix `calculate_entropy()` normalization)
**Step 1: Fix entropy normalization from log(3) to log(45)**
In `reward.rs`, find `calculate_entropy()` (around line 364-395). The function computes Shannon entropy of recent actions. Currently it normalizes by `log(3)` (3 legacy action categories). Change to `log(45)` to match the 45 factored actions.
Also change the entropy threshold from 0.5 to 0.3. With 45 actions, entropy of 0.3 (normalized) means ~4 distinct actions — a reasonable collapse threshold.
```rust
// Before:
let max_entropy = (3.0_f64).ln(); // log(3) = 1.099
let entropy_threshold = 0.5;
// After:
let max_entropy = (45.0_f64).ln(); // log(45) = 3.807
let entropy_threshold = 0.3;
```
**Step 2: Update the action counting to use 45 action indices**
If the entropy calculation currently maps actions to 3 categories (Buy/Sell/Hold), change it to use the full `to_index()` (0-44) for the frequency distribution.
**Step 3: Add test**
Test that:
- Uniform distribution over 45 actions → entropy ≈ 1.0 (normalized) → no penalty
- Single-action collapse (all A38) → entropy ≈ 0.0 → penalty fires (-0.1)
- 5 distinct actions → entropy ≈ 0.42 → penalty fires (< 0.3 threshold? depends on distribution)
**Step 4: Build and test**
```bash
SQLX_OFFLINE=true cargo test -p ml --lib -- dqn::reward
```
**Step 5: Commit**
```bash
git add crates/ml/src/dqn/reward.rs
git commit -m "fix(dqn): normalize diversity entropy for 45 actions (was calibrated for 3)"
```
---
## Task 3: Add epsilon floor for noisy nets
**Files:**
- Modify: `crates/ml/src/trainers/dqn/config.rs` (add `noisy_epsilon_floor` field)
- Modify: `crates/ml/src/dqn/dqn.rs` (use floor in `select_action()`)
- Modify: `crates/ml/src/hyperopt/adapters/dqn.rs` (add to search space)
**Step 1: Add config field**
In `config.rs`, add to `DQNHyperparameters`:
```rust
/// Minimum epsilon when noisy nets are enabled (guarantees some random exploration)
pub noisy_epsilon_floor: f64, // default: 0.05
```
Add default value in the `Default` impl: `noisy_epsilon_floor: 0.05`.
**Step 2: Use floor in select_action()**
In `dqn.rs`, change the noisy nets epsilon logic:
```rust
// Before:
let effective_epsilon = if self.config.use_noisy_nets {
0.0 // Learned exploration via parameter noise
} else {
self.epsilon
};
// After:
let effective_epsilon = if self.config.use_noisy_nets {
self.config.noisy_epsilon_floor // Minimum random exploration even with noisy nets
} else {
self.epsilon
};
```
**Step 3: Add to hyperopt search space**
In `dqn.rs` hyperopt adapter, add `noisy_epsilon_floor` to:
- `DQNParams` struct
- `continuous_bounds()` — range [0.02, 0.10]
- `from_continuous()` / `to_continuous()`
- `to_hyperparams()` mapping
**Step 4: Skip epsilon decay when noisy nets enabled**
In `trainer.rs`, guard the `agent.update_epsilon()` call:
```rust
// Before:
agent.update_epsilon();
// After:
if !self.hyperparams.use_noisy_nets {
agent.update_epsilon();
}
```
**Step 5: Build and test**
```bash
SQLX_OFFLINE=true cargo check -p ml
SQLX_OFFLINE=true cargo test -p ml --lib -- dqn::dqn::tests
```
**Step 6: Commit**
```bash
git add crates/ml/src/trainers/dqn/config.rs crates/ml/src/dqn/dqn.rs crates/ml/src/hyperopt/adapters/dqn.rs crates/ml/src/trainers/dqn/trainer.rs
git commit -m "feat(dqn): add epsilon floor for noisy nets (0.05 default, hyperopt-tunable)"
```
---
## Task 4: Enable noisy sigma scheduling with higher initial sigma
**Files:**
- Modify: `crates/ml/src/trainers/dqn/config.rs` (change defaults)
- Modify: `crates/ml/src/dqn/dqn.rs` (add sigma floor)
**Step 1: Change defaults for sigma scheduling**
In `config.rs`, update the defaults:
```rust
// Before:
enable_noisy_sigma_scheduler: false,
noisy_sigma_initial: 0.6,
noisy_sigma_final: 0.4,
// After:
enable_noisy_sigma_scheduler: true, // Enable by default
noisy_sigma_initial: 0.8, // Wider initial exploration
noisy_sigma_final: 0.4,
```
**Step 2: Add sigma floor**
In the sigma scheduling logic (wherever sigma is computed/annealed), add a minimum:
```rust
let sigma = computed_sigma.max(0.3); // Never let sigma drop below 0.3
```
**Step 3: Update hyperopt adapter defaults**
Make sure the hyperopt adapter uses the new defaults when constructing `DQNHyperparameters`.
**Step 4: Build and test**
```bash
SQLX_OFFLINE=true cargo check -p ml
```
**Step 5: Commit**
```bash
git add crates/ml/src/trainers/dqn/config.rs crates/ml/src/dqn/dqn.rs
git commit -m "feat(dqn): enable noisy sigma scheduling (0.8→0.4, floor=0.3)"
```
---
## Task 5: Add count-based exploration bonus (UCB)
**Files:**
- Create: `crates/ml/src/dqn/count_bonus.rs`
- Modify: `crates/ml/src/dqn/mod.rs` (add module)
- Modify: `crates/ml/src/trainers/dqn/config.rs` (add config fields)
- Modify: `crates/ml/src/dqn/dqn.rs` (apply bonus in action selection)
- Modify: `crates/ml/src/hyperopt/adapters/dqn.rs` (add to search space)
**Step 1: Create count_bonus.rs**
```rust
/// UCB-style count-based exploration bonus for action selection.
///
/// Adds bonus = β * sqrt(log(N) / (1 + n_a)) to Q-values during action selection,
/// where N = total actions taken this epoch, n_a = count of action a.
/// This gives direct exploration incentive to under-visited actions.
pub struct CountBonus {
action_counts: [u64; 45],
total_count: u64,
coefficient: f64,
}
impl CountBonus {
pub fn new(coefficient: f64) -> Self { ... }
/// Record that action was taken
pub fn record_action(&mut self, action_idx: usize) { ... }
/// Compute bonus for each of 45 actions, returns [f64; 45]
pub fn bonuses(&self) -> [f64; 45] {
if self.total_count == 0 {
return [self.coefficient; 45]; // Max bonus when no data
}
let log_n = (self.total_count as f64).ln();
let mut result = [0.0; 45];
for a in 0..45 {
result[a] = self.coefficient * (log_n / (1.0 + self.action_counts[a] as f64)).sqrt();
}
result
}
/// Reset counts (call at epoch boundary)
pub fn reset(&mut self) { ... }
}
```
**Step 2: Add config fields**
In `config.rs`:
```rust
pub use_count_bonus: bool, // default: true
pub count_bonus_coefficient: f64, // default: 0.1
```
**Step 3: Wire into action selection**
In `dqn.rs` `select_action()`, when choosing the greedy action:
```rust
// After computing q_values from forward pass:
if self.config.use_count_bonus {
let bonuses = self.count_bonus.bonuses();
let bonus_tensor = Tensor::from_vec(bonuses.to_vec(), 45, device)?;
let q_with_bonus = q_values.broadcast_add(&bonus_tensor)?;
// Use q_with_bonus for argmax instead of q_values
}
```
Record action after selection:
```rust
if self.config.use_count_bonus {
self.count_bonus.record_action(action.to_index());
}
```
**Step 4: Reset at epoch boundary**
In `trainer.rs`, at the start of each epoch:
```rust
agent.reset_count_bonus(); // Resets per-epoch exploration pressure
```
**Step 5: Add to hyperopt search space**
- `count_bonus_coefficient` — range [0.01, 0.5]
- `use_count_bonus` — always true (not tuned, it's a safety mechanism)
**Step 6: Add tests**
- Test that bonuses are high for unvisited actions and low for frequent actions
- Test that after recording 100 of action 38, action 38's bonus is near zero while others are high
- Test reset clears all counts
**Step 7: Build and test**
```bash
SQLX_OFFLINE=true cargo test -p ml --lib -- dqn::count_bonus
SQLX_OFFLINE=true cargo check -p ml
```
**Step 8: Commit**
```bash
git add crates/ml/src/dqn/count_bonus.rs crates/ml/src/dqn/mod.rs crates/ml/src/trainers/dqn/config.rs crates/ml/src/dqn/dqn.rs crates/ml/src/hyperopt/adapters/dqn.rs
git commit -m "feat(dqn): add UCB count-based exploration bonus for 45-action diversity"
```
---
## Task 6: Update monitoring to report exploration diagnostics
**Files:**
- Modify: `crates/ml/src/trainers/dqn/trainer.rs` (log exploration source stats)
**Step 1: Track exploration source in action diversity log**
In the per-epoch action diversity logging, add:
- Count of random (epsilon) vs greedy vs noisy-net actions
- Effective entropy over 45 actions (normalized by log(45))
- Count bonus range (min/max/mean)
**Step 2: Add Prometheus metrics**
In `crates/common/src/metrics/training_metrics.rs`, add:
```rust
foxhunt_training_action_entropy // Normalized entropy (0-1) of epoch action distribution
foxhunt_training_epsilon_actions // Count of epsilon-random actions this epoch
foxhunt_training_greedy_actions // Count of greedy actions this epoch
```
**Step 3: Commit**
```bash
git add crates/ml/src/trainers/dqn/trainer.rs crates/common/src/metrics/training_metrics.rs
git commit -m "feat(monitoring): add exploration diagnostics (entropy, epsilon/greedy split)"
```
---
## Task 7: Full build verification
**Step 1:** `SQLX_OFFLINE=true cargo check --workspace`
**Step 2:** `SQLX_OFFLINE=true cargo test -p ml --lib`
**Step 3:** `SQLX_OFFLINE=true cargo test -p common --lib`
**Step 4:** `SQLX_OFFLINE=true cargo clippy --workspace -- -D warnings`
Expected: 0 errors, 0 warnings, all tests pass.