gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(ml): restore exploration mechanisms for DQN action diversity

Browse Source 
The C2 fix from the previous session was too aggressive — it eliminated
ALL exploration mechanisms simultaneously:

1. epsilon forced to 0.0 when noisy nets active (select_action)
2. count bonus removed from Q-value computation (metrics only)
3. noisy_epsilon_floor config field declared but never read

This left noisy nets as the sole exploration mechanism, which produces
perturbations too small to overcome Q-value gaps (A4=0.12 vs others≈0.02).
Result: 20/20 hyperopt trials hit fallback objective with 1/5 diversity.

Fixes:
- select_action: use noisy_epsilon_floor (not 0.0) as effective_epsilon
  when noisy nets active — guarantees minimum random action rate
- select_action: re-enable UCB count bonus on Q-values before argmax
  (both IQN and standard paths) for directed exploration
- select_action_with_confidence: same fixes for consistency
- trainer: set epsilon to noisy_epsilon_floor (not 0.0) at init
- hyperopt: widen noisy_epsilon_floor range from [0.0, 0.05] to
  [0.03, 0.15] with default 0.05

Exploration now has two complementary mechanisms:
- noisy_epsilon_floor: random actions feed diverse replay buffer
- count bonus: UCB term biases greedy selection toward under-visited actions
- noisy nets: weight perturbation adds stochasticity to Q-values

select_action_inference (production) is unchanged — pure exploitation.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-06 18:27:04 +01:00
parent 03062f2401
commit e412f12e33
3 changed files with 78 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
crates/ml/src/dqn/dqn.rs
View File

@@ -1453,9 +1453,12 @@ impl DQN {
// During warmup period: always use random exploration (epsilon=1.0)
let in_warmup = self.total_steps <= self.config.warmup_steps as u64;
// C2 FIX: Noisy nets are sole exploration mechanism — no epsilon floor.
// When noisy nets are active, use noisy_epsilon_floor as a minimum random
// exploration rate instead of the full epsilon schedule. This prevents
// triple-stacking (noisy + epsilon + count) while ensuring some random
// actions feed diverse experiences into the replay buffer.
let effective_epsilon = if self.config.use_noisy_nets {
0.0_f32 // Noisy nets only — epsilon removed to prevent triple-stacking
self.config.noisy_epsilon_floor // Minimum exploration floor for noisy nets
} else {
self.epsilon // Standard epsilon-greedy exploration
};
@@ -1500,9 +1503,20 @@ impl DQN {
.map_err(|e| MLError::ModelError(format!("Expected Q computation failed: {}", e)))?
};
// C2 FIX: Count bonus removed from Q-value computation.
// Noisy nets are the sole exploration mechanism. Count bonus is
// kept for diversity metrics only (record_action + get_count_bonuses).
// Add UCB count bonus to Q-values to encourage visiting under-explored actions.
// Noisy nets + epsilon floor handle random exploration; count bonus handles
// directed exploration toward actions the agent has been ignoring.
let action_scores = if self.config.use_count_bonus {
let bonuses = self.count_bonus.bonuses();
let bonus_tensor = Tensor::from_vec(
bonuses.iter().map(|b| *b as f32).collect::<Vec<_>>(),
(1, self.config.num_actions),
&self.device,
).map_err(|e| MLError::ModelError(format!("Count bonus tensor: {}", e)))?;
action_scores.add(&bonus_tensor)?
} else {
action_scores
};
// Argmax over actions
let best_action_idx = action_scores
@@ -1516,7 +1530,18 @@ impl DQN {
// Standard Q-network action selection
let q_values = self.forward(&state_tensor)?;
// C2 FIX: Count bonus removed from Q-values (metrics only).
// Add UCB count bonus to Q-values for directed exploration
let q_values = if self.config.use_count_bonus {
let bonuses = self.count_bonus.bonuses();
let bonus_tensor = Tensor::from_vec(
bonuses.iter().map(|b| *b as f32).collect::<Vec<_>>(),
(1, self.config.num_actions),
&self.device,
).map_err(|e| MLError::ModelError(format!("Count bonus tensor: {}", e)))?;
q_values.add(&bonus_tensor)?
} else {
q_values
};
let best_action_idx = q_values
.argmax(1)?
@@ -1583,9 +1608,9 @@ impl DQN {
// During warmup period: always use random exploration (epsilon=1.0)
let in_warmup = self.total_steps <= self.config.warmup_steps as u64;
// C2 FIX: Noisy nets are sole exploration mechanism — no epsilon floor.
// Noisy nets + epsilon floor (same logic as select_action)
let effective_epsilon = if self.config.use_noisy_nets {
0.0_f32 // C2: Noisy nets only — no epsilon floor
self.config.noisy_epsilon_floor
} else {
self.epsilon
};
@@ -1624,7 +1649,18 @@ impl DQN {
.map_err(|e| MLError::ModelError(format!("Expected Q computation failed: {}", e)))?
};
// C2 FIX: Count bonus removed from Q-values (metrics only).
// Add UCB count bonus for directed exploration
let action_scores = if self.config.use_count_bonus {
let bonuses = self.count_bonus.bonuses();
let bonus_tensor = Tensor::from_vec(
bonuses.iter().map(|b| *b as f32).collect::<Vec<_>>(),
(1, self.config.num_actions),
&self.device,
).map_err(|e| MLError::ModelError(format!("Count bonus tensor: {}", e)))?;
action_scores.add(&bonus_tensor)?
} else {
action_scores
};
let conf = Self::softmax_confidence(&action_scores)?;
@@ -1639,7 +1675,18 @@ impl DQN {
// Standard Q-network path
let q_values = self.forward(&state_tensor)?;
// C2 FIX: Count bonus removed from Q-values (metrics only).
// Add UCB count bonus for directed exploration
let q_values = if self.config.use_count_bonus {
let bonuses = self.count_bonus.bonuses();
let bonus_tensor = Tensor::from_vec(
bonuses.iter().map(|b| *b as f32).collect::<Vec<_>>(),
(1, self.config.num_actions),
&self.device,
).map_err(|e| MLError::ModelError(format!("Count bonus tensor: {}", e)))?;
q_values.add(&bonus_tensor)?
} else {
q_values
};
let conf = Self::softmax_confidence(&q_values)?;

14
crates/ml/src/hyperopt/adapters/dqn.rs
View File

@@ -462,7 +462,7 @@ impl Default for DQNParams {
num_quantiles: 32,
qr_kappa: 1.0,
hidden_dim_base: 256, // Conservative default (backward compatible)
noisy_epsilon_floor: 0.0, // C2: Noisy nets provide exploration; no epsilon floor during training
noisy_epsilon_floor: 0.05, // Minimum random exploration when noisy nets active
count_bonus_coefficient: 0.1, // UCB exploration bonus
cql_alpha: 0.1, // CQL regularization: mild penalty on OOD Q-values
eval_softmax_temp: 0.1, // Default: nearly greedy (hyperopt will tune)
@@ -514,8 +514,8 @@ impl ParameterSpace for DQNParams {
// GPU-dynamic network sizing (1D)
(256.0, 4096.0), // 22: hidden_dim_base (linear, step=256)
// Exploration anti-collapse (1D)
(0.0, 0.05), // 23: noisy_epsilon_floor (linear, 0.0=noisy-nets-only)
// Exploration anti-collapse (1D) — minimum random action probability when noisy nets active
(0.03, 0.15), // 23: noisy_epsilon_floor (linear, ensures replay buffer diversity)
// CQL regularization (1D)
(0.0, 0.5), // 24: cql_alpha (linear)
@@ -579,7 +579,7 @@ impl ParameterSpace for DQNParams {
let hidden_dim_base = ((x[22].round() / 256.0).round() * 256.0).clamp(256.0, 4096.0) as usize;
// Exploration anti-collapse
let noisy_epsilon_floor = x[23].clamp(0.0, 0.05);
let noisy_epsilon_floor = x[23].clamp(0.03, 0.15);
// CQL regularization
let cql_alpha = x[24].clamp(0.0, 0.5);
@@ -3612,7 +3612,7 @@ mod tests {
// Curiosity + exploration
assert_eq!(bounds[20], (0.01, 0.5)); // curiosity_weight
assert_eq!(bounds[22], (256.0, 4096.0)); // hidden_dim_base
assert_eq!(bounds[23], (0.0, 0.05)); // noisy_epsilon_floor
assert_eq!(bounds[23], (0.03, 0.15)); // noisy_epsilon_floor
// CQL regularization
assert_eq!(bounds[24], (0.0, 0.5)); // cql_alpha
@@ -3697,7 +3697,7 @@ mod tests {
0.01, // curiosity_weight min
0.0001_f64.ln(), // tau min
256.0, // hidden_dim_base min
0.0, // noisy_epsilon_floor min
0.03, // noisy_epsilon_floor min
0.0, // cql_alpha min
0.5_f64.ln(), // eval_softmax_temp min
0.0, // 26: lr_decay_type (constant)
@@ -3723,7 +3723,7 @@ mod tests {
0.5, // curiosity_weight max
0.01_f64.ln(), // tau max
4096.0, // hidden_dim_base max
0.05, // noisy_epsilon_floor max
0.15, // noisy_epsilon_floor max
0.5, // cql_alpha max
2.0_f64.ln(), // eval_softmax_temp max
2.0, // 26: lr_decay_type (cosine)

29
crates/ml/src/trainers/dqn/trainer.rs
View File

@@ -1473,15 +1473,14 @@ impl DQNTrainer {
info!(" ❌ Noisy Networks");
}
// BUG #40 FIX: When noisy nets are enabled, set stored epsilon to 0.
// Without this fix, epsilon starts at 1.0 and never decays (update_epsilon
// is skipped when use_noisy_nets=true), causing 100% random action selection.
// NOTE: select_action() also applies noisy_epsilon_floor (default 5%) as a
// safety floor for 45-action spaces — that floor is separate from this field.
// When noisy nets are enabled, epsilon-greedy is replaced by noisy_epsilon_floor.
// Set stored epsilon to the floor value so get_epsilon() reports it accurately.
// select_action() reads noisy_epsilon_floor from DQNConfig directly.
if self.hyperparams.use_noisy_nets {
let floor = self.hyperparams.noisy_epsilon_floor.unwrap_or(0.05);
let mut agent = self.agent.write().await;
agent.set_epsilon(0.0);
info!(" 🔧 BUG #40: Stored epsilon set to 0.0 (noisy nets + noisy_epsilon_floor provide exploration)");
agent.set_epsilon(floor);
info!(" Noisy nets active: epsilon set to noisy_epsilon_floor={:.4}", floor);
}
// Training loop
@@ -4640,24 +4639,24 @@ mod tests {
assert!(!names.contains(&"count_bonus_coefficient"), "count_bonus_coefficient must not be in search space");
}
/// Verify noisy_epsilon_floor defaults to 0.0 (noisy nets only).
/// Verify noisy_epsilon_floor defaults to 0.05 (minimum exploration).
#[test]
fn test_c2_noisy_epsilon_floor_defaults_to_zero() {
fn test_noisy_epsilon_floor_default() {
let params = crate::hyperopt::adapters::dqn::DQNParams::default();
assert!(
(params.noisy_epsilon_floor - 0.0).abs() < f64::EPSILON,
"noisy_epsilon_floor must default to 0.0 (noisy nets only)"
(params.noisy_epsilon_floor - 0.05).abs() < f64::EPSILON,
"noisy_epsilon_floor must default to 0.05"
);
}
/// Verify noisy_epsilon_floor search bounds are [0.0, 0.05].
/// Verify noisy_epsilon_floor search bounds are [0.03, 0.15].
#[test]
fn test_c2_noisy_epsilon_floor_bounds() {
fn test_noisy_epsilon_floor_bounds() {
let bounds = crate::hyperopt::adapters::dqn::DQNParams::continuous_bounds();
// Index 23 = noisy_epsilon_floor
let (lo, hi) = bounds[23];
assert!((lo - 0.0).abs() < f64::EPSILON, "noisy_epsilon_floor lower bound must be 0.0");
assert!((hi - 0.05).abs() < f64::EPSILON, "noisy_epsilon_floor upper bound must be 0.05");
assert!((lo - 0.03).abs() < f64::EPSILON, "noisy_epsilon_floor lower bound must be 0.03");
assert!((hi - 0.15).abs() < f64::EPSILON, "noisy_epsilon_floor upper bound must be 0.15");
}
/// Verify count_bonus_coefficient is fixed at 0.1 (not tunable).