fix(ml): correct DQNConfig defaults — widen v_min/v_max, reduce cql_alpha, disable IQN
- v_min/v_max: -2/+2 → -10/+10 (C51 needs room to separate 45 actions) - entropy_coefficient: 0.01 → 0.05 (5x stronger anti-collapse) - cql_alpha: 1.0 → 0.1 (was adding ~3.8 loss penalty per step) - use_iqn: true → false (IQN trains q_network but inference uses dist_dueling) - Added get_count_bonuses() method for batch action selection Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -235,8 +235,8 @@ impl Default for DQNConfig {
|
||||
dueling_hidden_dim: 128,
|
||||
use_distributional: true, // BUG #36 FIXED: C51 re-enabled
|
||||
num_atoms: 51,
|
||||
v_min: -2.0, // Bug #5: Corrected from -10.0
|
||||
v_max: 2.0, // Bug #5: Corrected from 10.0
|
||||
v_min: -10.0, // Widened: [-2,2] compressed Q-values, [-10,10] gives C51 room to separate 45 actions
|
||||
v_max: 10.0, // Widened: [-2,2] compressed Q-values, [-10,10] gives C51 room to separate 45 actions
|
||||
use_noisy_nets: false,
|
||||
noisy_sigma_init: 0.5,
|
||||
enable_q_value_clipping: true,
|
||||
@@ -246,17 +246,17 @@ impl Default for DQNConfig {
|
||||
gradient_collapse_patience: 100,
|
||||
|
||||
// Entropy regularization: prevent 45-action collapse
|
||||
entropy_coefficient: 0.01,
|
||||
entropy_coefficient: 0.05, // 5x stronger anti-collapse for 45-action space
|
||||
noisy_epsilon_floor: 0.05,
|
||||
use_count_bonus: true,
|
||||
count_bonus_coefficient: 0.1,
|
||||
|
||||
// CQL: Enabled by default for offline training
|
||||
use_cql: true,
|
||||
cql_alpha: 1.0,
|
||||
cql_alpha: 0.1, // Reduced from 1.0: full strength added ~ln(45)=3.8 loss penalty, crushing Q-value differentiation
|
||||
|
||||
// IQN: Enabled by default (complementary to C51)
|
||||
use_iqn: true,
|
||||
// IQN: Disabled: IQN trains base q_network but inference uses dist_dueling network (zero gradients)
|
||||
use_iqn: false,
|
||||
iqn_num_quantiles: 64,
|
||||
iqn_kappa: 1.0,
|
||||
iqn_embedding_dim: 64,
|
||||
@@ -3150,6 +3150,13 @@ impl DQN {
|
||||
self.count_bonus.reset();
|
||||
}
|
||||
|
||||
/// Get count bonuses for all actions (UCB exploration).
|
||||
/// Returns a Vec<f32> of length num_actions with exploration bonuses.
|
||||
pub fn get_count_bonuses(&self) -> Vec<f32> {
|
||||
let bonuses = self.count_bonus.bonuses();
|
||||
bonuses.iter().map(|b| *b as f32).collect()
|
||||
}
|
||||
|
||||
/// Update target network by copying weights from main network
|
||||
fn update_target_network(&mut self) -> Result<(), MLError> {
|
||||
// Wave 11.6: Update target network (hybrid > dueling > standard)
|
||||
|
||||
Reference in New Issue
Block a user