diff --git a/config/training/dqn-hyperopt.toml b/config/training/dqn-hyperopt.toml index af9e33cc9..8d5933a4d 100644 --- a/config/training/dqn-hyperopt.toml +++ b/config/training/dqn-hyperopt.toml @@ -39,7 +39,7 @@ tau = [0.005, 0.01] # log scale in adapter hidden_dim_base = [128, 256] # capped at production default: 512 is 4x slower for marginal benefit # CQL regularization -cql_alpha = [0.0, 1.0] +cql_alpha = [0.5, 5.0] # Training dynamics lr_decay_type = [0, 2] # discrete: 0=constant, 1=linear, 2=cosine diff --git a/config/training/dqn-localdev.toml b/config/training/dqn-localdev.toml index 7f253a3df..6110c45c5 100644 --- a/config/training/dqn-localdev.toml +++ b/config/training/dqn-localdev.toml @@ -62,7 +62,7 @@ n_steps = 3 tau = 0.005 c51_warmup_epochs = 5 her_ratio = 0.2 -cql_alpha = 0.1 +cql_alpha = 1.0 curiosity_weight = 0.1 iqn_lambda = 0.25 spectral_norm_sigma_max = 1.5 diff --git a/config/training/dqn-production.toml b/config/training/dqn-production.toml index 9633c2e36..67ca3e1f6 100644 --- a/config/training/dqn-production.toml +++ b/config/training/dqn-production.toml @@ -67,7 +67,7 @@ n_steps = 3 tau = 0.005 c51_warmup_epochs = 5 her_ratio = 0.2 -cql_alpha = 0.1 +cql_alpha = 1.0 curiosity_weight = 0.1 iqn_lambda = 0.25 spectral_norm_sigma_max = 1.5 diff --git a/config/training/dqn-smoketest.toml b/config/training/dqn-smoketest.toml index 3ff400dc6..2cb69323b 100644 --- a/config/training/dqn-smoketest.toml +++ b/config/training/dqn-smoketest.toml @@ -66,7 +66,7 @@ n_steps = 3 tau = 0.005 c51_warmup_epochs = 5 her_ratio = 0.2 -cql_alpha = 0.1 +cql_alpha = 1.0 curiosity_weight = 0.1 iqn_lambda = 0.25 spectral_norm_sigma_max = 1.5 diff --git a/crates/ml/src/trainers/dqn/config.rs b/crates/ml/src/trainers/dqn/config.rs index 652b5adaa..d938b9503 100644 --- a/crates/ml/src/trainers/dqn/config.rs +++ b/crates/ml/src/trainers/dqn/config.rs @@ -1514,7 +1514,7 @@ impl DQNHyperparameters { adam_epsilon: 1e-8, // f32 Adam (master weights are f32) // Conservative Q-Learning (CQL) - cql_alpha: 0.1, // Default: mild conservatism (0.1 of 0.0-1.0 range) + cql_alpha: 1.0, // Strong conservatism — penalizes OOS-destructive Q-values // QR-DQN (complementary to C51 — IQN for quantile estimation) num_quantiles: 32, // Default: 32 quantiles diff --git a/crates/ml/src/trainers/dqn/fused_training.rs b/crates/ml/src/trainers/dqn/fused_training.rs index 9e3c5517c..70dd7600b 100644 --- a/crates/ml/src/trainers/dqn/fused_training.rs +++ b/crates/ml/src/trainers/dqn/fused_training.rs @@ -46,9 +46,9 @@ use super::DQNHyperparameters; /// Per-component gradient norm budget fractions for auxiliary objectives. /// C51 gets whatever remains: `1.0 - sum(active_auxiliary_budgets)`. -/// When all auxiliaries are active: C51=70%, CQL=15%, IQN=10%, Ens=5%. -/// When auxiliaries are disabled, their budget automatically goes to C51. -pub(crate) const CQL_GRAD_BUDGET: f32 = 0.15; +/// When all auxiliaries are active: C51=60%, CQL=25%, IQN=10%, Ens=5%. +/// CQL at 25% (was 15%) to enforce conservatism on OOS-destructive Q-values. +pub(crate) const CQL_GRAD_BUDGET: f32 = 0.25; pub(crate) const IQN_GRAD_BUDGET: f32 = 0.10; pub(crate) const ENS_GRAD_BUDGET: f32 = 0.05;