From ae995673621b3b705cc269ddd16d153e69ee50bf Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 29 Mar 2026 23:40:30 +0200 Subject: [PATCH] =?UTF-8?q?feat(generalization):=20CQL=20alpha=200.1?= =?UTF-8?q?=E2=86=921.0,=20gradient=20budget=2015%=E2=86=9225%?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CQL was effectively disabled (0.1 alpha × 0.15 budget = 1.5% of gradient). Now: alpha=1.0 × 0.25 budget = 25% of gradient enforces conservatism. C51 reduced from 70% to 60% to accommodate. CQL penalizes Q-values for actions not in the data — directly prevents the model from being "confident but wrong" on OOS state-action pairs. Hyperopt search range updated: [0.0, 1.0] → [0.5, 5.0]. Co-Authored-By: Claude Opus 4.6 (1M context) --- config/training/dqn-hyperopt.toml | 2 +- config/training/dqn-localdev.toml | 2 +- config/training/dqn-production.toml | 2 +- config/training/dqn-smoketest.toml | 2 +- crates/ml/src/trainers/dqn/config.rs | 2 +- crates/ml/src/trainers/dqn/fused_training.rs | 6 +++--- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/config/training/dqn-hyperopt.toml b/config/training/dqn-hyperopt.toml index af9e33cc9..8d5933a4d 100644 --- a/config/training/dqn-hyperopt.toml +++ b/config/training/dqn-hyperopt.toml @@ -39,7 +39,7 @@ tau = [0.005, 0.01] # log scale in adapter hidden_dim_base = [128, 256] # capped at production default: 512 is 4x slower for marginal benefit # CQL regularization -cql_alpha = [0.0, 1.0] +cql_alpha = [0.5, 5.0] # Training dynamics lr_decay_type = [0, 2] # discrete: 0=constant, 1=linear, 2=cosine diff --git a/config/training/dqn-localdev.toml b/config/training/dqn-localdev.toml index 7f253a3df..6110c45c5 100644 --- a/config/training/dqn-localdev.toml +++ b/config/training/dqn-localdev.toml @@ -62,7 +62,7 @@ n_steps = 3 tau = 0.005 c51_warmup_epochs = 5 her_ratio = 0.2 -cql_alpha = 0.1 +cql_alpha = 1.0 curiosity_weight = 0.1 iqn_lambda = 0.25 spectral_norm_sigma_max = 1.5 diff --git a/config/training/dqn-production.toml b/config/training/dqn-production.toml index 9633c2e36..67ca3e1f6 100644 --- a/config/training/dqn-production.toml +++ b/config/training/dqn-production.toml @@ -67,7 +67,7 @@ n_steps = 3 tau = 0.005 c51_warmup_epochs = 5 her_ratio = 0.2 -cql_alpha = 0.1 +cql_alpha = 1.0 curiosity_weight = 0.1 iqn_lambda = 0.25 spectral_norm_sigma_max = 1.5 diff --git a/config/training/dqn-smoketest.toml b/config/training/dqn-smoketest.toml index 3ff400dc6..2cb69323b 100644 --- a/config/training/dqn-smoketest.toml +++ b/config/training/dqn-smoketest.toml @@ -66,7 +66,7 @@ n_steps = 3 tau = 0.005 c51_warmup_epochs = 5 her_ratio = 0.2 -cql_alpha = 0.1 +cql_alpha = 1.0 curiosity_weight = 0.1 iqn_lambda = 0.25 spectral_norm_sigma_max = 1.5 diff --git a/crates/ml/src/trainers/dqn/config.rs b/crates/ml/src/trainers/dqn/config.rs index 652b5adaa..d938b9503 100644 --- a/crates/ml/src/trainers/dqn/config.rs +++ b/crates/ml/src/trainers/dqn/config.rs @@ -1514,7 +1514,7 @@ impl DQNHyperparameters { adam_epsilon: 1e-8, // f32 Adam (master weights are f32) // Conservative Q-Learning (CQL) - cql_alpha: 0.1, // Default: mild conservatism (0.1 of 0.0-1.0 range) + cql_alpha: 1.0, // Strong conservatism — penalizes OOS-destructive Q-values // QR-DQN (complementary to C51 — IQN for quantile estimation) num_quantiles: 32, // Default: 32 quantiles diff --git a/crates/ml/src/trainers/dqn/fused_training.rs b/crates/ml/src/trainers/dqn/fused_training.rs index 9e3c5517c..70dd7600b 100644 --- a/crates/ml/src/trainers/dqn/fused_training.rs +++ b/crates/ml/src/trainers/dqn/fused_training.rs @@ -46,9 +46,9 @@ use super::DQNHyperparameters; /// Per-component gradient norm budget fractions for auxiliary objectives. /// C51 gets whatever remains: `1.0 - sum(active_auxiliary_budgets)`. -/// When all auxiliaries are active: C51=70%, CQL=15%, IQN=10%, Ens=5%. -/// When auxiliaries are disabled, their budget automatically goes to C51. -pub(crate) const CQL_GRAD_BUDGET: f32 = 0.15; +/// When all auxiliaries are active: C51=60%, CQL=25%, IQN=10%, Ens=5%. +/// CQL at 25% (was 15%) to enforce conservatism on OOS-destructive Q-values. +pub(crate) const CQL_GRAD_BUDGET: f32 = 0.25; pub(crate) const IQN_GRAD_BUDGET: f32 = 0.10; pub(crate) const ENS_GRAD_BUDGET: f32 = 0.05;