From 796fbd01cf2afebc981c45e4189b1d449f5ce8e5 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Thu, 9 Apr 2026 20:02:48 +0200 Subject: [PATCH] =?UTF-8?q?fix:=20set=20c51=5Falpha=5Fmax=3D0.5=20?= =?UTF-8?q?=E2=80=94=20prevent=20C51=20gradient=20convergence=20on=20H100?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit H100 fold 2 showed grad_norm→0 when C51 fully replaced MSE (alpha=1.0). MSE provides a non-vanishing gradient floor. Default 0.5 = 50/50 blend. Hyperopt can search [0.3, 0.9]. Co-Authored-By: Claude Opus 4.6 (1M context) --- config/training/dqn-production.toml | 2 +- crates/ml/src/trainers/dqn/config.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/training/dqn-production.toml b/config/training/dqn-production.toml index ae3f39a20..2b3aaec08 100644 --- a/config/training/dqn-production.toml +++ b/config/training/dqn-production.toml @@ -82,7 +82,7 @@ q_clip_max = 50.0 n_steps = 5 tau = 0.005 c51_warmup_epochs = 5 -c51_alpha_max = 1.0 +c51_alpha_max = 0.5 her_ratio = 0.2 cql_alpha = 1.0 curiosity_weight = 0.1 diff --git a/crates/ml/src/trainers/dqn/config.rs b/crates/ml/src/trainers/dqn/config.rs index d8c9fc1a7..9a80a3f4c 100644 --- a/crates/ml/src/trainers/dqn/config.rs +++ b/crates/ml/src/trainers/dqn/config.rs @@ -1865,7 +1865,7 @@ impl DQNHyperparameters { // C51 warmup: use MSE loss for first 5 epochs (stronger gradient than C51 cross-entropy) c51_warmup_epochs: 5, - c51_alpha_max: 1.0, // Default: pure C51 after warmup. Hyperopt can search [0.3, 0.9]. + c51_alpha_max: 0.5, // MSE always contributes 50%. Hyperopt searches [0.3, 0.9]. // Decision Transformer pre-training: disabled by default dt_pretrain_epochs: 0, // 0 = disabled