gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cleanup(fflag): remove enable_q_value_clipping — always clip — [FFLAG-003]

Browse Source 
DQNConfig.enable_q_value_clipping was true at all 4 construction sites;
the else branch returned the pre-clamp tensor unchanged (dead identity
path). Q-value clipping is a BUG #37 fix — clipping happens after
forward pass so gradients still flow, and prevents Q-value explosions
from poisoning action selection. Per feedback_no_feature_flags.md, the
flag is gone — clipping is now unconditional. Kept q_value_clip_min /
q_value_clip_max since they're still passed to clamp().
This commit is contained in:
jgrusewski
2026-04-20 22:32:08 +02:00
parent 4f79572783
commit b78fd2f1f0
1 changed files with 2 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
crates/ml-dqn/src/dqn.rs
View File

@@ -111,10 +111,7 @@ pub struct DQNConfig {
/// Initial standard deviation for noisy layers
pub noisy_sigma_init: f64,
// BUG #37 FIX: Q-value clipping configuration (prevents step-level explosions)
/// Whether to enable Q-value clipping in forward pass
/// Default: true (prevents explosions without breaking gradients)
pub enable_q_value_clipping: bool,
// BUG #37 FIX: Q-value clipping is unconditional in forward pass (prevents step-level explosions)
/// Minimum Q-value bound (clipped during forward pass)
/// Default: -500.0 (based on portfolio normalization baseline ±375 × 1.33 safety margin)
pub q_value_clip_min: f32,
@@ -267,7 +264,6 @@ impl Default for DQNConfig {
v_min: -(10.0_f32 / (1.0 - 0.95) * 1.2).clamp(20.0, 300.0),
v_max: (10.0_f32 / (1.0 - 0.95) * 1.2).clamp(20.0, 300.0),
noisy_sigma_init: 0.5,
enable_q_value_clipping: true,
q_value_clip_min: -200.0, // Reward v6: tighter than old -500 but covers v_range + safety margin
q_value_clip_max: 200.0,
gradient_collapse_multiplier: 2.0,
@@ -579,7 +575,6 @@ impl DQNConfig {
noisy_sigma_init: 0.5,
// BUG #37 FIX: Q-value clipping (prevents step-level explosions)
enable_q_value_clipping: true,
q_value_clip_min: -1000.0, // Wider bounds for aggressive exploration
q_value_clip_max: 1000.0,
@@ -644,7 +639,6 @@ impl DQNConfig {
noisy_sigma_init: 0.5,
// BUG #37 FIX: Q-value clipping (prevents step-level explosions)
enable_q_value_clipping: true,
q_value_clip_min: -200.0, // Reward v6: tighter than old -500 but covers v_range + safety margin
q_value_clip_max: 200.0,
@@ -719,7 +713,6 @@ impl DQNConfig {
noisy_sigma_init: 0.5,
// BUG #37 FIX: Q-value clipping (prevents step-level explosions)
enable_q_value_clipping: true,
q_value_clip_min: -200.0, // Reward v6: tighter than old -500 but covers v_range + safety margin
q_value_clip_max: 200.0,
@@ -1217,7 +1210,7 @@ impl DQN {
// 1. Clipping happens AFTER forward pass (inference)
// 2. Gradients flow through during backward (no gradient death)
// 3. Prevents extreme Q-values from propagating through action selection
let q_values = if self.config.enable_q_value_clipping {
let q_values = {
let clamped = q_values.clamp(
self.config.q_value_clip_min,
self.config.q_value_clip_max,
@@ -1260,8 +1253,6 @@ impl DQN {
}
clamped
} else {
q_values
};
// Cast output back to F32 for API compatibility (callers expect f32 tensors)