diff --git a/crates/ml/src/cuda_pipeline/experience_kernels.cu b/crates/ml/src/cuda_pipeline/experience_kernels.cu index 230e54a82..30f453b65 100644 --- a/crates/ml/src/cuda_pipeline/experience_kernels.cu +++ b/crates/ml/src/cuda_pipeline/experience_kernels.cu @@ -822,34 +822,32 @@ extern "C" __global__ void experience_action_select( q_max_d = fmaxf(q_max_d, qv); q_min_d = fminf(q_min_d, qv); } - /* Direction temperature: 2× Q-range ensures Short/Long always get ≥15% - * probability even when Flat Q is highest. Without this, Flat dominance - * grows to 90%+ as training progresses (zero-cost Flat has inherent Q advantage). - * 2× gives max ratio e^0.5:1 ≈ 1.65:1 between best and worst direction. */ - float tau_d = fmaxf((q_max_d - q_min_d) * 2.0f, 0.01f); - - float sum_e = 0.0f; - float exps_d[3]; /* b0_size is always 3 (Short/Flat/Long) */ - for (int a = 0; a < b0_size; a++) { - float qv = __bfloat162float(q_b0[a]); - exps_d[a] = expf((qv - q_max_d) / tau_d); - sum_e += exps_d[a]; + /* Direction selection: Flat floor + Boltzmann for directional. + * + * 30% of the time: force Flat (prevents over-trading on production). + * 70% of the time: Boltzmann over Short/Flat/Long proportional to Q. + * This guarantees ≥30% Flat regardless of Q-values or hold enforcement, + * without the Q-gap conviction filter's snowball dynamics. */ + if (lcg_random(&rng) < 0.30f) { + dir_idx = 1; /* Flat floor: 30% unconditional */ + } else { + /* Boltzmann over all 3 directions for remaining 70% */ + float tau_d = fmaxf(q_max_d - q_min_d, 0.01f); + float sum_e = 0.0f; + float exps_d[3]; + for (int a = 0; a < b0_size; a++) { + float qv = __bfloat162float(q_b0[a]); + exps_d[a] = expf((qv - q_max_d) / tau_d); + sum_e += exps_d[a]; + } + float r = lcg_random(&rng) * sum_e; + float cum = 0.0f; + dir_idx = b0_size - 1; + for (int a = 0; a < b0_size; a++) { + cum += exps_d[a]; + if (r < cum) { dir_idx = a; break; } + } } - float r = lcg_random(&rng) * sum_e; - float cum = 0.0f; - dir_idx = b0_size - 1; - for (int a = 0; a < b0_size; a++) { - cum += exps_d[a]; - if (r < cum) { dir_idx = a; break; } - } - - /* Q-gap conviction filter REMOVED from training path. - * Boltzmann already encodes conviction through softmax probabilities. - * The filter was redundant and harmful: after a high-Sharpe epoch, Bellman - * max bootstraps raise Q(Flat) towards Q(directional), shrinking the gap. - * The filter then overrides Boltzmann's directional samples → Flat grows - * → less directional experience → model loses directional learning signal. - * Conviction gating is preserved in the backtest evaluator (eps=0 path). */ } /* Gem 3: Flat detection shortcut — when direction=Flat, magnitude is irrelevant.