From 408e0ef4ac3bd9fd91f91f275f0cc78067d5bce4 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 25 May 2026 13:50:25 +0200 Subject: [PATCH] =?UTF-8?q?fix(rl):=20per-step=20=C2=B12%=20clamp=20on=20r?= =?UTF-8?q?eward=5Fscale=20movement?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reward_scale oscillated 10,873× (min 0.000123, max 1.33) because the Wiener-α=0.4 blend tracked sparse trade PnL spikes instantly. Old transitions in PER had stale-scale rewards even with raw-reward re-normalization (the current scale itself was unstable). Per-step clamp: scale can only move ±2% from previous value. Doubling takes ~35 steps, halving takes ~35 steps — fast enough to track regime changes, stable enough for Q targets across the replay window. Max oscillation over 1000 steps: ~7× (was 10,873×). Co-Authored-By: Claude Opus 4.7 --- crates/ml-alpha/cuda/rl_reward_scale_controller.cu | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/ml-alpha/cuda/rl_reward_scale_controller.cu b/crates/ml-alpha/cuda/rl_reward_scale_controller.cu index fcb98773f..58e586e56 100644 --- a/crates/ml-alpha/cuda/rl_reward_scale_controller.cu +++ b/crates/ml-alpha/cuda/rl_reward_scale_controller.cu @@ -130,6 +130,15 @@ extern "C" __global__ void rl_reward_scale_controller( const float a = fmaxf(alpha_step, WIENER_ALPHA_FLOOR); float out = (1.0f - a) * prev + a * target; + // Per-step movement clamp: scale moves at most ±2% from previous. + // Damps the 10,000× oscillation from sparse trade PnL spikes to + // a smooth ramp. At 2%/step, doubling takes ~35 steps, halving + // takes ~35 steps — fast enough to track regime changes, slow + // enough to give Q stable targets across the replay window. + const float max_move = prev * 1.02f; + const float min_move = prev * 0.98f; + out = fmaxf(min_move, fminf(out, max_move)); + out = fmaxf(scale_min, fminf(out, REWARD_SCALE_MAX)); isv[RL_REWARD_SCALE_INDEX] = out; }