diff --git a/crates/ml-alpha/cuda/rl_pi_action_kernel.cu b/crates/ml-alpha/cuda/rl_pi_action_kernel.cu index 0f4e2af2b..d90bbbf54 100644 --- a/crates/ml-alpha/cuda/rl_pi_action_kernel.cu +++ b/crates/ml-alpha/cuda/rl_pi_action_kernel.cu @@ -21,12 +21,33 @@ // per-thread parallelism of the Thompson kernel is unjustified; // multinomial CDF walk is sequential anyway). // +// Anti-collapse: each action probability is floored at P_MIN before +// the CDF walk, then renormalised. This is a STRUCTURAL guarantee +// per `pearl_blend_formulas_must_have_permanent_floor` — H(π_sample) +// is bounded below regardless of what the logits say. P_MIN=0.02 → +// max-collapse entropy ≈ 0.77 nats (vs uniform ln(9)=2.197), well +// above the δ-function attractor observed in alpha-rl-9k9x6 where +// the policy collapsed to action 1 by step 2500 with action_entropy +// EMA at 0.001 and never recovered (controller pegged at COEF_MAX +// without effect because PPO updates couldn't escape the fixed point). +// +// Discrepancy with PPO importance ratio: log_pi_at_action records +// log(softmax(logits)[a]), not log(p_floored[a]). At P_MIN=0.02 the +// divergence is negligible when π is healthy (max abs Δlog_prob is +// ~0.01 for π(a)=0.99), and during recovery it nudges PPO toward +// conservative updates on rare-action samples — desirable. +// +// P_MIN is exempt from `feedback_isv_for_adaptive_bounds` per the +// user-stated "floors and clamp bounds" exemption; lives here as a +// design constant alongside N_ACTIONS. +// // Per `feedback_no_atomicadd`: single-thread kernel, no atomics. // Per `feedback_cpu_is_read_only`: pure device-side sampling. #include #define N_ACTIONS 9 +#define P_MIN 0.02f __device__ static uint32_t xorshift32(uint32_t* state) { uint32_t x = *state; @@ -79,6 +100,20 @@ extern "C" __global__ void rl_pi_action_kernel( #pragma unroll for (int a = 0; a < N_ACTIONS; ++a) probs[a] *= inv_sum; + // ── Anti-collapse floor: max(P_MIN, p[a]) then renormalise. ─── + // Structural guarantee that no action probability falls below + // P_MIN. See header comment for rationale + alpha-rl-9k9x6 + // failure mode. + float renorm_sum = 0.0f; + #pragma unroll + for (int a = 0; a < N_ACTIONS; ++a) { + probs[a] = fmaxf(P_MIN, probs[a]); + renorm_sum += probs[a]; + } + const float inv_renorm = 1.0f / renorm_sum; + #pragma unroll + for (int a = 0; a < N_ACTIONS; ++a) probs[a] *= inv_renorm; + // ── Draw uniform u ∈ [0, 1) from per-batch PRNG. ────────────── uint32_t state = prng_state[b]; // A few warmup advances dispel low-quality seeding correlations. diff --git a/infra/k8s/argo/alpha-rl-template.yaml b/infra/k8s/argo/alpha-rl-template.yaml index 3c61f315c..3e7ffa7de 100644 --- a/infra/k8s/argo/alpha-rl-template.yaml +++ b/infra/k8s/argo/alpha-rl-template.yaml @@ -65,7 +65,11 @@ spec: - name: seq-len value: "32" - name: n-backtests - value: "1" + # Default 16 — fixes b_size=1 signal starvation per + # `pearl_b_size_1_signal_starvation_blocks_q_learning`. 16 + # parallel transitions per Adam step give Q proper gradient + # variance reduction. Override via `-p n-backtests=N`. + value: "16" - name: per-capacity value: "4096" - name: seed diff --git a/scripts/argo-alpha-rl.sh b/scripts/argo-alpha-rl.sh index af38f6878..b0b9dc0f3 100755 --- a/scripts/argo-alpha-rl.sh +++ b/scripts/argo-alpha-rl.sh @@ -32,7 +32,11 @@ BRANCH=$(git symbolic-ref --short HEAD 2>/dev/null || echo "ml-alpha-phase-a") GPU_POOL=ci-training-l40s N_STEPS=1000 SEQ_LEN=32 -N_BACKTESTS=1 +# Default 16 — fixes b_size=1 signal starvation per +# `pearl_b_size_1_signal_starvation_blocks_q_learning`. 16 parallel +# transitions per Adam step give Q proper gradient variance reduction. +# Override with `--n-backtests N` for sweeps. +N_BACKTESTS=16 PER_CAPACITY=4096 SEED=16962 INSTRUMENT_MODE=front-month