feat(v8): PopArt normalize + curriculum difficulty + hindsight relabel CUDA kernels

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

crates/ml/src/cuda_pipeline/dqn_utility_kernels.cu

@@ -1431,3 +1431,91 @@ extern "C" __global__ void exposure_aux_grad_kernel(
         }
     }
 }
+
+/* ══════════════════════════════════════════════════════════════════════
+ * POPART REWARD NORMALIZATION KERNEL (v8 D2)
+ *
+ * Normalizes rewards in-place using running mean/variance (Welford).
+ * Updates statistics from the current batch, then normalizes.
+ * Eliminates manual v_min/v_max tuning for C51.
+ *
+ * Launch: grid=(1), block=(256). Single block for atomic stat update.
+ * ══════════════════════════════════════════════════════════════════════ */
+extern "C" __global__ void popart_normalize_kernel(
+    float* __restrict__ rewards,
+    float* __restrict__ mean_buf,
+    float* __restrict__ var_buf,
+    float* __restrict__ count_buf,
+    int N, int warmup_count
+) {
+    __shared__ float s_sum;
+    __shared__ float s_sq_sum;
+    __shared__ float s_mu;
+    __shared__ float s_sigma;
+
+    int tid = threadIdx.x;
+
+    /* Phase 1: compute batch sum and sum-of-squares via reduction */
+    float local_sum = 0.0f;
+    float local_sq = 0.0f;
+    for (int i = tid; i < N; i += blockDim.x) {
+        float r = rewards[i];
+        local_sum += r;
+        local_sq += r * r;
+    }
+
+    /* Warp reduction */
+    for (int offset = 16; offset > 0; offset >>= 1) {
+        local_sum += __shfl_xor_sync(0xFFFFFFFF, local_sum, offset);
+        local_sq += __shfl_xor_sync(0xFFFFFFFF, local_sq, offset);
+    }
+
+    __shared__ float warp_sums[8];
+    __shared__ float warp_sqs[8];
+    int warp_id = tid / 32;
+    int lane = tid % 32;
+    if (lane == 0) { warp_sums[warp_id] = local_sum; warp_sqs[warp_id] = local_sq; }
+    __syncthreads();
+
+    if (warp_id == 0) {
+        float val = (lane < blockDim.x / 32) ? warp_sums[lane] : 0.0f;
+        float sq = (lane < blockDim.x / 32) ? warp_sqs[lane] : 0.0f;
+        for (int offset = 16; offset > 0; offset >>= 1) {
+            val += __shfl_xor_sync(0xFFFFFFFF, val, offset);
+            sq += __shfl_xor_sync(0xFFFFFFFF, sq, offset);
+        }
+        if (lane == 0) { s_sum = val; s_sq_sum = sq; }
+    }
+    __syncthreads();
+
+    /* Phase 2: thread 0 updates running statistics */
+    if (tid == 0) {
+        float old_count = *count_buf;
+        float new_count = old_count + (float)N;
+        float batch_mean = s_sum / fmaxf((float)N, 1.0f);
+        float new_mean = (old_count > 0.0f)
+            ? (*mean_buf * old_count + s_sum) / new_count
+            : batch_mean;
+        float new_var = (new_count > 1.0f)
+            ? (s_sq_sum + *var_buf * old_count) / new_count - new_mean * new_mean
+            : 1.0f;
+        new_var = fmaxf(new_var, 0.0001f);
+        *mean_buf = new_mean;
+        *var_buf = new_var;
+        *count_buf = new_count;
+        s_mu = new_mean;
+        s_sigma = sqrtf(new_var);
+    }
+    __syncthreads();
+
+    /* Phase 3: normalize rewards in-place */
+    float mu = s_mu;
+    float sigma = fmaxf(s_sigma, 0.01f);
+    float total = *count_buf;
+
+    if (total >= (float)warmup_count) {
+        for (int i = tid; i < N; i += blockDim.x) {
+            rewards[i] = (rewards[i] - mu) / sigma;
+        }
+    }
+}

crates/ml/src/cuda_pipeline/experience_kernels.cu

@@ -2015,3 +2015,80 @@ extern "C" __global__ void classify_regime_gpu(
 extern "C" __global__ void step_counter_advance(int* counter) {
     counter[0] += 1;
 }
+
+/* ══════════════════════════════════════════════════════════════════════
+ * CURRICULUM DIFFICULTY SCORING KERNEL (v8 D3)
+ *
+ * Per-bar difficulty: inverse of 5-bar directional clarity.
+ * Low score = strong trend (easy). High score = choppy (hard).
+ * Used to order episode starts from easy → hard across training.
+ *
+ * Launch: grid=(ceil(total_bars/256)), block=(256).
+ * ══════════════════════════════════════════════════════════════════════ */
+extern "C" __global__ void compute_difficulty_scores(
+    const __nv_bfloat16* __restrict__ targets,
+    float* __restrict__ scores,
+    int total_bars
+) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i >= total_bars) return;
+    if (i >= total_bars - 5) { scores[i] = 1e6f; return; }
+
+    float close_0 = __bfloat162float(targets[i * 4 + 2]);
+    float close_5 = __bfloat162float(targets[(i + 5) * 4 + 2]);
+    float ret_5 = fabsf(close_5 - close_0) / fmaxf(close_0, 1.0f);
+    scores[i] = 1.0f / (ret_5 + 0.001f);
+}
+
+/* ══════════════════════════════════════════════════════════════════════
+ * HINDSIGHT OPTIMAL EXIT RELABELING KERNEL (v8 D4)
+ *
+ * For a fraction of replay samples, replaces the reward with the
+ * optimal exit PnL within a lookahead window. Teaches the model
+ * "what was the best possible outcome for this entry?"
+ *
+ * Launch: grid=(ceil(batch_size/256)), block=(256).
+ * ══════════════════════════════════════════════════════════════════════ */
+extern "C" __global__ void hindsight_relabel_kernel(
+    const __nv_bfloat16* __restrict__ targets,
+    const int*   __restrict__ actions,
+    const int*   __restrict__ bar_indices,
+    float*       __restrict__ rewards,
+    int b0_size, int b1_size, int b2_size,
+    float max_position, int lookahead,
+    int total_bars, float hindsight_fraction,
+    int batch_size
+) {
+    int i = blockIdx.x * blockDim.x + threadIdx.x;
+    if (i >= batch_size) return;
+
+    /* Deterministic mask */
+    unsigned int hash = (unsigned int)(i * 48271 + bar_indices[i] * 7919);
+    hash ^= (hash >> 16); hash *= 0x45d9f3bu; hash ^= (hash >> 16);
+    if ((float)(hash & 0xFFFF) / 65535.0f >= hindsight_fraction) return;
+
+    int bar = bar_indices[i];
+    if (bar < 0 || bar + lookahead >= total_bars) return;
+
+    int exposure_idx = actions[i] / (b1_size * b2_size);
+    float step_size = (b0_size > 1) ? 2.0f / (float)(b0_size - 1) : 0.0f;
+    float fraction = -1.0f + (float)exposure_idx * step_size;
+    float pos = fraction * max_position;
+    if (fabsf(pos) < 0.001f) return;
+
+    float entry_price = __bfloat162float(targets[bar * 4 + 2]);
+    if (entry_price < 1.0f) return;
+
+    float best_pnl = 0.0f;
+    for (int k = 1; k <= lookahead && (bar + k) < total_bars; k++) {
+        float future_price = __bfloat162float(targets[(bar + k) * 4 + 2]);
+        float pnl = pos * (future_price - entry_price);
+        if (pnl > best_pnl) best_pnl = pnl;
+    }
+
+    if (best_pnl > 0.0f) {
+        float opt_return = best_pnl / entry_price;
+        float opt_reward = asymmetric_soft_clamp(10.0f * opt_return / 0.005f);
+        rewards[i] = opt_reward;
+    }
+}