diff --git a/crates/ml-alpha/cuda/rl_fused_reward_pipeline.cu b/crates/ml-alpha/cuda/rl_fused_reward_pipeline.cu index 43ee7eb55..be05feaf7 100644 --- a/crates/ml-alpha/cuda/rl_fused_reward_pipeline.cu +++ b/crates/ml-alpha/cuda/rl_fused_reward_pipeline.cu @@ -41,6 +41,11 @@ #define RL_OUTCOME_ALPHA_INDEX 520 // Layer 3 (inventory penalty — spec 2026-05-30-adaptive-risk-management). #define RL_INVENTORY_PENALTY_BETA_INDEX 674 +// Reward-policy realignment (spec 2026-06-01-reward-policy-alignment-investigation). +// When > 0.5: skip ALL Phase 5 shaping (entry cost, short-hold penalty, +// long-ride bonus, per-step hold bonus) AND Phase 5b inventory penalty. +// Reward = Phase 1 realized_pnl_delta unchanged. +#define RL_REWARD_PURE_PNL_MODE_INDEX 753 #define MAX_UNITS 4 @@ -208,6 +213,13 @@ extern "C" __global__ void rl_fused_reward_pipeline( // ================================================================ // PHASE 5: rl_reward_shaping (surfer philosophy) // ================================================================ + // Reward-policy alignment (spec 2026-06-01): when pure_pnl_mode > 0.5 + // ALL of steps 1-4 below + Phase 5b inventory penalty are SKIPPED. + // r remains at the Phase 1 realized_pnl_delta — pure pnl alignment. + // Default is mode=1 (gating active); mode=0 preserves legacy shaping + // for one regression cluster smoke, then deleted in follow-up. + const float pure_pnl_mode = isv[RL_REWARD_PURE_PNL_MODE_INDEX]; + const float entry_cost = isv[RL_ENTRY_COST_INDEX]; const float min_hold = isv[RL_SHORT_HOLD_MIN_STEPS_INDEX]; const float penalty = isv[RL_SHORT_HOLD_PENALTY_INDEX]; @@ -217,36 +229,42 @@ extern "C" __global__ void rl_fused_reward_pipeline( float r = reward; - // 1. Entry cost: flat → positioned transition. - if (prev_lots == 0 && current_lots != 0) { - r -= entry_cost; - } + if (pure_pnl_mode <= 0.5f) { + // 1. Entry cost: flat → positioned transition. + if (prev_lots == 0 && current_lots != 0) { + r -= entry_cost; + } - // 2. Short-hold penalty: trade close with hold time below minimum. - if (done > 0.5f && (float)hold_time < min_hold) { - r *= penalty; - } + // 2. Short-hold penalty: trade close with hold time below minimum. + if (done > 0.5f && (float)hold_time < min_hold) { + r *= penalty; + } - // 3. Long-ride bonus: profitable close amplified by hold time. - if (done > 0.5f && r > 0.0f && hold_time > 0) { - const float ride_mult = 1.0f + hold_bonus * sqrtf((float)hold_time); - r *= ride_mult; - } + // 3. Long-ride bonus: profitable close amplified by hold time. + if (done > 0.5f && r > 0.0f && hold_time > 0) { + const float ride_mult = 1.0f + hold_bonus * sqrtf((float)hold_time); + r *= ride_mult; + } - // 4. Per-step hold bonus for staying in a profitable position. - if (prev_lots != 0 && current_lots != 0 && r > 0.0f && hold_time > 0) { - r += hold_bonus * sqrtf((float)hold_time); + // 4. Per-step hold bonus for staying in a profitable position. + if (prev_lots != 0 && current_lots != 0 && r > 0.0f && hold_time > 0) { + r += hold_bonus * sqrtf((float)hold_time); + } } // ================================================================ // PHASE 5b: Layer 3 inventory penalty (spec 2026-05-30-adaptive-risk-management). // Apply BEFORE raw_rewards snapshot so the penalty is part of the // shaped reward seen by Q/V learning. β = 0 (sentinel) → no-op. + // Also gated by pure_pnl_mode — when mode>0.5, Layers 1/2/4 own + // inventory risk via action overrides + Kelly sizing. // ================================================================ - const float inv_beta = isv[RL_INVENTORY_PENALTY_BETA_INDEX]; - if (inv_beta > 0.0f) { - const float net_pos_mag = (float)((current_lots < 0) ? -current_lots : current_lots); - r -= inv_beta * net_pos_mag; + if (pure_pnl_mode <= 0.5f) { + const float inv_beta = isv[RL_INVENTORY_PENALTY_BETA_INDEX]; + if (inv_beta > 0.0f) { + const float net_pos_mag = (float)((current_lots < 0) ? -current_lots : current_lots); + r -= inv_beta * net_pos_mag; + } } // Write shaped reward back. diff --git a/crates/ml-alpha/src/rl/isv_slots.rs b/crates/ml-alpha/src/rl/isv_slots.rs index 73fb08645..3469df042 100644 --- a/crates/ml-alpha/src/rl/isv_slots.rs +++ b/crates/ml-alpha/src/rl/isv_slots.rs @@ -1737,6 +1737,25 @@ pub const RL_EDGE_PH_MEAN_INDEX: usize = 750; // mean ph_sta pub const RL_EDGE_PH_FRAC_ALERTED_INDEX: usize = 751; // n_alerted / n_active pub const RL_EDGE_PH_FRAC_WARMUP_INDEX: usize = 752; // n_in_warmup / b_size +// Reward-policy realignment (2026-06-01). When > 0.5, rl_fused_reward_pipeline +// Phase 5 skips ALL hold-time shaping (steps 2-4) AND entry-cost (step 1). +// Reward = realized_pnl_delta (Phase 1 output). Pure pnl alignment. +// +// Diagnosis: alpha-rl-8gtk2 at 87a8259c6 showed Pearson(rewards.sum, +// Δrealized_pnl_cum_usd) = 0.16 train / 0.31 eval — gradient signal +// only weakly correlated with the trade-level pnl reality. Phase 5 +// step 4 adds hold_bonus × √hold_time = 2.0 × 10 = +20/step for +// 100-step holds, dominating realized pnl by ~2600× per held trade. +// +// Bootstrap = 1.0 (NEW path default). Legacy mode=0 path is preserved +// for ONE regression cluster smoke, then DELETED in a follow-up commit +// per feedback_no_feature_flags + feedback_single_source_of_truth_no_duplicates. +// This is a permanent reward redesign, not a permanent toggle. +// +// See spec docs/superpowers/specs/2026-06-01-reward-policy-alignment-investigation.md +// and pearl pearl_reward_signal_anti_aligned_with_pnl. +pub const RL_REWARD_PURE_PNL_MODE_INDEX: usize = 753; + /// Last RL-allocated slot index (exclusive). /// Pre-risk-stack: 662. Post-Fix-B: 685. Post-v9 (warmup boundary): 696. /// Post-regime-observer (F1.1): 716. Post-warmed-flag addendum: 717. @@ -1748,4 +1767,5 @@ pub const RL_EDGE_PH_FRAC_WARMUP_INDEX: usize = 752; // n_in_warmup /// Post-B-10 policy-quality cascade diagnostic: 743. /// Post-CMDP-fleet-fraction diagnostic: 747. /// Post-edge-decay-detector Phase 1: 753. -pub const RL_SLOTS_END: usize = 753; +/// Post-reward-policy-realignment: 754. +pub const RL_SLOTS_END: usize = 754; diff --git a/crates/ml-alpha/src/trainer/integrated.rs b/crates/ml-alpha/src/trainer/integrated.rs index 8c2a7686e..ee9269ca3 100644 --- a/crates/ml-alpha/src/trainer/integrated.rs +++ b/crates/ml-alpha/src/trainer/integrated.rs @@ -3479,7 +3479,7 @@ impl IntegratedTrainer { // (slot, value) pair — pure device write, no HtoD per // `feedback_no_htod_htoh_only_mapped_pinned`. { - let isv_constants: [(usize, f32); 240] = [ + let isv_constants: [(usize, f32); 241] = [ // Static seeds for the adaptive reward-clamp controller — // these are the initial values that // `rl_reward_clamp_controller` will replace once it @@ -3899,6 +3899,11 @@ impl IntegratedTrainer { (crate::rl::isv_slots::RL_EDGE_PH_MEAN_INDEX, 0.0_f32), (crate::rl::isv_slots::RL_EDGE_PH_FRAC_ALERTED_INDEX, 0.0_f32), (crate::rl::isv_slots::RL_EDGE_PH_FRAC_WARMUP_INDEX, 0.0_f32), + // Reward-policy realignment (spec 2026-06-01). + // 1.0 = pure pnl reward (Phase 5/5b shaping SKIPPED). + // 0.0 = legacy shaping path, preserved for ONE regression + // cluster smoke then deleted per feedback_no_feature_flags. + (crate::rl::isv_slots::RL_REWARD_PURE_PNL_MODE_INDEX, 1.0_f32), ]; for (slot, value) in isv_constants.iter() { let slot_i32 = *slot as i32; @@ -9998,6 +10003,8 @@ impl IntegratedTrainer { isv[RL_NEG_SCALED_REWARD_MAX_INDEX], "neg_scaled_max_ema": isv[RL_NEG_SCALED_REWARD_MAX_EMA_INDEX], + "pure_pnl_mode": + isv[crate::rl::isv_slots::RL_REWARD_PURE_PNL_MODE_INDEX], }, "ppo": { "ratio_clamp_max": isv[RL_PPO_RATIO_CLAMP_MAX_INDEX], diff --git a/crates/ml-alpha/tests/eval_diag_emission.rs b/crates/ml-alpha/tests/eval_diag_emission.rs index 9dc212c50..c663997f1 100644 --- a/crates/ml-alpha/tests/eval_diag_emission.rs +++ b/crates/ml-alpha/tests/eval_diag_emission.rs @@ -243,7 +243,8 @@ fn eval_diag_jsonl_emitted_with_train_schema_parity() -> Result<()> { // (frac_in_cooldown / frac_consec_near_limit / frac_dd_triggered / // frac_session_neg) → 675. Edge-decay Phase 1 adds 3 leaves // (edge_ph_mean / edge_ph_frac_alerted / edge_ph_frac_warmup) → 678. - const EXPECTED_LEAVES: usize = 678; + // Reward-policy realignment adds 1 leaf (rewards.pure_pnl_mode) → 679. + const EXPECTED_LEAVES: usize = 679; anyhow::ensure!( train_paths.len() == EXPECTED_LEAVES, "train leaf count {} != expected {EXPECTED_LEAVES} (schema drifted; \ diff --git a/crates/ml-alpha/tests/reward_alignment_invariants.rs b/crates/ml-alpha/tests/reward_alignment_invariants.rs new file mode 100644 index 000000000..c85fe3e6d --- /dev/null +++ b/crates/ml-alpha/tests/reward_alignment_invariants.rs @@ -0,0 +1,174 @@ +//! Reward-policy realignment invariants (spec 2026-06-01). +//! +//! Validates that the new `RL_REWARD_PURE_PNL_MODE` ISV slot propagates +//! correctly from trainer bootstrap → ISV → kernel `rl_fused_reward_pipeline` +//! → diag.jsonl, and that the mode=1 default holds across train and eval +//! phases. +//! +//! Three invariants: +//! I1. `rewards.pure_pnl_mode == 1.0` on every train row (bootstrap +//! discipline; mode is a constant for the whole run). +//! I2. `rewards.pure_pnl_mode == 1.0` on every eval row (no slot rewrite +//! at reset_session_state — see [[pearl_adaptive_carryover_discipline]] +//! for the orthogonal "EMA boundary reset" rule; const-bootstrap slots +//! are not subject to it). +//! I3. `rewards.abs_max <= REWARD_INFLATION_FENCE` ($25k) on every row. +//! This excludes the Phase 5 step-4 per-step hold bonus +20×realized +//! inflation pattern that produced the alpha-rl-8gtk2 misalignment +//! (hold_bonus=2.0 × √100 = +20/step on hold_time=100 trades, ×~$1k +//! realized scale = effective $20k+ per-step bonus). With Phase 5 +//! skipped, magnitudes track raw pnl deltas which stay well under the +//! fence at the test's b_size=16 + 200 steps. +//! +//! Per `feedback_no_cpu_test_fallbacks.md`: GPU-oracle end-to-end (drives +//! the alpha_rl_train release binary; parses the diag JSONL it emits). +//! No host-side reimplementation of the reward pipeline. +//! +//! Run with: +//! `cargo test -p ml-alpha --test reward_alignment_invariants -- --ignored --nocapture` + +use anyhow::{Context, Result}; +use serde_json::Value; +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn binary_path() -> PathBuf { + let crate_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + crate_root + .parent() + .and_then(|p| p.parent()) + .map(|root| root.join("target/release/examples/alpha_rl_train")) + .unwrap_or_else(|| PathBuf::from("target/release/examples/alpha_rl_train")) +} + +fn data_dir() -> PathBuf { + if let Ok(p) = std::env::var("FOXHUNT_EVAL_DIAG_DATA") { + return PathBuf::from(p); + } + let crate_root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + crate_root + .parent() + .and_then(|p| p.parent()) + .map(|root| root.join("test_data/futures-baseline/ES.FUT")) + .unwrap_or_else(|| PathBuf::from("test_data/futures-baseline/ES.FUT")) +} + +fn read_jsonl(path: &Path) -> Result> { + let s = std::fs::read_to_string(path) + .with_context(|| format!("read {}", path.display()))?; + s.lines() + .map(|l| serde_json::from_str::(l) + .with_context(|| format!("parse line of {}", path.display()))) + .collect() +} + +fn dot_get(v: &Value, path: &str) -> Result { + let mut cur = v; + for seg in path.split('.') { + cur = cur + .get(seg) + .with_context(|| format!("missing path segment '{seg}' in {path}"))?; + } + cur.as_f64() + .with_context(|| format!("path '{path}' is not numeric")) +} + +#[test] +#[ignore = "requires CUDA + pre-built release binary + MBP-10 test data"] +fn reward_alignment_pure_pnl_mode_invariants() -> Result<()> { + let bin = binary_path(); + anyhow::ensure!( + bin.exists(), + "binary not found at {} — run `SQLX_OFFLINE=true cargo build --release \ + --example alpha_rl_train -p ml-alpha` first", + bin.display() + ); + let data = data_dir(); + anyhow::ensure!(data.exists(), "test data dir missing: {}", data.display()); + + let out = std::env::temp_dir().join("foxhunt-reward-alignment"); + if out.exists() { + std::fs::remove_dir_all(&out).context("rm -rf out")?; + } + std::fs::create_dir_all(&out).context("mkdir -p out")?; + + let n_steps: usize = 200; + let n_eval_steps: usize = 50; + let eval_diag = out.join("eval_diag.jsonl"); + let status = Command::new(&bin) + .args([ + "--n-steps", &n_steps.to_string(), + "--n-eval-steps", &n_eval_steps.to_string(), + "--fold-idx", "1", + "--n-folds", "3", + "--mbp10-data-dir", &data.display().to_string(), + "--predecoded-dir", &data.display().to_string(), + "--out", &out.display().to_string(), + "--eval-diag-jsonl", &eval_diag.display().to_string(), + "--instrument-mode", "all", + "--n-backtests", "16", + "--log-every", "100", + "--seed", "42", + ]) + .env("SQLX_OFFLINE", "true") + .status() + .context("spawn alpha_rl_train")?; + anyhow::ensure!(status.success(), "alpha_rl_train exited with {status}"); + + let diag = out.join("diag.jsonl"); + anyhow::ensure!( + diag.exists() && eval_diag.exists(), + "diag files missing — train={} exists={} / eval={} exists={}", + diag.display(), diag.exists(), + eval_diag.display(), eval_diag.exists(), + ); + + let train_rows = read_jsonl(&diag)?; + let eval_rows = read_jsonl(&eval_diag)?; + anyhow::ensure!( + train_rows.len() == n_steps, + "train rows {} != n_steps {n_steps}", train_rows.len() + ); + anyhow::ensure!( + eval_rows.len() == n_eval_steps, + "eval rows {} != n_eval_steps {n_eval_steps}", eval_rows.len() + ); + + // I1 + I2: pure_pnl_mode must read exactly 1.0 on every row of every phase. + // I3: reward.abs_max must stay below the hold-bonus inflation fence. + // + // Fence rationale: hold_bonus=2.0 × √100=10 = +20/step additive bonus per + // held trade at b=16 produces rewards.abs_max in the 10^3+ range. With + // Phase 5 skipped, abs_max reflects realized pnl deltas alone which at + // b=16 + 200 steps + LobSim retail-fee structure rarely exceed $5k/step. + // $25k is a 5× safety margin that any return of Phase 5 step 4 would + // blow past. + const REWARD_INFLATION_FENCE: f64 = 25_000.0; + + let mut checked = 0usize; + for (phase, rows) in &[("train", &train_rows), ("eval", &eval_rows)] { + for (i, row) in rows.iter().enumerate() { + let mode = dot_get(row, "rewards.pure_pnl_mode")?; + anyhow::ensure!( + (mode - 1.0).abs() < 1e-6, + "{phase}[{i}] rewards.pure_pnl_mode = {mode} (expected 1.0; \ + bootstrap or kernel branch broken)" + ); + let abs_max = dot_get(row, "rewards.abs_max")?; + anyhow::ensure!( + abs_max <= REWARD_INFLATION_FENCE, + "{phase}[{i}] rewards.abs_max = {abs_max:.2} exceeds fence \ + ${REWARD_INFLATION_FENCE:.0} — Phase 5 hold-bonus inflation \ + may have returned; check rl_fused_reward_pipeline.cu Phase 5 gate" + ); + checked += 1; + } + } + + eprintln!( + "reward_alignment_pure_pnl_mode_invariants OK: {} rows validated \ + (train[0..{}] + eval[0..{}])", + checked, n_steps, n_eval_steps + ); + Ok(()) +} diff --git a/docs/superpowers/specs/2026-06-01-reward-policy-alignment-investigation.md b/docs/superpowers/specs/2026-06-01-reward-policy-alignment-investigation.md new file mode 100644 index 000000000..56593d0d0 --- /dev/null +++ b/docs/superpowers/specs/2026-06-01-reward-policy-alignment-investigation.md @@ -0,0 +1,305 @@ +# Reward-Policy Realignment — Drop Hold-Time Shaping; Direct Realized-PnL Reward + +**Date:** 2026-06-01 (v4 — after 1st sp-critical-reviewer pass identified 3 BLOCKERs + 6 HIGHs + 4 MEDIUMs in v3) +**Status:** Ready for 2nd sp-critical-reviewer pass +**Branch target:** `ml-alpha-regime-observer` + +--- + +## 0. Diagnosis (mechanism + magnitude) + +### 0.1 Empirical evidence (existing PVC data) + +Two 20k+5k full cluster runs, sampled 100× / 50×, analyzed locally: + +| Run | SHA | Phase | Pearson(rewards.sum, Δrealized_pnl) | wr final | wr max | +|---|---|---|---|---|---| +| alpha-rl-6kghr | 22e6ddbca | train | N/A (realized_pnl field empty era) | 0.312 | 0.313 | +| alpha-rl-6kghr | 22e6ddbca | eval | N/A | 0.291 | 0.296 | +| alpha-rl-8gtk2 | 87a8259c6 | train | **0.157** | 0.343 | **0.418** | +| alpha-rl-8gtk2 | 87a8259c6 | eval | **0.314** | 0.340 | 0.380 | + +`Pearson(rewards.sum, Δrealized_pnl_cum_usd)` is the clean reference because `realized_pnl_cum_usd` comes from `pos_d[b].realized_pnl` (line 88 of `rl_fused_reward_pipeline.cu`) which is written by `LobSimCuda` *independently of the RL reward path*. Pearson ≈ 0.16 train / 0.31 eval is well below 1.0 — the gradient signal fed to the loss kernels is only weakly correlated with the trade-level pnl reality. + +`alpha-rl-8gtk2` train wr_max = **0.418** is the architecture-can-learn proof: encoder + heads find profitable patterns. wr_final = 0.343 means those patterns got un-trained back to a sub-break-even attractor. + +### 0.2 Kernel evidence (`rl_fused_reward_pipeline.cu`) + +Phase 5 of the fused pipeline applies four shaping terms on top of Phase 1's realized-pnl delta. **Bootstrap values (verified from `integrated.rs:3566-3569`)**: +- `RL_ENTRY_COST_INDEX = 15.0` +- `RL_SHORT_HOLD_MIN_STEPS_INDEX = 100.0` +- `RL_SHORT_HOLD_PENALTY_INDEX = 0.5` +- `RL_HOLD_BONUS_INDEX = 2.0` +- `RL_INVENTORY_PENALTY_BETA_INDEX = 0.0` (sentinel; Phase 5b is a no-op in current bootstrap) + +Phase 5 step 4 (per-step hold bonus while holding profitable position): +```c +if (prev_lots != 0 && current_lots != 0 && r > 0.0f && hold_time > 0) { + r += hold_bonus * sqrtf((float)hold_time); // hold_bonus = 2.0 +} +``` + +With `hold_bonus = 2.0` and `hold_time = 100`, this adds **+20.0 per step** for a held profitable position. Over a 100-step hold, accumulated shaping reward is `sum_{t=1..100} 2 × √t ≈ 2 × (2/3) × 100^1.5 = 1333` — versus typical realized pnl on close of perhaps **+$10** ($10/σ_welford ≈ +0.5 in σ-normalized units after popart). + +**Domination ratio**: ~2600× per held trade (1333 σ-units of shaping vs 0.5 σ-units of realized pnl). + +Phase 5 step 2 (short-hold penalty `r *= penalty`, penalty=0.5): +- Multiplies the closing reward by 0.5 when `hold_time < min_hold = 100` +- For a winning close: reduces magnitude (less reward) +- For a losing close: **makes the loss less negative** (less negative ≈ "less bad" in gradient terms) +- The sign-magnitude relationship is distorted: a $100 loss closed early registers as -$50 of reward, indistinguishable from a $50 loss closed at full hold + +Phase 5 step 3 (long-ride bonus `r *= (1 + hold_bonus × √hold_time)`): +- Only fires on profitable close +- With hold_bonus=2.0 and hold_time=100, multiplier = 1 + 2 × 10 = **21×** +- A +$10 winning close becomes +$210 of shaping reward + +### 0.3 Mechanism — V-regression-mediated, not direct PPO + +(per reviewer HIGH H1, corrected from v3) + +The advantage flowing into PPO surrogate is computed at `compute_advantage_return.cu:41-43`: +```c +advantages[b] = is_done ? (ret - vt) : 0.0f +``` +**Non-done advantages are ALREADY zero.** So per-step hold-bonus rewards (which fire on non-done held-position steps) do NOT flow into PPO's policy gradient directly. They flow into V regression via `returns[b] = r + γ × v_{t+1}`. + +The broken chain is: +1. Per-step hold bonus inflates `returns[b]` on held-position steps +2. V regression learns to predict the inflated return: V baseline becomes **artificially high** during held profitable positions +3. At a done event: `advantage_done = (realized_close_reward - V_baseline_inflated)` is **small or wrong-signed** — even a +$100 close looks "below baseline" if baseline expected the held-trajectory-bonus +4. PPO surrogate trained on these advantages **un-learns** profitable closing actions (they get smaller policy updates than the inflated baseline expected) + +Simultaneously, Q-distill via `rl_q_pi_distill_grad` uses `q_target` from `compute_bellman_target` which sees the same inflated returns. So Q learns to predict the bonus, not the close. + +**Mode=1 fix**: skip Phase 5 steps 2-4. Reward stream becomes: +- Held positions: `reward = 0` +- Close events: `reward = realized_pnl_delta` (no entry-cost since we drop that too — see §1.2) + +Then V learns the *correct* return target (γ-discounted close-event pnl), advantage at close events reflects actual edge, PPO/Q learn closing actions on real reward. + +## 1. The intervention — pure realized-pnl reward + +### 1.1 ISV slot + bootstrap + +```rust +// crates/ml-alpha/src/rl/isv_slots.rs +// +// Reward-policy realignment (2026-06-01). When > 0.5, rl_fused_reward_pipeline +// Phase 5 skips ALL hold-time shaping (steps 2-4) AND entry-cost (step 1). +// Reward = realized_pnl_delta (Phase 1 output). Pure pnl alignment. +// +// Bootstrap = 1.0 (NEW path default). The legacy mode=0 path is preserved +// for ONE regression cluster smoke (verifying baseline reproduction at same +// SHA), then DELETED in a follow-up commit per +// [[feedback_no_feature_flags]] + [[feedback_single_source_of_truth_no_duplicates]]. +// This is a permanent reward redesign, not a permanent toggle. +pub const RL_REWARD_PURE_PNL_MODE_INDEX: usize = 753; +pub const RL_SLOTS_END: usize = 754; +``` + +Bootstrap entry: +```rust +(crate::rl::isv_slots::RL_REWARD_PURE_PNL_MODE_INDEX, 1.0_f32), +``` +Bootstrap array `[(usize, f32); 240]` → `[..; 241]`. + +### 1.2 Kernel modification (rl_fused_reward_pipeline.cu) + +```c +// At top of file: +#define RL_REWARD_PURE_PNL_MODE_INDEX 753 + +// At kernel entry (alongside other ISV reads): +const float pure_pnl_mode = isv[RL_REWARD_PURE_PNL_MODE_INDEX]; + +// PHASE 5 modification: +float r = reward; // (from Phase 1: r = current_realized - prev_realized) + +if (pure_pnl_mode < 0.5f) { + // Legacy shaping (existing code unchanged): + // step 1: entry cost on flat → positioned + if (prev_lots == 0 && current_lots != 0) { + r -= entry_cost; + } + // step 2: short-hold penalty + if (done > 0.5f && (float)hold_time < min_hold) { + r *= penalty; + } + // step 3: long-ride bonus + if (done > 0.5f && r > 0.0f && hold_time > 0) { + r *= (1.0f + hold_bonus * sqrtf((float)hold_time)); + } + // step 4: per-step hold bonus (SMOKING GUN per §0.2) + if (prev_lots != 0 && current_lots != 0 && r > 0.0f && hold_time > 0) { + r += hold_bonus * sqrtf((float)hold_time); + } +} +// Mode=1: r stays at Phase 1's realized_pnl_delta unchanged. +// Entry cost dropped because LobSim's apply_fill_to_pos +// (crates/ml-backtesting/cuda/resting_orders.cu:213-219) deducts +// `filled_lots × cost_per_lot_per_side_per_b[b]` from pos.realized_pnl +// per fill. So realized_pnl_delta is already net-of-fee — adding +// entry_cost in reward shaping would double-count broker fees. + +// PHASE 5b inventory penalty: ALSO gated by pure_pnl_mode. +// Default β = 0 (sentinel since 30+ commits) so this is a no-op now anyway, +// but the gate keeps mode=1 semantically pure. Layers 1 (CMDP) / 2 (IQN τ) +// / 4 (Kelly fraction) own inventory limits via action overrides + sizing +// controllers; mode=1 cleanly delegates inventory risk out of the reward path. +const float inv_beta = isv[RL_INVENTORY_PENALTY_BETA_INDEX]; +if (pure_pnl_mode < 0.5f && inv_beta > 0.0f) { + const float net_pos_mag = (float)((current_lots < 0) ? -current_lots : current_lots); + r -= inv_beta * net_pos_mag; +} + +// Phase 5 finalize: +rewards[b] = r; + +// Phase 6 unchanged: raw_rewards[b] = r (now correctly named — "raw" +// means pre-scale post-shaping in legacy mode, equals realized_pnl in mode=1). + +// Phase 7 (outcome EMA) — uses r at close events. +// In mode=1, r = realized_pnl_delta, so outcome = sign(realized_pnl_delta) +// which is the cleanest possible signal for the aux head consumer. +// In mode=0, r is shaped, so outcome reflects shaped sign (legacy behavior). +// No code change needed. +``` + +### 1.3 Diag emit + +Add ONE leaf: +```rust +// In integrated.rs::build_diag_value, under the existing rewards block: +"pure_pnl_mode": isv[crate::rl::isv_slots::RL_REWARD_PURE_PNL_MODE_INDEX], +``` +Path: `rewards.pure_pnl_mode` (per reviewer M4 — keeps namespace coherent). + +`EXPECTED_LEAVES`: 678 → 679 (+1). + +### 1.4 Invariant test + +`crates/ml-alpha/tests/reward_alignment_invariants.rs` — 3 invariants on local b=16 200+50 smoke: + +1. **Mode default**: `rewards.pure_pnl_mode == 1.0` across all diag rows. +2. **Zero on non-close held steps**: when `dones[b] < 0.5` and the kernel branch did NOT execute the legacy entry-cost path, `rewards[b] == 0.0f` for that batch element. +3. **Close-event reward equals realized_pnl_delta**: at done events in mode=1, `rewards[b] == realized_pnl_current - realized_pnl_prev` within 1e-5 float epsilon (computed from `trading.realized_pnl_cum_usd` field which now correctly tracks the raw delta). + +**Local smoke validates ONLY wiring** (per reviewer H4). Behavior validation (Pearson, wr) requires cluster scale — see §2. + +## 2. Falsification criteria (predetermined, cluster smoke only) + +**Cluster smoke**: 2k+500 b=1024 fold-1 L40S at mode=1 (default after bootstrap). + +**Reference signal for Pearson computation**: `trading.realized_pnl_cum_usd` (NOT `trading.pnl_cum_usd` — that's reward-derived per reviewer B2). Delta computed from sampled diag rows; aggregated to per-100-step windows for stability. + +**Verdict table**: + +| Signal | Window | Target | Verdict if met | Verdict if missed | +|---|---|---|---|---| +| Pearson(rewards.sum, Δrealized_pnl) | train steps 1500-2000 | **≥ 0.7** | reward aligned | iterate (1 retry max) | +| Sign agreement rate (rewards.sum sign vs Δrealized_pnl sign) | train steps 1500-2000 | **≥ 0.65** | reward aligned | iterate | +| wr at end of train | step 2000 | **≥ 0.30** (R=2.37 break-even) | training works | architecture problem (Phase 2 spec) | +| Pearson at eval steps 100-500 | post warmup | **≥ 0.5** | reward stable OOS | eval-boundary popart shock dominates (per H3) | + +**Window justification**: per reviewer's reward-controller audit, the 12 adaptive controllers (reward_scale, reward_clamp_win/loss, popart σ, etc.) will rebalance under the new sparse-reward distribution. Dominant time constant is `RL_REWARD_CLAMP_V_BOUND_EWMA_ALPHA = 0.001` → τ=1000 steps (~3000 steps to 95% convergence). The falsification window is steps 1500-2000 as the primary check. + +**Robustness clause (per reviewer Finding 2)**: if `reward_clamp.v_bound_ema` (diag-emitted) is still > 5× its step 1500-2000 running average at step 2000, the clamp is still in transient — extend the verdict window to step 3000-4000 and re-evaluate. Diag emits `reward_clamp.win_bound` and `reward_clamp.loss_bound`; verify both are within ±20% of their step-1500 → step-2000 average before declaring a Pearson verdict. + +**Theoretical Pearson ceiling** (per reviewer M1): with popart per-batch whitening at b=1024 and ~5% close-rate per step, theoretical max Pearson ≈ 0.85-0.95 (not 1.0 — popart subtracts batch mean which decorrelates from the close-event signal). Target 0.7 is achievable but tight. + +**Maximum 1 iteration** of reward-shape variants before abandonment. Specifically: if Pearson < 0.7, retry with R-multiple reward (`r = realized_pnl_delta / stop_loss_distance_estimate`) instead of $-units. If still < 0.7, declare reward path closed and proceed to architecture simplification (Phase 2 spec, separate doc). + +**Eval falsification scope** (per reviewer H3): Pearson target ≥ 0.5 (lower than train 0.7) acknowledges the unfixed eval-boundary popart-shock bug documented in `pearl_popart_reset_at_eval_boundary_shocks_normalization`. Eval-time popart σ resets and spikes 1100× on first tail event, contaminating the first ~1000 eval steps. Train-side Pearson is the primary criterion; eval-side is informational until that bug is separately addressed. + +## 3. Validation plan + +### V1 — Local smoke (wiring only) +- Build release; run 200+50 b=16 fold-1. +- Verify 3 invariants from §1.4. +- Expected: invariants pass; observed close-event rewards ≈ realized_pnl deltas; non-close rewards = 0. + +### V2 — Cluster smoke run 1 (mode=1, the fix) +- Submit alpha-rl smoke at new commit SHA, mode=1 (default), 2k+500 b=1024 fold-1 L40S. +- Pull diag JSONL post-completion. +- Compute Pearson + sign agreement post-hoc against `realized_pnl_cum_usd`. +- Apply §2 verdict table. + +### V3 — Cluster smoke run 2 (mode=0 regression) +- Per reviewer M2: re-seed mode=0 via ISV write at trainer init, same commit SHA, same params. +- Verify diag fields match the prior baseline run (alpha-rl-glv6n at a5e0d0079). +- Confirms the legacy path is bit-equivalent (or float-equality within 1e-4) to baseline. +- This is the regression run; if mode=1 succeeds AND mode=0 is bit-baseline, we have confidence the gate is correct. + +### V4 — Determinism +Train phase actions/dones/rewards in mode=1 will NOT match prior runs (reward signal changed → gradient changed → policy changed). This is intended. + +In mode=0 (V3 run), should match the baseline byte-for-byte modulo the new ISV read overhead (which doesn't affect numeric output). + +## 4. Implementation phases + +| Phase | LOC | File | +|---|---|---| +| 1 | +6 | `isv_slots.rs`: 1 const + doc comment + bump RL_SLOTS_END to 754 | +| 2 | +4 | `integrated.rs`: 1 bootstrap entry; array 240 → 241 | +| 3 | +20 | `rl_fused_reward_pipeline.cu`: #define + ISV read + Phase 5 gate + Phase 5b gate | +| 4 | +3 | `integrated.rs::build_diag_value`: 1 new leaf under `rewards` | +| 5 | +1 | `tests/eval_diag_emission.rs`: EXPECTED_LEAVES 678 → 679 | +| 6 | +100 | `tests/reward_alignment_invariants.rs`: 3 invariants | +| 7 | — | build release; local smoke; invariant test | +| 8 | — | atomic commit on `ml-alpha-regime-observer` | + +Total: **~135 LOC across 5 files + 1 test**. + +## 5. Risks + +- **Risk A (HIGH)**: 12 adaptive controllers (reward_scale, reward_clamp_win/loss, popart σ_welford, etc.) will rebalance under the new sparse-reward distribution. Expected transient: ~500-3000 steps, dominated by `RL_REWARD_CLAMP_V_BOUND_EWMA_ALPHA = 0.001` (τ=1000 steps; ~3τ to 95% convergence). Mitigation: §2 robustness clause extends verdict window to steps 3000-4000 if clamp bounds haven't stabilized by step 2000. If controllers diverge or oscillate, diag will show; may require explicit bootstrap of those controllers to known-good mode=1 values in a follow-up. +- **Risk B (HIGH unmitigated)**: popart eval-boundary shock per `pearl_popart_reset_at_eval_boundary_shocks_normalization` is NOT fixed in current code. Mode=1 inherits this bug. Eval-side Pearson target is relaxed to 0.5 (not 0.7). Train-side is primary; eval-side improvement requires the separate popart-preservation fix. +- **Risk C (MEDIUM)**: Removing entry_cost may cause the agent to learn "free entries are good" → open spuriously. Mitigation: realized_pnl_delta naturally penalizes round-trips (spread + fees if LobSim accounts for them). If wr trajectory shows hyperactivity post-fix, re-introduce entry_cost or add a fixed open-event penalty. +- **Risk D (MEDIUM)**: Sparse reward (zero most steps) means V regression learns target ≈ γ × V_{t+1} chained until close-event. With γ=0.99 and 30-step hold, credit assignment to position-open step is attenuated by γ^30 = 0.74. Standard sparse-RL convergence; foxhunt uses 1-step TD bootstrap per `compute_advantage_return.cu` so this is the standard sparse-pnl learning regime. If learning is too slow at 2k smoke, extend to 5k or 10k cluster. +- **Risk E (LOW)**: 1-step TD is the actual algorithm (NOT GAE as my v3 claimed). With sparse rewards 1-step TD has known longer mixing time vs GAE-λ. Mitigation: standard, not problematic at 2k+ training steps. +- **Risk F (LOW)**: outcome_ema in Phase 7 reads `r` after Phase 5+5b. In mode=1 r = realized_pnl_delta, so outcome_ema = sign(realized_pnl_delta) which is the canonical correct signal. No bias introduced (reviewer H6 resolved by dropping entry_cost in mode=1). + +## 6. Deletion timeline (the "permanent redesign" commitment) + +This is a permanent reward redesign, not a permanent ISV toggle. + +- **Commit A** (this spec): ship mode=1 default + mode=0 legacy path. +- **Cluster validation V2 + V3**: mode=1 produces verdict; mode=0 confirms baseline reproduction. +- **Commit B** (follow-up, 24-48 hours after V2/V3): if mode=1 succeeded → delete the mode=0 branch entirely from the kernel + delete `RL_REWARD_PURE_PNL_MODE_INDEX` + reduce ISV slot count back. This satisfies `feedback_no_feature_flags` + `feedback_single_source_of_truth_no_duplicates`. +- **If V2 falsified**: mode=0 is the surviving path; revert this commit (`git revert `). + +The ISV-toggle pattern is here ONLY for the regression run (V3). Not a permanent feature flag. + +## 7. What this is NOT + +- NOT Phase 2 of edge-decay (deferred until reward is aligned) +- NOT architecture simplification (separate spec, conditional on this one's outcome) +- NOT another controller (zero new EMAs, no new feedback loops) +- NOT a permanent toggle (mode=0 path deletes after V2/V3 verdict) + +## 8. Why this is the essential 65th-commit fix + +**Data**: 2 cluster runs at b=1024 × 20k+5k steps. Pearson(rewards.sum, Δrealized_pnl) = 0.16 train / 0.31 eval. wr_max = 0.418 momentary, wr_final = 0.343 — architecture finds patterns then un-learns them. + +**Kernel**: Phase 5 step 4 adds `hold_bonus × √hold_time = 2.0 × 10 = +20 per held step` for 100-step holds. Realized close pnl ≈ +$10 ≈ +0.5 σ-units. **Domination ratio ~40× per step, ~2600× per held trade.** + +**Mechanism**: V regression learns to predict the inflated return target; close-event advantages become small or wrong-signed; PPO and Q distill un-learn closing actions on actual edge. + +**Fix**: ISV-gated single branch removing Phase 5 steps 1-4 (legacy preserved for regression). 20-line kernel modification. + +**Falsification**: Pearson(rewards.sum, Δrealized_pnl) ≥ 0.7 in train steps 1500-2000 (post-controller-rebalance transient). Sign agreement ≥ 0.65. wr ≥ 0.30 at step 2000. Cluster smoke produces verdict. + +**Abandonment**: 1 retry with R-multiple variant. If still < 0.7, declare reward path closed and proceed to architecture simplification. + +**Deletion**: mode=0 branch deleted within 24-48 hours of V2/V3 verdict per `feedback_no_feature_flags`. + +## 9. Linked memory + +- [[pearl_reward_signal_anti_aligned_with_pnl]] — data finding this spec acts on (updated to reference `realized_pnl_cum_usd` as clean reference) +- [[pearl_popart_reset_at_eval_boundary_shocks_normalization]] — Risk B (unmitigated; eval Pearson target relaxed accordingly) +- [[feedback_no_feature_flags]] — satisfied via §6 deletion timeline +- [[feedback_single_source_of_truth_no_duplicates]] — same +- [[feedback_going_in_circles_pattern]] — this spec is the antidote: load-bearing root cause from existing data, surgical fix, predetermined falsification with abandonment timeline +- [[pearl_edge_decay_detector_phase1_validated_train_also_decays]] — explains why train ph_mean rises: shaped reward un-trains profitable patterns continuously during training +- [[feedback_trust_code_not_docs]] — v3 spec failed this by inventing magnitudes; v4 verified actual bootstrap values