diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs index 72aea4666..059219c29 100644 --- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs @@ -903,7 +903,7 @@ const ISV_NETWORK_DIM: usize = 23; /// (shifted 112→116 in Plan 4 Task 6 Commit A). /// Written by the constructor; checked at checkpoint load. Fail-fast only — no migration /// path exists. See spec §4.A.2 and `LAYOUT_FINGERPRINT_CURRENT` for structural-hash rationale. -pub(crate) const ISV_TOTAL_DIM: usize = 340; // SP5 + Layer D D1+D2+D3 + SP7 + SP8 + SP9 + SP10: 173 + 167 (164 SP5 slots @ 174..278/280..340, with 2-slot gap before cross-fold-persistent Kelly block; Layer D D1 PnL outputs at [286..290); Layer D D2 health composition outputs at [290..294); Layer D D3 training metrics EMA at [294..297); SP7 T1 loss-balance Wiener stats at [297..313); SP7 activation-flag fix per-(head,branch) flags at [313..321); SP8 Fix 36 train_active_frac canary @ [321..322) + LB_MAX_BUDGET per-(head,branch) at [322..330); SP9 Fix 37 Kelly warmup floor at [330..331) + Q_VAR_MAG_EMA at [331..332) + INTENT_EVAL_DIVERGENCE at [332..333) + 3 EMA targets at [333..336) + 3 eval_dist mag bins at [336..339); SP10 Fix 38 EVAL_THOMPSON_TEMP at [339..340)) +pub(crate) const ISV_TOTAL_DIM: usize = 360; // SP5 + Layer D D1+D2+D3 + SP7 + SP8 + SP9 + SP10 + SP11: 173 + 187 (164 SP5 slots @ 174..278/280..340, with 2-slot gap before cross-fold-persistent Kelly block; Layer D D1 PnL outputs at [286..290); Layer D D2 health composition outputs at [290..294); Layer D D3 training metrics EMA at [294..297); SP7 T1 loss-balance Wiener stats at [297..313); SP7 activation-flag fix per-(head,branch) flags at [313..321); SP8 Fix 36 train_active_frac canary @ [321..322) + LB_MAX_BUDGET per-(head,branch) at [322..330); SP9 Fix 37 Kelly warmup floor at [330..331) + Q_VAR_MAG_EMA at [331..332) + INTENT_EVAL_DIVERGENCE at [332..333) + 3 EMA targets at [333..336) + 3 eval_dist mag bins at [336..339); SP10 Fix 38 EVAL_THOMPSON_TEMP at [339..340); SP11 Fix 39 reward-subsystem controller at [340..360) — 6 component weights [340..346) + 4 controller scalar outputs [346..350) + 2 val-sharpe canaries [350..352) + 6 mag-ratio canaries [352..358) + saboteur engagement [358] + PnL magnitude EMA [359]) /// Legacy alias preserved for call sites that haven't been audited for the /// network-vs-total split. New code should pick `ISV_NETWORK_DIM` (for weight /// tensor sizing) or `ISV_TOTAL_DIM` (for the broadcast bus buffer). @@ -1838,7 +1838,14 @@ const fn layout_fingerprint_seed() -> &'static [u8] { KELLY_DIVERGENCE_TARGET=334;KELLY_TEMPORAL_TARGET=335;\ EVAL_DIST_Q=336;EVAL_DIST_H=337;EVAL_DIST_F=338;\ EVAL_THOMPSON_TEMP=339;\ - ISV_TOTAL_DIM=340;\ + REWARD_POPART_WEIGHT=340;REWARD_CF_WEIGHT=341;REWARD_TRAIL_WEIGHT=342;\ + REWARD_MICRO_WEIGHT=343;REWARD_OPP_COST_WEIGHT=344;REWARD_BONUS_WEIGHT=345;\ + CURIOSITY_PRESSURE=346;SABOTEUR_INTENSITY_MULT=347;\ + REWARD_WEIGHT_FLOOR=348;CURIOSITY_BOUND=349;\ + VAL_SHARPE_DELTA_EMA=350;VAL_SHARPE_VAR_EMA=351;\ + REWARD_COMPONENT_MAG_RATIO_BASE=352;\ + SABOTEUR_ENGAGEMENT_RATE=358;PNL_REWARD_MAGNITUDE_EMA=359;\ + ISV_TOTAL_DIM=360;\ PARAM_W_A_H_S1=0;PARAM_B_A_H_S1=1;PARAM_W_B_H_S1=2;PARAM_B_B_H_S1=3;\ PARAM_W_RESIDUAL_H_S1=4;PARAM_GAMMA_H_S1=5;PARAM_BETA_H_S1=6;\ PARAM_W_A_H_S2=7;PARAM_B_A_H_S2=8;PARAM_W_B_H_S2=9;PARAM_B_B_H_S2=10;\ diff --git a/crates/ml/src/cuda_pipeline/mod.rs b/crates/ml/src/cuda_pipeline/mod.rs index f853603eb..414f51a3d 100644 --- a/crates/ml/src/cuda_pipeline/mod.rs +++ b/crates/ml/src/cuda_pipeline/mod.rs @@ -60,6 +60,7 @@ pub mod q_snapshot; pub mod meta_q_network; pub mod sp4_isv_slots; pub mod sp5_isv_slots; +pub mod sp11_isv_slots; pub use sp4_isv_slots::{ TARGET_Q_BOUND_INDEX, ATOM_POS_BOUND_BASE, WEIGHT_BOUND_BASE, ADAM_M_BOUND_BASE, ADAM_V_BOUND_BASE, WD_RATE_BASE, diff --git a/crates/ml/src/cuda_pipeline/sp11_isv_slots.rs b/crates/ml/src/cuda_pipeline/sp11_isv_slots.rs new file mode 100644 index 000000000..5e203f947 --- /dev/null +++ b/crates/ml/src/cuda_pipeline/sp11_isv_slots.rs @@ -0,0 +1,73 @@ +//! SP11 — Reward-subsystem controller ISV slot constants. +//! +//! Slots [340..360) populated by the SP11 controller chain: +//! - 6 component weights [340..346) +//! - 4 controller scalar outputs [346..350) +//! - 2 val-sharpe canaries [350..352) +//! - 6 mag-ratio canaries [352..358) +//! - 1 saboteur engagement (358) +//! - 1 PnL magnitude EMA (359) +//! +//! Spec: docs/superpowers/specs/2026-05-04-sp11-reward-as-controlled-subsystem.md +//! Pearl: pearl_reward_as_controlled_subsystem.md + +// Component weights (controller outputs, consumed by reward shaping kernels) +pub const REWARD_POPART_WEIGHT_INDEX: usize = 340; +pub const REWARD_CF_WEIGHT_INDEX: usize = 341; +pub const REWARD_TRAIL_WEIGHT_INDEX: usize = 342; +pub const REWARD_MICRO_WEIGHT_INDEX: usize = 343; +pub const REWARD_OPP_COST_WEIGHT_INDEX: usize = 344; +pub const REWARD_BONUS_WEIGHT_INDEX: usize = 345; + +// Controller scalar outputs +pub const CURIOSITY_PRESSURE_INDEX: usize = 346; +pub const SABOTEUR_INTENSITY_MULT_INDEX: usize = 347; +pub const REWARD_WEIGHT_FLOOR_INDEX: usize = 348; +pub const CURIOSITY_BOUND_INDEX: usize = 349; + +// Canaries — val-sharpe trend + variance +pub const VAL_SHARPE_DELTA_EMA_INDEX: usize = 350; +pub const VAL_SHARPE_VAR_EMA_INDEX: usize = 351; + +// Canaries — per-component reward-magnitude ratios (6 contiguous slots). +// Source: REWARD_POPART_EMA_INDEX..+6 (existing per-component reward +// magnitude EMAs from reward_component_ema_kernel). Spec §3.3.1 said +// "grad EMA" but the SP4 grad-balancer is per-branch (4 slots), not +// per-reward-component. Per `feedback_trust_code_not_docs`: code wins, +// terminology corrected. Semantics for the controller are equivalent — +// "which component contributes most to total reward" maps the same way +// in the winner_weight / diversifier_weight blend. +pub const REWARD_COMPONENT_MAG_RATIO_BASE: usize = 352; +pub const REWARD_COMPONENT_MAG_RATIO_COUNT: usize = 6; + +// Canaries — saboteur + PnL signal scale +pub const SABOTEUR_ENGAGEMENT_RATE_INDEX: usize = 358; +pub const PNL_REWARD_MAGNITUDE_EMA_INDEX: usize = 359; + +// Invariant-1 anchors (rate-limiters, NOT regime thresholds). +// +// Per `feedback_isv_for_adaptive_bounds`: every adaptive bound lives on +// ISV. The constants below are the few exceptions allowed by Invariant-1 +// — they are numerical-stability anchors (epsilon floors), structural +// fractions of signal-driven quantities (curiosity-bound = 0.3 × PnL EMA), +// and rate-limiter clamps that bound the saboteur multiplier output. Each +// is documented in the SP11 spec §3.4.2 (Invariant-1 carve-out table). +pub const SP11_EPS_DIV: f32 = 1e-6; +pub const SP11_WEIGHT_HARD_FLOOR: f32 = 0.01; +pub const SP11_SABOTEUR_MIN: f32 = 0.5; +pub const SP11_SABOTEUR_MAX: f32 = 2.0; +pub const SP11_CURIOSITY_PERMANENT_FRACTION: f32 = 0.2; +pub const SP11_CURIOSITY_BOUND_FRACTION: f32 = 0.3; +pub const SP11_WEIGHT_FLOOR_FRACTION: f32 = 0.5; +pub const SP11_ENGAGEMENT_FLOOR: f32 = 0.1; + +pub const SP11_SLOT_BASE: usize = 340; +pub const SP11_SLOT_END: usize = 360; +pub const SP11_PRODUCER_COUNT: usize = SP11_SLOT_END - SP11_SLOT_BASE; + +/// Salt added to `config.seed` when seeding the novelty SimHash projection +/// matrix init kernel. Co-located here so the seed-derivation path is +/// reproducible from a single source of truth. Consumed by the launcher +/// `launch_novelty_simhash_proj_init` (added in Task A2 alongside the +/// projection buffer). +pub const SP11_PROJ_SEED_SALT: u64 = 0x_5511_0001; diff --git a/crates/ml/src/cuda_pipeline/sp5_isv_slots.rs b/crates/ml/src/cuda_pipeline/sp5_isv_slots.rs index dad556541..0c499449b 100644 --- a/crates/ml/src/cuda_pipeline/sp5_isv_slots.rs +++ b/crates/ml/src/cuda_pipeline/sp5_isv_slots.rs @@ -18,8 +18,15 @@ //! 321..330 SP8 (Fix 36) train_active_frac canary + LB_MAX_BUDGET (9 slots, fold-reset) //! 330..339 SP9 (Fix 37) Kelly warmup floor + EMA targets + eval_dist (9 slots, fold-reset) //! 339..340 SP10 (Fix 38) eval Thompson temperature (1 slot, fold-reset) +//! 340..360 SP11 (Fix 39) reward-subsystem controller (20 slots, fold-reset) — see sp11_isv_slots //! -//! Total: 164 new SP5 ISV slots (52 + 24 + 20 + 4 + 4 + 6 + 4 + 4 + 3 + 16 + 8 + 9 + 9 + 1). +//! Total: 184 new SP5 ISV slots (52 + 24 + 20 + 4 + 4 + 6 + 4 + 4 + 3 + 16 + 8 + 9 + 9 + 1 + 20). + +// Re-export SP11 slot constants so existing consumers can import every +// slot constant from a single location (`crate::cuda_pipeline::sp5_isv_slots::*`). +// SP11 owns its file but participates in the SP5 linear-span window +// (SP5_SLOT_BASE..SP5_SLOT_END covers all SP5/SP7/SP8/SP9/SP10/SP11 slots). +pub use crate::cuda_pipeline::sp11_isv_slots::*; pub const SP5_SLOT_BASE: usize = 174; @@ -281,7 +288,21 @@ pub const EVAL_DIST_F_INDEX: usize = 338; // [1] eval-side Full m // from the first observation. pub const EVAL_THOMPSON_TEMP_INDEX: usize = 339; // [1] eval Thompson selector temperature -pub const SP5_SLOT_END: usize = 340; +// ── SP11 (Fix 39, 2026-05-04): reward-subsystem controller ──────────── +// +// Per `pearl_reward_as_controlled_subsystem.md` + `pearl_controller_anchors_isv_driven.md`: +// the reward subsystem is migrated from a fixed-weight composition (host-side +// constants in `unified_reward_kernel.cu` and friends) to a fully ISV-driven +// controller producing 6 component weights + 4 controller scalars + 10 canaries. +// +// All 20 slot constants live in `crate::cuda_pipeline::sp11_isv_slots` +// (re-exported above). They occupy ISV[340..360) and are FoldReset (sentinel 0; +// Pearl A bootstrap on first observation). SP5_SLOT_END bumps 340 → 360 so the +// linear-span window covers the new block; the wiener_state_buf lockstep is +// handled by the existing bulk memset of `wiener_state_buf` covered by the +// `sp4_wiener_state` registry entry's dispatch arm. + +pub const SP5_SLOT_END: usize = 360; /// Wiener-buffer producer-count constant. Sizes `wiener_state_buf` via /// `(SP4_PRODUCER_COUNT + SP5_PRODUCER_COUNT) * SP4_WIENER_FLOATS_PER_SLOT`. @@ -359,9 +380,20 @@ pub const SP5_SLOT_END: usize = 340; /// the existing `intent_eval_divergence_compute_kernel` (single producer, /// two outputs); FoldReset (sentinel 0; Pearl A bootstrap on first /// observation). -pub const SP5_PRODUCER_COUNT: usize = 166; -// linear span = SP5_SLOT_END - SP5_SLOT_BASE = 340 - 174 = 166 wiener triples -// unique-slot count = 164 (52 per-branch + 24 Adam + 20 IQN τ + 4 trail +/// +/// SP11 (2026-05-04, Fix 39): allocates 20 new SP11 ISV slots at +/// ISV[340..360) — 6 reward component weights + 4 controller scalar +/// outputs + 10 canaries (val-sharpe Δ/var, 6 mag-ratios, saboteur +/// engagement, PnL EMA). Unique-slot count grows 164 → 184; the linear +/// span (and `SP5_PRODUCER_COUNT`) grows 166 → 186. Drives the reward +/// subsystem controller per `pearl_reward_as_controlled_subsystem.md` + +/// `pearl_controller_anchors_isv_driven.md`. All 20 slots are FoldReset +/// (sentinel 0; Pearl A bootstrap on first observation). Constants for +/// the SP11 block live in `crate::cuda_pipeline::sp11_isv_slots` and are +/// re-exported above. +pub const SP5_PRODUCER_COUNT: usize = 186; +// linear span = SP5_SLOT_END - SP5_SLOT_BASE = 360 - 174 = 186 wiener triples +// unique-slot count = 184 (52 per-branch + 24 Adam + 20 IQN τ + 4 trail // + 4 num_atoms + 6 Kelly + 4 Layer D D1 PnL aggregation // + 4 Layer D D2 health composition // + 3 Layer D D3 training metrics EMA @@ -370,7 +402,8 @@ pub const SP5_PRODUCER_COUNT: usize = 166; // + 1 SP8 train_active_frac canary // + 8 SP8 LB_MAX_BUDGET per-(head,branch) // + 9 SP9 Kelly cold-start warmup floor + targets + eval_dist -// + 1 SP10 eval Thompson temperature) +// + 1 SP10 eval Thompson temperature +// + 20 SP11 reward-subsystem controller) // ── Convenience accessors ──────────────────────────────────────────── #[inline] pub const fn atom_v_center(b: usize) -> usize { ATOM_V_CENTER_BASE + b } @@ -426,7 +459,14 @@ pub const SP5_LAYOUT_FINGERPRINT_FRAGMENT: &str = KELLY_DIVERGENCE_TARGET=334;KELLY_TEMPORAL_TARGET=335;\ EVAL_DIST_Q=336;EVAL_DIST_H=337;EVAL_DIST_F=338;\ EVAL_THOMPSON_TEMP=339;\ - ISV_TOTAL_DIM=340"; + REWARD_POPART_WEIGHT=340;REWARD_CF_WEIGHT=341;REWARD_TRAIL_WEIGHT=342;\ + REWARD_MICRO_WEIGHT=343;REWARD_OPP_COST_WEIGHT=344;REWARD_BONUS_WEIGHT=345;\ + CURIOSITY_PRESSURE=346;SABOTEUR_INTENSITY_MULT=347;\ + REWARD_WEIGHT_FLOOR=348;CURIOSITY_BOUND=349;\ + VAL_SHARPE_DELTA_EMA=350;VAL_SHARPE_VAR_EMA=351;\ + REWARD_COMPONENT_MAG_RATIO_BASE=352;\ + SABOTEUR_ENGAGEMENT_RATE=358;PNL_REWARD_MAGNITUDE_EMA=359;\ + ISV_TOTAL_DIM=360"; #[cfg(test)] mod tests { @@ -534,29 +574,34 @@ mod tests { // SP10 (Fix 38): eval Thompson selector temperature slots.insert(EVAL_THOMPSON_TEMP_INDEX); - // 1. Exactly 164 unique slots. - assert_eq!(slots.len(), 164, "expected 164 unique slots, got {}", slots.len()); + // SP11 (Fix 39): include the 20 new reward-subsystem controller slots. + for s in SP11_SLOT_BASE..SP11_SLOT_END { + slots.insert(s); + } + + // 1. Exactly 184 unique slots (164 pre-SP11 + 20 SP11). + assert_eq!(slots.len(), 184, "expected 184 unique slots, got {}", slots.len()); // 2. Min slot is SP5_SLOT_BASE = 174. assert_eq!(*slots.iter().min().unwrap(), 174); - // 3. Max slot is SP5_SLOT_END - 1 = 339. - assert_eq!(*slots.iter().max().unwrap(), 339); + // 3. Max slot is SP5_SLOT_END - 1 = 359. + assert_eq!(*slots.iter().max().unwrap(), 359); // 4. Intentional carve-out gap (278, 279) is absent. assert!(!slots.contains(&278), "slot 278 must be absent (carve-out gap)"); assert!(!slots.contains(&279), "slot 279 must be absent (carve-out gap)"); - // 5. Set equals {174..278} ∪ {280..340} — no holes (other than the + // 5. Set equals {174..278} ∪ {280..360} — no holes (other than the // carve-out gap 278..280), no overlaps. Layer D D1 extended the // upper end 286 → 290; D2 extended it 290 → 294; D3 extends it // 294 → 297; SP7 T1 extended it 297 → 313; SP7 activation-flag // fix extends it 313 → 321; SP8 (Fix 36) extends it 321 → 330; // SP9 (Fix 37) extends it 330 → 339; SP10 (Fix 38) extends it - // 339 → 340. - let expected: HashSet = (174..278).chain(280..340).collect(); + // 339 → 340; SP11 (Fix 39) extends it 340 → 360. + let expected: HashSet = (174..278).chain(280..360).collect(); assert_eq!(slots, expected, - "slot set does not match expected {{174..278}} ∪ {{280..340}}"); + "slot set does not match expected {{174..278}} ∪ {{280..360}}"); } #[test] @@ -583,11 +628,11 @@ mod tests { // is cross-fold-persistent — the contracts must remain disjoint). assert!(PNL_TOTAL_INDEX > LOSS_RATE_SMOOTH_INDEX); // SP5_SLOT_END must reflect the post-SP8 end-of-block. - assert_eq!(SP5_SLOT_END, 340); + assert_eq!(SP5_SLOT_END, 360); // SP5_PRODUCER_COUNT is the wiener-buffer linear span (slot-range // width including the 2-slot carve-out gap), NOT the unique-slot // count. See SP5_PRODUCER_COUNT docstring for the rationale. - assert_eq!(SP5_PRODUCER_COUNT, 166); + assert_eq!(SP5_PRODUCER_COUNT, 186); assert_eq!(SP5_PRODUCER_COUNT, SP5_SLOT_END - SP5_SLOT_BASE); } @@ -606,7 +651,7 @@ mod tests { // 4-slot block is internally contiguous. assert_eq!(GRAD_NORM_NORM_INDEX - HEALTH_SCORE_INDEX, 3); // SP5_SLOT_END must reflect the post-SP8 end-of-block. - assert_eq!(SP5_SLOT_END, 340); + assert_eq!(SP5_SLOT_END, 360); // SP5_PRODUCER_COUNT linear-span check matches the new end. assert_eq!(SP5_PRODUCER_COUNT, SP5_SLOT_END - SP5_SLOT_BASE); } @@ -627,12 +672,12 @@ mod tests { assert_eq!(MAX_DD_EMA_INDEX - TRAINING_SHARPE_EMA_INDEX, 1); assert_eq!(LOW_DD_RATIO_INDEX - MAX_DD_EMA_INDEX, 1); // SP5_SLOT_END must reflect the post-SP8 end-of-block. - assert_eq!(SP5_SLOT_END, 340); + assert_eq!(SP5_SLOT_END, 360); // SP5_PRODUCER_COUNT linear-span check matches the new end. The // wiener buffer must cover the entire linear span — including the // 2-slot carve-out gap (278..280) and the Pearl 6 reserved-but- // unused 6-float block (slots 280..286 don't call apply_pearls). - assert_eq!(SP5_PRODUCER_COUNT, 166); + assert_eq!(SP5_PRODUCER_COUNT, 186); assert_eq!(SP5_PRODUCER_COUNT, SP5_SLOT_END - SP5_SLOT_BASE); } @@ -652,7 +697,7 @@ mod tests { assert_eq!(lb_c51_active(b), LB_C51_ACTIVE_BASE + b); } // SP5_SLOT_END / SP5_PRODUCER_COUNT cover the new range. - assert_eq!(SP5_SLOT_END, 340); + assert_eq!(SP5_SLOT_END, 360); assert_eq!(SP5_PRODUCER_COUNT, SP5_SLOT_END - SP5_SLOT_BASE); } @@ -672,7 +717,7 @@ mod tests { assert_eq!(lb_max_budget_c51(b), LB_MAX_BUDGET_C51_BASE + b); } // Layout: 1 + 4 + 4 = 9 contiguous slots (321..330). - assert_eq!(SP5_SLOT_END, 340); + assert_eq!(SP5_SLOT_END, 360); assert_eq!(SP5_PRODUCER_COUNT, SP5_SLOT_END - SP5_SLOT_BASE); } @@ -705,7 +750,7 @@ mod tests { // Strictly above the SP8 MAX_BUDGET block. assert!(KELLY_WARMUP_FLOOR_INDEX > LB_MAX_BUDGET_C51_BASE + 3); // Block end at ISV[340) post-SP10 (Fix 38). - assert_eq!(SP5_SLOT_END, 340); + assert_eq!(SP5_SLOT_END, 360); // Linear span matches producer count (no gaps in SP9 block). assert_eq!(SP5_PRODUCER_COUNT, SP5_SLOT_END - SP5_SLOT_BASE); } @@ -719,9 +764,9 @@ mod tests { // Strictly above the SP9 eval_dist block (last entry at 338). assert!(EVAL_THOMPSON_TEMP_INDEX > EVAL_DIST_F_INDEX); // SP5_SLOT_END reflects the post-SP10 end-of-block. - assert_eq!(SP5_SLOT_END, 340); + assert_eq!(SP5_SLOT_END, 360); // Linear span matches producer count (single new slot at top). assert_eq!(SP5_PRODUCER_COUNT, SP5_SLOT_END - SP5_SLOT_BASE); - assert_eq!(SP5_PRODUCER_COUNT, 166); + assert_eq!(SP5_PRODUCER_COUNT, 186); } } diff --git a/crates/ml/src/trainers/dqn/state_reset_registry.rs b/crates/ml/src/trainers/dqn/state_reset_registry.rs index 6d8f5a24a..dadae205d 100644 --- a/crates/ml/src/trainers/dqn/state_reset_registry.rs +++ b/crates/ml/src/trainers/dqn/state_reset_registry.rs @@ -765,6 +765,121 @@ impl StateResetRegistry { category: ResetCategory::FoldReset, description: "ISV[EVAL_THOMPSON_TEMP_INDEX=339] — SP10 Fix 38 eval Thompson selector temperature. Produced as a side-output of `intent_eval_divergence_compute_kernel` (`temp = clamp(divergence/divergence_target, 0.5, 2.0)`). Consumed in `experience_action_select` direction-branch unconditionally (training AND eval) for the temperature-blended Thompson sample `q_eff[d] = E[Q][d] + temp · (q_sample[d] − E[Q][d])`. MIN_TEMP=0.5 enforces permanent stochasticity per `pearl_blend_formulas_must_have_permanent_floor.md`. Eliminates the val-Flat-collapse pathology (T10 train-multi-seed-khr7c on `8a25b330f`: dir_entropy=0, trade_count=1 in 214k bars) caused by the prior `eval_mode ⇒ argmax(E[Q])` selector branch. FoldReset sentinel 0; Pearl A bootstraps from the first observation; the consumer's defensive clamp keeps the selector at MIN_TEMP between fold-reset and first producer launch.", }, + // ── SP11 (Fix 39, 2026-05-04): reward-subsystem controller ───── + // + // Per `pearl_reward_as_controlled_subsystem.md` + + // `pearl_controller_anchors_isv_driven.md`: 20 new ISV slots at + // [340..360) populated by the SP11 controller chain — 6 component + // weights + 4 controller scalar outputs + 10 canaries (val-sharpe + // delta/var, 6 mag-ratios, saboteur engagement, PnL EMA). All + // slots are FoldReset sentinel 0; Pearl A bootstraps from the + // first producer launch on the new fold (per + // `pearl_first_observation_bootstrap.md`). Producers and + // consumers are introduced in subsequent A1/A2/B tasks; A0 is + // additive — no consumer reads any of these slots yet, so the + // trainer continues to behave identically. + // + // The 20 entries below mirror the registry layout established by + // the SP9/SP10 entries above (one entry per uniquely-zeroed slot + // / per-related-block). The 6 mag-ratio canaries occupy a + // contiguous 6-slot block at [352..358) and share a single + // entry — the dispatch arm zeroes the whole range with one loop, + // matching the `sp5_pnl_aggregation`/`sp5_health_composition` + // pattern. + // + // Wiener-state companion: the bulk memset of `wiener_state_buf` + // covered by the `sp4_wiener_state` registry entry's dispatch + // arm includes the new SP11 triples (the buffer linear span + // grew from `(SP4_PRODUCER_COUNT + 166) × 3` to + // `(SP4_PRODUCER_COUNT + 186) × 3` = +60 floats). Both halves + // of the Pearl A sentinel contract reset together per + // `feedback_no_partial_refactor.md`. + // + // The novelty-hash device buffer (`novelty_hash_buf`) is NOT an + // ISV slot but a separate device-side hash table sized for ~1M + // 16-bit codes. Its registry entry + dispatch arm + // (`fill_zero_async` at fold boundary) lands in Task A2 + // alongside the buffer field's introduction on `GpuDqnTrainer`. + // Adding the entry here would be an orphan against a + // non-existent buffer field, violating + // `feedback_wire_everything_up.md`. A2 owns the contract end- + // to-end: buffer field, registry entry, dispatch arm, and + // launcher invocation in the same commit. + RegistryEntry { + name: "sp11_reward_popart_weight", + category: ResetCategory::FoldReset, + description: "ISV[REWARD_POPART_WEIGHT_INDEX=340] — SP11 Fix 39 PopArt reward component weight. Produced by `reward_subsystem_controller_kernel` from improvement_z + per-component mag-ratio canaries (winner/diversifier blend). Consumed in unified-reward composition (Layer B). FoldReset sentinel 0; Pearl A bootstraps from the first controller emit (per `pearl_first_observation_bootstrap.md`). Spec: docs/superpowers/specs/2026-05-04-sp11-reward-as-controlled-subsystem.md §3.3.", + }, + RegistryEntry { + name: "sp11_reward_cf_weight", + category: ResetCategory::FoldReset, + description: "ISV[REWARD_CF_WEIGHT_INDEX=341] — SP11 Fix 39 counterfactual (CF) reward component weight. Produced by `reward_subsystem_controller_kernel`. Consumed in `mse_loss_kernel` and `c51_loss_kernel` (Layer B replaces the hardcoded `cf_weight=0.3f`). FoldReset sentinel 0; Pearl A bootstraps from the first controller emit (per `pearl_first_observation_bootstrap.md`). Spec §3.3.", + }, + RegistryEntry { + name: "sp11_reward_trail_weight", + category: ResetCategory::FoldReset, + description: "ISV[REWARD_TRAIL_WEIGHT_INDEX=342] — SP11 Fix 39 trail-stop reward component weight. Produced by `reward_subsystem_controller_kernel`. Consumed in trail-shaping reward path (Layer B). FoldReset sentinel 0; Pearl A bootstraps from the first controller emit. Spec §3.3.", + }, + RegistryEntry { + name: "sp11_reward_micro_weight", + category: ResetCategory::FoldReset, + description: "ISV[REWARD_MICRO_WEIGHT_INDEX=343] — SP11 Fix 39 microstructure reward component weight. Produced by `reward_subsystem_controller_kernel`. Consumed in microstructure-shaping reward path (Layer B). FoldReset sentinel 0; Pearl A bootstraps from the first controller emit. Spec §3.3.", + }, + RegistryEntry { + name: "sp11_reward_opp_cost_weight", + category: ResetCategory::FoldReset, + description: "ISV[REWARD_OPP_COST_WEIGHT_INDEX=344] — SP11 Fix 39 opportunity-cost reward component weight. Produced by `reward_subsystem_controller_kernel`. Consumed in opportunity-cost shaping reward path (Layer B). FoldReset sentinel 0; Pearl A bootstraps from the first controller emit. Spec §3.3.", + }, + RegistryEntry { + name: "sp11_reward_bonus_weight", + category: ResetCategory::FoldReset, + description: "ISV[REWARD_BONUS_WEIGHT_INDEX=345] — SP11 Fix 39 exploration-bonus reward component weight. Produced by `reward_subsystem_controller_kernel`. Consumed in bonus-shaping reward path (Layer B). FoldReset sentinel 0; Pearl A bootstraps from the first controller emit. Spec §3.3.", + }, + RegistryEntry { + name: "sp11_curiosity_pressure", + category: ResetCategory::FoldReset, + description: "ISV[CURIOSITY_PRESSURE_INDEX=346] — SP11 Fix 39 curiosity-pressure scalar. Produced by `reward_subsystem_controller_kernel` from val-sharpe trend + mag-ratio dispersion. Consumed at REPLAY-time (not collect-time) as `r_with_curiosity = r_base + curiosity_pressure · novelty_signal(state, action)`. Curiosity-bound capped at `SP11_CURIOSITY_BOUND_FRACTION=0.3 × pnl_reward_magnitude_ema`, with a permanent-fraction floor of `SP11_CURIOSITY_PERMANENT_FRACTION=0.2 × bound` per `pearl_blend_formulas_must_have_permanent_floor.md`. FoldReset sentinel 0; Pearl A bootstraps. Spec §3.3, §3.4.2.", + }, + RegistryEntry { + name: "sp11_saboteur_intensity_mult", + category: ResetCategory::FoldReset, + description: "ISV[SABOTEUR_INTENSITY_MULT_INDEX=347] — SP11 Fix 39 saboteur curriculum intensity multiplier. Produced by `reward_subsystem_controller_kernel` from saboteur_engagement_rate canary. Consumed in `gpu_experience_collector` (Layer B multiplies `saboteur_perturbation_scale` by this slot). Clamped to `[SP11_SABOTEUR_MIN=0.5, SP11_SABOTEUR_MAX=2.0]` (Invariant-1 rate-limiter anchor per `feedback_isv_for_adaptive_bounds.md` §3.4.2). FoldReset sentinel 0; Pearl A bootstraps. Spec §3.3, §3.4.2.", + }, + RegistryEntry { + name: "sp11_reward_weight_floor", + category: ResetCategory::FoldReset, + description: "ISV[REWARD_WEIGHT_FLOOR_INDEX=348] — SP11 Fix 39 controller-output minimum-weight floor (`SP11_WEIGHT_FLOOR_FRACTION=0.5 × hard_floor`). Produced by `reward_subsystem_controller_kernel`. Consumed by the controller itself (re-read on next step) to enforce that no reward component drops below `SP11_WEIGHT_HARD_FLOOR=0.01` after blending — Invariant-1 rate-limiter floor preventing winner-take-all collapse. FoldReset sentinel 0; Pearl A bootstraps. Spec §3.4.2.", + }, + RegistryEntry { + name: "sp11_curiosity_bound", + category: ResetCategory::FoldReset, + description: "ISV[CURIOSITY_BOUND_INDEX=349] — SP11 Fix 39 curiosity-pressure structural cap (`SP11_CURIOSITY_BOUND_FRACTION=0.3 × pnl_reward_magnitude_ema`). Produced by `reward_subsystem_controller_kernel`. Consumed at the curiosity-pressure clamp — keeps curiosity scaled to the run's PnL magnitude rather than a tuned constant per `feedback_isv_for_adaptive_bounds.md`. FoldReset sentinel 0; Pearl A bootstraps. Spec §3.4.2.", + }, + RegistryEntry { + name: "sp11_val_sharpe_delta_ema", + category: ResetCategory::FoldReset, + description: "ISV[VAL_SHARPE_DELTA_EMA_INDEX=350] — SP11 Fix 39 val-sharpe trend canary. Produced by `val_sharpe_delta_compute_kernel` (two-pass delta_ema + delta_var_ema, A1). Smoothed by Pearls A+D via chained `apply_pearls_ad_kernel` per `feedback_no_cpu_compute_strict.md`. Consumed by the SP11 controller as the improvement_z numerator. FoldReset sentinel 0; Pearl A bootstraps from the first val-sharpe sample of the new fold. Spec §3.3.1.", + }, + RegistryEntry { + name: "sp11_val_sharpe_var_ema", + category: ResetCategory::FoldReset, + description: "ISV[VAL_SHARPE_VAR_EMA_INDEX=351] — SP11 Fix 39 val-sharpe variance canary (squared-deviation EMA against the running delta_ema). Produced alongside `sp11_val_sharpe_delta_ema` by `val_sharpe_delta_compute_kernel` (A1). Smoothed by Pearls A+D. Consumed by the SP11 controller as the improvement_z denominator (`improvement_z = delta_ema / sqrt(var_ema + SP11_EPS_DIV)`). FoldReset sentinel 0; Pearl A bootstraps. Spec §3.3.1.", + }, + RegistryEntry { + name: "sp11_reward_component_mag_ratios", + category: ResetCategory::FoldReset, + description: "ISV[REWARD_COMPONENT_MAG_RATIO_BASE=352..358) — SP11 Fix 39 per-component reward-magnitude ratios (6 contiguous slots: popart, cf, trail, micro, opp_cost, bonus). Produced by `reward_component_mag_ratio_compute_kernel` (A1) reading existing per-component reward-magnitude EMAs (`REWARD_POPART_EMA_INDEX..+6`) and normalising to a unit simplex. Smoothed by Pearls A+D via chained `apply_pearls_ad_kernel`. Consumed by the SP11 controller for the winner_weight / diversifier_weight blend. The single registry entry covers the whole 6-slot block — the dispatch arm zeroes all 6 with one loop, matching the `sp5_pnl_aggregation` block-reset pattern. Per `feedback_trust_code_not_docs.md`: spec §3.3.1 said `grad EMA` but the SP4 grad-balancer is per-branch (4 slots), not per-reward-component — code wins, terminology corrected to `mag-ratio` (semantics for the controller are equivalent). FoldReset sentinel 0; Pearl A bootstraps. Spec §3.3.1.", + }, + RegistryEntry { + name: "sp11_saboteur_engagement_rate", + category: ResetCategory::FoldReset, + description: "ISV[SABOTEUR_ENGAGEMENT_RATE_INDEX=358] — SP11 Fix 39 saboteur engagement-rate canary (per-bar fraction of bars where the saboteur curriculum was actively perturbing experience tuples). Produced by `saboteur_engagement_compute_kernel` (A1) — block tree-reduce over the per-bar engagement flag, then EMA. Smoothed by Pearls A+D. Consumed by the SP11 controller to drive `saboteur_intensity_mult` toward the engagement floor (`SP11_ENGAGEMENT_FLOOR=0.1`). FoldReset sentinel 0; Pearl A bootstraps. Spec §3.3.1.", + }, + RegistryEntry { + name: "sp11_pnl_reward_magnitude_ema", + category: ResetCategory::FoldReset, + description: "ISV[PNL_REWARD_MAGNITUDE_EMA_INDEX=359] — SP11 Fix 39 PnL-reward magnitude EMA (running `|reward_total|` smoothed with Wiener-α). Produced by the unified-reward kernel chain (existing producer; SP11 wires the EMA output to this slot in A1 alongside the val-sharpe and engagement canaries). Consumed by the SP11 controller as the curiosity-bound scale factor (`curiosity_bound = SP11_CURIOSITY_BOUND_FRACTION=0.3 × this slot`) per `feedback_isv_for_adaptive_bounds.md` §3.4.2. FoldReset sentinel 0; Pearl A bootstraps. Spec §3.3.1, §3.4.2.", + }, // SP5 Task A1: Wiener-state companion reset. The wiener_state_buf // now covers SP4 (71 producers × 3 = 213 floats) + SP5 (110 × 3 = 330 // floats) = 543 floats total. The SP5 triples start at offset 213. diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 6cc55a6a6..d19912a68 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -6968,6 +6968,122 @@ impl DQNTrainer { fused.trainer().write_isv_signal_at(EVAL_THOMPSON_TEMP_INDEX, 0.0); } } + // ── SP11 (Fix 39, 2026-05-04): reward-subsystem controller ───── + // + // 15 registry entries covering 20 slots [340..360). All are + // FoldReset sentinel 0 so Pearl A's first-observation + // replacement fires on the new fold's first producer launch + // (per `pearl_first_observation_bootstrap.md`). Wiener-state + // companion is reset by the existing bulk memset of + // `wiener_state_buf` covered by the `sp4_wiener_state` + // registry entry's dispatch arm — `SP5_PRODUCER_COUNT` linear + // span grew 166 → 186, the buffer expanded in lockstep. + // + // No consumer reads any of these slots in A0 — the trainer + // continues to behave identically. Layer A1/A2 introduce the + // producer kernels and Layer B atomically migrates consumers. + "sp11_reward_popart_weight" => { + if let Some(ref fused) = self.fused_ctx { + use crate::cuda_pipeline::sp5_isv_slots::REWARD_POPART_WEIGHT_INDEX; + fused.trainer().write_isv_signal_at(REWARD_POPART_WEIGHT_INDEX, 0.0); + } + } + "sp11_reward_cf_weight" => { + if let Some(ref fused) = self.fused_ctx { + use crate::cuda_pipeline::sp5_isv_slots::REWARD_CF_WEIGHT_INDEX; + fused.trainer().write_isv_signal_at(REWARD_CF_WEIGHT_INDEX, 0.0); + } + } + "sp11_reward_trail_weight" => { + if let Some(ref fused) = self.fused_ctx { + use crate::cuda_pipeline::sp5_isv_slots::REWARD_TRAIL_WEIGHT_INDEX; + fused.trainer().write_isv_signal_at(REWARD_TRAIL_WEIGHT_INDEX, 0.0); + } + } + "sp11_reward_micro_weight" => { + if let Some(ref fused) = self.fused_ctx { + use crate::cuda_pipeline::sp5_isv_slots::REWARD_MICRO_WEIGHT_INDEX; + fused.trainer().write_isv_signal_at(REWARD_MICRO_WEIGHT_INDEX, 0.0); + } + } + "sp11_reward_opp_cost_weight" => { + if let Some(ref fused) = self.fused_ctx { + use crate::cuda_pipeline::sp5_isv_slots::REWARD_OPP_COST_WEIGHT_INDEX; + fused.trainer().write_isv_signal_at(REWARD_OPP_COST_WEIGHT_INDEX, 0.0); + } + } + "sp11_reward_bonus_weight" => { + if let Some(ref fused) = self.fused_ctx { + use crate::cuda_pipeline::sp5_isv_slots::REWARD_BONUS_WEIGHT_INDEX; + fused.trainer().write_isv_signal_at(REWARD_BONUS_WEIGHT_INDEX, 0.0); + } + } + "sp11_curiosity_pressure" => { + if let Some(ref fused) = self.fused_ctx { + use crate::cuda_pipeline::sp5_isv_slots::CURIOSITY_PRESSURE_INDEX; + fused.trainer().write_isv_signal_at(CURIOSITY_PRESSURE_INDEX, 0.0); + } + } + "sp11_saboteur_intensity_mult" => { + if let Some(ref fused) = self.fused_ctx { + use crate::cuda_pipeline::sp5_isv_slots::SABOTEUR_INTENSITY_MULT_INDEX; + fused.trainer().write_isv_signal_at(SABOTEUR_INTENSITY_MULT_INDEX, 0.0); + } + } + "sp11_reward_weight_floor" => { + if let Some(ref fused) = self.fused_ctx { + use crate::cuda_pipeline::sp5_isv_slots::REWARD_WEIGHT_FLOOR_INDEX; + fused.trainer().write_isv_signal_at(REWARD_WEIGHT_FLOOR_INDEX, 0.0); + } + } + "sp11_curiosity_bound" => { + if let Some(ref fused) = self.fused_ctx { + use crate::cuda_pipeline::sp5_isv_slots::CURIOSITY_BOUND_INDEX; + fused.trainer().write_isv_signal_at(CURIOSITY_BOUND_INDEX, 0.0); + } + } + "sp11_val_sharpe_delta_ema" => { + if let Some(ref fused) = self.fused_ctx { + use crate::cuda_pipeline::sp5_isv_slots::VAL_SHARPE_DELTA_EMA_INDEX; + fused.trainer().write_isv_signal_at(VAL_SHARPE_DELTA_EMA_INDEX, 0.0); + } + } + "sp11_val_sharpe_var_ema" => { + if let Some(ref fused) = self.fused_ctx { + use crate::cuda_pipeline::sp5_isv_slots::VAL_SHARPE_VAR_EMA_INDEX; + fused.trainer().write_isv_signal_at(VAL_SHARPE_VAR_EMA_INDEX, 0.0); + } + } + "sp11_reward_component_mag_ratios" => { + // Block-reset the 6 contiguous mag-ratio canaries at + // ISV[REWARD_COMPONENT_MAG_RATIO_BASE=352..358). Matches + // the `sp5_pnl_aggregation`/`sp5_health_composition` + // block-reset pattern (one registry entry, one dispatch + // arm, multi-slot zero-write). + if let Some(ref fused) = self.fused_ctx { + use crate::cuda_pipeline::sp5_isv_slots::{ + REWARD_COMPONENT_MAG_RATIO_BASE, REWARD_COMPONENT_MAG_RATIO_COUNT, + }; + for i in 0..REWARD_COMPONENT_MAG_RATIO_COUNT { + fused.trainer().write_isv_signal_at( + REWARD_COMPONENT_MAG_RATIO_BASE + i, + 0.0, + ); + } + } + } + "sp11_saboteur_engagement_rate" => { + if let Some(ref fused) = self.fused_ctx { + use crate::cuda_pipeline::sp5_isv_slots::SABOTEUR_ENGAGEMENT_RATE_INDEX; + fused.trainer().write_isv_signal_at(SABOTEUR_ENGAGEMENT_RATE_INDEX, 0.0); + } + } + "sp11_pnl_reward_magnitude_ema" => { + if let Some(ref fused) = self.fused_ctx { + use crate::cuda_pipeline::sp5_isv_slots::PNL_REWARD_MAGNITUDE_EMA_INDEX; + fused.trainer().write_isv_signal_at(PNL_REWARD_MAGNITUDE_EMA_INDEX, 0.0); + } + } "sp5_pnl_aggregation" => { // Layer D Task D1: zero ISV[286..290) — total/mean/var/max_dd — // so the new fold's first `launch_sp5_pnl_aggregation` triggers diff --git a/docs/isv-slots.md b/docs/isv-slots.md index 98ccf8624..c43996507 100644 --- a/docs/isv-slots.md +++ b/docs/isv-slots.md @@ -183,3 +183,50 @@ gradient contribution = `iqn_budget` (same as SP5 Layer B). The averaged **Files changed:** `crates/ml/src/cuda_pipeline/gpu_iqn_head.rs`, `crates/ml/src/trainers/dqn/fused_training.rs`. + +## SP11: Reward as controlled subsystem (Task A0 reservation) + +SP11 Task A0 (Fix 39, 2026-05-04) extends `ISV_TOTAL_DIM` from 340 → 360 by +allocating 20 contiguous slots at `[340..360)` for the reward-subsystem +controller. Constants live in `crates/ml/src/cuda_pipeline/sp11_isv_slots.rs` +and are re-exported via `cuda_pipeline::sp5_isv_slots::*`. **A0 is reservation- +only — no producer kernels or consumers are wired yet. All 20 slots remain +zero-initialized at construction and behave as no-ops until A1/A2/Layer-B +land producers and atomic consumer migration.** + +| Range | Name constant | Family | Purpose | +|---|---|---|---| +| [340] | `REWARD_POPART_WEIGHT_INDEX` | scalar | Component weight: PopArt reward | +| [341] | `REWARD_CF_WEIGHT_INDEX` | scalar | Component weight: counterfactual (replaces hardcoded `cf_weight=0.3f`) | +| [342] | `REWARD_TRAIL_WEIGHT_INDEX` | scalar | Component weight: trail-stop | +| [343] | `REWARD_MICRO_WEIGHT_INDEX` | scalar | Component weight: microstructure | +| [344] | `REWARD_OPP_COST_WEIGHT_INDEX` | scalar | Component weight: opportunity cost | +| [345] | `REWARD_BONUS_WEIGHT_INDEX` | scalar | Component weight: exploration bonus | +| [346] | `CURIOSITY_PRESSURE_INDEX` | scalar | Controller output: curiosity scale (replay-time bonus) | +| [347] | `SABOTEUR_INTENSITY_MULT_INDEX` | scalar | Controller output: saboteur intensity multiplier ∈ [0.5, 2.0] | +| [348] | `REWARD_WEIGHT_FLOOR_INDEX` | scalar | Controller output: minimum-weight floor | +| [349] | `CURIOSITY_BOUND_INDEX` | scalar | Controller output: curiosity hard cap (= 0.3 × pnl_magnitude_ema) | +| [350] | `VAL_SHARPE_DELTA_EMA_INDEX` | scalar canary | val-sharpe trend EMA (improvement_z numerator) | +| [351] | `VAL_SHARPE_VAR_EMA_INDEX` | scalar canary | val-sharpe variance EMA (improvement_z denominator) | +| [352..358) | `REWARD_COMPONENT_MAG_RATIO_BASE` (+6) | per-component canary | Per-component reward-magnitude ratio (winner/diversifier blend input) | +| [358] | `SABOTEUR_ENGAGEMENT_RATE_INDEX` | scalar canary | Per-bar saboteur engagement rate EMA | +| [359] | `PNL_REWARD_MAGNITUDE_EMA_INDEX` | scalar canary | `|reward_total|` EMA (curiosity-bound scale factor) | + +Invariant-1 fraction constants (rate-limiter anchors per spec §3.4.2): +`SP11_EPS_DIV=1e-6`, `SP11_WEIGHT_HARD_FLOOR=0.01`, +`SP11_SABOTEUR_MIN=0.5`, `SP11_SABOTEUR_MAX=2.0`, +`SP11_CURIOSITY_PERMANENT_FRACTION=0.2`, `SP11_CURIOSITY_BOUND_FRACTION=0.3`, +`SP11_WEIGHT_FLOOR_FRACTION=0.5`, `SP11_ENGAGEMENT_FLOOR=0.1`. Plus +`SP11_PROJ_SEED_SALT=0x_5511_0001` for the A2 novelty-SimHash projection-init +launcher's seed derivation. + +All 20 slots are FoldReset (sentinel 0; Pearl A bootstraps from first +producer launch on each fold per `pearl_first_observation_bootstrap.md`). +Producers and consumers wire in subsequent A1/A2/Layer-B tasks. The +novelty-hash device buffer (separate from the ISV bus, sized for ~1M 16-bit +codes) lands in A2 alongside its registry entry and dispatch arm — adding +the entry in A0 would orphan it against a non-existent buffer field, so +A2 owns the contract end-to-end. + +**Spec:** `docs/superpowers/specs/2026-05-04-sp11-reward-as-controlled-subsystem.md` +**Plan:** `docs/superpowers/plans/2026-05-04-sp11-reward-as-controlled-subsystem.md`