diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs index 2006c33ea..12fa6a182 100644 --- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs @@ -32102,6 +32102,8 @@ impl GpuDqnTrainer { fn launch_adam_update(&self) -> Result<(), MLError> { use crate::cuda_pipeline::sp4_isv_slots::{ ParamGroup as SP4Group, L1_LAMBDA_TRUNK_INDEX, SP4_ENGAGE_OFFSET_DISABLED, + SP4_ENGAGE_OFFSET_BRANCH_DIR, SP4_ENGAGE_OFFSET_BRANCH_MAG, + SP4_ENGAGE_OFFSET_BRANCH_ORDER, SP4_ENGAGE_OFFSET_BRANCH_URGENCY, }; use crate::cuda_pipeline::sp5_isv_slots::{ ADAM_BETA1_BASE, ADAM_BETA2_BASE, ADAM_EPS_BASE, @@ -32195,29 +32197,32 @@ impl GpuDqnTrainer { // (group_tag, byte_off, count, engage_offset, lr_scale). All // non-branch entries pass lr_scale=1.0 (no-op for trunk / value - // / trunk-extras / aux trainers). The 4 branch sub-launches use - // SP4_ENGAGE_OFFSET_DISABLED for Pearl C — splitting the single - // DqnBranches engagement-tracking buffer across 4 sub-launches - // would require adding 3 ParamGroup variants OR a sub-block - // offsetting scheme; both deferred to a Phase 4.5 follow-up - // commit. Per `feedback_no_partial_refactor` Pearl C is a - // diagnostic system (not a load-bearing controller), so its - // temporary unavailability for branches is acceptable as long - // as Trunk/Value/Trunk-extras Pearl C still fires. + // / trunk-extras / aux trainers). + // + // SP21 Phase 4.5 (2026-05-11): the 4 branch sub-launches now have + // their OWN distinct engage_offsets within `clamp_engage_per_block_buf` + // (Dir reuses canonical DqnBranches slot 2; Mag/Order/Urgency get + // extra slots 11/12/13 immediately after Curiosity's tail at 10). + // `pearl_c_post_adam_engagement_check(DqnBranches, ...)` aggregates + // all 4 sub-ranges into a single per-group rate-deficit EMA — + // semantically equivalent to the pre-Phase-4 single-launch + // tracking, and follows the same pattern Curiosity uses (4 + // sub-launches W1/b1/W2/b2 sharing one Pearl C bucket). let trunk_engage_off: i32 = (SP4Group::DqnTrunk.idx() * MAX_BLOCKS_PER_ADAM) as i32; let value_engage_off: i32 = (SP4Group::DqnValue.idx() * MAX_BLOCKS_PER_ADAM) as i32; let groups: [(SP4Group, u64, usize, i32, f32); 7] = [ - (SP4Group::DqnTrunk, trunk_byte_off, trunk_count, trunk_engage_off, 1.0), - (SP4Group::DqnValue, value_byte_off, value_count, value_engage_off, 1.0), - // SP21 Phase 4: 4 per-branch sub-launches. - (SP4Group::DqnBranches, b_dir_off, b_dir_count, SP4_ENGAGE_OFFSET_DISABLED, lr_scale_dir), - (SP4Group::DqnBranches, b_mag_off, b_mag_count, SP4_ENGAGE_OFFSET_DISABLED, lr_scale_mag), - (SP4Group::DqnBranches, b_order_off, b_order_count, SP4_ENGAGE_OFFSET_DISABLED, lr_scale_order), - (SP4Group::DqnBranches, b_urgency_off, b_urgency_count, SP4_ENGAGE_OFFSET_DISABLED, lr_scale_urgency), + (SP4Group::DqnTrunk, trunk_byte_off, trunk_count, trunk_engage_off, 1.0), + (SP4Group::DqnValue, value_byte_off, value_count, value_engage_off, 1.0), + // SP21 Phase 4: 4 per-branch sub-launches with per-branch + // lr_scale; Phase 4.5: per-branch Pearl C engagement offsets. + (SP4Group::DqnBranches, b_dir_off, b_dir_count, SP4_ENGAGE_OFFSET_BRANCH_DIR, lr_scale_dir), + (SP4Group::DqnBranches, b_mag_off, b_mag_count, SP4_ENGAGE_OFFSET_BRANCH_MAG, lr_scale_mag), + (SP4Group::DqnBranches, b_order_off, b_order_count, SP4_ENGAGE_OFFSET_BRANCH_ORDER, lr_scale_order), + (SP4Group::DqnBranches, b_urgency_off, b_urgency_count, SP4_ENGAGE_OFFSET_BRANCH_URGENCY, lr_scale_urgency), // Trunk-extras tail — shares DqnTrunk bounds; Pearl C disabled // for this sub-launch (engagement folds into trunk's // accounting since the same WEIGHT_BOUND drives both clamps). - (SP4Group::DqnTrunk, trunk_extras_byte_off, trunk_extras_count, SP4_ENGAGE_OFFSET_DISABLED, 1.0), + (SP4Group::DqnTrunk, trunk_extras_byte_off, trunk_extras_count, SP4_ENGAGE_OFFSET_DISABLED, 1.0), ]; // Coverage invariant: every of the 163 weight tensors must be @@ -32387,9 +32392,12 @@ impl GpuDqnTrainer { } let group_idx = group.idx(); - // Curiosity sums 4 sub-launch ranges (see launch_adam_step in - // gpu_curiosity_trainer.rs). All other groups read a single - // contiguous range starting at `group_idx × MAX_BLOCKS_PER_ADAM`. + // Curiosity sums 4 sub-launch ranges (W1/b1/W2/b2). SP21 Phase 4.5 + // (2026-05-11): DqnBranches now also sums 4 sub-launch ranges + // (Dir/Mag/Order/Urgency) — Dir at canonical group_idx=2 offset, + // Mag/Order/Urgency at extra slots 11/12/13. All other groups + // read a single contiguous range starting at + // `group_idx × MAX_BLOCKS_PER_ADAM`. let block_offsets: &[usize] = match group { ParamGroup::Curiosity => &[ 7 * MAX_BLOCKS_PER_ADAM, // W1 @@ -32397,6 +32405,12 @@ impl GpuDqnTrainer { 9 * MAX_BLOCKS_PER_ADAM, // W2 10 * MAX_BLOCKS_PER_ADAM, // b2 ], + ParamGroup::DqnBranches => &[ + 2 * MAX_BLOCKS_PER_ADAM, // Dir (canonical DqnBranches slot) + 11 * MAX_BLOCKS_PER_ADAM, // Mag (extra) + 12 * MAX_BLOCKS_PER_ADAM, // Order (extra) + 13 * MAX_BLOCKS_PER_ADAM, // Urgency (extra) + ], _ => &[ /* group_idx * MAX_BLOCKS_PER_ADAM — fall back to * runtime computation in the loop body. */ @@ -32411,9 +32425,10 @@ impl GpuDqnTrainer { // lifetime of the trainer. Reads are zero-copy via the device- // mapped page; the kernel writes were ordered by Adam-launch // synchronization upstream of this call. + let multi_range = matches!(group, ParamGroup::Curiosity | ParamGroup::DqnBranches); unsafe { let p = self.clamp_engage_per_block_buf.host_ptr as *const i32; - if matches!(group, ParamGroup::Curiosity) { + if multi_range { for &base in block_offsets { for b in 0..MAX_BLOCKS_PER_ADAM { total_engage += *p.add(base + b) as i64; @@ -32472,10 +32487,11 @@ impl GpuDqnTrainer { } // Zero per-block counts post-read so next step starts clean. - // Curiosity must zero all 4 sub-ranges; others zero one range. + // Curiosity + DqnBranches (SP21 Phase 4.5) zero all 4 sub-ranges; + // others zero one range. unsafe { let p = self.clamp_engage_per_block_buf.host_ptr as *mut i32; - if matches!(group, ParamGroup::Curiosity) { + if multi_range { for &base in block_offsets { std::ptr::write_bytes(p.add(base), 0, MAX_BLOCKS_PER_ADAM); } diff --git a/crates/ml/src/cuda_pipeline/sp4_isv_slots.rs b/crates/ml/src/cuda_pipeline/sp4_isv_slots.rs index 17575628b..df5591278 100644 --- a/crates/ml/src/cuda_pipeline/sp4_isv_slots.rs +++ b/crates/ml/src/cuda_pipeline/sp4_isv_slots.rs @@ -113,23 +113,48 @@ pub const MAX_BLOCKS_PER_ADAM: usize = 4096; /// collisions between sub-launches, each gets its own offset slot. /// Sub-launch 0 reuses Curiosity's main group offset (group_idx=7). /// Sub-launches 1/2/3 occupy extra slots — 3 extra logical "groups" -/// beyond the canonical 8. Total length: (8 + 3) × MAX_BLOCKS_PER_ADAM -/// = 11 × 4096 = 45056 i32s. Host-side +/// beyond the canonical 8. +/// +/// **SP21 Phase 4.5 (2026-05-11)**: DqnBranches gains 4 sub-launches +/// too (one per action-branch: Dir/Mag/Order/Urgency) for per-branch +/// LR scaling. Sub-launch 0 (Dir) reuses DqnBranches' main group +/// offset (group_idx=2); sub-launches 1/2/3 (Mag/Order/Urgency) +/// occupy 3 more extra slots beyond Curiosity's tail. Total length +/// becomes (8 + 3 [Curiosity] + 3 [Branches]) × MAX_BLOCKS_PER_ADAM +/// = 14 × 4096 = 57344 i32s. Host-side /// `pearl_c_post_adam_engagement_check` for `ParamGroup::Curiosity` -/// sums all 4 sub-launch ranges. +/// sums Curiosity's 4 sub-launch ranges; for `ParamGroup::DqnBranches` +/// sums DqnBranches' 4 sub-launch ranges. Same aggregation pattern. pub const SP4_ENGAGE_EXTRA_CURIOSITY_SUBLAUNCHES: usize = 3; -pub const SP4_ENGAGE_BUF_LEN: usize = - (SP4_PARAM_GROUP_COUNT + SP4_ENGAGE_EXTRA_CURIOSITY_SUBLAUNCHES) * MAX_BLOCKS_PER_ADAM; +pub const SP4_ENGAGE_EXTRA_BRANCHES_SUBLAUNCHES: usize = 3; +pub const SP4_ENGAGE_BUF_LEN: usize = ( + SP4_PARAM_GROUP_COUNT + + SP4_ENGAGE_EXTRA_CURIOSITY_SUBLAUNCHES + + SP4_ENGAGE_EXTRA_BRANCHES_SUBLAUNCHES +) * MAX_BLOCKS_PER_ADAM; /// Curiosity sub-launch offsets in `clamp_engage_per_block_buf`. -/// Sub-launch 0 (W1) reuses the Curiosity main-group offset (7×256=1792). -/// Sub-launches 1/2/3 (b1/W2/b2) get extra slots 8/9/10 × 256 -/// (= 2048/2304/2560). +/// Sub-launch 0 (W1) reuses the Curiosity main-group offset (7×4096). +/// Sub-launches 1/2/3 (b1/W2/b2) get extra slots 8/9/10 × 4096. pub const SP4_ENGAGE_OFFSET_CURIOSITY_W1: i32 = 7 * (MAX_BLOCKS_PER_ADAM as i32); pub const SP4_ENGAGE_OFFSET_CURIOSITY_B1: i32 = 8 * (MAX_BLOCKS_PER_ADAM as i32); pub const SP4_ENGAGE_OFFSET_CURIOSITY_W2: i32 = 9 * (MAX_BLOCKS_PER_ADAM as i32); pub const SP4_ENGAGE_OFFSET_CURIOSITY_B2: i32 = 10 * (MAX_BLOCKS_PER_ADAM as i32); +/// SP21 Phase 4.5 (2026-05-11) — DqnBranches sub-launch offsets in +/// `clamp_engage_per_block_buf`. Sub-launch 0 (Dir) reuses DqnBranches' +/// main-group offset (group_idx=2 → 2×4096); sub-launches 1/2/3 +/// (Mag/Order/Urgency) get extra slots 11/12/13 × 4096 (immediately +/// after Curiosity's extras at slots 8-10). Each branch's +/// per-block engagement counts go to its own range, so the 4 +/// sub-launches don't collide. Pearl C check for DqnBranches sums all +/// 4 ranges into a single per-group rate-deficit EMA — semantically +/// equivalent to the pre-Phase-4 single-launch tracking. +pub const SP4_ENGAGE_OFFSET_BRANCH_DIR: i32 = 2 * (MAX_BLOCKS_PER_ADAM as i32); +pub const SP4_ENGAGE_OFFSET_BRANCH_MAG: i32 = 11 * (MAX_BLOCKS_PER_ADAM as i32); +pub const SP4_ENGAGE_OFFSET_BRANCH_ORDER: i32 = 12 * (MAX_BLOCKS_PER_ADAM as i32); +pub const SP4_ENGAGE_OFFSET_BRANCH_URGENCY: i32 = 13 * (MAX_BLOCKS_PER_ADAM as i32); + /// Sentinel `engage_buf_offset` value passed by aux trainers outside /// the SP4 8-group taxonomy (DT, OFI embed, denoise, recursive_conf, /// sel). Causes the kernel's per-block writeback to be skipped — Pearl @@ -261,15 +286,17 @@ mod tests { #[test] fn pearl_c_engage_buf_layout() { - // 8 main groups + 3 extra curiosity sub-launch slots. + // 8 main groups + 3 extra curiosity sub-launch slots + // + 3 extra DqnBranches sub-launch slots (SP21 Phase 4.5). // Layer B fix-up (2026-05-01): MAX_BLOCKS_PER_ADAM grown 256 → 4096 // to fit trunk-extras tail (≈2400 blocks at production cfg). assert_eq!(MAX_BLOCKS_PER_ADAM, 4096); assert_eq!(SP4_PARAM_GROUP_COUNT, 8); assert_eq!(SP4_ENGAGE_EXTRA_CURIOSITY_SUBLAUNCHES, 3); - assert_eq!(SP4_ENGAGE_BUF_LEN, 11 * MAX_BLOCKS_PER_ADAM); - assert_eq!(SP4_ENGAGE_BUF_LEN, 11 * 4096); - assert_eq!(SP4_ENGAGE_BUF_LEN, 45056); + assert_eq!(SP4_ENGAGE_EXTRA_BRANCHES_SUBLAUNCHES, 3); + assert_eq!(SP4_ENGAGE_BUF_LEN, 14 * MAX_BLOCKS_PER_ADAM); + assert_eq!(SP4_ENGAGE_BUF_LEN, 14 * 4096); + assert_eq!(SP4_ENGAGE_BUF_LEN, 57344); // Curiosity sub-launch offsets are distinct + contiguous starting // from the canonical Curiosity group offset. They're derived from @@ -279,8 +306,16 @@ mod tests { assert_eq!(SP4_ENGAGE_OFFSET_CURIOSITY_W2, 9 * MAX_BLOCKS_PER_ADAM as i32); assert_eq!(SP4_ENGAGE_OFFSET_CURIOSITY_B2, 10 * MAX_BLOCKS_PER_ADAM as i32); + // SP21 Phase 4.5: DqnBranches sub-launch offsets — Dir reuses the + // canonical DqnBranches main-group offset (slot 2); Mag/Order/Urgency + // get extra slots 11/12/13 immediately after Curiosity's tail. + assert_eq!(SP4_ENGAGE_OFFSET_BRANCH_DIR, 2 * MAX_BLOCKS_PER_ADAM as i32); + assert_eq!(SP4_ENGAGE_OFFSET_BRANCH_MAG, 11 * MAX_BLOCKS_PER_ADAM as i32); + assert_eq!(SP4_ENGAGE_OFFSET_BRANCH_ORDER, 12 * MAX_BLOCKS_PER_ADAM as i32); + assert_eq!(SP4_ENGAGE_OFFSET_BRANCH_URGENCY, 13 * MAX_BLOCKS_PER_ADAM as i32); + // Highest sub-launch end must fit within the buffer. - let highest_end = (SP4_ENGAGE_OFFSET_CURIOSITY_B2 as usize) + MAX_BLOCKS_PER_ADAM; + let highest_end = (SP4_ENGAGE_OFFSET_BRANCH_URGENCY as usize) + MAX_BLOCKS_PER_ADAM; assert_eq!(highest_end, SP4_ENGAGE_BUF_LEN); // Sentinel disabled value. diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index a7242977f..a3b5e986b 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -14044,3 +14044,114 @@ T2.2 multi-phase scope continues with Phases 5-7 + Phase 8: - Phase 8: signal-drive remaining controller GAINS. - Phase 4.5 (deferred): re-instate Pearl C engagement tracking for branches via ParamGroup expansion or sub-block offsetting. + +## 2026-05-11 — SP21 T2.2 Phase 4.5: re-instate Pearl C engagement tracking for branches + +### Scope (atomic single commit) + +Closes the Pearl C engagement-tracking deferral from Phase 4. The 4 +DqnBranches sub-launches now write per-block engagement counts to 4 +distinct ranges in `clamp_engage_per_block_buf`; the host-side +`pearl_c_post_adam_engagement_check(DqnBranches, ...)` aggregates +all 4 ranges into one per-group rate-deficit EMA. + +### Design choice + +Two options were considered in the Phase 4 deferral note: +- **(a)** Grow `ParamGroup` enum to add 4 sub-variants + (`DqnBranchDir/Mag/Order/Urgency`). Cascades into ISV slot + allocation (28 new slots: 4 each × WEIGHT_BOUND, ADAM_M_BOUND, + ADAM_V_BOUND, WD_RATE, ADAM_BETA1, ADAM_BETA2, ADAM_EPS) and + fingerprint changes. +- **(b)** Sub-block offsetting within `clamp_engage_per_block_buf`. + Reuses existing `ParamGroup` (DqnBranches stays group_idx=2); + branches share Adam β1/β2/ε / WEIGHT_BOUND / WD_RATE; only the + engagement counter gets per-branch granularity via 3 extra + buffer slots beyond the canonical 8. + +**Resolution: (b)**. Mirrors the existing Curiosity pattern exactly +— Curiosity uses `SP4_ENGAGE_EXTRA_CURIOSITY_SUBLAUNCHES = 3` to +reserve slots 8/9/10 × MAX_BLOCKS_PER_ADAM beyond the canonical 8 +groups; sub-launch 0 (W1) reuses the canonical Curiosity slot 7. +SP21 Phase 4.5 follows the same pattern: `SP4_ENGAGE_EXTRA_BRANCHES_ +SUBLAUNCHES = 3` reserves slots 11/12/13 × MAX_BLOCKS_PER_ADAM; +sub-launch 0 (Dir) reuses the canonical DqnBranches slot 2. + +Total `SP4_ENGAGE_BUF_LEN`: 11 → 14 × MAX_BLOCKS_PER_ADAM +(45056 → 57344 i32 slots, +12 KiB on a non-hot-path mapped-pinned +buffer — negligible). + +### Branch-to-offset mapping + +| Sub-launch | Offset constant | Slot | Buffer offset | +|------------|-----------------|------|---------------| +| Dir (b0) | `SP4_ENGAGE_OFFSET_BRANCH_DIR` | 2 (canonical DqnBranches) | 8 192 | +| Mag (b1) | `SP4_ENGAGE_OFFSET_BRANCH_MAG` | 11 (extra) | 45 056 | +| Order (b2) | `SP4_ENGAGE_OFFSET_BRANCH_ORDER` | 12 (extra) | 49 152 | +| Urgency (b3) | `SP4_ENGAGE_OFFSET_BRANCH_URGENCY` | 13 (extra) | 53 248 | + +Each branch's per-block engagement counts go to its own range, so +the 4 sub-launches don't collide on writes. Pearl C aggregates all +4 ranges into one rate-deficit EMA — semantically equivalent to +the pre-Phase-4 single-launch tracking (same total engagement +count, same param_count denominator). + +### Pearl C aggregation symmetry + +`pearl_c_post_adam_engagement_check` previously had a special case +for Curiosity (`matches!(group, ParamGroup::Curiosity)`) that +read 4 sub-ranges and zeroed them post-read. Phase 4.5 generalizes: + +```rust +let multi_range = matches!(group, ParamGroup::Curiosity | ParamGroup::DqnBranches); +let block_offsets = match group { + ParamGroup::Curiosity => &[7, 8, 9, 10] × MAX_BLOCKS_PER_ADAM, + ParamGroup::DqnBranches => &[2, 11, 12, 13] × MAX_BLOCKS_PER_ADAM, + _ => single canonical group_idx range, +}; +``` + +Both read AND zero-out paths use the same `multi_range` flag; no +duplication of branching logic between read and zero phases. + +### Files changed + +| File | Status | Purpose | +|------|--------|---------| +| `crates/ml/src/cuda_pipeline/sp4_isv_slots.rs` | +offsets + buf bump | `SP4_ENGAGE_EXTRA_BRANCHES_SUBLAUNCHES = 3`; `SP4_ENGAGE_BUF_LEN` formula extended; `SP4_ENGAGE_OFFSET_BRANCH_{DIR,MAG,ORDER,URGENCY}` constants; layout test updated to assert 14×MAX_BLOCKS_PER_ADAM | +| `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs` | Branch engage offsets + Pearl C aggregation | `launch_adam_update`'s 4 branch sub-launches use the new offsets (no longer SP4_ENGAGE_OFFSET_DISABLED); `pearl_c_post_adam_engagement_check` adds DqnBranches multi-range case | +| `docs/dqn-wire-up-audit.md` | This entry | 2026-05-11 audit log | + +### Pearls + invariants honoured + +- `feedback_no_partial_refactor` — engage offset definitions, buffer + size bump, launcher migration, and Pearl C reader migrate + atomically. +- `feedback_no_atomicadd` — per-branch engagement still uses + per-block tree-reduce within each sub-launch; the multi-range + pattern only spans across launches (host-side reduction). +- `feedback_no_stubs` — `SP4_ENGAGE_OFFSET_DISABLED` no longer used + for branches; all 4 sub-launches now write real engagement + counts. +- `pearl_no_host_branches_in_captured_graph` — Pearl C check is + host-side only (called outside CUDA Graph capture); no + in-graph-stream effects. + +### Verification + +``` +SQLX_OFFLINE=true cargo check -p ml --tests --features cuda # 0 errors +cargo test -p ml --lib sp4_isv_slots --features cuda # 2/2 (engage buf layout asserts 14× + slot offsets correct) +cargo test -p ml --lib sp21_isv_slots --features cuda # 3/3 (unchanged) +cargo test -p ml --test sp20_aggregate_inputs_test ... # 12/12 +cargo test -p ml --test sp20_phase1_4_wireup_test ... # 2/2 +cargo test -p ml --test sp20_emas_compute_test ... # 4/4 +cargo test -p ml --test sp20_controllers_compute_test ... # 7/7 +cargo test -p ml --test sp21_per_trade_predicted_q_test ... # 3/3 +``` + +Total: 33 tests, 0 failures. Behavioral gate: a smoke training run +will surface per-branch engagement-rate-deficit EMAs in HEALTH_DIAG +(emit threshold `> 0.005`) for the first time — Phase 4.5 +re-instates the per-group Pearl C signal that Phase 4 temporarily +disabled.