From 56a4627bb2d3d305e0ac550f60f313d39403bb3f Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 24 May 2026 18:08:16 +0200 Subject: [PATCH] feat(rl): reserve FRD head ISV slots + structural consts (F.1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foundation patch for the Forward-Return-Distribution head (SP20 P3). No new behavior — kernels arrive in the next commit (F.2). This commit just establishes the ISV vocabulary and structural dims so the kernel code can reference named slots/consts from day one. ISV slots 498-503 (RL_SLOTS_END bumped 498 → 504): * RL_FRD_LAMBDA_INDEX = 498 seed 0.5 * RL_FRD_LR_INDEX = 499 seed 1e-3 * RL_FRD_HORIZON_1_TICKS_INDEX = 500 seed 60.0 * RL_FRD_HORIZON_2_TICKS_INDEX = 501 seed 300.0 * RL_FRD_HORIZON_3_TICKS_INDEX = 502 seed 1800.0 * RL_FRD_BUCKET_RANGE_SIGMA_INDEX = 503 seed 3.0 (±3σ) Bootstraps written via the existing isv_constants table in IntegratedTrainer::new — same path as the SP20 P5 trail bounds. No HtoD path opened (rl_isv_write does device-side scalar writes). Structural consts (crates/ml-alpha/src/rl/common.rs): * FRD_HIDDEN_DIM = 64 (MLP hidden layer width) * FRD_N_HORIZONS = 3 (h1/h2/h3 forward returns) * FRD_N_ATOMS = 21 (return-bucket atoms per horizon) Atom count is the only structural compile-time dim per §0.1 of the SP20 spec; range_σ is ISV-driven (slot 503) so the head can adapt as realised σ drifts. --- crates/ml-alpha/src/rl/common.rs | 14 ++++++++++ crates/ml-alpha/src/rl/isv_slots.rs | 34 +++++++++++++++++++++-- crates/ml-alpha/src/trainer/integrated.rs | 9 +++++- 3 files changed, 54 insertions(+), 3 deletions(-) diff --git a/crates/ml-alpha/src/rl/common.rs b/crates/ml-alpha/src/rl/common.rs index fd466de10..47a2b1458 100644 --- a/crates/ml-alpha/src/rl/common.rs +++ b/crates/ml-alpha/src/rl/common.rs @@ -14,6 +14,20 @@ pub const N_ACTIONS: usize = 11; /// learned support `[v_min, v_max]`). pub const Q_N_ATOMS: usize = 21; +/// Forward-Return-Distribution (FRD) head — hidden-layer width of the +/// `[HIDDEN_DIM → FRD_HIDDEN_DIM → FRD_N_HORIZONS × FRD_N_ATOMS]` MLP. +/// Per SP20 P3 design. +pub const FRD_HIDDEN_DIM: usize = 64; + +/// FRD head — number of forward-return horizons supervised per step +/// (h1 = 60 ticks, h2 = 300, h3 = 1800; tick offsets ISV-driven). +pub const FRD_N_HORIZONS: usize = 3; + +/// FRD head — return-bucket atom count per horizon. Span is ISV-driven +/// (`RL_FRD_BUCKET_RANGE_SIGMA_INDEX`, default ±3σ), atom count is the +/// only structural compile-time dim per §0.1 audit. +pub const FRD_N_ATOMS: usize = 21; + /// C51 atom support `v_min`. After standardisation by the reward-scale /// controller (`RL_REWARD_SCALE_INDEX`, Phase F), per-step reward + /// γ-discounted returns are expected to fit inside `[V_MIN, V_MAX]`. diff --git a/crates/ml-alpha/src/rl/isv_slots.rs b/crates/ml-alpha/src/rl/isv_slots.rs index 1b598e387..2dbe1bfad 100644 --- a/crates/ml-alpha/src/rl/isv_slots.rs +++ b/crates/ml-alpha/src/rl/isv_slots.rs @@ -47,8 +47,9 @@ //! //! | 493 | Q→π distill KL_EMA blend alpha | rl_isv_write (seed) + rl_q_pi_distill_grad | //! | 494-497 | SP20 P5 trail-stop bounds (MIN/MAX/K_INIT/ADJUST_RATE) | rl_isv_write (seed) + rl_trail_mutate + rl_unit_state_update | +//! | 498-503 | SP20 P3 FRD head (λ/LR/h1-3 ticks/bucket-range σ) | rl_isv_write (seed) + rl_frd_fwd + rl_frd_bwd + loader label gen | //! -//! Total: 98 slots; `RL_SLOTS_END = 498`. +//! Total: 104 slots; `RL_SLOTS_END = 504`. /// Discount factor γ. Controller input: mean trade duration / anchor. /// Bootstrap 0.99. @@ -762,6 +763,35 @@ pub const RL_TRAIL_K_INIT_INDEX: usize = 496; /// a8 (loosen). Symmetric reciprocal per SP20 §4.12. Default 0.9. pub const RL_TRAIL_ADJUST_RATE_INDEX: usize = 497; +// ─── Forward-Return-Distribution (FRD) head — supervised forecaster +// over 3 horizons × 21 return-bucket atoms, per +// `pearl_glm_fitter_link_must_match_inference`. Slots seed the head's +// loss weight, LR, bucket-span scale, and per-horizon tick offsets used +// by both label generation in the loader and the forward kernel. + +/// FRD loss-weight (`λ_frd`) — multiplier on `CE(h1)+CE(h2)+CE(h3)` +/// before backward. Default 0.5 (seed; downstream controller may modulate). +pub const RL_FRD_LAMBDA_INDEX: usize = 498; + +/// FRD head learning rate. Default 1e-3 (seed; LR controller modulates). +pub const RL_FRD_LR_INDEX: usize = 499; + +/// FRD horizon 1 in tick offsets — `r_h1 = (mid[i+h1] - mid[i]) / σ`. +/// Default 60.0 ticks. +pub const RL_FRD_HORIZON_1_TICKS_INDEX: usize = 500; + +/// FRD horizon 2 in tick offsets. Default 300.0 ticks. +pub const RL_FRD_HORIZON_2_TICKS_INDEX: usize = 501; + +/// FRD horizon 3 in tick offsets. Default 1800.0 ticks. +pub const RL_FRD_HORIZON_3_TICKS_INDEX: usize = 502; + +/// FRD bucket range in σ units — atoms span `[-range_σ, +range_σ]`. +/// 21 atoms → `Δ_atom = 2 × range_σ / 20`. Default 3.0 (±3σ). +/// Per §0.1 audit: range is ISV-tunable so the head can adapt as +/// realised σ drifts; only the 21-atom count is structural compile-time. +pub const RL_FRD_BUCKET_RANGE_SIGMA_INDEX: usize = 503; + /// Last RL-allocated slot index (exclusive). The integrated trainer /// extends `ISV_TOTAL_DIM` to at least this value at trainer init time. -pub const RL_SLOTS_END: usize = 498; +pub const RL_SLOTS_END: usize = 504; diff --git a/crates/ml-alpha/src/trainer/integrated.rs b/crates/ml-alpha/src/trainer/integrated.rs index fbab67e1c..c37faba9a 100644 --- a/crates/ml-alpha/src/trainer/integrated.rs +++ b/crates/ml-alpha/src/trainer/integrated.rs @@ -1346,7 +1346,7 @@ impl IntegratedTrainer { // (slot, value) pair — pure device write, no HtoD per // `feedback_no_htod_htoh_only_mapped_pinned`. { - let isv_constants: [(usize, f32); 40] = [ + let isv_constants: [(usize, f32); 46] = [ // Static seeds for the adaptive reward-clamp controller — // these are the initial values that // `rl_reward_clamp_controller` will replace once it @@ -1386,6 +1386,13 @@ impl IntegratedTrainer { (crate::rl::isv_slots::RL_TRAIL_MAX_INDEX, 100.0), (crate::rl::isv_slots::RL_TRAIL_K_INIT_INDEX, 2.0), (crate::rl::isv_slots::RL_TRAIL_ADJUST_RATE_INDEX, 0.9), + // SP20 P3 FRD head seeds — λ, LR, h1-3 tick offsets, ±σ atom range. + (crate::rl::isv_slots::RL_FRD_LAMBDA_INDEX, 0.5), + (crate::rl::isv_slots::RL_FRD_LR_INDEX, 1e-3), + (crate::rl::isv_slots::RL_FRD_HORIZON_1_TICKS_INDEX, 60.0), + (crate::rl::isv_slots::RL_FRD_HORIZON_2_TICKS_INDEX, 300.0), + (crate::rl::isv_slots::RL_FRD_HORIZON_3_TICKS_INDEX, 1800.0), + (crate::rl::isv_slots::RL_FRD_BUCKET_RANGE_SIGMA_INDEX, 3.0), (crate::rl::isv_slots::RL_KL_TARGET_INDEX, 0.01), (crate::rl::isv_slots::RL_IMPROVEMENT_THRESHOLD_INDEX, 0.99), (crate::rl::isv_slots::RL_PLATEAU_PATIENCE_INDEX, 1000.0),