feat(rl): reserve FRD head ISV slots + structural consts (F.1)
Foundation patch for the Forward-Return-Distribution head (SP20 P3). No new behavior — kernels arrive in the next commit (F.2). This commit just establishes the ISV vocabulary and structural dims so the kernel code can reference named slots/consts from day one. ISV slots 498-503 (RL_SLOTS_END bumped 498 → 504): * RL_FRD_LAMBDA_INDEX = 498 seed 0.5 * RL_FRD_LR_INDEX = 499 seed 1e-3 * RL_FRD_HORIZON_1_TICKS_INDEX = 500 seed 60.0 * RL_FRD_HORIZON_2_TICKS_INDEX = 501 seed 300.0 * RL_FRD_HORIZON_3_TICKS_INDEX = 502 seed 1800.0 * RL_FRD_BUCKET_RANGE_SIGMA_INDEX = 503 seed 3.0 (±3σ) Bootstraps written via the existing isv_constants table in IntegratedTrainer::new — same path as the SP20 P5 trail bounds. No HtoD path opened (rl_isv_write does device-side scalar writes). Structural consts (crates/ml-alpha/src/rl/common.rs): * FRD_HIDDEN_DIM = 64 (MLP hidden layer width) * FRD_N_HORIZONS = 3 (h1/h2/h3 forward returns) * FRD_N_ATOMS = 21 (return-bucket atoms per horizon) Atom count is the only structural compile-time dim per §0.1 of the SP20 spec; range_σ is ISV-driven (slot 503) so the head can adapt as realised σ drifts.
This commit is contained in:
jgrusewski
@@ -14,6 +14,20 @@ pub const N_ACTIONS: usize = 11;
|
||||
/// learned support `[v_min, v_max]`).
|
||||
pub const Q_N_ATOMS: usize = 21;
|
||||
|
||||
/// Forward-Return-Distribution (FRD) head — hidden-layer width of the
|
||||
/// `[HIDDEN_DIM → FRD_HIDDEN_DIM → FRD_N_HORIZONS × FRD_N_ATOMS]` MLP.
|
||||
/// Per SP20 P3 design.
|
||||
pub const FRD_HIDDEN_DIM: usize = 64;
|
||||
|
||||
/// FRD head — number of forward-return horizons supervised per step
|
||||
/// (h1 = 60 ticks, h2 = 300, h3 = 1800; tick offsets ISV-driven).
|
||||
pub const FRD_N_HORIZONS: usize = 3;
|
||||
|
||||
/// FRD head — return-bucket atom count per horizon. Span is ISV-driven
|
||||
/// (`RL_FRD_BUCKET_RANGE_SIGMA_INDEX`, default ±3σ), atom count is the
|
||||
/// only structural compile-time dim per §0.1 audit.
|
||||
pub const FRD_N_ATOMS: usize = 21;
|
||||
|
||||
/// C51 atom support `v_min`. After standardisation by the reward-scale
|
||||
/// controller (`RL_REWARD_SCALE_INDEX`, Phase F), per-step reward +
|
||||
/// γ-discounted returns are expected to fit inside `[V_MIN, V_MAX]`.
|
||||
|
||||
@@ -47,8 +47,9 @@
|
||||
//!
|
||||
//! | 493 | Q→π distill KL_EMA blend alpha | rl_isv_write (seed) + rl_q_pi_distill_grad |
|
||||
//! | 494-497 | SP20 P5 trail-stop bounds (MIN/MAX/K_INIT/ADJUST_RATE) | rl_isv_write (seed) + rl_trail_mutate + rl_unit_state_update |
|
||||
//! | 498-503 | SP20 P3 FRD head (λ/LR/h1-3 ticks/bucket-range σ) | rl_isv_write (seed) + rl_frd_fwd + rl_frd_bwd + loader label gen |
|
||||
//!
|
||||
//! Total: 98 slots; `RL_SLOTS_END = 498`.
|
||||
//! Total: 104 slots; `RL_SLOTS_END = 504`.
|
||||
|
||||
/// Discount factor γ. Controller input: mean trade duration / anchor.
|
||||
/// Bootstrap 0.99.
|
||||
@@ -762,6 +763,35 @@ pub const RL_TRAIL_K_INIT_INDEX: usize = 496;
|
||||
/// a8 (loosen). Symmetric reciprocal per SP20 §4.12. Default 0.9.
|
||||
pub const RL_TRAIL_ADJUST_RATE_INDEX: usize = 497;
|
||||
|
||||
// ─── Forward-Return-Distribution (FRD) head — supervised forecaster
|
||||
// over 3 horizons × 21 return-bucket atoms, per
|
||||
// `pearl_glm_fitter_link_must_match_inference`. Slots seed the head's
|
||||
// loss weight, LR, bucket-span scale, and per-horizon tick offsets used
|
||||
// by both label generation in the loader and the forward kernel.
|
||||
|
||||
/// FRD loss-weight (`λ_frd`) — multiplier on `CE(h1)+CE(h2)+CE(h3)`
|
||||
/// before backward. Default 0.5 (seed; downstream controller may modulate).
|
||||
pub const RL_FRD_LAMBDA_INDEX: usize = 498;
|
||||
|
||||
/// FRD head learning rate. Default 1e-3 (seed; LR controller modulates).
|
||||
pub const RL_FRD_LR_INDEX: usize = 499;
|
||||
|
||||
/// FRD horizon 1 in tick offsets — `r_h1 = (mid[i+h1] - mid[i]) / σ`.
|
||||
/// Default 60.0 ticks.
|
||||
pub const RL_FRD_HORIZON_1_TICKS_INDEX: usize = 500;
|
||||
|
||||
/// FRD horizon 2 in tick offsets. Default 300.0 ticks.
|
||||
pub const RL_FRD_HORIZON_2_TICKS_INDEX: usize = 501;
|
||||
|
||||
/// FRD horizon 3 in tick offsets. Default 1800.0 ticks.
|
||||
pub const RL_FRD_HORIZON_3_TICKS_INDEX: usize = 502;
|
||||
|
||||
/// FRD bucket range in σ units — atoms span `[-range_σ, +range_σ]`.
|
||||
/// 21 atoms → `Δ_atom = 2 × range_σ / 20`. Default 3.0 (±3σ).
|
||||
/// Per §0.1 audit: range is ISV-tunable so the head can adapt as
|
||||
/// realised σ drifts; only the 21-atom count is structural compile-time.
|
||||
pub const RL_FRD_BUCKET_RANGE_SIGMA_INDEX: usize = 503;
|
||||
|
||||
/// Last RL-allocated slot index (exclusive). The integrated trainer
|
||||
/// extends `ISV_TOTAL_DIM` to at least this value at trainer init time.
|
||||
pub const RL_SLOTS_END: usize = 498;
|
||||
pub const RL_SLOTS_END: usize = 504;
|
||||
|
||||
@@ -1346,7 +1346,7 @@ impl IntegratedTrainer {
|
||||
// (slot, value) pair — pure device write, no HtoD per
|
||||
// `feedback_no_htod_htoh_only_mapped_pinned`.
|
||||
{
|
||||
let isv_constants: [(usize, f32); 40] = [
|
||||
let isv_constants: [(usize, f32); 46] = [
|
||||
// Static seeds for the adaptive reward-clamp controller —
|
||||
// these are the initial values that
|
||||
// `rl_reward_clamp_controller` will replace once it
|
||||
@@ -1386,6 +1386,13 @@ impl IntegratedTrainer {
|
||||
(crate::rl::isv_slots::RL_TRAIL_MAX_INDEX, 100.0),
|
||||
(crate::rl::isv_slots::RL_TRAIL_K_INIT_INDEX, 2.0),
|
||||
(crate::rl::isv_slots::RL_TRAIL_ADJUST_RATE_INDEX, 0.9),
|
||||
// SP20 P3 FRD head seeds — λ, LR, h1-3 tick offsets, ±σ atom range.
|
||||
(crate::rl::isv_slots::RL_FRD_LAMBDA_INDEX, 0.5),
|
||||
(crate::rl::isv_slots::RL_FRD_LR_INDEX, 1e-3),
|
||||
(crate::rl::isv_slots::RL_FRD_HORIZON_1_TICKS_INDEX, 60.0),
|
||||
(crate::rl::isv_slots::RL_FRD_HORIZON_2_TICKS_INDEX, 300.0),
|
||||
(crate::rl::isv_slots::RL_FRD_HORIZON_3_TICKS_INDEX, 1800.0),
|
||||
(crate::rl::isv_slots::RL_FRD_BUCKET_RANGE_SIGMA_INDEX, 3.0),
|
||||
(crate::rl::isv_slots::RL_KL_TARGET_INDEX, 0.01),
|
||||
(crate::rl::isv_slots::RL_IMPROVEMENT_THRESHOLD_INDEX, 0.99),
|
||||
(crate::rl::isv_slots::RL_PLATEAU_PATIENCE_INDEX, 1000.0),
|
||||
|
||||
Reference in New Issue
Block a user