From 79ac46c672ff6c9ebbb9a5d6e86a58a5d810ca6c Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 25 May 2026 11:21:39 +0200 Subject: [PATCH] =?UTF-8?q?fix(rl):=20C51=20atom=20span=20[-1,+1]=20?= =?UTF-8?q?=E2=86=92=20[-0.5,+0.5],=20PER=20capacity=204096=20=E2=86=92=20?= =?UTF-8?q?32768?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of Q non-convergence: at reward_scale ≈ 0.003, a typical ±$5 trade maps to ±0.015 scaled reward. With 21 atoms spanning [-1,+1] (step=0.1), the entire reward distribution falls within a SINGLE atom. Q cannot distinguish wins from losses — confirmed by q_pi_agree ≈ 0. Fix 1: Tighten atom span to [-0.5, +0.5]. Now atom_step = 0.05. A ±$5 trade at scale=0.01 spans 2 atoms; at scale=0.05 spans 10. The reward clamp controller ratchets the span at runtime if rewards exceed the bounds. Fix 2: PER capacity 4096 → 32768. At b=16, old capacity retained only 256 unique steps — Q replayed each transition 1-2× before eviction. New capacity retains 2048 steps, giving Q proper replay depth for convergence. Also aligns reward clamp bounds (WIN/LOSS) with the new span. Co-Authored-By: Claude Opus 4.7 --- crates/ml-alpha/src/rl/common.rs | 13 ++++++++----- crates/ml-alpha/src/trainer/integrated.rs | 16 ++++++++-------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/crates/ml-alpha/src/rl/common.rs b/crates/ml-alpha/src/rl/common.rs index 6fbe85476..20b3453a0 100644 --- a/crates/ml-alpha/src/rl/common.rs +++ b/crates/ml-alpha/src/rl/common.rs @@ -41,12 +41,15 @@ pub const FRD_HORIZON_TICKS: [usize; FRD_N_HORIZONS] = [60, 300, 1800]; pub const FRD_BUCKET_RANGE_SIGMA: f32 = 3.0; /// C51 atom support `v_min`. After standardisation by the reward-scale -/// controller (`RL_REWARD_SCALE_INDEX`, Phase F), per-step reward + -/// γ-discounted returns are expected to fit inside `[V_MIN, V_MAX]`. -/// The atom delta is `Δ_z = (V_MAX - V_MIN) / (Q_N_ATOMS - 1) = 0.1`. -pub const Q_V_MIN: f32 = -1.0; +/// controller (`RL_REWARD_SCALE_INDEX`), per-step reward + γ-discounted +/// returns are expected to fit inside `[V_MIN, V_MAX]`. The atom delta +/// `Δ_z = (V_MAX - V_MIN) / (Q_N_ATOMS - 1)` determines the minimum +/// reward difference Q can resolve. At ±0.5 with 21 atoms: Δ_z = 0.05, +/// so a ±$5 trade at scale=0.01 spans 2 atoms (resolvable). +/// The reward clamp controller (ISV 484/485) ratchets these at runtime. +pub const Q_V_MIN: f32 = -0.5; /// C51 atom support `v_max`. See [`Q_V_MIN`]. -pub const Q_V_MAX: f32 = 1.0; +pub const Q_V_MAX: f32 = 0.5; /// Discrete 11-action grid identifiers. Was 9 pre-SP20 P4; extended /// with HalfFlatLong/HalfFlatShort to enable per-unit partial-close diff --git a/crates/ml-alpha/src/trainer/integrated.rs b/crates/ml-alpha/src/trainer/integrated.rs index 1dff71100..03a18f15b 100644 --- a/crates/ml-alpha/src/trainer/integrated.rs +++ b/crates/ml-alpha/src/trainer/integrated.rs @@ -270,9 +270,9 @@ pub struct IntegratedTrainerConfig { pub dqn_seed: u64, /// Seed for PPO heads (policy + value share derived seeds). pub ppo_seed: u64, - /// PER buffer capacity (Phase R7d). Per the canonical replay.rs - /// doc, naive O(N) sampling is acceptable up to ≈ 4096; production - /// scale-up to 100k transitions needs a GPU sum-tree (R-future). + /// PER buffer capacity. At b_size=16, capacity/16 = unique steps of + /// history retained. 32768/16 = 2048 steps (~34 minutes of ES data + /// at event rate). Naive O(N) sampling is fast enough at this scale. pub per_capacity: usize, /// PER seed for deterministic sampling per /// `pearl_scoped_init_seed_for_reproducibility`. Fed to @@ -290,7 +290,7 @@ impl Default for IntegratedTrainerConfig { perception: PerceptionTrainerConfig::default(), dqn_seed: 0xDA_DA_DA_DA, ppo_seed: 0xBE_BE_BE_BE, - per_capacity: 4096, + per_capacity: 32768, per_seed: 0x9E37_79B9_7F4A_7C15, } } @@ -1560,13 +1560,13 @@ impl IntegratedTrainer { // outliers." V_MAX/V_MIN seeded to the original C51 // [-1, +1] span as the canonical floor — the ratchet // in `rl_reward_clamp_controller` only grows magnitude. - (crate::rl::isv_slots::RL_REWARD_CLAMP_WIN_INDEX, 1.0), - (crate::rl::isv_slots::RL_REWARD_CLAMP_LOSS_INDEX, 3.0), + (crate::rl::isv_slots::RL_REWARD_CLAMP_WIN_INDEX, 0.5), + (crate::rl::isv_slots::RL_REWARD_CLAMP_LOSS_INDEX, 0.5), (crate::rl::isv_slots::RL_REWARD_CLAMP_MARGIN_INDEX, 1.5), (crate::rl::isv_slots::RL_REWARD_CLAMP_RATIO_INDEX, 3.0), (crate::rl::isv_slots::RL_REWARD_CLAMP_CLIP_RATE_TARGET_INDEX, 0.05), - (crate::rl::isv_slots::RL_C51_V_MAX_INDEX, 1.0), - (crate::rl::isv_slots::RL_C51_V_MIN_INDEX, -1.0), + (crate::rl::isv_slots::RL_C51_V_MAX_INDEX, 0.5), + (crate::rl::isv_slots::RL_C51_V_MIN_INDEX, -0.5), // Q→π distillation (vj5f6 followup) — λ now controller- // driven by `rl_q_distill_lambda_controller` from // KL_EMA vs KL_TARGET. Seed 0.05 is the starting point