From fcb8222a6072afef8b7b443a54ae6b54e4f6563e Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 25 May 2026 15:26:58 +0200 Subject: [PATCH] =?UTF-8?q?feat(rl):=20tier=202=20wave-scale=20=E2=80=94?= =?UTF-8?q?=20=CE=B3=3D0.995,=20PER=2032k,=20min-hold=20100?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prepared for next run after the γ=0.99 1M run completes: - γ floor 0.99 → 0.995: horizon 100 → 200 steps (50 seconds). Real ES directional moves (2-5 points) happen at this scale. - PER 16384 → 32768: 2048 unique steps of replay depth. Supports the 200-step γ horizon with margin. - Min-hold 50 → 100 steps (25 seconds): commit to the full wave. Short-hold penalty threshold matches. Safe to push because raw-reward re-normalization eliminates scale drift in the deeper buffer, and the ±2% scale clamp keeps targets stable across the 2048-step replay window. Co-Authored-By: Claude Opus 4.7 --- crates/ml-alpha/cuda/rl_gamma_controller.cu | 2 +- crates/ml-alpha/examples/alpha_rl_train.rs | 2 +- crates/ml-alpha/src/trainer/integrated.rs | 6 +++--- infra/k8s/argo/alpha-rl-template.yaml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/ml-alpha/cuda/rl_gamma_controller.cu b/crates/ml-alpha/cuda/rl_gamma_controller.cu index 287e411a9..2b14d9b7f 100644 --- a/crates/ml-alpha/cuda/rl_gamma_controller.cu +++ b/crates/ml-alpha/cuda/rl_gamma_controller.cu @@ -45,7 +45,7 @@ // the policy myopic). #define RL_GAMMA_INDEX 400 -#define GAMMA_MIN 0.99f +#define GAMMA_MIN 0.995f #define GAMMA_MAX 0.999f #define WIENER_ALPHA_FLOOR 0.4f diff --git a/crates/ml-alpha/examples/alpha_rl_train.rs b/crates/ml-alpha/examples/alpha_rl_train.rs index 123f923c6..83ace56b1 100644 --- a/crates/ml-alpha/examples/alpha_rl_train.rs +++ b/crates/ml-alpha/examples/alpha_rl_train.rs @@ -145,7 +145,7 @@ struct Cli { /// PER buffer capacity. At b_size=16, capacity/16 = unique steps /// retained. 32768/16 = 2048 steps of replay depth. - #[arg(long, default_value_t = 16384)] + #[arg(long, default_value_t = 32768)] per_capacity: usize, /// Random seed pinning RNG + Xavier init + ISV controller bootstrap diff --git a/crates/ml-alpha/src/trainer/integrated.rs b/crates/ml-alpha/src/trainer/integrated.rs index 29f861e7e..1a3bb2b5d 100644 --- a/crates/ml-alpha/src/trainer/integrated.rs +++ b/crates/ml-alpha/src/trainer/integrated.rs @@ -294,7 +294,7 @@ impl Default for IntegratedTrainerConfig { perception: PerceptionTrainerConfig::default(), dqn_seed: 0xDA_DA_DA_DA, ppo_seed: 0xBE_BE_BE_BE, - per_capacity: 16384, + per_capacity: 32768, per_seed: 0x9E37_79B9_7F4A_7C15, } } @@ -1652,10 +1652,10 @@ impl IntegratedTrainer { (crate::rl::isv_slots::RL_GATE_FRD_MAX_INDEX, 0.50), (crate::rl::isv_slots::RL_GATE_ADJUST_RATE_INDEX, 0.95), (crate::rl::isv_slots::RL_ENTRY_COST_INDEX, 15.0), - (crate::rl::isv_slots::RL_SHORT_HOLD_MIN_STEPS_INDEX, 50.0), + (crate::rl::isv_slots::RL_SHORT_HOLD_MIN_STEPS_INDEX, 100.0), (crate::rl::isv_slots::RL_SHORT_HOLD_PENALTY_INDEX, 0.5), (crate::rl::isv_slots::RL_HOLD_BONUS_INDEX, 2.0), - (crate::rl::isv_slots::RL_MIN_HOLD_STEPS_INDEX, 50.0), + (crate::rl::isv_slots::RL_MIN_HOLD_STEPS_INDEX, 100.0), (crate::rl::isv_slots::RL_MULTIRES_HORIZON_1_INDEX, 1.0), (crate::rl::isv_slots::RL_MULTIRES_HORIZON_2_INDEX, 10.0), (crate::rl::isv_slots::RL_MULTIRES_HORIZON_3_INDEX, 600.0), diff --git a/infra/k8s/argo/alpha-rl-template.yaml b/infra/k8s/argo/alpha-rl-template.yaml index 58d297a59..938eb0509 100644 --- a/infra/k8s/argo/alpha-rl-template.yaml +++ b/infra/k8s/argo/alpha-rl-template.yaml @@ -48,7 +48,7 @@ spec: - name: seq-len value: "32" - name: per-capacity - value: "16384" + value: "32768" - name: instrument-mode value: "all" - name: log-every