feat(rl): tier 2 wave-scale — γ=0.995, PER 32k, min-hold 100

Prepared for next run after the γ=0.99 1M run completes:

- γ floor 0.99 → 0.995: horizon 100 → 200 steps (50 seconds).
  Real ES directional moves (2-5 points) happen at this scale.
- PER 16384 → 32768: 2048 unique steps of replay depth. Supports
  the 200-step γ horizon with margin.
- Min-hold 50 → 100 steps (25 seconds): commit to the full wave.
  Short-hold penalty threshold matches.

Safe to push because raw-reward re-normalization eliminates scale
drift in the deeper buffer, and the ±2% scale clamp keeps targets
stable across the 2048-step replay window.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-05-25 15:26:58 +02:00
parent b0dbb70989
commit fcb8222a60
4 changed files with 6 additions and 6 deletions

View File

@@ -45,7 +45,7 @@
// the policy myopic).
#define RL_GAMMA_INDEX 400
#define GAMMA_MIN 0.99f
#define GAMMA_MIN 0.995f
#define GAMMA_MAX 0.999f
#define WIENER_ALPHA_FLOOR 0.4f

View File

@@ -145,7 +145,7 @@ struct Cli {
/// PER buffer capacity. At b_size=16, capacity/16 = unique steps
/// retained. 32768/16 = 2048 steps of replay depth.
#[arg(long, default_value_t = 16384)]
#[arg(long, default_value_t = 32768)]
per_capacity: usize,
/// Random seed pinning RNG + Xavier init + ISV controller bootstrap

View File

@@ -294,7 +294,7 @@ impl Default for IntegratedTrainerConfig {
perception: PerceptionTrainerConfig::default(),
dqn_seed: 0xDA_DA_DA_DA,
ppo_seed: 0xBE_BE_BE_BE,
per_capacity: 16384,
per_capacity: 32768,
per_seed: 0x9E37_79B9_7F4A_7C15,
}
}
@@ -1652,10 +1652,10 @@ impl IntegratedTrainer {
(crate::rl::isv_slots::RL_GATE_FRD_MAX_INDEX, 0.50),
(crate::rl::isv_slots::RL_GATE_ADJUST_RATE_INDEX, 0.95),
(crate::rl::isv_slots::RL_ENTRY_COST_INDEX, 15.0),
(crate::rl::isv_slots::RL_SHORT_HOLD_MIN_STEPS_INDEX, 50.0),
(crate::rl::isv_slots::RL_SHORT_HOLD_MIN_STEPS_INDEX, 100.0),
(crate::rl::isv_slots::RL_SHORT_HOLD_PENALTY_INDEX, 0.5),
(crate::rl::isv_slots::RL_HOLD_BONUS_INDEX, 2.0),
(crate::rl::isv_slots::RL_MIN_HOLD_STEPS_INDEX, 50.0),
(crate::rl::isv_slots::RL_MIN_HOLD_STEPS_INDEX, 100.0),
(crate::rl::isv_slots::RL_MULTIRES_HORIZON_1_INDEX, 1.0),
(crate::rl::isv_slots::RL_MULTIRES_HORIZON_2_INDEX, 10.0),
(crate::rl::isv_slots::RL_MULTIRES_HORIZON_3_INDEX, 600.0),

View File

@@ -48,7 +48,7 @@ spec:
- name: seq-len
value: "32"
- name: per-capacity
value: "16384"
value: "32768"
- name: instrument-mode
value: "all"
- name: log-every