feat(rl): tier 2 wave-scale — γ=0.995, PER 32k, min-hold 100
Prepared for next run after the γ=0.99 1M run completes: - γ floor 0.99 → 0.995: horizon 100 → 200 steps (50 seconds). Real ES directional moves (2-5 points) happen at this scale. - PER 16384 → 32768: 2048 unique steps of replay depth. Supports the 200-step γ horizon with margin. - Min-hold 50 → 100 steps (25 seconds): commit to the full wave. Short-hold penalty threshold matches. Safe to push because raw-reward re-normalization eliminates scale drift in the deeper buffer, and the ±2% scale clamp keeps targets stable across the 2048-step replay window. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -45,7 +45,7 @@
|
||||
// the policy myopic).
|
||||
|
||||
#define RL_GAMMA_INDEX 400
|
||||
#define GAMMA_MIN 0.99f
|
||||
#define GAMMA_MIN 0.995f
|
||||
#define GAMMA_MAX 0.999f
|
||||
#define WIENER_ALPHA_FLOOR 0.4f
|
||||
|
||||
|
||||
@@ -145,7 +145,7 @@ struct Cli {
|
||||
|
||||
/// PER buffer capacity. At b_size=16, capacity/16 = unique steps
|
||||
/// retained. 32768/16 = 2048 steps of replay depth.
|
||||
#[arg(long, default_value_t = 16384)]
|
||||
#[arg(long, default_value_t = 32768)]
|
||||
per_capacity: usize,
|
||||
|
||||
/// Random seed pinning RNG + Xavier init + ISV controller bootstrap
|
||||
|
||||
@@ -294,7 +294,7 @@ impl Default for IntegratedTrainerConfig {
|
||||
perception: PerceptionTrainerConfig::default(),
|
||||
dqn_seed: 0xDA_DA_DA_DA,
|
||||
ppo_seed: 0xBE_BE_BE_BE,
|
||||
per_capacity: 16384,
|
||||
per_capacity: 32768,
|
||||
per_seed: 0x9E37_79B9_7F4A_7C15,
|
||||
}
|
||||
}
|
||||
@@ -1652,10 +1652,10 @@ impl IntegratedTrainer {
|
||||
(crate::rl::isv_slots::RL_GATE_FRD_MAX_INDEX, 0.50),
|
||||
(crate::rl::isv_slots::RL_GATE_ADJUST_RATE_INDEX, 0.95),
|
||||
(crate::rl::isv_slots::RL_ENTRY_COST_INDEX, 15.0),
|
||||
(crate::rl::isv_slots::RL_SHORT_HOLD_MIN_STEPS_INDEX, 50.0),
|
||||
(crate::rl::isv_slots::RL_SHORT_HOLD_MIN_STEPS_INDEX, 100.0),
|
||||
(crate::rl::isv_slots::RL_SHORT_HOLD_PENALTY_INDEX, 0.5),
|
||||
(crate::rl::isv_slots::RL_HOLD_BONUS_INDEX, 2.0),
|
||||
(crate::rl::isv_slots::RL_MIN_HOLD_STEPS_INDEX, 50.0),
|
||||
(crate::rl::isv_slots::RL_MIN_HOLD_STEPS_INDEX, 100.0),
|
||||
(crate::rl::isv_slots::RL_MULTIRES_HORIZON_1_INDEX, 1.0),
|
||||
(crate::rl::isv_slots::RL_MULTIRES_HORIZON_2_INDEX, 10.0),
|
||||
(crate::rl::isv_slots::RL_MULTIRES_HORIZON_3_INDEX, 600.0),
|
||||
|
||||
@@ -48,7 +48,7 @@ spec:
|
||||
- name: seq-len
|
||||
value: "32"
|
||||
- name: per-capacity
|
||||
value: "16384"
|
||||
value: "32768"
|
||||
- name: instrument-mode
|
||||
value: "all"
|
||||
- name: log-every
|
||||
|
||||
Reference in New Issue
Block a user