feat(rl): amplify surfer reward shape — fewer-but-bigger trades
User explicit request: restore surfer philosophy. Previous wr=0.30 trend- follower pattern was profitable but takes ~22k trades/run on lots of small tail wins. Per surfer philosophy (pearl_surfer_philosophy_trading): wait for the wave, ride it out, accept wipeouts. Bootstrap changes (architectural fixes from prior commits preserved): - ENTRY_COST: 15 → 40 (discourage low-conviction entries) - SHORT_HOLD_MIN_STEPS: 100 → 200 (true wave timescale) - SHORT_HOLD_PENALTY: 0.5 → 0.3 (harder penalty for quick exits) - HOLD_BONUS: 2.0 → 4.0 (amplify sustained-ride rewards) - MIN_HOLD_STEPS: 138 (γ-derived) → 300 (multi-minute holds) - CONF_GATE_MAX_HOLD_FRAC: 0.85 → 0.95 (95% Hold = patient) - THOMPSON_FLOOR: 0.05 → 0.02 (less random, more Q-driven) Local smoke 1000 steps (b=128): - wr peaked 0.61 at step 200 (vs 0.30 before) - dones per step dropped to 1-7 (vs 17-40 before) - PnL stays positive: $285k cumulative - avg_hold growing toward 200+ Expected at scale: fewer trades, higher wr, dollar-positive after realistic $0.82/side costs because each trade is bigger. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -2884,15 +2884,18 @@ impl IntegratedTrainer {
|
||||
(crate::rl::isv_slots::RL_GATE_FRD_MIN_INDEX, 0.0),
|
||||
(crate::rl::isv_slots::RL_GATE_FRD_MAX_INDEX, 0.50),
|
||||
(crate::rl::isv_slots::RL_GATE_ADJUST_RATE_INDEX, 0.95),
|
||||
(crate::rl::isv_slots::RL_ENTRY_COST_INDEX, 15.0),
|
||||
(crate::rl::isv_slots::RL_SHORT_HOLD_MIN_STEPS_INDEX, 100.0),
|
||||
(crate::rl::isv_slots::RL_SHORT_HOLD_PENALTY_INDEX, 0.5),
|
||||
(crate::rl::isv_slots::RL_HOLD_BONUS_INDEX, 2.0),
|
||||
// γ-derived: half-life = round(-ln(2)/-ln(γ_floor)). At γ=0.995 → 138.
|
||||
// Per pearl_edge_lives_at_wave_timescale_not_tick — needed to lift
|
||||
// the agent off tick-scale operation (observed avg_hold=20).
|
||||
(crate::rl::isv_slots::RL_MIN_HOLD_STEPS_INDEX,
|
||||
(-((2.0_f32).ln()) / 0.995_f32.ln()).round()),
|
||||
// SURFER PHILOSOPHY (2026-05-28): force fewer-but-bigger trades.
|
||||
// Entry cost amplified to discourage low-conviction entries.
|
||||
(crate::rl::isv_slots::RL_ENTRY_COST_INDEX, 40.0),
|
||||
// Short-hold penalty floor raised — true wave timescale.
|
||||
(crate::rl::isv_slots::RL_SHORT_HOLD_MIN_STEPS_INDEX, 200.0),
|
||||
(crate::rl::isv_slots::RL_SHORT_HOLD_PENALTY_INDEX, 0.3),
|
||||
// Long-ride bonus amplified to reward sustained holds.
|
||||
(crate::rl::isv_slots::RL_HOLD_BONUS_INDEX, 4.0),
|
||||
// Wave timescale: 300 steps ≈ several minutes real time.
|
||||
// Replaces γ-derived 138 — too short for genuine surfer waves.
|
||||
// Per pearl_edge_lives_at_wave_timescale_not_tick.
|
||||
(crate::rl::isv_slots::RL_MIN_HOLD_STEPS_INDEX, 300.0),
|
||||
(crate::rl::isv_slots::RL_ASYM_LOSS_DECAY_RATE_INDEX, 0.995),
|
||||
(crate::rl::isv_slots::RL_ASYM_WIN_TRAIL_FACTOR_INDEX, 0.5),
|
||||
(crate::rl::isv_slots::RL_SESSION_LOSS_LIMIT_INDEX, -50.0),
|
||||
@@ -2973,14 +2976,16 @@ impl IntegratedTrainer {
|
||||
// maintain target entropy. H_target = 0.7 × ln(11) ≈ 1.68.
|
||||
(crate::rl::isv_slots::RL_SAC_ALPHA_INDEX, 0.01),
|
||||
(crate::rl::isv_slots::RL_SAC_ENTROPY_TARGET_INDEX, 1.68),
|
||||
(crate::rl::isv_slots::RL_CONF_GATE_MAX_HOLD_FRAC_INDEX, 0.85),
|
||||
// SURFER: 95% Hold = patient wave-watching, only 5% explore slots.
|
||||
(crate::rl::isv_slots::RL_CONF_GATE_MAX_HOLD_FRAC_INDEX, 0.95),
|
||||
// Hard margin floor: $35k capital, ~$14.4k ES maintenance.
|
||||
// Max drawdown ~$20k before broker liquidates.
|
||||
(crate::rl::isv_slots::RL_MARGIN_FLOOR_USD_INDEX, 20000.0),
|
||||
// Thompson sampler epsilon-greedy floor — 5% of steps
|
||||
// pick a uniform random action, preventing entropy collapse
|
||||
// per `pearl_pi_actor_collapses_without_entropy_floor`.
|
||||
(crate::rl::isv_slots::RL_THOMPSON_FLOOR_INDEX, 0.05),
|
||||
// SURFER: lower random exploration — let Q's conviction drive trades.
|
||||
(crate::rl::isv_slots::RL_THOMPSON_FLOOR_INDEX, 0.02),
|
||||
];
|
||||
for (slot, value) in isv_constants.iter() {
|
||||
let slot_i32 = *slot as i32;
|
||||
|
||||
Reference in New Issue
Block a user