From 60c714c8d12565aa7723b868d053916cd182655c Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 25 May 2026 12:12:49 +0200 Subject: [PATCH] =?UTF-8?q?feat(rl):=20surfer-philosophy=20reward=20shapin?= =?UTF-8?q?g=20=E2=80=94=20entry=20cost=20+=20hold=20bonus?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three ISV-driven reward shaping components discourage churning and reward patience: 1. Entry cost ($15 default): subtracted when opening a new position. Agent must expect profit > cost to justify entry. Slightly above ES 1-tick spread ($12.50) so marginal trades are net-negative. 2. Short-hold penalty (0.5× for holds < 20 steps): multiplicative penalty at trade close for quick flips. "Don't bail on the first bump" — halves the reward for sub-5-second holds. 3. Hold bonus ($0.50/step × sqrt(hold_time)): per-step reward for staying in a profitable position. "Ride the wave" — incentivizes patience when the trade is working. Runs BEFORE reward_scale so all costs/bonuses are in raw USD terms. ISV slots 532-535. RL_SLOTS_END → 536. Co-Authored-By: Claude Opus 4.7 --- crates/ml-alpha/build.rs | 1 + crates/ml-alpha/cuda/rl_reward_shaping.cu | 78 +++++++++++++++++++++++ crates/ml-alpha/src/rl/isv_slots.rs | 19 +++++- crates/ml-alpha/src/trainer/integrated.rs | 46 ++++++++++++- 4 files changed, 142 insertions(+), 2 deletions(-) create mode 100644 crates/ml-alpha/cuda/rl_reward_shaping.cu diff --git a/crates/ml-alpha/build.rs b/crates/ml-alpha/build.rs index e511f7d48..5f04110e7 100644 --- a/crates/ml-alpha/build.rs +++ b/crates/ml-alpha/build.rs @@ -87,6 +87,7 @@ const KERNELS: &[&str] = &[ "rl_multires_features_update", // P0: per-batch multi-resolution streaming features (3 horizons × 4 features) "rl_encoder_context_broadcast", // P2: broadcast per-batch context (16 dims) into encoder input [B,K,56] at cols 40-55 "rl_gate_threshold_controller", // adaptive gate thresholds from trade frequency (dones EMA) + "rl_reward_shaping", // surfer-philosophy: entry cost + short-hold penalty + hold bonus ]; // Cache bust v31 — five new reduce / derive kernels populate the input diff --git a/crates/ml-alpha/cuda/rl_reward_shaping.cu b/crates/ml-alpha/cuda/rl_reward_shaping.cu new file mode 100644 index 000000000..d920089a6 --- /dev/null +++ b/crates/ml-alpha/cuda/rl_reward_shaping.cu @@ -0,0 +1,78 @@ +// rl_reward_shaping.cu — surfer-philosophy reward shaping. +// +// Three components applied AFTER extract_realized_pnl_delta, BEFORE +// apply_reward_scale: +// +// 1. ENTRY COST: subtract a fixed penalty when a new position opens. +// "Don't paddle for ripples" — the agent must believe the trade +// will profit more than the cost to justify entering. ISV-driven. +// +// 2. SHORT-HOLD PENALTY: multiply reward by a factor < 1 when a trade +// closes with hold_time < min_hold. "Don't bail on the first bump." +// Discourages churning by making quick flips less rewarding. +// +// 3. HOLD BONUS: small per-step bonus when in a profitable position +// (unrealized_r > 0). "Ride the wave" — rewards patience when the +// trade is working. Scales with sqrt(hold_time) so the incentive +// grows as the ride extends. +// +// All thresholds and coefficients are ISV-driven for runtime tuning. +// Runs BEFORE reward_scale so the shaping is in raw dollar terms. +// +// Per `feedback_no_atomicadd`: per-batch element-wise. +// Per `feedback_cpu_is_read_only`: pure device-side. + +#include + +#define RL_ENTRY_COST_INDEX 532 +#define RL_SHORT_HOLD_MIN_STEPS_INDEX 533 +#define RL_SHORT_HOLD_PENALTY_INDEX 534 +#define RL_HOLD_BONUS_INDEX 535 + +extern "C" __global__ void rl_reward_shaping( + float* __restrict__ rewards, // [B] IN/OUT (raw PnL delta) + const int* __restrict__ prev_pos_lots, // [B] position BEFORE this step + const unsigned char* __restrict__ pos_state, // [B × pos_bytes] position AFTER fill + const float* __restrict__ dones, // [B] (1.0 if done, 0.0 otherwise) + const int* __restrict__ steps_since_done, // [B] hold time counter + const float* __restrict__ isv, + int b_size, + int pos_bytes +) { + const int b = blockIdx.x * blockDim.x + threadIdx.x; + if (b >= b_size) return; + + const int prev_pos = prev_pos_lots[b]; + const int curr_pos = *(const int*)(pos_state + b * pos_bytes); + const float entry_cost = isv[RL_ENTRY_COST_INDEX]; + const float min_hold = isv[RL_SHORT_HOLD_MIN_STEPS_INDEX]; + const float penalty = isv[RL_SHORT_HOLD_PENALTY_INDEX]; + const float hold_bonus = isv[RL_HOLD_BONUS_INDEX]; + const int hold_time = steps_since_done[b]; + + float r = rewards[b]; + + // 1. Entry cost: fires when transitioning from flat to positioned. + const bool was_flat = (prev_pos == 0); + const bool is_positioned = (curr_pos != 0); + if (was_flat && is_positioned) { + r -= entry_cost; + } + + // 2. Short-hold penalty: fires at trade close (done) if hold time + // is below minimum. Multiplicative so it scales with trade size. + if (dones[b] > 0.5f && (float)hold_time < min_hold) { + r *= penalty; + } + + // 3. Hold bonus: per-step reward for staying in a profitable + // position. "Ride the wave" incentive. Only fires when the + // mark-to-market PnL delta is positive AND the agent is + // holding (not flat). Scales with sqrt(hold_time) so longer + // rides are rewarded more. + if (!was_flat && is_positioned && r > 0.0f && hold_time > 0) { + r += hold_bonus * sqrtf((float)hold_time); + } + + rewards[b] = r; +} diff --git a/crates/ml-alpha/src/rl/isv_slots.rs b/crates/ml-alpha/src/rl/isv_slots.rs index 2f1ed77ef..e5a862a04 100644 --- a/crates/ml-alpha/src/rl/isv_slots.rs +++ b/crates/ml-alpha/src/rl/isv_slots.rs @@ -909,6 +909,23 @@ pub const RL_GATE_FRD_MAX_INDEX: usize = 530; /// Gate threshold controller: Schulman step rate. Default 0.95. pub const RL_GATE_ADJUST_RATE_INDEX: usize = 531; +/// Entry cost in raw USD subtracted when opening a new position. +/// "Don't paddle for ripples" — agent must expect profit > cost. +/// Default: 15.0 (slightly above ES 1-tick spread of $12.50). +pub const RL_ENTRY_COST_INDEX: usize = 532; + +/// Minimum hold time (steps) below which the short-hold penalty fires. +/// Default: 20 (~5 seconds of ES data at typical event rate). +pub const RL_SHORT_HOLD_MIN_STEPS_INDEX: usize = 533; + +/// Reward multiplier for trades closed below min hold time. +/// Default: 0.5 (halve reward for quick flips). +pub const RL_SHORT_HOLD_PENALTY_INDEX: usize = 534; + +/// Per-step bonus (USD) for holding a profitable position. +/// Scales with sqrt(hold_time). Default: 0.50. +pub const RL_HOLD_BONUS_INDEX: usize = 535; + /// Last RL-allocated slot index (exclusive). The integrated trainer /// extends `ISV_TOTAL_DIM` to at least this value at trainer init time. -pub const RL_SLOTS_END: usize = 532; +pub const RL_SLOTS_END: usize = 536; diff --git a/crates/ml-alpha/src/trainer/integrated.rs b/crates/ml-alpha/src/trainer/integrated.rs index b25481dd6..341a53276 100644 --- a/crates/ml-alpha/src/trainer/integrated.rs +++ b/crates/ml-alpha/src/trainer/integrated.rs @@ -202,6 +202,8 @@ const RL_RECENT_OUTCOME_UPDATE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/rl_recent_outcome_update.cubin")); const RL_GATE_THRESHOLD_CONTROLLER_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/rl_gate_threshold_controller.cubin")); +const RL_REWARD_SHAPING_CUBIN: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/rl_reward_shaping.cubin")); const RL_TRADE_CONTEXT_UPDATE_CUBIN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/rl_trade_context_update.cubin")); const RL_MULTIRES_FEATURES_UPDATE_CUBIN: &[u8] = @@ -463,6 +465,8 @@ pub struct IntegratedTrainer { rl_recent_outcome_update_fn: CudaFunction, _rl_gate_threshold_controller_module: Arc, rl_gate_threshold_controller_fn: CudaFunction, + _rl_reward_shaping_module: Arc, + rl_reward_shaping_fn: CudaFunction, _rl_trade_context_update_module: Arc, rl_trade_context_update_fn: CudaFunction, _rl_multires_features_update_module: Arc, @@ -980,6 +984,12 @@ impl IntegratedTrainer { let rl_gate_threshold_controller_fn = rl_gate_threshold_controller_module .load_function("rl_gate_threshold_controller") .context("load rl_gate_threshold_controller")?; + let rl_reward_shaping_module = ctx + .load_cubin(RL_REWARD_SHAPING_CUBIN.to_vec()) + .context("load rl_reward_shaping cubin")?; + let rl_reward_shaping_fn = rl_reward_shaping_module + .load_function("rl_reward_shaping") + .context("load rl_reward_shaping")?; let rl_trade_context_update_module = ctx .load_cubin(RL_TRADE_CONTEXT_UPDATE_CUBIN.to_vec()) .context("load rl_trade_context_update cubin")?; @@ -1407,6 +1417,8 @@ impl IntegratedTrainer { rl_recent_outcome_update_fn, _rl_gate_threshold_controller_module: rl_gate_threshold_controller_module, rl_gate_threshold_controller_fn, + _rl_reward_shaping_module: rl_reward_shaping_module, + rl_reward_shaping_fn, _rl_trade_context_update_module: rl_trade_context_update_module, rl_trade_context_update_fn, _rl_multires_features_update_module: rl_multires_features_update_module, @@ -1552,7 +1564,7 @@ impl IntegratedTrainer { // (slot, value) pair — pure device write, no HtoD per // `feedback_no_htod_htoh_only_mapped_pinned`. { - let isv_constants: [(usize, f32); 68] = [ + let isv_constants: [(usize, f32); 72] = [ // Static seeds for the adaptive reward-clamp controller — // these are the initial values that // `rl_reward_clamp_controller` will replace once it @@ -1627,6 +1639,10 @@ impl IntegratedTrainer { (crate::rl::isv_slots::RL_GATE_FRD_MIN_INDEX, 0.0), (crate::rl::isv_slots::RL_GATE_FRD_MAX_INDEX, 0.50), (crate::rl::isv_slots::RL_GATE_ADJUST_RATE_INDEX, 0.95), + (crate::rl::isv_slots::RL_ENTRY_COST_INDEX, 15.0), + (crate::rl::isv_slots::RL_SHORT_HOLD_MIN_STEPS_INDEX, 20.0), + (crate::rl::isv_slots::RL_SHORT_HOLD_PENALTY_INDEX, 0.5), + (crate::rl::isv_slots::RL_HOLD_BONUS_INDEX, 0.50), (crate::rl::isv_slots::RL_MULTIRES_HORIZON_1_INDEX, 1.0), (crate::rl::isv_slots::RL_MULTIRES_HORIZON_2_INDEX, 10.0), (crate::rl::isv_slots::RL_MULTIRES_HORIZON_3_INDEX, 600.0), @@ -4324,6 +4340,34 @@ impl IntegratedTrainer { // grid-stride loop walks the rest. Shared mem now 2× block × // f32 to fit the parallel max(|scaled|) + max(positive) // reductions. + // Reward shaping: entry cost + short-hold penalty + hold bonus. + // Runs on raw rewards BEFORE scaling so costs are in USD terms. + { + let pos_d_ref: &CudaSlice = lobsim.pos_d(); + let cfg = LaunchConfig { + grid_dim: (((b_size as u32) + 31) / 32, 1, 1), + block_dim: (32, 1, 1), + shared_mem_bytes: 0, + }; + let b_size_i = b_size as i32; + let pos_bytes_i = lobsim.pos_bytes() as i32; + let mut launch = self.stream.launch_builder(&self.rl_reward_shaping_fn); + launch + .arg(&mut self.rewards_d) + .arg(&self.prev_position_lots_d) + .arg(pos_d_ref) + .arg(&self.dones_d) + .arg(&self.steps_since_done_d) + .arg(&self.isv_d) + .arg(&b_size_i) + .arg(&pos_bytes_i); + unsafe { + launch + .launch(cfg) + .context("rl_reward_shaping launch")?; + } + } + // Snapshot raw rewards before scaling — replay buffer stores // raw_reward for re-normalization at sample time. {