gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(rl): surfer-philosophy reward shaping — entry cost + hold bonus

Browse Source 
Three ISV-driven reward shaping components discourage churning and
reward patience:

1. Entry cost ($15 default): subtracted when opening a new position.
   Agent must expect profit > cost to justify entry. Slightly above
   ES 1-tick spread ($12.50) so marginal trades are net-negative.

2. Short-hold penalty (0.5× for holds < 20 steps): multiplicative
   penalty at trade close for quick flips. "Don't bail on the first
   bump" — halves the reward for sub-5-second holds.

3. Hold bonus ($0.50/step × sqrt(hold_time)): per-step reward for
   staying in a profitable position. "Ride the wave" — incentivizes
   patience when the trade is working.

Runs BEFORE reward_scale so all costs/bonuses are in raw USD terms.
ISV slots 532-535. RL_SLOTS_END → 536.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-05-25 12:12:49 +02:00
parent a1d35cd6e6
commit 60c714c8d1
4 changed files with 142 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
crates/ml-alpha/build.rs
View File

@@ -87,6 +87,7 @@ const KERNELS: &[&str] = &[
"rl_multires_features_update", // P0: per-batch multi-resolution streaming features (3 horizons × 4 features)
"rl_encoder_context_broadcast", // P2: broadcast per-batch context (16 dims) into encoder input [B,K,56] at cols 40-55
"rl_gate_threshold_controller", // adaptive gate thresholds from trade frequency (dones EMA)
"rl_reward_shaping", // surfer-philosophy: entry cost + short-hold penalty + hold bonus
];
// Cache bust v31 — five new reduce / derive kernels populate the input

78
crates/ml-alpha/cuda/rl_reward_shaping.cu Normal file
View File

@@ -0,0 +1,78 @@
// rl_reward_shaping.cu — surfer-philosophy reward shaping.
//
// Three components applied AFTER extract_realized_pnl_delta, BEFORE
// apply_reward_scale:
//
// 1. ENTRY COST: subtract a fixed penalty when a new position opens.
// "Don't paddle for ripples" — the agent must believe the trade
// will profit more than the cost to justify entering. ISV-driven.
//
// 2. SHORT-HOLD PENALTY: multiply reward by a factor < 1 when a trade
// closes with hold_time < min_hold. "Don't bail on the first bump."
// Discourages churning by making quick flips less rewarding.
//
// 3. HOLD BONUS: small per-step bonus when in a profitable position
// (unrealized_r > 0). "Ride the wave" — rewards patience when the
// trade is working. Scales with sqrt(hold_time) so the incentive
// grows as the ride extends.
//
// All thresholds and coefficients are ISV-driven for runtime tuning.
// Runs BEFORE reward_scale so the shaping is in raw dollar terms.
//
// Per `feedback_no_atomicadd`: per-batch element-wise.
// Per `feedback_cpu_is_read_only`: pure device-side.
#include <stdint.h>
#define RL_ENTRY_COST_INDEX 532
#define RL_SHORT_HOLD_MIN_STEPS_INDEX 533
#define RL_SHORT_HOLD_PENALTY_INDEX 534
#define RL_HOLD_BONUS_INDEX 535
extern "C" __global__ void rl_reward_shaping(
float* __restrict__ rewards, // [B] IN/OUT (raw PnL delta)
const int* __restrict__ prev_pos_lots, // [B] position BEFORE this step
const unsigned char* __restrict__ pos_state, // [B × pos_bytes] position AFTER fill
const float* __restrict__ dones, // [B] (1.0 if done, 0.0 otherwise)
const int* __restrict__ steps_since_done, // [B] hold time counter
const float* __restrict__ isv,
int b_size,
int pos_bytes
) {
const int b = blockIdx.x * blockDim.x + threadIdx.x;
if (b >= b_size) return;
const int prev_pos = prev_pos_lots[b];
const int curr_pos = *(const int*)(pos_state + b * pos_bytes);
const float entry_cost = isv[RL_ENTRY_COST_INDEX];
const float min_hold = isv[RL_SHORT_HOLD_MIN_STEPS_INDEX];
const float penalty = isv[RL_SHORT_HOLD_PENALTY_INDEX];
const float hold_bonus = isv[RL_HOLD_BONUS_INDEX];
const int hold_time = steps_since_done[b];
float r = rewards[b];
// 1. Entry cost: fires when transitioning from flat to positioned.
const bool was_flat = (prev_pos == 0);
const bool is_positioned = (curr_pos != 0);
if (was_flat && is_positioned) {
r -= entry_cost;
}
// 2. Short-hold penalty: fires at trade close (done) if hold time
// is below minimum. Multiplicative so it scales with trade size.
if (dones[b] > 0.5f && (float)hold_time < min_hold) {
r *= penalty;
}
// 3. Hold bonus: per-step reward for staying in a profitable
// position. "Ride the wave" incentive. Only fires when the
// mark-to-market PnL delta is positive AND the agent is
// holding (not flat). Scales with sqrt(hold_time) so longer
// rides are rewarded more.
if (!was_flat && is_positioned && r > 0.0f && hold_time > 0) {
r += hold_bonus * sqrtf((float)hold_time);
}
rewards[b] = r;
}

19
crates/ml-alpha/src/rl/isv_slots.rs
View File

@@ -909,6 +909,23 @@ pub const RL_GATE_FRD_MAX_INDEX: usize = 530;
/// Gate threshold controller: Schulman step rate. Default 0.95.
pub const RL_GATE_ADJUST_RATE_INDEX: usize = 531;
/// Entry cost in raw USD subtracted when opening a new position.
/// "Don't paddle for ripples" — agent must expect profit > cost.
/// Default: 15.0 (slightly above ES 1-tick spread of $12.50).
pub const RL_ENTRY_COST_INDEX: usize = 532;
/// Minimum hold time (steps) below which the short-hold penalty fires.
/// Default: 20 (~5 seconds of ES data at typical event rate).
pub const RL_SHORT_HOLD_MIN_STEPS_INDEX: usize = 533;
/// Reward multiplier for trades closed below min hold time.
/// Default: 0.5 (halve reward for quick flips).
pub const RL_SHORT_HOLD_PENALTY_INDEX: usize = 534;
/// Per-step bonus (USD) for holding a profitable position.
/// Scales with sqrt(hold_time). Default: 0.50.
pub const RL_HOLD_BONUS_INDEX: usize = 535;
/// Last RL-allocated slot index (exclusive). The integrated trainer
/// extends `ISV_TOTAL_DIM` to at least this value at trainer init time.
pub const RL_SLOTS_END: usize = 532;
pub const RL_SLOTS_END: usize = 536;

46
crates/ml-alpha/src/trainer/integrated.rs
View File

@@ -202,6 +202,8 @@ const RL_RECENT_OUTCOME_UPDATE_CUBIN: &[u8] =
include_bytes!(concat!(env!("OUT_DIR"), "/rl_recent_outcome_update.cubin"));
const RL_GATE_THRESHOLD_CONTROLLER_CUBIN: &[u8] =
include_bytes!(concat!(env!("OUT_DIR"), "/rl_gate_threshold_controller.cubin"));
const RL_REWARD_SHAPING_CUBIN: &[u8] =
include_bytes!(concat!(env!("OUT_DIR"), "/rl_reward_shaping.cubin"));
const RL_TRADE_CONTEXT_UPDATE_CUBIN: &[u8] =
include_bytes!(concat!(env!("OUT_DIR"), "/rl_trade_context_update.cubin"));
const RL_MULTIRES_FEATURES_UPDATE_CUBIN: &[u8] =
@@ -463,6 +465,8 @@ pub struct IntegratedTrainer {
rl_recent_outcome_update_fn: CudaFunction,
_rl_gate_threshold_controller_module: Arc<CudaModule>,
rl_gate_threshold_controller_fn: CudaFunction,
_rl_reward_shaping_module: Arc<CudaModule>,
rl_reward_shaping_fn: CudaFunction,
_rl_trade_context_update_module: Arc<CudaModule>,
rl_trade_context_update_fn: CudaFunction,
_rl_multires_features_update_module: Arc<CudaModule>,
@@ -980,6 +984,12 @@ impl IntegratedTrainer {
let rl_gate_threshold_controller_fn = rl_gate_threshold_controller_module
.load_function("rl_gate_threshold_controller")
.context("load rl_gate_threshold_controller")?;
let rl_reward_shaping_module = ctx
.load_cubin(RL_REWARD_SHAPING_CUBIN.to_vec())
.context("load rl_reward_shaping cubin")?;
let rl_reward_shaping_fn = rl_reward_shaping_module
.load_function("rl_reward_shaping")
.context("load rl_reward_shaping")?;
let rl_trade_context_update_module = ctx
.load_cubin(RL_TRADE_CONTEXT_UPDATE_CUBIN.to_vec())
.context("load rl_trade_context_update cubin")?;
@@ -1407,6 +1417,8 @@ impl IntegratedTrainer {
rl_recent_outcome_update_fn,
_rl_gate_threshold_controller_module: rl_gate_threshold_controller_module,
rl_gate_threshold_controller_fn,
_rl_reward_shaping_module: rl_reward_shaping_module,
rl_reward_shaping_fn,
_rl_trade_context_update_module: rl_trade_context_update_module,
rl_trade_context_update_fn,
_rl_multires_features_update_module: rl_multires_features_update_module,
@@ -1552,7 +1564,7 @@ impl IntegratedTrainer {
// (slot, value) pair — pure device write, no HtoD per
// `feedback_no_htod_htoh_only_mapped_pinned`.
{
let isv_constants: [(usize, f32); 68] = [
let isv_constants: [(usize, f32); 72] = [
// Static seeds for the adaptive reward-clamp controller —
// these are the initial values that
// `rl_reward_clamp_controller` will replace once it
@@ -1627,6 +1639,10 @@ impl IntegratedTrainer {
(crate::rl::isv_slots::RL_GATE_FRD_MIN_INDEX, 0.0),
(crate::rl::isv_slots::RL_GATE_FRD_MAX_INDEX, 0.50),
(crate::rl::isv_slots::RL_GATE_ADJUST_RATE_INDEX, 0.95),
(crate::rl::isv_slots::RL_ENTRY_COST_INDEX, 15.0),
(crate::rl::isv_slots::RL_SHORT_HOLD_MIN_STEPS_INDEX, 20.0),
(crate::rl::isv_slots::RL_SHORT_HOLD_PENALTY_INDEX, 0.5),
(crate::rl::isv_slots::RL_HOLD_BONUS_INDEX, 0.50),
(crate::rl::isv_slots::RL_MULTIRES_HORIZON_1_INDEX, 1.0),
(crate::rl::isv_slots::RL_MULTIRES_HORIZON_2_INDEX, 10.0),
(crate::rl::isv_slots::RL_MULTIRES_HORIZON_3_INDEX, 600.0),
@@ -4324,6 +4340,34 @@ impl IntegratedTrainer {
// grid-stride loop walks the rest. Shared mem now 2× block ×
// f32 to fit the parallel max(|scaled|) + max(positive)
// reductions.
// Reward shaping: entry cost + short-hold penalty + hold bonus.
// Runs on raw rewards BEFORE scaling so costs are in USD terms.
{
let pos_d_ref: &CudaSlice<u8> = lobsim.pos_d();
let cfg = LaunchConfig {
grid_dim: (((b_size as u32) + 31) / 32, 1, 1),
block_dim: (32, 1, 1),
shared_mem_bytes: 0,
};
let b_size_i = b_size as i32;
let pos_bytes_i = lobsim.pos_bytes() as i32;
let mut launch = self.stream.launch_builder(&self.rl_reward_shaping_fn);
launch
.arg(&mut self.rewards_d)
.arg(&self.prev_position_lots_d)
.arg(pos_d_ref)
.arg(&self.dones_d)
.arg(&self.steps_since_done_d)
.arg(&self.isv_d)
.arg(&b_size_i)
.arg(&pos_bytes_i);
unsafe {
launch
.launch(cfg)
.context("rl_reward_shaping launch")?;
}
}
// Snapshot raw rewards before scaling — replay buffer stores
// raw_reward for re-normalization at sample time.
{