From 0d534f1dc0347c714101d12759837b1edde4bca3 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 12 Apr 2026 16:17:26 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20dense=20OFI-alignment=20reward=20?= =?UTF-8?q?=E2=80=94=20state-conditional=20direction=20signal=20every=20ba?= =?UTF-8?q?r?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces dead Gem 1 micro-reward (used future price = hindsight leak) and moves OFI-alignment from sparse (trade completion) to dense (every positioned bar). Uses CURRENT-BAR observable OFI features: - depth_imbalance (70%): bid vs ask volume ratio from MBP-10 - trade_imbalance (30%): inferred trade direction flow Rewards Long when order flow says buy pressure, Short when sell pressure. This gives the direction advantage head a STATE-CONDITIONAL signal: "Long is better than Short WHEN order book shows buy pressure." Without this, direction rewards are 50/50 (price is random) and the advantage gradient cancels via dueling mean-subtraction. Combined with dense DSR, every positioned bar now gets: 1. DSR: "is holding improving Sharpe?" (consistency signal) 2. OFI-alignment: "does order flow confirm your direction?" (microstructure signal) 3. Inventory penalty: "holding costs money" (urgency signal) Co-Authored-By: Claude Opus 4.6 (1M context) --- .../src/cuda_pipeline/experience_kernels.cu | 64 ++++++++----------- 1 file changed, 28 insertions(+), 36 deletions(-) diff --git a/crates/ml/src/cuda_pipeline/experience_kernels.cu b/crates/ml/src/cuda_pipeline/experience_kernels.cu index 1bba445e1..b25a51fba 100644 --- a/crates/ml/src/cuda_pipeline/experience_kernels.cu +++ b/crates/ml/src/cuda_pipeline/experience_kernels.cu @@ -1570,18 +1570,7 @@ extern "C" __global__ void experience_env_step( reward += exit_timing_weight * asymmetric_soft_clamp(timing_quality * 10.0f) * 0.1f; } - /* ── Layer 8: OFI-weighted reward confidence ── */ - if (ofi_reward_weight > 0.0f) { - float price_move = (raw_next - raw_close) / fmaxf(raw_close, 1.0f); - float move_magnitude = fabsf(price_move) / vol_proxy; - /* Use dir_idx (current action) not position (stale state) */ - float alignment = 0.0f; - if (dir_idx == 2 && price_move > 0.0f) alignment = 1.0f; - if (dir_idx == 0 && price_move < 0.0f) alignment = 1.0f; - float ofi_bonus = alignment * fminf(move_magnitude, 0.5f); - /* Additive: doesn't compound magnitude bias (was multiplicative) */ - reward += ofi_reward_weight * ofi_bonus; - } + /* (OFI-alignment moved to dense per-bar path below) */ /* ── Layer 9: Kelly-optimal position sizing signal ── */ if (kelly_sizing_weight > 0.0f) { @@ -1605,30 +1594,33 @@ extern "C" __global__ void experience_env_step( } } - /* ── Gem 1: Direction-specific dense micro-reward ── - * Uses decoded dir_idx instead of position sign for clearer gradient. - * Short (dir_idx==0) profits when price falls; Long (dir_idx==2) profits - * when price rises; Flat (dir_idx==1) gets no micro-reward. */ - if (fabsf(position) > 0.001f && micro_reward_scale > 0.0f) { - float micro_vol_proxy = 0.005f; - if (features != NULL && bar_idx < total_bars && market_dim > 9) { - float atr_n = (features[(long long)bar_idx * market_dim + 9]); - float log_a = atr_n * 16.0f - 7.0f; - micro_vol_proxy = fmaxf(expf(log_a) / fmaxf(raw_close, 1.0f), 0.0001f); - } - float price_change = (raw_next - raw_close) / fmaxf(raw_close, 1.0f); - float dir_reward = 0.0f; - if (dir_idx == 0) dir_reward = -price_change; /* Short profits when price falls */ - if (dir_idx == 2) dir_reward = price_change; /* Long profits when price rises */ - /* dir_idx == 1 (Flat): no micro-reward — position should be 0 */ - float vol_ratio = micro_vol_proxy / 0.005f; - float adaptive_scale = micro_reward_scale / fmaxf(sqrtf(vol_ratio), 0.5f); - /* sqrt-compressed: Small=0.50, Half=0.71, Full=1.00. Max ratio 2:1 (was 4:1). */ - float mag_fraction; - if (mag_idx == 0) mag_fraction = 0.50f; - else if (mag_idx == 1) mag_fraction = 0.707f; - else mag_fraction = 1.00f; - reward += adaptive_scale * dir_reward * mag_fraction / micro_vol_proxy; + /* ── v9: Dense OFI-alignment reward (state-conditional direction signal) ── + * Replaces old Gem 1 micro-reward (which used future price = hindsight leak). + * Uses CURRENT-BAR order flow imbalance (observable, causal). + * + * depth_imbalance ∈ [-1, +1]: +1=bid-heavy (buy pressure), -1=ask-heavy. + * Rewards Long when depth says buy, Short when depth says sell. + * Gives the direction advantage head a STATE-CONDITIONAL signal every bar: + * "Long is better than Short WHEN order book shows buy pressure." + * + * Without this, direction rewards are 50/50 (price direction is random at bar + * timescale) and the advantage gradient cancels via dueling mean-subtraction. + * The OFI signal is NOT random — it reflects real order book microstructure. */ + if (ofi_reward_weight > 0.0f && fabsf(position) > 0.001f && market_dim > 44) { + /* OFI feature [2] = depth_imbalance at feature index 42 + 2 = 44 */ + float depth_imb = (features[(long long)bar_idx * market_dim + 44]); + /* OFI feature [7] = trade_imbalance at feature index 42 + 7 = 49 */ + float trade_imb = (market_dim > 49) + ? (features[(long long)bar_idx * market_dim + 49]) + : 0.0f; + /* Combined flow signal: depth imbalance (70%) + trade flow (30%) */ + float flow_signal = 0.7f * depth_imb + 0.3f * trade_imb; + + float alignment = 0.0f; + if (dir_idx == 2) alignment = flow_signal; /* Long: reward aligns with buy pressure */ + if (dir_idx == 0) alignment = -flow_signal; /* Short: reward aligns with sell pressure */ + /* dir_idx == 1 (Flat): alignment = 0, no OFI bonus — safe default */ + reward += ofi_reward_weight * fminf(fmaxf(alignment, -1.0f), 1.0f); } /* ── v9: Inventory penalty — penalize holding positions (Sharpe killer) ── */