From 55d70b7cc83824e53e71e34819d8163b208505f7 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 20 Apr 2026 00:03:51 +0200 Subject: [PATCH] feat: dense micro-reward + DSR fix + counterfactual sign fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dense micro-reward: OFI momentum × price confirmation (MBP-10 mid-price mark-to-market) × adaptive cost tolerance (capital_ratio × Sharpe_ema) + book aggression + retrospective hold quality bonus. Replaces flat -0.0001 holding cost. Scale: micro_reward_scale=0.1. DSR Sharpe EMA: was hardcoded price_change_dsr=0.0 — adaptive cost tolerance was permanently floored at 0.1. Now uses actual per-bar returns from ps[PREV_CLOSE_SLOT]. Counterfactual: cf_cycle==1 (magnitude) and cf_cycle==2 (order) now undo do_flip before computing CF reward, then re-apply. Previously 2/3 of counterfactual experiences had wrong sign when do_flip=true. Rank normalization threshold: 0.001 → 1e-5 for dense micro-rewards. Co-Authored-By: Claude Opus 4.6 (1M context) --- config/training/dqn-production.toml | 2 +- .../src/cuda_pipeline/experience_kernels.cu | 93 ++++++++++++++++--- .../cuda_pipeline/reward_shaping_kernel.cu | 2 +- 3 files changed, 82 insertions(+), 15 deletions(-) diff --git a/config/training/dqn-production.toml b/config/training/dqn-production.toml index d7414e320..7f8c4967a 100644 --- a/config/training/dqn-production.toml +++ b/config/training/dqn-production.toml @@ -141,7 +141,7 @@ causal_intensity = 1.0 [reward] # v8 comprehensive training overhaul popart_enabled = true -micro_reward_scale = 0.0 +micro_reward_scale = 0.1 td_lambda = 0.9 max_trace_length = 7 hindsight_fraction = 0.1 diff --git a/crates/ml/src/cuda_pipeline/experience_kernels.cu b/crates/ml/src/cuda_pipeline/experience_kernels.cu index 3959fe93a..cc8de1b1b 100644 --- a/crates/ml/src/cuda_pipeline/experience_kernels.cu +++ b/crates/ml/src/cuda_pipeline/experience_kernels.cu @@ -1718,11 +1718,67 @@ extern "C" __global__ void experience_env_step( * Dense per-bar noise (OFI alignment, DSR, inventory penalty) REMOVED. * C51 atoms now model distribution of TRADE OUTCOMES, not per-bar noise. * ──────────────────────────────────────────────────────────────────── */ - if (!segment_complete && fabsf(position) > 0.001f) { - /* Holding cost: tiny negative proportional to |position|. - * Prevents infinite holding without drowning out trade signal. - * -0.0001 per contract → ~200 bars to accumulate -0.02 at 1 contract. - * Small enough that a profitable trade P&L (~0.5-5.0) dominates. */ + if (!segment_complete && fabsf(position) > 0.001f && micro_reward_scale > 0.0f + && full_batch_states != NULL && full_state_dim >= 74) { + /* ── Dense micro-reward: OFI momentum + MBP-10 price confirmation ── + * Replaces flat holding cost with informed per-bar signal using + * order flow imbalance deltas and mid-price mark-to-market. */ + const float* ofi_cur = full_batch_states + (long long)i * full_state_dim + 66; + float sign_pos = (position > 0.0f) ? 1.0f : -1.0f; + + /* OFI momentum: delta from previous bar (stored in ps[30..37]) */ + float delta_depth = ofi_cur[1] - ps[31]; /* depth_imbalance_L1 */ + float delta_vpin = ofi_cur[6] - ps[36]; /* VPIN */ + float delta_queue = ofi_cur[2] - ps[32]; /* queue_pressure */ + float delta_tarr = ofi_cur[5] - ps[35]; /* trade_arrival_rate */ + float flow_momentum = sign_pos * 0.25f * (delta_depth + delta_vpin + delta_queue + delta_tarr); + + /* ATR for normalization (same decode as vol normalization above) */ + float atr_abs = 1.0f; + if (features != NULL && bar_idx < total_bars && market_dim > 9) { + float atr_n = features[(long long)bar_idx * market_dim + 9]; + float log_a = atr_n * 16.0f - 7.0f; + atr_abs = fmaxf(expf(log_a), 0.01f); + } + float atr_pct = atr_abs / fmaxf(raw_close, 1.0f); + + /* Price confirmation: MBP-10 mid-price mark-to-market at decision points */ + float raw_open = tgt[4]; + float mid_open = tgt[5]; + float bar_move = (raw_close - raw_open) / fmaxf(atr_abs, 1e-6f); + float price_confirm = sign_pos * bar_move; + + /* Adaptive cost tolerance: profitable models trade more freely */ + float spread_at_entry = fabsf(raw_open - mid_open); + float capital_ratio = equity / fmaxf(peak_equity, 1.0f); + float sharpe_ema = ps[DSR_A_SLOT]; + float cost_tolerance = fmaxf(capital_ratio * fmaxf(sharpe_ema, 0.0f), 0.1f); + float spread_penalty = spread_at_entry / (fmaxf(atr_abs, 1e-6f) * cost_tolerance); + + /* Book aggression from state vector (precomputed from 10-level MBP-10) */ + float book_aggression = (full_state_dim >= 83) ? full_batch_states[(long long)i * full_state_dim + 82] : 0.0f; + + /* Informed flow intensity */ + float vol_informed = ofi_cur[6] * ofi_cur[7]; /* VPIN x Kyle's_lambda */ + + /* Retrospective hold quality: was previous bar's hold correct? */ + float prev_mid = ps[PREV_MID_SLOT]; + float mid_now = tgt[5]; + float hold_quality = 0.0f; + if (prev_mid > 0.0f && mid_now > 0.0f) { + float hold_return = (mid_now - prev_mid) / fmaxf(prev_mid, 1.0f); + hold_quality = sign_pos * hold_return / fmaxf(atr_pct, 1e-6f); + } + + /* Composite quality */ + float quality = flow_momentum * (1.0f + vol_informed) + + 0.5f * price_confirm + + 0.3f * sign_pos * book_aggression + + 0.2f * hold_quality + - spread_penalty; + reward = micro_reward_scale * tanhf(quality / 3.0f); + } else if (!segment_complete && fabsf(position) > 0.001f) { + /* Fallback: original holding cost when micro_reward_scale=0 or no OFI */ reward = -0.0001f * fabsf(position); } /* Flat (no position, no trade): reward stays 0.0f from initialization. @@ -1732,11 +1788,13 @@ extern "C" __global__ void experience_env_step( /* DSR EMA statistics still updated for positioned bars (used by metrics * reporting) but NO LONGER added to reward. */ if (w_dsr > 0.0f && fabsf(position) > 0.001f) { - /* DSR EMA uses current-bar return only (not raw_next). Since we - * don't have the previous bar's close in this kernel invocation, - * we approximate with a zero return — DSR is for metrics only - * and does NOT feed into reward. */ - float price_change_dsr = 0.0f; + /* DSR EMA uses current-bar return from previous close (stored in + * ps[PREV_CLOSE_SLOT] at end of each step). First bar has prev=0 + * so we skip it (price_change_dsr stays 0). */ + float prev_close_val = ps[PREV_CLOSE_SLOT]; + float price_change_dsr = (prev_close_val > 0.0f) + ? (raw_close - prev_close_val) / prev_close_val + : 0.0f; float dsr_vol = 0.005f; if (features != NULL && bar_idx < total_bars && market_dim > 9) { float atr_n = (features[(long long)bar_idx * market_dim + 9]); @@ -2039,7 +2097,12 @@ extern "C" __global__ void experience_env_step( cf_action = dir_idx * b1_size * b2_size * b3_size + alt_mag * b2_size * b3_size + orig_order * b3_size + orig_urgency; - cf_reward = reward * (alt_mag_frac / actual_mag_frac); + /* Undo do_flip before magnitude scaling, then re-apply. + * Without this, flipped reward gets scaled as if it were + * the original direction, producing wrong-sign gradients. */ + float base_reward = do_flip ? -reward : reward; + cf_reward = base_reward * (alt_mag_frac / actual_mag_frac); + if (do_flip) cf_reward = -cf_reward; cf_reward = fminf(fmaxf(cf_reward, -10.0f), 10.0f); } else { /* Near-zero reward: magnitude scaling uninformative, use directional mirror */ @@ -2068,8 +2131,12 @@ extern "C" __global__ void experience_env_step( cf_action = dir_idx * b1_size * b2_size * b3_size + mag_idx * b2_size * b3_size + alt_order * b3_size + orig_urgency; - /* Amplify 50×: raw cost delta ~0.0001-0.0005, directional ~0.01-0.1 */ - cf_reward = reward + 50.0f * cost_delta; + /* Undo do_flip before adding cost delta, then re-apply. + * Same fix as magnitude branch — cost delta is direction-invariant + * but the base reward sign must match the original trade direction. */ + float base_reward_ord = do_flip ? -reward : reward; + cf_reward = base_reward_ord + 50.0f * cost_delta; + if (do_flip) cf_reward = -cf_reward; cf_reward = fminf(fmaxf(cf_reward, -10.0f), 10.0f); } else { /* Flat position: order irrelevant, directional mirror fallback */ diff --git a/crates/ml/src/cuda_pipeline/reward_shaping_kernel.cu b/crates/ml/src/cuda_pipeline/reward_shaping_kernel.cu index cd4d6b1a7..9d67778bc 100644 --- a/crates/ml/src/cuda_pipeline/reward_shaping_kernel.cu +++ b/crates/ml/src/cuda_pipeline/reward_shaping_kernel.cu @@ -100,7 +100,7 @@ extern "C" __global__ void reward_scatter_rank( float sharpe = (std_ema > 1e-6f) ? (raw - return_mean_ema) / std_ema : raw; /* Holding bars: passthrough zero, don't rank */ - int is_trade = (fabsf(sharpe) > 0.001f) ? 1 : 0; + int is_trade = (fabsf(sharpe) > 1e-5f) ? 1 : 0; /* lowered for dense micro-rewards */ if (!is_trade) { rewards_out[orig_idx] = 0.0f; return;