gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: dense micro-reward + DSR fix + counterfactual sign fix

Browse Source 
Dense micro-reward: OFI momentum × price confirmation (MBP-10 mid-price
mark-to-market) × adaptive cost tolerance (capital_ratio × Sharpe_ema)
+ book aggression + retrospective hold quality bonus. Replaces flat
-0.0001 holding cost. Scale: micro_reward_scale=0.1.

DSR Sharpe EMA: was hardcoded price_change_dsr=0.0 — adaptive cost
tolerance was permanently floored at 0.1. Now uses actual per-bar
returns from ps[PREV_CLOSE_SLOT].

Counterfactual: cf_cycle==1 (magnitude) and cf_cycle==2 (order) now
undo do_flip before computing CF reward, then re-apply. Previously
2/3 of counterfactual experiences had wrong sign when do_flip=true.

Rank normalization threshold: 0.001 → 1e-5 for dense micro-rewards.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-20 00:03:51 +02:00
parent c2c1802e58
commit 55d70b7cc8
3 changed files with 82 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
config/training/dqn-production.toml
View File

@@ -141,7 +141,7 @@ causal_intensity = 1.0
[reward]
# v8 comprehensive training overhaul
popart_enabled = true
micro_reward_scale = 0.0
micro_reward_scale = 0.1
td_lambda = 0.9
max_trace_length = 7
hindsight_fraction = 0.1

93
crates/ml/src/cuda_pipeline/experience_kernels.cu
View File

@@ -1718,11 +1718,67 @@ extern "C" __global__ void experience_env_step(
* Dense per-bar noise (OFI alignment, DSR, inventory penalty) REMOVED.
* C51 atoms now model distribution of TRADE OUTCOMES, not per-bar noise.
* ──────────────────────────────────────────────────────────────────── */
if (!segment_complete && fabsf(position) > 0.001f) {
/* Holding cost: tiny negative proportional to |position|.
* Prevents infinite holding without drowning out trade signal.
* -0.0001 per contract → ~200 bars to accumulate -0.02 at 1 contract.
* Small enough that a profitable trade P&L (~0.5-5.0) dominates. */
if (!segment_complete && fabsf(position) > 0.001f && micro_reward_scale > 0.0f
&& full_batch_states != NULL && full_state_dim >= 74) {
/* ── Dense micro-reward: OFI momentum + MBP-10 price confirmation ──
* Replaces flat holding cost with informed per-bar signal using
* order flow imbalance deltas and mid-price mark-to-market. */
const float* ofi_cur = full_batch_states + (long long)i * full_state_dim + 66;
float sign_pos = (position > 0.0f) ? 1.0f : -1.0f;
/* OFI momentum: delta from previous bar (stored in ps[30..37]) */
float delta_depth = ofi_cur[1] - ps[31]; /* depth_imbalance_L1 */
float delta_vpin = ofi_cur[6] - ps[36]; /* VPIN */
float delta_queue = ofi_cur[2] - ps[32]; /* queue_pressure */
float delta_tarr = ofi_cur[5] - ps[35]; /* trade_arrival_rate */
float flow_momentum = sign_pos * 0.25f * (delta_depth + delta_vpin + delta_queue + delta_tarr);
/* ATR for normalization (same decode as vol normalization above) */
float atr_abs = 1.0f;
if (features != NULL && bar_idx < total_bars && market_dim > 9) {
float atr_n = features[(long long)bar_idx * market_dim + 9];
float log_a = atr_n * 16.0f - 7.0f;
atr_abs = fmaxf(expf(log_a), 0.01f);
}
float atr_pct = atr_abs / fmaxf(raw_close, 1.0f);
/* Price confirmation: MBP-10 mid-price mark-to-market at decision points */
float raw_open = tgt[4];
float mid_open = tgt[5];
float bar_move = (raw_close - raw_open) / fmaxf(atr_abs, 1e-6f);
float price_confirm = sign_pos * bar_move;
/* Adaptive cost tolerance: profitable models trade more freely */
float spread_at_entry = fabsf(raw_open - mid_open);
float capital_ratio = equity / fmaxf(peak_equity, 1.0f);
float sharpe_ema = ps[DSR_A_SLOT];
float cost_tolerance = fmaxf(capital_ratio * fmaxf(sharpe_ema, 0.0f), 0.1f);
float spread_penalty = spread_at_entry / (fmaxf(atr_abs, 1e-6f) * cost_tolerance);
/* Book aggression from state vector (precomputed from 10-level MBP-10) */
float book_aggression = (full_state_dim >= 83) ? full_batch_states[(long long)i * full_state_dim + 82] : 0.0f;
/* Informed flow intensity */
float vol_informed = ofi_cur[6] * ofi_cur[7]; /* VPIN x Kyle's_lambda */
/* Retrospective hold quality: was previous bar's hold correct? */
float prev_mid = ps[PREV_MID_SLOT];
float mid_now = tgt[5];
float hold_quality = 0.0f;
if (prev_mid > 0.0f && mid_now > 0.0f) {
float hold_return = (mid_now - prev_mid) / fmaxf(prev_mid, 1.0f);
hold_quality = sign_pos * hold_return / fmaxf(atr_pct, 1e-6f);
}
/* Composite quality */
float quality = flow_momentum * (1.0f + vol_informed)
+ 0.5f * price_confirm
+ 0.3f * sign_pos * book_aggression
+ 0.2f * hold_quality
- spread_penalty;
reward = micro_reward_scale * tanhf(quality / 3.0f);
} else if (!segment_complete && fabsf(position) > 0.001f) {
/* Fallback: original holding cost when micro_reward_scale=0 or no OFI */
reward = -0.0001f * fabsf(position);
}
/* Flat (no position, no trade): reward stays 0.0f from initialization.
@@ -1732,11 +1788,13 @@ extern "C" __global__ void experience_env_step(
/* DSR EMA statistics still updated for positioned bars (used by metrics
* reporting) but NO LONGER added to reward. */
if (w_dsr > 0.0f && fabsf(position) > 0.001f) {
/* DSR EMA uses current-bar return only (not raw_next). Since we
* don't have the previous bar's close in this kernel invocation,
* we approximate with a zero return — DSR is for metrics only
* and does NOT feed into reward. */
float price_change_dsr = 0.0f;
/* DSR EMA uses current-bar return from previous close (stored in
* ps[PREV_CLOSE_SLOT] at end of each step). First bar has prev=0
* so we skip it (price_change_dsr stays 0). */
float prev_close_val = ps[PREV_CLOSE_SLOT];
float price_change_dsr = (prev_close_val > 0.0f)
? (raw_close - prev_close_val) / prev_close_val
: 0.0f;
float dsr_vol = 0.005f;
if (features != NULL && bar_idx < total_bars && market_dim > 9) {
float atr_n = (features[(long long)bar_idx * market_dim + 9]);
@@ -2039,7 +2097,12 @@ extern "C" __global__ void experience_env_step(
cf_action = dir_idx * b1_size * b2_size * b3_size
+ alt_mag * b2_size * b3_size
+ orig_order * b3_size + orig_urgency;
cf_reward = reward * (alt_mag_frac / actual_mag_frac);
/* Undo do_flip before magnitude scaling, then re-apply.
* Without this, flipped reward gets scaled as if it were
* the original direction, producing wrong-sign gradients. */
float base_reward = do_flip ? -reward : reward;
cf_reward = base_reward * (alt_mag_frac / actual_mag_frac);
if (do_flip) cf_reward = -cf_reward;
cf_reward = fminf(fmaxf(cf_reward, -10.0f), 10.0f);
} else {
/* Near-zero reward: magnitude scaling uninformative, use directional mirror */
@@ -2068,8 +2131,12 @@ extern "C" __global__ void experience_env_step(
cf_action = dir_idx * b1_size * b2_size * b3_size
+ mag_idx * b2_size * b3_size
+ alt_order * b3_size + orig_urgency;
/* Amplify 50×: raw cost delta ~0.0001-0.0005, directional ~0.01-0.1 */
cf_reward = reward + 50.0f * cost_delta;
/* Undo do_flip before adding cost delta, then re-apply.
* Same fix as magnitude branch — cost delta is direction-invariant
* but the base reward sign must match the original trade direction. */
float base_reward_ord = do_flip ? -reward : reward;
cf_reward = base_reward_ord + 50.0f * cost_delta;
if (do_flip) cf_reward = -cf_reward;
cf_reward = fminf(fmaxf(cf_reward, -10.0f), 10.0f);
} else {
/* Flat position: order irrelevant, directional mirror fallback */

2
crates/ml/src/cuda_pipeline/reward_shaping_kernel.cu
View File

@@ -100,7 +100,7 @@ extern "C" __global__ void reward_scatter_rank(
float sharpe = (std_ema > 1e-6f) ? (raw - return_mean_ema) / std_ema : raw;
/* Holding bars: passthrough zero, don't rank */
int is_trade = (fabsf(sharpe) > 0.001f) ? 1 : 0;
int is_trade = (fabsf(sharpe) > 1e-5f) ? 1 : 0; /* lowered for dense micro-rewards */
if (!is_trade) {
rewards_out[orig_idx] = 0.0f;
return;