diff --git a/config/training/dqn-production.toml b/config/training/dqn-production.toml index 53dc9103d..c3e9fb2b1 100644 --- a/config/training/dqn-production.toml +++ b/config/training/dqn-production.toml @@ -97,6 +97,7 @@ spectral_decoupling_lambda = 0.01 # This clip is now the SOLE safety mechanism — set high enough to not fire # during normal training, only on genuine divergence spikes. gradient_clip_norm = 10000.0 +dsr_eta = 0.01 [generalization] shrink_perturb_interval = 20 @@ -139,7 +140,7 @@ causal_intensity = 1.0 [reward] # v8 comprehensive training overhaul popart_enabled = true -micro_reward_scale = 0.01 +micro_reward_scale = 0.0 td_lambda = 0.9 max_trace_length = 7 hindsight_fraction = 0.1 @@ -149,10 +150,11 @@ q_gap_threshold = 0.1 w_dd = 1.0 dd_threshold = 0.02 cea_weight = 0.3 -order_credit_weight = 0.1 +order_credit_weight = 1.0 risk_efficiency_weight = 0.1 -urgency_credit_weight = 0.1 -exit_timing_weight = 0.05 +urgency_credit_weight = 0.5 +exit_timing_weight = 0.1 +w_dsr = 5.0 ofi_reward_weight = 0.2 kelly_sizing_weight = 0.1 reward_noise_scale = 0.05 diff --git a/config/training/dqn-smoketest.toml b/config/training/dqn-smoketest.toml index bf160ed84..9772d9666 100644 --- a/config/training/dqn-smoketest.toml +++ b/config/training/dqn-smoketest.toml @@ -86,6 +86,7 @@ iqn_lambda = 0.25 spectral_norm_sigma_max = 1.5 spectral_decoupling_lambda = 0.01 gradient_clip_norm = 10000.0 +dsr_eta = 0.01 [generalization] # Toned-down generalization for tiny smoketest network [64,64] @@ -128,7 +129,7 @@ causal_intensity = 1.0 [reward] # v8 comprehensive training overhaul popart_enabled = true -micro_reward_scale = 0.01 +micro_reward_scale = 0.0 td_lambda = 0.9 max_trace_length = 7 hindsight_fraction = 0.1 @@ -138,10 +139,11 @@ q_gap_threshold = 0.1 w_dd = 1.0 dd_threshold = 0.02 cea_weight = 0.3 -order_credit_weight = 0.1 +order_credit_weight = 1.0 risk_efficiency_weight = 0.1 -urgency_credit_weight = 0.1 -exit_timing_weight = 0.05 +urgency_credit_weight = 0.5 +exit_timing_weight = 0.1 +w_dsr = 5.0 ofi_reward_weight = 0.2 kelly_sizing_weight = 0.1 reward_noise_scale = 0.05 diff --git a/crates/ml/src/cuda_pipeline/experience_kernels.cu b/crates/ml/src/cuda_pipeline/experience_kernels.cu index e4d077491..2aeb304a4 100644 --- a/crates/ml/src/cuda_pipeline/experience_kernels.cu +++ b/crates/ml/src/cuda_pipeline/experience_kernels.cu @@ -24,10 +24,10 @@ * [0] position — current contract position (signed) * [1] cash — cash balance * [2] portfolio_value — mark-to-market total value (cash + position * price) - * [3] (reserved) — was clustering last_trade_t, unused by reward v8 - * [4] (reserved) — was clustering interval_sum, unused by reward v8 - * [5] (reserved) — was clustering interval_sq_sum, unused by reward v8 - * [6] (reserved) — was clustering interval_count, unused by reward v8 + * [3] dsr_A — EMA of realized trade returns (v9 DSR) + * [4] dsr_B — EMA of squared trade returns (v9 DSR) + * [5] dsr_trade_count — completed trades counter (for DSR warmup) + * [6] (reserved) * [7] peak_equity — high-water mark (init to initial_capital) * [8] flat_counter — consecutive flat steps * [9] prev_equity — equity at previous step (init to initial_capital) @@ -56,6 +56,9 @@ /* portfolio_sim_kernel uses its own stride of 8 — do NOT change it. */ /* ------------------------------------------------------------------ */ #define PORTFOLIO_STRIDE 23 +#define DSR_A_SLOT 3 +#define DSR_B_SLOT 4 +#define DSR_TRADE_COUNT_SLOT 5 /* ── Reward v7: Asymmetric soft-clamp ────────────────────────────────── * Gains: linear, capped at +10 (full gradient for positive rewards) @@ -1078,6 +1081,8 @@ extern "C" __global__ void experience_env_step( float kelly_sizing_weight, /* v7 gem: Kelly-optimal sizing signal */ float reward_noise_scale, /* v7 gem: reward label smoothing noise */ float micro_reward_scale, /* v8: dense directional micro-reward scale */ + float w_dsr, /* v9: Differential Sharpe Ratio weight */ + float dsr_eta, /* v9: DSR EMA decay rate (0.01 = ~100-trade window) */ /* #33 Per-episode saboteur params [N, 3]: (spread_mult, fill_prob, slippage_mult). * NULL = disabled (use global scalars). When non-NULL, overrides * spread_cost, fill_ioc_fill_prob, tx_cost_multiplier per episode. */ @@ -1521,8 +1526,32 @@ extern "C" __global__ void experience_env_step( float vol_norm = vol_proxy * sqrtf(fmaxf(segment_hold_time, 1.0f)); float vol_normalized_return = segment_return / vol_norm; + /* ── v9: Differential Sharpe Ratio (Moody & Saffell, 2001) ── + * Reward = how much this trade improves the running Sharpe ratio. + * Small consistent gains → high DSR. Large volatile wins → low DSR. */ + float R_t = vol_normalized_return; + float dsr_a = ps[DSR_A_SLOT]; + float dsr_b = ps[DSR_B_SLOT]; + float dsr_count = ps[DSR_TRADE_COUNT_SLOT]; + + float new_a = dsr_a + dsr_eta * (R_t - dsr_a); + float new_b = dsr_b + dsr_eta * (R_t * R_t - dsr_b); + ps[DSR_A_SLOT] = new_a; + ps[DSR_B_SLOT] = new_b; + ps[DSR_TRADE_COUNT_SLOT] = dsr_count + 1.0f; + + if (w_dsr > 0.0f && dsr_count >= 10.0f) { + float variance = dsr_b - dsr_a * dsr_a; + if (variance > 1e-8f) { + float dsr = (dsr_b * R_t - 0.5f * dsr_a * R_t * R_t) + / powf(variance, 1.5f); + dsr = fmaxf(fminf(dsr, 5.0f), -5.0f); + reward += w_dsr * dsr; + } + } + /* ── Layer 2: Asymmetric soft-clamp (replaces loss_aversion + tanh) ── */ - float base_reward = 10.0f * vol_normalized_return; + float base_reward = 2.0f * vol_normalized_return; reward = asymmetric_soft_clamp(base_reward); /* ── Layer 3: CEA counterfactual loop REMOVED (4-branch: direction(3) replaces exposure(9)). @@ -1624,6 +1653,12 @@ extern "C" __global__ void experience_env_step( reward += adaptive_scale * dir_reward * mag_fraction / micro_vol_proxy; } + /* ── v9: Inventory penalty — penalize holding positions (Sharpe killer) ── */ + if (fabsf(position) > 0.001f) { + float inventory_cost = -0.005f * fabsf(position) / fmaxf(max_position, 0.001f); + reward += inventory_cost; + } + /* ── Reward label smoothing (magnitude-relative) ── */ if (reward_noise_scale > 0.0f && reward != 0.0f) { unsigned int hash = (unsigned int)(i * 31337 + bar_idx * 7919 + current_t * 1013); diff --git a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs index b64fc8ff4..ea0a6f730 100644 --- a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs +++ b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs @@ -279,6 +279,8 @@ pub struct ExperienceCollectorConfig { /// #23 Feature mask: per-feature binary mask. None = no masking. /// Generated per-epoch on CPU, uploaded to GPU. pub feature_mask: Option>, + /// v9: Differential Sharpe Ratio weight (0.0 = disabled). + pub w_dsr: f32, /// v8: Dense directional micro-reward scale (adaptive vol). pub micro_reward_scale: f32, /// v8: Current training epoch (for GPU cosine epsilon schedule). @@ -368,6 +370,7 @@ impl Default for ExperienceCollectorConfig { feature_noise_scale: 0.0, vol_normalizer: 0.0, feature_mask: None, + w_dsr: 5.0, micro_reward_scale: 0.0, current_epoch: 0, total_epochs: 20, @@ -2112,6 +2115,8 @@ impl GpuExperienceCollector { .arg(&config.kelly_sizing_weight) .arg(&config.reward_noise_scale) .arg(&config.micro_reward_scale) // v8: dense directional micro-reward (Gem 1) + .arg(&config.w_dsr) // v9: Differential Sharpe Ratio weight + .arg(&config.dsr_eta) // v9: DSR EMA decay rate // #33 Per-episode saboteur params (0 = NULL = disabled) .arg(&{ if self.saboteur_active { diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index d7ef47796..515575d39 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -1079,6 +1079,7 @@ impl DQNTrainer { feature_noise_scale: self.hyperparams.feature_noise_scale as f32, vol_normalizer: self.epoch_vol_normalizer, feature_mask: self.epoch_feature_mask.clone(), + w_dsr: self.hyperparams.w_dsr as f32, micro_reward_scale: self.hyperparams.micro_reward_scale as f32, current_epoch: self.current_epoch as i32, total_epochs: self.hyperparams.epochs as i32, diff --git a/crates/ml/src/training_profile.rs b/crates/ml/src/training_profile.rs index 8d65a93ad..97507ceb9 100644 --- a/crates/ml/src/training_profile.rs +++ b/crates/ml/src/training_profile.rs @@ -1433,7 +1433,7 @@ mod tests { assert!((hp.loss_aversion - 1.0).abs() < 0.01); // [reward] section — all composite weights - assert!((hp.w_dsr - 1.0).abs() < 0.01); + assert!((hp.w_dsr - 5.0).abs() < 0.01); assert!((hp.w_pnl - 0.3).abs() < 0.01); assert!((hp.w_dd - 1.0).abs() < 0.01); assert!((hp.w_idle - 0.01).abs() < 0.001); diff --git a/docs/superpowers/plans/2026-04-12-dsr-reward-restructure.md b/docs/superpowers/plans/2026-04-12-dsr-reward-restructure.md new file mode 100644 index 000000000..bc0993289 --- /dev/null +++ b/docs/superpowers/plans/2026-04-12-dsr-reward-restructure.md @@ -0,0 +1,293 @@ +# Differential Sharpe Ratio Reward Restructure + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace directional PnL as the dominant reward with Differential Sharpe Ratio (DSR) — the model optimizes for Sharpe consistency instead of PnL magnitude, targeting Sharpe 5+. + +**Architecture:** DSR is computed at trade completion in the `experience_env_step` CUDA kernel using per-episode EMA statistics stored in portfolio state slots [3] (dsr_A) and [4] (dsr_B). The reward hierarchy flips: DSR (w=5.0) becomes primary, directional PnL demoted (10→2), order credit amplified (0.1→1.0), inventory penalty added, dense micro-reward removed. Config fields `w_dsr` and `dsr_eta` already exist — just need kernel implementation + TOML weight changes. + +**Tech Stack:** CUDA 12.4 (experience_kernels.cu), Rust config wiring, TOML profiles + +--- + +## File Map + +| File | Action | Changes | +|------|--------|---------| +| `crates/ml/src/cuda_pipeline/experience_kernels.cu` | Modify | Add DSR computation + inventory penalty in reward kernel, add `w_dsr`/`dsr_eta` kernel params | +| `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs` | Modify | Pass `w_dsr`/`dsr_eta` to kernel launch | +| `config/training/dqn-production.toml` | Modify | New reward weights | +| `config/training/dqn-smoketest.toml` | Modify | Same | + +--- + +### Task 1: Add DSR computation to reward kernel + inventory penalty + +**Files:** +- Modify: `crates/ml/src/cuda_pipeline/experience_kernels.cu` + +- [ ] **Step 1: Add `w_dsr`, `dsr_eta` kernel parameters** + +In the `experience_env_step` kernel signature (after `float micro_reward_scale` at line 1080), add two new parameters: + +```c + float micro_reward_scale, /* v8: dense directional micro-reward scale */ + float w_dsr, /* v9: Differential Sharpe Ratio weight */ + float dsr_eta, /* v9: DSR EMA decay rate (0.01 = ~100-trade window) */ +``` + +- [ ] **Step 2: Define DSR portfolio state slot constants** + +Near the top of the file (after `#define PORTFOLIO_STRIDE 23` at line 58), add slot documentation: + +```c +/* v9: Differential Sharpe Ratio running statistics (Moody & Saffell, 2001). + * Stored in per-episode portfolio state — persistent across bars within an episode. + * dsr_A = EMA of realized trade returns (updated at trade completion). + * dsr_B = EMA of squared realized trade returns. + * DSR_t = (B*R - 0.5*A*R²) / (B - A²)^1.5 measures how much each trade improves Sharpe. */ +#define DSR_A_SLOT 3 +#define DSR_B_SLOT 4 +#define DSR_TRADE_COUNT_SLOT 5 +``` + +Update the portfolio state layout comment (lines 23-38) to reflect: +```c + * [3] dsr_A — EMA of realized trade returns (v9 DSR) + * [4] dsr_B — EMA of squared trade returns (v9 DSR) + * [5] dsr_trade_count — number of completed trades (for DSR warmup) + * [6] (reserved) +``` + +- [ ] **Step 3: Implement DSR reward at trade completion** + +Inside the `if (segment_complete && segment_hold_time > 0)` block (after the vol normalization at line 1522 and BEFORE the base_reward computation at line 1525), add DSR computation: + +```c + /* ── v9: Differential Sharpe Ratio ── + * Moody & Saffell (2001): reward = how much this trade improves the running Sharpe. + * Small consistent gains → high DSR. Large volatile wins → low DSR. + * Directly optimizes for risk-adjusted returns, not raw PnL. */ + float R_t = vol_normalized_return; /* already computed above */ + float dsr_a = ps[DSR_A_SLOT]; + float dsr_b = ps[DSR_B_SLOT]; + float dsr_count = ps[DSR_TRADE_COUNT_SLOT]; + + /* Update running EMA statistics */ + float new_a = dsr_a + dsr_eta * (R_t - dsr_a); + float new_b = dsr_b + dsr_eta * (R_t * R_t - dsr_b); + ps[DSR_A_SLOT] = new_a; + ps[DSR_B_SLOT] = new_b; + ps[DSR_TRADE_COUNT_SLOT] = dsr_count + 1.0f; + + /* Compute DSR after warmup (need >= 10 trades for stable statistics) */ + if (w_dsr > 0.0f && dsr_count >= 10.0f) { + float variance = dsr_b - dsr_a * dsr_a; + if (variance > 1e-8f) { + float dsr = (dsr_b * R_t - 0.5f * dsr_a * R_t * R_t) + / powf(variance, 1.5f); + /* Soft-clamp DSR to [-5, +5] to prevent single-trade domination */ + dsr = fmaxf(fminf(dsr, 5.0f), -5.0f); + reward += w_dsr * dsr; + } + } +``` + +- [ ] **Step 4: Add inventory penalty (dense, every bar)** + +After the existing micro-reward block (after line 1625), add an inventory penalty that fires EVERY bar (not just trade completion): + +```c + /* ── v9: Inventory penalty — penalize holding positions (reduces variance) ── + * Market makers minimize inventory. Holding = directional risk = Sharpe killer. + * Scales with position size: Full(1.0) penalized 4× more than Quarter(0.25). */ + if (fabsf(position) > 0.001f) { + float inventory_cost = -0.005f * fabsf(position) / fmaxf(max_position, 0.001f); + reward += inventory_cost; + } +``` + +- [ ] **Step 5: Demote directional PnL base reward** + +Change line 1525 from: +```c + float base_reward = 10.0f * vol_normalized_return; +``` +to: +```c + float base_reward = 2.0f * vol_normalized_return; +``` + +This demotes directional PnL from 10× to 2× (DSR at 5× is now the primary signal). + +- [ ] **Step 6: Verify kernel compiles** + +```bash +touch crates/ml/build.rs && SQLX_OFFLINE=true cargo build -p ml --lib 2>&1 | grep -E "experience_kernels|error" +``` + +Expected: compilation fails (Rust launch site doesn't pass new params yet). + +- [ ] **Step 7: Commit kernel changes** + +```bash +git add crates/ml/src/cuda_pipeline/experience_kernels.cu +git commit -m "feat: DSR reward + inventory penalty in experience kernel (v9 reward restructure)" +``` + +--- + +### Task 2: Wire DSR parameters from Rust to kernel + +**Files:** +- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs` + +- [ ] **Step 1: Add `w_dsr` to ExperienceConfig** + +In the `ExperienceConfig` struct (around line 240), add after `dsr_eta`: + +```rust + pub w_dsr: f32, +``` + +Add default in the Default impl: +```rust + w_dsr: 5.0, +``` + +- [ ] **Step 2: Pass `w_dsr` and `dsr_eta` to kernel launch** + +In the `env_step_kernel` launch (around line 2114), after `.arg(&config.micro_reward_scale)`, add: + +```rust + .arg(&config.w_dsr) + .arg(&config.dsr_eta) +``` + +- [ ] **Step 3: Wire from hyperparams to ExperienceConfig** + +In `training_loop.rs` where `ExperienceConfig` is constructed (search for `micro_reward_scale:`), add: + +```rust + w_dsr: self.hyperparams.w_dsr as f32, + dsr_eta: self.hyperparams.dsr_eta as f32, +``` + +- [ ] **Step 4: Verify compilation and run tests** + +```bash +touch crates/ml/build.rs && SQLX_OFFLINE=true cargo build -p ml --lib +SQLX_OFFLINE=true cargo test -p ml-dqn --lib +SQLX_OFFLINE=true cargo test -p ml --lib -- gradient_budget config monitoring +``` + +- [ ] **Step 5: Commit** + +```bash +git add crates/ml/src/cuda_pipeline/gpu_experience_collector.rs crates/ml/src/trainers/dqn/trainer/training_loop.rs +git commit -m "feat: wire w_dsr + dsr_eta from config to experience kernel" +``` + +--- + +### Task 3: Update TOML profiles with new reward weights + +**Files:** +- Modify: `config/training/dqn-production.toml` +- Modify: `config/training/dqn-smoketest.toml` + +- [ ] **Step 1: Update production TOML** + +In the `[reward]` section of `config/training/dqn-production.toml`, update: + +```toml +[reward] +# v9: DSR-primary reward restructure — optimize for Sharpe, not PnL +popart_enabled = true +micro_reward_scale = 0.0 # REMOVED: noisy directional micro-reward +td_lambda = 0.9 +max_trace_length = 7 +hindsight_fraction = 0.1 +hindsight_lookahead = 10 +loss_aversion = 1.0 +q_gap_threshold = 0.1 +w_dd = 1.0 +dd_threshold = 0.02 +cea_weight = 0.3 +order_credit_weight = 1.0 # AMPLIFIED: spread capture is primary alpha (was 0.1) +risk_efficiency_weight = 0.1 +urgency_credit_weight = 0.5 # AMPLIFIED: fill quality matters for consistency (was 0.1) +exit_timing_weight = 0.1 # Slightly increased (was 0.05) +ofi_reward_weight = 0.2 +kelly_sizing_weight = 0.1 +reward_noise_scale = 0.05 +w_dsr = 5.0 # NEW: Differential Sharpe Ratio (primary reward) +b3_size = 3 + +[advanced] +dsr_eta = 0.01 # DSR EMA decay (~100-trade window) +``` + +- [ ] **Step 2: Update smoketest TOML** + +Same changes in `config/training/dqn-smoketest.toml`: + +```toml +micro_reward_scale = 0.0 +order_credit_weight = 1.0 +urgency_credit_weight = 0.5 +exit_timing_weight = 0.1 +w_dsr = 5.0 +``` + +- [ ] **Step 3: Run all tests** + +```bash +SQLX_OFFLINE=true cargo test -p ml --lib -- training_profile +SQLX_OFFLINE=true cargo test -p ml-dqn --lib +``` + +- [ ] **Step 4: Commit** + +```bash +git add config/training/dqn-production.toml config/training/dqn-smoketest.toml +git commit -m "feat: v9 reward weights — DSR primary, directional PnL demoted, spread capture amplified" +``` + +--- + +### Task 4: Smoke test + H100 baseline + +**Files:** (no changes — verification only) + +- [ ] **Step 1: Run smoke test** + +```bash +FOXHUNT_TEST_DATA=test_data/futures-baseline SQLX_OFFLINE=true cargo test -p ml --lib -- smoke_tests::feature_coverage --ignored --test-threads=1 --nocapture 2>&1 | grep -E "Sharpe|WinRate|Action div|Q-value" +``` + +Expected: WinRate should start shifting (maybe not immediately in 10-epoch smoketest, but the reward structure change should be visible in the Sharpe metrics). + +- [ ] **Step 2: Run full smoke test suite** + +```bash +FOXHUNT_TEST_DATA=test_data/futures-baseline SQLX_OFFLINE=true cargo test -p ml --lib -- smoke_tests --ignored --test-threads=1 +``` + +Expected: 19 passed, 0 failed. + +- [ ] **Step 3: Commit and push** + +```bash +git add -A +git commit -m "feat: v9 DSR reward restructure — Sharpe-optimal training for market-making alpha" +git push origin main +``` + +- [ ] **Step 4: Launch H100 baseline** + +```bash +argo submit -n foxhunt --from=wftmpl/train -p model=dqn -p gpu-pool=ci-training-h100 -p hyperopt-trials=0 -p train-epochs=200 +``` + +Monitor: watch for WinRate climbing above 50% and Sharpe trajectory.