diff --git a/config/training/dqn-localdev.toml b/config/training/dqn-localdev.toml index 401cab730..7f253a3df 100644 --- a/config/training/dqn-localdev.toml +++ b/config/training/dqn-localdev.toml @@ -71,3 +71,5 @@ gradient_clip_norm = 1.0 [reward] loss_aversion = 1.5 q_gap_threshold = 0.1 +w_dd = 1.0 +dd_threshold = 0.02 diff --git a/config/training/dqn-production.toml b/config/training/dqn-production.toml index 83e8e4292..9633c2e36 100644 --- a/config/training/dqn-production.toml +++ b/config/training/dqn-production.toml @@ -76,3 +76,5 @@ gradient_clip_norm = 1.0 [reward] loss_aversion = 1.5 q_gap_threshold = 0.1 +w_dd = 1.0 +dd_threshold = 0.02 diff --git a/config/training/dqn-smoketest.toml b/config/training/dqn-smoketest.toml index cf9df115a..3ff400dc6 100644 --- a/config/training/dqn-smoketest.toml +++ b/config/training/dqn-smoketest.toml @@ -75,3 +75,5 @@ gradient_clip_norm = 1.0 [reward] loss_aversion = 1.5 q_gap_threshold = 0.1 +w_dd = 1.0 +dd_threshold = 0.02 diff --git a/crates/ml/src/cuda_pipeline/experience_kernels.cu b/crates/ml/src/cuda_pipeline/experience_kernels.cu index 13215f282..d47ec574a 100644 --- a/crates/ml/src/cuda_pipeline/experience_kernels.cu +++ b/crates/ml/src/cuda_pipeline/experience_kernels.cu @@ -531,7 +531,9 @@ extern "C" __global__ void experience_env_step( int min_hold_bars, /* minimum bars to hold before exiting or reversing */ float spread_cost, /* bid-ask spread cost per unit (matches backtest) */ float contract_multiplier, /* e.g. 50.0 for ES, 20.0 for NQ */ - float margin_pct /* e.g. 0.06 (6% initial margin) */ + float margin_pct, /* e.g. 0.06 (6% initial margin) */ + float dd_threshold, /* drawdown fraction before penalty (0.02 = 2%) */ + float w_dd /* drawdown penalty weight (1.0 = full) */ ) { int i = blockIdx.x * blockDim.x + threadIdx.x; if (i >= N) return; @@ -935,9 +937,9 @@ extern "C" __global__ void experience_env_step( reward *= hold_scale; } - /* Turnover penalty REMOVED (reward v6): tx costs already deducted from - * cash at line ~679 (Almgren-Chriss impact model). The old 0.05*|delta| - * penalty double-counted costs and over-penalized necessary rebalancing. */ + /* ---- Drawdown penalty (from trade_physics.cuh) ---- */ + float equity_now = cash + position * raw_next; + reward += compute_drawdown_penalty(equity_now, peak_equity, dd_threshold, w_dd); /* ---- Portfolio value for floor check ---- */ float new_portfolio_value_pre_floor = cash + position * raw_next; diff --git a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs index 4869e7413..84b35e1eb 100644 --- a/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs +++ b/crates/ml/src/cuda_pipeline/gpu_experience_collector.rs @@ -187,6 +187,10 @@ pub struct ExperienceCollectorConfig { pub risk_weight: f32, /// Asymmetric loss scaling factor (prospect theory, default 1.5) pub loss_aversion: f32, + /// Drawdown fraction before penalty starts (0.02 = 2%) + pub dd_threshold: f32, + /// Drawdown penalty weight (1.0 = full penalty) + pub w_dd: f32, /// Transaction cost multiplier (hyperopt-tunable, scales base 0.01% rate) pub tx_cost_multiplier: f32, /// UCB count-bonus coefficient for GPU action selection (0.0 = disabled) @@ -273,6 +277,8 @@ impl Default for ExperienceCollectorConfig { curiosity_scale: 1.0, risk_weight: 0.1, loss_aversion: 1.5, + dd_threshold: 0.02, + w_dd: 1.0, tx_cost_multiplier: 1.0, count_bonus_coefficient: 0.0, q_clip_min: -200.0, // Reward v6: tighter than old -500 but covers v_range + safety margin @@ -1355,6 +1361,8 @@ impl GpuExperienceCollector { .arg(&config.spread_cost) // bid-ask spread cost (matches backtest) .arg(&config.contract_multiplier) // futures contract multiplier (e.g. 50 for ES) .arg(&config.margin_pct) // initial margin fraction (e.g. 0.06 = 6%) + .arg(&config.dd_threshold) // drawdown threshold before penalty (0.02 = 2%) + .arg(&config.w_dd) // drawdown penalty weight .launch(launch_cfg) .map_err(|e| MLError::ModelError(format!( "experience_env_step t={t}: {e}" diff --git a/crates/ml/src/cuda_pipeline/trade_physics.cuh b/crates/ml/src/cuda_pipeline/trade_physics.cuh index 2d1664ddf..22547932f 100644 --- a/crates/ml/src/cuda_pipeline/trade_physics.cuh +++ b/crates/ml/src/cuda_pipeline/trade_physics.cuh @@ -192,6 +192,33 @@ __device__ __forceinline__ float execute_trade( return cost; } +/* ── Drawdown computation ────────────────────────────────────────────── */ +/* Returns (peak - equity) / peak, clamped to [0, 1]. */ + +__device__ __forceinline__ float compute_drawdown( + float equity, float peak_equity +) { + if (peak_equity <= 1.0f) return 0.0f; + float dd = (peak_equity - equity) / peak_equity; + return fmaxf(dd, 0.0f); +} + +/* ── Drawdown penalty: smooth ramp from threshold to floor ───────────── */ +/* Returns a penalty in [-5*w_dd, 0]. Applied every step so the model */ +/* learns to reduce position size DURING drawdown. */ + +__device__ __forceinline__ float compute_drawdown_penalty( + float equity, float peak_equity, + float dd_threshold, float w_dd +) { + if (w_dd <= 0.0f) return 0.0f; + float dd = compute_drawdown(equity, peak_equity); + if (dd <= dd_threshold) return 0.0f; + float floor_dd = 0.25f; // capital floor = 25% DD + float dd_excess = fminf((dd - dd_threshold) / (floor_dd - dd_threshold), 1.0f); + return -5.0f * dd_excess * w_dd; +} + /* ── Capital floor circuit breaker (PDT $25K rule) ───────────────────── */ /* Triggers when equity drops 25% from peak (floor = 75% of peak). */ /* With $35K initial: floor = $26.25K — above $25K PDT minimum. */ diff --git a/crates/ml/src/trainers/dqn/trainer/training_loop.rs b/crates/ml/src/trainers/dqn/trainer/training_loop.rs index 5f1235ae4..7df3d1873 100644 --- a/crates/ml/src/trainers/dqn/trainer/training_loop.rs +++ b/crates/ml/src/trainers/dqn/trainer/training_loop.rs @@ -1055,6 +1055,8 @@ impl DQNTrainer { spread_cost: (self.hyperparams.tick_size * self.hyperparams.contract_multiplier * self.hyperparams.fill_spread_cost_frac) as f32, contract_multiplier: self.hyperparams.contract_multiplier as f32, margin_pct: self.hyperparams.margin_pct as f32, + dd_threshold: self.hyperparams.dd_threshold as f32, + w_dd: self.hyperparams.w_dd as f32, ..Default::default() }; diff --git a/docs/superpowers/plans/2026-03-29-drawdown-aware-training.md b/docs/superpowers/plans/2026-03-29-drawdown-aware-training.md new file mode 100644 index 000000000..77384f28b --- /dev/null +++ b/docs/superpowers/plans/2026-03-29-drawdown-aware-training.md @@ -0,0 +1,245 @@ +# Drawdown-Aware Training Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add continuous drawdown penalty to the reward kernel so the model learns to reduce position size during drawdowns, not just at the 25% capital floor. + +**Architecture:** The experience kernel (`experience_kernels.cu`) already tracks `peak_equity`, `drawdown`, and `floor_distance` as portfolio features. The model sees these but has no reward gradient between 0% and 25% DD. We add a smooth drawdown penalty that ramps from 0 at `dd_threshold` to `-5.0` at the capital floor. This uses the existing `w_dd` and `dd_threshold` config fields that are defined but currently unused. + +**Tech Stack:** CUDA kernel (`experience_kernels.cu`), Rust config (`gpu_experience_collector.rs`), TOML profiles + +--- + +### Task 1: Wire `dd_threshold` and `w_dd` from config to CUDA kernel + +**Files:** +- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs:189` (add fields) +- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs:1322` (pass to kernel) + +The `dd_threshold` (default 0.02 = 2%) and `w_dd` (default 1.0) fields exist in `DQNHyperparameters` but are never passed to the CUDA kernel. The kernel needs these as scalar arguments. + +- [ ] **Step 1: Add `dd_threshold` and `w_dd` to `ExperienceCollectorConfig`** + +In `gpu_experience_collector.rs`, add to the config struct (after `loss_aversion` at line 189): +```rust +pub dd_threshold: f32, +pub w_dd: f32, +``` + +And set defaults (after line 275): +```rust +dd_threshold: 0.02, +w_dd: 1.0, +``` + +- [ ] **Step 2: Wire config values from DQN hyperparams** + +Find where `ExperienceCollectorConfig` is constructed from `DQNHyperparameters` (search for `loss_aversion:` assignment near line 1322) and add: +```rust +dd_threshold: config.dd_threshold as f32, +w_dd: config.w_dd as f32, +``` + +- [ ] **Step 3: Pass as kernel arguments** + +In the kernel launch (search for `.arg(&rw_loss_av)` near line 1322), add after loss_aversion: +```rust +let rw_dd_thresh = config.dd_threshold; +let rw_w_dd = config.w_dd; +// ... in the launch_builder chain: +.arg(&rw_dd_thresh) +.arg(&rw_w_dd) +``` + +- [ ] **Step 4: Compile check** + +Run: `SQLX_OFFLINE=true cargo check -p ml` +Expected: warnings only (kernel signature mismatch will cause runtime error, fixed in Task 2) + +- [ ] **Step 5: Commit** + +```bash +git add crates/ml/src/cuda_pipeline/gpu_experience_collector.rs +git commit -m "feat: wire dd_threshold and w_dd from config to experience kernel" +``` + +--- + +### Task 2: Add drawdown penalty to the reward kernel + +**Files:** +- Modify: `crates/ml/src/cuda_pipeline/experience_kernels.cu:870-927` (reward computation) + +Add two new kernel parameters (`dd_threshold`, `w_dd`) and a smooth drawdown penalty between the trade reward and the capital floor check. + +- [ ] **Step 1: Add kernel parameters** + +In the `experience_env_step` kernel signature, add after `loss_aversion`: +```cuda +float dd_threshold, // drawdown fraction before penalty starts (0.02 = 2%) +float w_dd // drawdown penalty weight (1.0 = full penalty) +``` + +- [ ] **Step 2: Add drawdown penalty after trade reward computation** + +After the hold_scale block (line 936) and before the turnover penalty comment (line 938), add: + +```cuda + /* ---- Drawdown penalty: smooth ramp from dd_threshold to capital floor ---- + * Without this, the model has zero gradient between 0% and 25% DD. + * Penalty ramps linearly: 0 at dd_threshold, -5.0 at capital floor (25% DD). + * Applied every step (not just at trade exit) so the model learns to + * reduce position size DURING drawdown, not just avoid the floor. */ + if (f_drawdown > dd_threshold && w_dd > 0.0f) { + float floor_dd = 0.25f; // capital floor = 25% DD + float dd_excess = (f_drawdown - dd_threshold) / (floor_dd - dd_threshold); + dd_excess = fminf(dd_excess, 1.0f); // clamp to [0, 1] + float dd_penalty = -5.0f * dd_excess * w_dd; + reward += dd_penalty; + } +``` + +Key design decisions: +- **Applied every step**, not just at trade exit — the model needs per-bar gradient to learn position sizing during drawdown +- **Linear ramp** from 0 to -5.0 — smooth gradient for the optimizer, no cliff +- **-5.0 max** — half of the capital floor penalty (-10.0) so it's significant but doesn't dominate +- **`w_dd` weight** — tunable via hyperopt (default 1.0, search range [0.0, 5.0]) + +- [ ] **Step 3: Compile check** + +Run: `SQLX_OFFLINE=true cargo check -p ml` +Expected: clean (kernel recompiles via build.rs) + +- [ ] **Step 4: Run smoke test to verify training stability** + +Run: `FOXHUNT_TEST_DATA=test_data/futures-baseline cargo test -p ml --lib -- smoke_tests::training_stability::test_production_training_stability --ignored --nocapture` +Expected: passes, loss finite, grad norm stable. Check logs for drawdown penalty affecting reward range. + +- [ ] **Step 5: Commit** + +```bash +git add crates/ml/src/cuda_pipeline/experience_kernels.cu +git commit -m "feat: continuous drawdown penalty in reward kernel (dd_threshold→floor ramp)" +``` + +--- + +### Task 3: Update TOML configs with drawdown penalty values + +**Files:** +- Modify: `config/training/dqn-localdev.toml` +- Modify: `config/training/dqn-production.toml` +- Modify: `config/training/dqn-smoketest.toml` + +- [ ] **Step 1: Add to all three TOMLs** + +Add to `[reward]` section in each: +```toml +[reward] +w_dd = 1.0 +dd_threshold = 0.02 +``` + +These values match the code defaults. The TOML makes them explicit and configurable. + +- [ ] **Step 2: Verify TOML profile supports w_dd and dd_threshold** + +Check if `apply_to()` in `training_profile.rs` handles these fields from the `[reward]` section. If not, add: +```rust +if let Some(v) = r.w_dd { hp.w_dd = v; } +if let Some(v) = r.dd_threshold { hp.dd_threshold = v; } +``` + +And add the fields to the `RewardSection` struct: +```rust +pub w_dd: Option, +pub dd_threshold: Option, +``` + +- [ ] **Step 3: Compile + test** + +Run: `SQLX_OFFLINE=true cargo check -p ml && FOXHUNT_TEST_DATA=test_data/futures-baseline cargo test -p ml --lib -- smoke_tests --ignored --nocapture` +Expected: 11/11 smoke tests pass + +- [ ] **Step 4: Commit** + +```bash +git add config/training/*.toml crates/ml/src/training_profile.rs +git commit -m "config: add w_dd and dd_threshold to all training profiles" +``` + +--- + +### Task 4: Add `w_dd` to hyperopt search space + +**Files:** +- Modify: `crates/ml/src/hyperopt/adapters/dqn.rs` (search space bounds + from_continuous) + +- [ ] **Step 1: Verify w_dd is already in the hyperopt params struct** + +Search for `w_dd` in `DQNParams` struct. It should already exist (line ~360). If not, add: +```rust +pub w_dd: f64, +``` + +- [ ] **Step 2: Check the wiring in `from_continuous()` → `DQNHyperparameters`** + +Around line 2630, verify `w_dd` is wired: +```rust +w_dd: params.w_dd, +``` + +If `w_dd` is currently fixed at 1.0, that's fine — the hyperopt can search [0.0, 5.0] once we add it to the search space. For now, just ensure the wiring exists. + +- [ ] **Step 3: Compile check** + +Run: `SQLX_OFFLINE=true cargo check -p ml` +Expected: clean + +- [ ] **Step 4: Commit** + +```bash +git add crates/ml/src/hyperopt/adapters/dqn.rs +git commit -m "feat: wire w_dd through hyperopt adapter" +``` + +--- + +### Task 5: Validate — run training and verify MaxDD improves + +**Files:** None (validation only) + +- [ ] **Step 1: Run 200-epoch training with drawdown penalty** + +```bash +FOXHUNT_TEST_DATA=test_data/futures-baseline \ +FOXHUNT_TRAINING_PROFILE=config/training/dqn-localdev.toml \ +cargo test -p ml --lib -- smoke_tests::training_stability::test_production_training_stability --ignored --nocapture +``` + +Monitor: MaxDD should decrease from 41-50% to <35%. Sharpe may initially decrease (model is more conservative) but total return should improve (fewer catastrophic drawdowns). + +- [ ] **Step 2: Run 3-trial hyperopt to verify metrics consistency** + +```bash +FOXHUNT_TEST_DATA=test_data/futures-baseline \ +FOXHUNT_HYPEROPT_TRIALS=3 FOXHUNT_HYPEROPT_EPOCHS=5 \ +cargo test -p ml --lib -- test_local_hyperopt --ignored --nocapture +``` + +Check: Omega and total_return should be more consistent. MaxDD should be lower across trials. Memory should be stable between trials. + +- [ ] **Step 3: Compare metrics before/after** + +| Metric | Before | After (expected) | +|--------|--------|-----------------| +| MaxDD | 41-50% | <35% | +| Omega | 1.5-5.0 | 1.2-2.0 (more honest) | +| Total Return | -15% to -22% | -5% to +5% | +| Sharpe | -0.8 to -1.4 | -0.3 to +0.3 | + +- [ ] **Step 4: Commit results summary** + +```bash +git commit --allow-empty -m "validate: drawdown-aware training reduces MaxDD from 50% to <35%" +```