From a69174e99f782102af20ea02f749f1c74711f31c Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 24 Mar 2026 23:57:04 +0100 Subject: [PATCH] fix: segment-based trade lifecycle + reversal P&L computation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Trade detection now counts reversals (S100→L50) as completed segments - old_pos_pnl saved before position update for correct reversal P&L - realized_pnl writeback uses old position PnL on reversal bars - 0% win rate persists — needs deeper investigation (likely tx cost interaction) WIP: The trade_return formula produces correct sign for raw market moves, but every trade still shows as a loss. Suspect tx costs on both entry AND exit of each reversal segment exceed the 1-bar price movement. Co-Authored-By: Claude Opus 4.6 (1M context) --- .serena/project.yml | 14 + .../src/cuda_pipeline/experience_kernels.cu | 152 +++-- .../2026-03-24-financial-metrics-pipeline.md | 288 +++++++++ .../2026-03-24-remaining-gpu-features.md | 567 ++++++++++++++++++ ...-43-16.144446538+00-00_sharpe_-0.0177.json | 78 +++ ...3-44-00.953235416+00-00_sharpe_0.0182.json | 78 +++ ...7-22-11.285931152+00-00_sharpe_0.0091.json | 78 +++ ...9-21-47.273661543+00-00_sharpe_0.0047.json | 79 +++ ...0-01-01.349397710+00-00_sharpe_0.0419.json | 79 +++ ...-27-44.051295574+00-00_sharpe_-0.0068.json | 79 +++ ...-53-49.539858557+00-00_sharpe_-0.0116.json | 79 +++ ...5-12-06.755573679+00-00_sharpe_0.0838.json | 79 +++ ...5-19-27.798184419+00-00_sharpe_0.0355.json | 79 +++ ...5-22-41.871557162+00-00_sharpe_0.0075.json | 79 +++ 14 files changed, 1773 insertions(+), 35 deletions(-) create mode 100644 docs/superpowers/plans/2026-03-24-financial-metrics-pipeline.md create mode 100644 docs/superpowers/plans/2026-03-24-remaining-gpu-features.md create mode 100644 ml/hyperopt_results/dqn_best_trial_2026-03-22T23-43-16.144446538+00-00_sharpe_-0.0177.json create mode 100644 ml/hyperopt_results/dqn_best_trial_2026-03-22T23-44-00.953235416+00-00_sharpe_0.0182.json create mode 100644 ml/hyperopt_results/dqn_best_trial_2026-03-23T07-22-11.285931152+00-00_sharpe_0.0091.json create mode 100644 ml/hyperopt_results/dqn_best_trial_2026-03-24T09-21-47.273661543+00-00_sharpe_0.0047.json create mode 100644 ml/hyperopt_results/dqn_best_trial_2026-03-24T10-01-01.349397710+00-00_sharpe_0.0419.json create mode 100644 ml/hyperopt_results/dqn_best_trial_2026-03-24T14-27-44.051295574+00-00_sharpe_-0.0068.json create mode 100644 ml/hyperopt_results/dqn_best_trial_2026-03-24T14-53-49.539858557+00-00_sharpe_-0.0116.json create mode 100644 ml/hyperopt_results/dqn_best_trial_2026-03-24T15-12-06.755573679+00-00_sharpe_0.0838.json create mode 100644 ml/hyperopt_results/dqn_best_trial_2026-03-24T15-19-27.798184419+00-00_sharpe_0.0355.json create mode 100644 ml/hyperopt_results/dqn_best_trial_2026-03-24T15-22-41.871557162+00-00_sharpe_0.0075.json diff --git a/.serena/project.yml b/.serena/project.yml index 13e1ed57a..d72947c0e 100644 --- a/.serena/project.yml +++ b/.serena/project.yml @@ -134,3 +134,17 @@ line_ending: # list of regex patterns which, when matched, mark a memory entry as read‑only. # Extends the list from the global configuration, merging the two lists. read_only_memory_patterns: [] + +# list of regex patterns for memories to completely ignore. +# Matching memories will not appear in list_memories or activate_project output +# and cannot be accessed via read_memory or write_memory. +# To access ignored memory files, use the read_file tool on the raw file path. +# Extends the list from the global configuration, merging the two lists. +# Example: ["_archive/.*", "_episodes/.*"] +ignored_memory_patterns: [] + +# advanced configuration option allowing to configure language server-specific options. +# Maps the language key to the options. +# Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available. +# No documentation on options means no options are available. +ls_specific_settings: {} diff --git a/crates/ml/src/cuda_pipeline/experience_kernels.cu b/crates/ml/src/cuda_pipeline/experience_kernels.cu index 7af1fb054..a9805c8e3 100644 --- a/crates/ml/src/cuda_pipeline/experience_kernels.cu +++ b/crates/ml/src/cuda_pipeline/experience_kernels.cu @@ -679,23 +679,68 @@ extern "C" __global__ void experience_env_step( /* ---- Mark-to-market PnL for this timestep ---- */ float raw_pnl = position * (raw_next - raw_close); + /* Old position's PnL — needed for reversal trade tracking. + * raw_pnl uses the NEW position (after action). On reversals, + * the old position's PnL determines trade segment return. */ + float old_pos_pnl = ps[0] * (raw_next - raw_close); /* ---- Mark-to-market equity (needed for peak_equity and reward) ---- */ float equity = cash + position * raw_next; peak_equity = (equity > peak_equity) ? equity : peak_equity; - /* ==== Trade lifecycle tracking (v3) ==== */ - /* ps[12] = entry_price, ps[13] = trade_start_pnl — dedicated slots, - * no dual-purpose hacking. PORTFOLIO_STRIDE=14 gives us room. */ + /* ==== Trade lifecycle tracking (v4: segment-based, reversal-aware) ==== */ + /* + * A "trade segment" = contiguous period with the same position sign. + * Segment ends when: (1) exit to flat, (2) reversal (sign change), (3) episode end. + * + * A reversal (e.g. S100->L50) is TWO half-trades: + * - Close old direction: book P&L, update Kelly, fire sparse reward + * - Open new direction: fresh entry_price, fresh trade_start_pnl + * + * This matches institutional P&L accounting (round-trip segments). + */ + float is_flat = (fabsf(position) < 0.001f) ? 1.0f : 0.0f; - float prev_pos_abs = fabsf(ps[0]); /* position BEFORE this step's delta */ - int was_flat = (prev_pos_abs < 0.001f) ? 1 : 0; - /* Lifecycle transitions */ - int entering_trade = (was_flat && !((int)(is_flat > 0.5f))); - int exiting_trade = (!was_flat && (int)(is_flat > 0.5f)); + /* Sign of position: -1 (short), 0 (flat), +1 (long) */ + int prev_sign = (ps[0] > 0.001f) ? 1 : ((ps[0] < -0.001f) ? -1 : 0); + int curr_sign = (position > 0.001f) ? 1 : ((position < -0.001f) ? -1 : 0); - /* On entry: snapshot price and cumulative PnL */ + int entering_trade = (prev_sign == 0 && curr_sign != 0); + int exiting_trade = (prev_sign != 0 && curr_sign == 0); + int reversing_trade = (prev_sign != 0 && curr_sign != 0 && prev_sign != curr_sign); + + /* A segment completes on exit OR reversal */ + int segment_complete = exiting_trade || reversing_trade; + + /* On reversal: close old segment, open new one. + * reversal_return is saved for the sparse reward block below + * (trade_start_pnl is overwritten here, so recomputing would yield 0). */ + float reversal_return = 0.0f; + if (reversing_trade) { + /* Closing P&L for the completed segment. + * CRITICAL: raw_pnl uses the NEW position (already updated at line 677). + * old_pos_pnl (saved above) has the OLD position's final-bar PnL. */ + float closing_pnl = ps[11] + old_pos_pnl - trade_start_pnl; + reversal_return = closing_pnl / (prev_equity > 1.0f ? prev_equity : 1.0f); + + /* Kelly stats for completed segment */ + if (reversal_return > 0.0f) { + win_count += 1.0f; + sum_wins += reversal_return; + } else { + loss_count += 1.0f; + sum_losses += fabsf(reversal_return); + } + sum_returns += reversal_return; + sum_sq_returns += reversal_return * reversal_return; + + /* Reset for new segment — realized_pnl snapshot after booking the close */ + entry_price = raw_close; + trade_start_pnl = ps[11] + old_pos_pnl; + } + + /* On entry from flat (unchanged) */ if (entering_trade) { entry_price = raw_close; trade_start_pnl = ps[11]; /* cumulative realized_pnl at entry */ @@ -708,7 +753,7 @@ extern "C" __global__ void experience_env_step( * Trail distance adapts to regime via ADX (trend) and CUSUM (vol). * ════════════════════════════════════════════════════════════════════ */ float unrealized_trade_pnl = 0.0f; - if (!was_flat && hold_time > 0.0f) { + if (prev_sign != 0 && hold_time > 0.0f) { unrealized_trade_pnl = (ps[11] + raw_pnl - trade_start_pnl) / (prev_equity > 1.0f ? prev_equity : 1.0f); } @@ -726,7 +771,7 @@ extern "C" __global__ void experience_env_step( float trail_distance = 0.005f * vol_scale * trend_scale; /* 0.5% base, widens in vol/trend */ /* Trailing stop: only when profitable and held > 2 bars */ - if (!was_flat && hold_time > 2.0f && peak_equity > 1.0f) { + if (prev_sign != 0 && hold_time > 2.0f && peak_equity > 1.0f) { float peak_return = (peak_equity - prev_equity) / (prev_equity > 1.0f ? prev_equity : 1.0f); if (peak_return > trail_distance) { float trail_floor = peak_return - trail_distance; @@ -741,11 +786,12 @@ extern "C" __global__ void experience_env_step( } } - /* Early-exit override: model tries to exit before 5 bars — force hold. + /* Early-exit override: model tries to exit to FLAT before 5 bars — force hold. + * Only blocks flat exits, NOT reversals — reversals always allowed. * CRITICAL: fix the stored action to match what actually executed. * Without this, the replay buffer stores (state, action=Flat, reward_from_Hold) * which corrupts Q-values for Flat — classic action aliasing bug. */ - if (!was_flat && is_flat > 0.5f && hold_time < 5.0f && hold_time > 0.0f && !exiting_trade) { + if (prev_sign != 0 && curr_sign == 0 && hold_time < 5.0f && hold_time > 0.0f && !exiting_trade) { position = ps[0]; cash = ps[1]; is_flat = 0.0f; @@ -760,10 +806,24 @@ extern "C" __global__ void experience_env_step( out_actions[out_off] = action_idx; } + /* Recompute segment_complete after trailing stop / early-exit may have + * modified exiting_trade. Trailing stop sets exiting_trade=1; early-exit + * override sets exiting_trade=0. reversing_trade is never modified. */ + segment_complete = exiting_trade || reversing_trade; + + /* Save hold_time BEFORE reset — sparse reward needs the pre-reset value + * for the patience multiplier. Without this, hold_time is 0 at exit + * and the sparse reward condition (hold_time > 0) never fires. */ + float segment_hold_time = hold_time; + /* Hold time tracking: counts TOTAL bars in position (not just losing). - * Reset to 0 when flat. This feeds the patience multiplier in reward v5. */ - if (is_flat < 0.5f) { + * Reset to 0 when flat OR on reversal (new segment starts fresh). + * This feeds the patience multiplier in reward v5. */ + if (is_flat < 0.5f && !reversing_trade) { hold_time += 1.0f; + } else if (reversing_trade) { + /* New segment starts: count this bar as bar 1 of the new direction */ + hold_time = 1.0f; } else { hold_time = 0.0f; } @@ -783,7 +843,8 @@ extern "C" __global__ void experience_env_step( * Keeps gradients flowing — the network sees direction each bar. * When flat: reward_dense = 0 (no trade, no signal). * - * SPARSE (at trade exit, large weight): trade return × patience. + * SPARSE (at segment completion, large weight): trade return x patience. + * Fires on BOTH exits to flat AND reversals (sign changes). * Primary learning signal. A trade held longer than expected * gets a patience bonus (sqrt scaling). Regime-adaptive expected * hold time via ADX. @@ -800,28 +861,41 @@ extern "C" __global__ void experience_env_step( reward_dense = 0.0f; } - /* ---- Sparse component: trade return × patience multiplier ---- */ + /* ---- Sparse component: segment return x patience multiplier ---- */ + /* segment_complete = exiting_trade || reversing_trade. + * For reversals, Kelly stats were already updated above (in the reversal + * block). For exits, Kelly stats are updated here. Both paths compute + * segment_return for the patience-scaled sparse reward. */ float reward_sparse = 0.0f; - if (exiting_trade && hold_time > 0.0f) { - float trade_pnl = ps[11] + raw_pnl - trade_start_pnl; - float trade_return = trade_pnl / (prev_equity > 1.0f ? prev_equity : 1.0f); + if (segment_complete && segment_hold_time > 0.0f) { + float segment_pnl = ps[11] + raw_pnl - trade_start_pnl; + float segment_return = segment_pnl / (prev_equity > 1.0f ? prev_equity : 1.0f); - /* Kelly statistics (GPU-native, no CPU). + /* Kelly statistics for EXIT only (reversal Kelly already done above). * Tracked here AFTER trailing stop may have set exiting_trade=1, * so trail-triggered exits are counted in Kelly stats. */ - if (trade_return > 0.0f) { - win_count += 1.0f; - sum_wins += trade_return; - } else { - loss_count += 1.0f; - sum_losses += fabsf(trade_return); + if (exiting_trade) { + if (segment_return > 0.0f) { + win_count += 1.0f; + sum_wins += segment_return; + } else { + loss_count += 1.0f; + sum_losses += fabsf(segment_return); + } + sum_returns += segment_return; + sum_sq_returns += segment_return * segment_return; + } + + /* For reversals, use the saved reversal_return — trade_start_pnl was + * already overwritten in the reversal block, so recomputing would + * yield 0. The reversal_return variable holds the correct value. */ + if (reversing_trade) { + segment_return = reversal_return; } - sum_returns += trade_return; - sum_sq_returns += trade_return * trade_return; /* Regime-adaptive expected hold time via ADX. - * Trending (ADX > 30): hold longer → expected_hold = 20 bars. - * Ranging (ADX < 20): shorter trades → expected_hold = 8 bars. + * Trending (ADX > 30): hold longer -> expected_hold = 20 bars. + * Ranging (ADX < 20): shorter trades -> expected_hold = 8 bars. * Default: expected_hold = 12 bars. */ float adx_val = 0.0f; if (bar_idx < total_bars && market_dim >= 42) { @@ -831,13 +905,14 @@ extern "C" __global__ void experience_env_step( if (adx_val > 30.0f) expected_hold = 20.0f; else if (adx_val < 20.0f) expected_hold = 8.0f; - /* Patience multiplier: sqrt(hold_time / expected_hold). + /* Patience multiplier: sqrt(segment_hold_time / expected_hold). + * Uses pre-reset hold_time so the multiplier reflects actual duration. * Held exactly expected: 1.0. Held 2x: 1.41. Held 0.25x: 0.5. * Capped at 3.0 to prevent extreme outliers. */ - float patience_mult = sqrtf(hold_time / expected_hold); + float patience_mult = sqrtf(segment_hold_time / expected_hold); patience_mult = fminf(patience_mult, 3.0f); - reward_sparse = trade_return * patience_mult; + reward_sparse = segment_return * patience_mult; } /* ---- Combined reward with loss aversion ---- */ @@ -883,7 +958,14 @@ extern "C" __global__ void experience_env_step( ps[8] = flat_counter; ps[9] = new_portfolio_value; /* prev_equity = current equity for next step */ ps[10] = hold_time; - ps[11] = ps[11] + raw_pnl; /* realized_pnl accumulates */ + /* realized_pnl accumulates. On reversal bars, use old_pos_pnl + * (saved before position was overwritten at ps[0] = position). + * For non-reversal bars, raw_pnl (= current position's PnL) is correct. */ + if (reversing_trade) { + ps[11] = ps[11] + old_pos_pnl; /* old position's PnL (saved at line ~700) */ + } else { + ps[11] = ps[11] + raw_pnl; /* normal: current position's PnL */ + } ps[12] = entry_price; /* preserved across bars of a trade */ ps[13] = trade_start_pnl; /* realized_pnl snapshot at trade entry */ ps[14] = win_count; /* Kelly: accumulated across episode */ diff --git a/docs/superpowers/plans/2026-03-24-financial-metrics-pipeline.md b/docs/superpowers/plans/2026-03-24-financial-metrics-pipeline.md new file mode 100644 index 000000000..4bb143c00 --- /dev/null +++ b/docs/superpowers/plans/2026-03-24-financial-metrics-pipeline.md @@ -0,0 +1,288 @@ +# Financial Metrics Pipeline Fix — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make Sharpe/WinRate/MaxDD/Trades report real per-trade performance from the GPU experience collector's portfolio simulation, not epoch-averaged mean rewards. + +**Architecture:** Add a GPU reduction kernel that reads per-episode Kelly stats from `portfolio_states[N, 14:19]` (win_count, loss_count, sum_wins, sum_losses, sum_returns, sum_sq_returns) and reduces them into a single epoch-level `TradeStats` struct. Download this 48-byte summary once per epoch. Feed real trade-level data to `compute_epoch_financials()`. + +**Tech Stack:** CUDA kernel (inline in gpu_experience_collector.rs), Rust structs, existing monitoring download path. + +--- + +## Root Cause + +``` +CURRENT (broken): + experience_env_step → out_rewards[N,L] → monitoring_reduce → mean(N*L rewards) → ONE float per epoch + pnl_history gets 1 entry per epoch → Sharpe=0.00, Trades=1 + +FIXED: + experience_env_step → portfolio_states[N, 14:19] → trade_stats_reduce → TradeStats per epoch + TradeStats has: total_trades, wins, losses, sum_pnl, sum_wins, sum_losses, per-step returns + compute_epoch_financials uses REAL trade data → meaningful Sharpe/WinRate/Trades +``` + +The per-trade Kelly stats already exist in `portfolio_states` — the experience kernel populates `ps[14]` (win_count), `ps[15]` (loss_count), `ps[16]` (sum_wins), `ps[17]` (sum_losses), `ps[18]` (sum_returns), `ps[19]` (sum_sq_returns) at every trade completion event. These are never read. + +## File Structure + +| File | Action | Responsibility | +|------|--------|---------------| +| `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs` | Modify | Add `trade_stats_reduce` kernel, `TradeStats` struct, `collect_trade_stats()` method | +| `crates/ml/src/trainers/dqn/financials.rs` | Modify | Rewrite `compute_epoch_financials()` to accept `TradeStats` instead of `VecDeque` | +| `crates/ml/src/trainers/dqn/trainer/training_loop.rs` | Modify | Call `collect_trade_stats()` after experience collection, feed to financials | +| `crates/ml/src/trainers/dqn/trainer/mod.rs` | Modify | Add `trade_stats_history: Vec` field | +| `crates/ml/src/trainers/dqn/trainer/metrics.rs` | Modify | Use TradeStats in final metrics | +| `crates/ml/src/trainers/dqn/smoke_tests/training_stability.rs` | Modify | Assert Trades > 1, WinRate > 0 | + +--- + +## Task 1: TradeStats Struct + GPU Reduction Kernel (20 min) + +**Files:** +- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs` + +The kernel reads `portfolio_states[N, PORTFOLIO_STRIDE]` and sums the per-episode Kelly fields into a single output vector. + +- [ ] **Step 1:** Add `TradeStats` struct to `gpu_experience_collector.rs`: +```rust +/// Per-epoch trading statistics reduced from GPU portfolio states. +/// Downloaded once per epoch (48 bytes — 6 f64s). +#[derive(Debug, Clone, Default)] +pub struct TradeStats { + /// Total completed trades (entries + exits) across all episodes + pub total_trades: usize, + /// Winning trades (positive trade return at exit) + pub winning_trades: usize, + /// Losing trades (negative trade return at exit) + pub losing_trades: usize, + /// Sum of P&L from winning trades (for profit factor) + pub sum_wins: f64, + /// Sum of |P&L| from losing trades (for profit factor) + pub sum_losses: f64, + /// Sum of all trade returns (for mean return) + pub sum_returns: f64, + /// Sum of squared trade returns (for Sharpe std computation) + pub sum_sq_returns: f64, + /// Per-step returns from all episodes (for drawdown/equity curve) + /// These are the actual out_rewards[N*L] values, not averaged. + pub step_returns: Vec, +} +``` + +- [ ] **Step 2:** Add inline CUDA kernel `trade_stats_reduce`: +```cuda +// Grid: 1 block, Block: 256 threads +// Parallel reduction over N episodes, summing portfolio_states[i, 14:19] +extern "C" __global__ void trade_stats_reduce( + const float* __restrict__ portfolio_states, // [N, PORTFOLIO_STRIDE] + float* __restrict__ output, // [6] win_count, loss_count, sum_wins, sum_losses, sum_returns, sum_sq + int N, + int stride +) { + __shared__ float shmem[6 * 32]; // 6 fields × 32 warps max + int tid = threadIdx.x; + + // Each thread accumulates over a range of episodes + float local[6] = {0}; + for (int i = tid; i < N; i += blockDim.x) { + const float* ps = portfolio_states + i * stride; + local[0] += ps[14]; // win_count + local[1] += ps[15]; // loss_count + local[2] += ps[16]; // sum_wins + local[3] += ps[17]; // sum_losses + local[4] += ps[18]; // sum_returns + local[5] += ps[19]; // sum_sq_returns + } + + // Warp-level reduction + for (int offset = 16; offset > 0; offset >>= 1) { + for (int f = 0; f < 6; f++) + local[f] += __shfl_down_sync(0xffffffff, local[f], offset); + } + + // Write warp results to shared memory + int warp_id = tid / 32; + int lane = tid % 32; + if (lane == 0) { + for (int f = 0; f < 6; f++) + shmem[warp_id * 6 + f] = local[f]; + } + __syncthreads(); + + // Final reduction by first warp + if (warp_id == 0) { + int num_warps = (blockDim.x + 31) / 32; + float sum[6] = {0}; + if (lane < num_warps) { + for (int f = 0; f < 6; f++) + sum[f] = shmem[lane * 6 + f]; + } + for (int offset = 16; offset > 0; offset >>= 1) { + for (int f = 0; f < 6; f++) + sum[f] += __shfl_down_sync(0xffffffff, sum[f], offset); + } + if (lane == 0) { + for (int f = 0; f < 6; f++) + output[f] = sum[f]; + } + } +} +``` + +- [ ] **Step 3:** Add `trade_stats_kernel: CudaFunction` field to `GpuExperienceCollector`. Compile the kernel in the constructor alongside the existing kernels. Add a `trade_stats_buf: CudaSlice` output buffer (6 floats). + +- [ ] **Step 4:** Add `pub fn collect_trade_stats(&mut self) -> Result` method: + - Launch the `trade_stats_reduce` kernel with `portfolio_states`, N=alloc_episodes, stride=PORTFOLIO_STRIDE + - Download the 6-float output buffer to host (24 bytes — single DtoH transfer) + - Also download `rewards_out[N*L]` for per-step returns (this IS a larger transfer but only once per epoch, ~12KB for 3200 experiences) + - Construct and return `TradeStats` + +- [ ] **Step 5:** Run: `SQLX_OFFLINE=true cargo check -p ml` — verify clean build. + +- [ ] **Step 6:** Commit: "feat: GPU trade stats reduction kernel — extracts per-trade metrics from portfolio states" + +--- + +## Task 2: Rewrite compute_epoch_financials() (10 min) + +**Files:** +- Modify: `crates/ml/src/trainers/dqn/financials.rs` + +Replace the `pnl_history: &VecDeque` input with `TradeStats`. The financial metrics should use REAL trade data: + +- [ ] **Step 1:** Change signature: +```rust +pub(crate) fn compute_epoch_financials( + trade_stats: &TradeStats, + action_counts: &[usize; 9], + initial_capital: f64, +) -> EpochFinancials +``` + +- [ ] **Step 2:** Rewrite the body: + - `total_trades = trade_stats.total_trades` (real trade count from GPU) + - `win_rate = trade_stats.winning_trades as f64 / total_trades.max(1) as f64` + - `profit_factor = trade_stats.sum_wins / trade_stats.sum_losses.max(1e-10)` + - `total_return = trade_stats.sum_returns` (sum of all trade returns) + - `avg_return = total_return / total_trades.max(1) as f64` + - Sharpe: compute from `step_returns` (per-bar returns, annualized with √252) + - Max drawdown: compute equity curve from `step_returns` starting at `initial_capital` + - Sortino: from negative step_returns only + - Action distribution: unchanged (from `action_counts`) + +- [ ] **Step 3:** Update the existing tests to use `TradeStats` input: + - `test_empty_history` → `TradeStats::default()` should produce zeros + - `test_all_winning` → `TradeStats { winning_trades: 5, total_trades: 5, sum_wins: 100.0, ... }` + - `test_mixed_pnl` → set win/loss counts and sums from trade data + +- [ ] **Step 4:** Run: `SQLX_OFFLINE=true cargo test -p ml --lib -- financials` — all tests pass. + +- [ ] **Step 5:** Commit: "refactor: compute_epoch_financials uses real TradeStats from GPU" + +--- + +## Task 3: Wire TradeStats into Training Loop (10 min) + +**Files:** +- Modify: `crates/ml/src/trainers/dqn/trainer/mod.rs` +- Modify: `crates/ml/src/trainers/dqn/trainer/training_loop.rs` +- Modify: `crates/ml/src/trainers/dqn/trainer/metrics.rs` + +- [ ] **Step 1:** Add `trade_stats_history: Vec` to `DQNTrainer` struct in `mod.rs`. Initialize to `Vec::new()` in constructor. + +- [ ] **Step 2:** In `training_loop.rs`, after GPU experience collection (where `summary.mean_reward` is currently pushed to `pnl_history`), add: +```rust +// Collect real per-trade stats from GPU portfolio states +if let Some(ref mut collector) = self.gpu_experience_collector { + match collector.collect_trade_stats() { + Ok(stats) => { + info!( + "Epoch {}/{}: GPU trades={} wins={} losses={} PF={:.2}", + epoch + 1, self.hyperparams.epochs, + stats.total_trades, stats.winning_trades, stats.losing_trades, + if stats.sum_losses > 1e-10 { stats.sum_wins / stats.sum_losses } else { 0.0 } + ); + self.trade_stats_history.push(stats); + } + Err(e) => warn!("Failed to collect trade stats: {e}"), + } +} +``` + +- [ ] **Step 3:** Replace the `compute_epoch_financials(&self.pnl_history, ...)` call with: +```rust +let trade_stats = self.trade_stats_history.last() + .cloned() + .unwrap_or_default(); +let financials = compute_epoch_financials( + &trade_stats, + &monitor.action_counts, + self.hyperparams.initial_capital, +); +``` + +- [ ] **Step 4:** In `metrics.rs`, update `create_final_metrics()` to aggregate trade stats across all epochs: + - Total trades = sum of all epoch trade counts + - Overall win rate = total wins / total trades + - Overall Sharpe = from concatenated step_returns across all epochs + +- [ ] **Step 5:** Remove the `pnl_history: VecDeque` field and all references. It's replaced by `trade_stats_history`. + +- [ ] **Step 6:** Run: `SQLX_OFFLINE=true cargo check -p ml` — verify clean build. + +- [ ] **Step 7:** Commit: "feat: wire GPU trade stats into training loop financials" + +--- + +## Task 4: Update Smoke Tests to Assert Real Trading Metrics (5 min) + +**Files:** +- Modify: `crates/ml/src/trainers/dqn/smoke_tests/training_stability.rs` + +- [ ] **Step 1:** In `test_trading_model_behavior`, add: +```rust +// With 3200 experiences per epoch and position changes, we should see trades +let total_trades = metrics.additional_metrics.get("total_trades").copied().unwrap_or(0.0); +assert!( + total_trades > 0.0, + "BEHAVIORAL: Model must execute trades during experience collection. Got 0 trades." +); +``` + +- [ ] **Step 2:** In `test_50_epoch_convergence`, add: +```rust +// After 50 epochs × 3200 experiences, model must have executed real trades +let total_trades = metrics.additional_metrics.get("total_trades").copied().unwrap_or(0.0); +assert!( + total_trades > 10.0, + "CONVERGENCE: Model must execute >10 trades over 50 epochs. Got {total_trades}" +); +``` + +- [ ] **Step 3:** Run all smoke tests: +```bash +FOXHUNT_TEST_DATA=test_data/futures-baseline SQLX_OFFLINE=true cargo test -p ml --lib -- smoke_tests --ignored --nocapture +``` + +- [ ] **Step 4:** Commit: "test: assert real trade count in smoke tests" + +--- + +## Key Design Decisions + +### Why Reduce on GPU Instead of Downloading portfolio_states? + +`portfolio_states` is `[N_episodes × PORTFOLIO_STRIDE=20]` = `N × 80 bytes`. With N=32, that's 2560 bytes — small enough to download. But the reduction kernel pattern is: +1. Future-proof: when N=1024 on H100, downloading 80KB per epoch wastes PCIe bandwidth +2. Consistent: follows the existing `monitoring_reduce` pattern +3. Zero-copy: the portfolio_states buffer is already on GPU, no staging needed + +### Why Download step_returns (out_rewards)? + +For Sharpe/MaxDD/Sortino, we need the per-step return distribution, not just the aggregates. With 3200 experiences, that's 12.8KB per epoch — acceptable for a once-per-epoch download. The alternative (computing Sharpe entirely on GPU) would require a second reduction kernel for mean/variance/drawdown, which is overengineering for a diagnostic metric. + +### What Counts as a "Trade"? + +A trade is an entry→exit cycle detected by the experience kernel's trade lifecycle logic. When `exiting_trade` fires (position goes from non-zero to zero), `win_count` or `loss_count` increments. Each episode can have 0-N trades depending on market conditions and model behavior. `total_trades = win_count + loss_count` summed across all episodes. diff --git a/docs/superpowers/plans/2026-03-24-remaining-gpu-features.md b/docs/superpowers/plans/2026-03-24-remaining-gpu-features.md new file mode 100644 index 000000000..533a7f4bf --- /dev/null +++ b/docs/superpowers/plans/2026-03-24-remaining-gpu-features.md @@ -0,0 +1,567 @@ +# Remaining GPU Features — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Wire and implement the 4 remaining unwired GPU features: multi-head attention integration, Decision Transformer CUDA kernels, ensemble training with diversity loss, and HER Future strategy with GPU episode boundaries. + +**Architecture:** Each feature integrates into the existing fused CUDA training pipeline (`GpuDqnTrainer` + CUDA Graph). All kernels are GPU-native (zero CPU in hot path). Post-graph operations use the `EventTrackingGuard` RAII pattern for safe cudarc buffer access. Kernels are compiled via nvcc → cubin (cached at `/tmp/.cubin_cache/`). + +**CUDA Graph Integration Strategy:** The CUDA Graph captures the full forward+loss+backward+Adam sequence. Features that modify intermediate activations (attention, ensemble heads) operate with a **1-step lag**: they modify `save_h_s2` AFTER graph replay, so the modification takes effect on the NEXT graph replay's unflatten step. This is the same pattern as the IQN trunk gradient and spectral normalization — standard in async gradient methods. The alternative (splitting the graph into trunk/head phases) is deferred until profiling shows the lag impacts convergence. + +**VRAM Budget:** +| Feature | RTX 3050 (4GB) | H100 (80GB) | +|---------|---------------|-------------| +| Attention (params + activations) | ~0.5MB | ~2MB | +| Decision Transformer (params + context) | ~4MB | ~60MB | +| Ensemble (K=3 head copies) | ~1.5MB | ~6MB | +| HER episode tracking | ~0.5MB | ~4MB | +| **Total** | **~6.5MB** | **~72MB** | + +**Tech Stack:** Rust (cudarc 0.19.3), CUDA C (nvcc/cubin), cuBLAS SGEMM + +--- + +## File Structure + +### New Files +| File | Responsibility | +|------|---------------| +| `crates/ml/src/cuda_pipeline/attention_backward_kernel.cu` | Attention backward pass: LayerNorm backward + output projection backward + attention weight gradients | +| `crates/ml/src/cuda_pipeline/dt_kernels.cu` | Decision Transformer forward: causal attention, softmax, positional encoding, token embedding, FFN, cross-entropy loss. Backward: softmax_backward, attention_backward (dQ/dK/dV), ffn_backward, layernorm_backward, embed_backward — 10 kernels total | +| `crates/ml/src/cuda_pipeline/ensemble_kernels.cu` | Ensemble: multi-head Q-aggregation, KL-divergence diversity loss, Thompson sampling action selection | +| `crates/ml/src/cuda_pipeline/her_episode_kernel.cu` | HER: episode boundary tracking, Future strategy donor sampling, episode-end detection | + +### Modified Files +| File | Changes | +|------|---------| +| `crates/ml/src/cuda_pipeline/gpu_attention.rs` | Add `backward()`, `params_buf()`, Adam state; wire into `GpuDqnTrainer` | +| `crates/ml/src/cuda_pipeline/batched_forward.rs` | Add attention call between h_s2 and value/advantage heads | +| `crates/ml/src/cuda_pipeline/batched_backward.rs` | Add attention backward in the gradient chain | +| `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs` | Conditional attention in forward/backward; ensemble head management | +| `crates/ml/src/cuda_pipeline/decision_transformer.rs` | Implement `pretrain_step()` with real CUDA kernels | +| `crates/ml/src/cuda_pipeline/gpu_her.rs` | Add Final/Future strategies, episode boundary buffer | +| `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs` | Add episode_ids to GpuExperienceBatch; fill via CUDA kernel (not CPU) | +| `crates/ml/src/trainers/dqn/fused_training.rs` | Wire attention, ensemble, HER strategies | +| `crates/ml/src/trainers/dqn/config.rs` | Add `use_attention: bool` to `DQNHyperparameters` (field exists in `DQNParams` hyperopt struct but not in the training config); wire from `DQNParams` during hyperopt conversion | +| `crates/ml/src/trainers/dqn/trainer/training_loop.rs` | Ensemble head rotation, DT pre-training phase, HER episode tracking | +| `crates/ml/src/hyperopt/adapters/dqn.rs` | Add DT pre-training epochs, ensemble params to search space | + +--- + +## Task 1: Attention Forward Pass Integration + +**Files:** +- Modify: `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs` +- Modify: `crates/ml/src/cuda_pipeline/gpu_attention.rs` +- Modify: `crates/ml/src/trainers/dqn/fused_training.rs` + +- [ ] **Step 1: Add `GpuAttention` as optional field in `FusedTrainingCtx`** + +In `fused_training.rs`, add to `FusedTrainingCtx`: +```rust +pub(crate) gpu_attention: Option, +``` + +Initialize it in `new()` when `hyperparams.use_attention` is true (similar to `gpu_iqn` initialization pattern at line 233). + +- [ ] **Step 2: Wire attention forward BETWEEN trunk h_s2 and value/advantage heads** + +In `gpu_dqn_trainer.rs`, add a method: +```rust +pub fn apply_attention_forward( + &self, + attention: &mut GpuAttention, + h_s2: &CudaSlice, + batch_size: usize, +) -> Result<(), MLError> +``` + +This copies `h_s2` into attention input, runs `attention.forward()`, then copies the attended output BACK into `save_h_s2` (in-place replacement). The CUDA Graph captures the trunk forward FIRST, then attention runs OUTSIDE the graph (same pattern as IQN/spectral norm), then value/advantage heads run. + +- [ ] **Step 3: Call attention in `run_full_step()` after CUDA Graph replay** + +After Step 2 (graph replay) and before Step 5 (IQN): +```rust +if let Some(ref mut attn) = self.gpu_attention { + let _evt_guard = EventTrackingGuard::new(self.stream.context()); + self.trainer.apply_attention_forward(attn, ...)?; +} +``` + +- [ ] **Step 4: Compile check** + +```bash +SQLX_OFFLINE=true cargo check -p ml 2>&1 | tail -5 +``` + +- [ ] **Step 5: Commit** + +```bash +git add crates/ml/src/cuda_pipeline/gpu_attention.rs crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs crates/ml/src/trainers/dqn/fused_training.rs +git commit -m "feat: wire attention forward pass into DQN training (post-graph, frozen weights)" +``` + +--- + +## Task 2: Attention Backward Pass Kernel + +**Files:** +- Create: `crates/ml/src/cuda_pipeline/attention_backward_kernel.cu` +- Modify: `crates/ml/src/cuda_pipeline/gpu_attention.rs` + +- [ ] **Step 1: Write the attention backward CUDA kernel** + +The kernel computes `dL/d(h_s2_input)` given `dL/d(h_s2_attended)`: + +```cuda +extern "C" __global__ void attention_backward_kernel( + const float* d_output, // [B, D] gradient from downstream + const float* states_input, // [B, D] saved input to attention + const float* params, // attention weights + float* d_input, // [B, D] gradient to upstream (trunk) + float* d_params, // [total_params] weight gradients (atomicAdd) + int B +); +``` + +The backward flows through: +1. LayerNorm backward → d_prenorm +2. Residual: d_prenorm splits into d_projection + d_residual +3. Output projection backward: d_projection → d_concat, dW_O, db_O +4. Multi-head attention backward → dQ, dK, dV per head +5. Q/K/V projection backward → d_input, dW_Q, dW_K, dW_V + +Since the kernel is complex (matching the 237-line forward kernel), implement in phases: +- Phase A: Residual passthrough only (d_input = d_output) — gradient flows, weights frozen +- Phase B: Full backward with weight gradients (unfreezes attention weights) + +Start with Phase A (trivial kernel, verifiable). + +- [ ] **Step 2: Add `backward()` method to `GpuAttention`** + +```rust +pub fn backward( + &self, + d_output: &CudaSlice, + d_input: &mut CudaSlice, + batch_size: usize, +) -> Result<(), MLError> +``` + +For Phase A: simply `memcpy_dtod(d_input, d_output)` — residual gradient passthrough. + +- [ ] **Step 3: Wire backward into `apply_iqn_trunk_gradient`** + +After attention forward rewrites `save_h_s2`, the IQN trunk gradient flows through the attention backward before reaching the trunk layers: +```rust +// Before trunk backward: d_h_s2 → attention_backward → d_h_s2_input +if let Some(ref attn) = self.gpu_attention { + attn.backward(&bw_d_h_s2, &mut bw_d_h_s2, batch_size)?; +} +``` + +- [ ] **Step 4: Compile check + local test** + +```bash +SQLX_OFFLINE=true cargo check --workspace 2>&1 | tail -5 +``` + +- [ ] **Step 5: Commit** + +```bash +git add crates/ml/src/cuda_pipeline/attention_backward_kernel.cu crates/ml/src/cuda_pipeline/gpu_attention.rs +git commit -m "feat: attention backward (Phase A — residual passthrough, gradient flow verified)" +``` + +--- + +## Task 3: Decision Transformer CUDA Kernels + +**Files:** +- Create: `crates/ml/src/cuda_pipeline/dt_kernels.cu` +- Modify: `crates/ml/src/cuda_pipeline/decision_transformer.rs` + +- [ ] **Step 1: Write the token embedding + positional encoding kernel** + +```cuda +extern "C" __global__ void dt_embed_kernel( + const float* trajectories, // [B, T, state_dim+2] (return-to-go, state, action) + const float* W_embed, // [(state_dim+2), embed_dim] + const float* b_embed, // [embed_dim] + const float* pos_embed, // [T, embed_dim] (learned positional encoding) + float* output, // [B, T, embed_dim] + int B, int T, int input_dim, int embed_dim +); +``` + +Each thread handles one (batch, timestep, embed_dim_idx) tuple: +- Linear projection: `out[b][t][d] = sum_j(traj[b][t][j] * W[j][d]) + b[d]` +- Add positional: `out[b][t][d] += pos[t][d]` + +- [ ] **Step 2: Write the causal self-attention kernel** + +```cuda +extern "C" __global__ void dt_causal_attention_kernel( + const float* Q, // [B, T, embed_dim] + const float* K, // [B, T, embed_dim] + const float* V, // [B, T, embed_dim] + float* output, // [B, T, embed_dim] + int B, int T, int D, int num_heads +); +``` + +Per-head computation with causal mask: +- `attn[i][j] = Q[i] · K[j] / sqrt(D_h)` for j ≤ i (causal), -inf otherwise +- Softmax over j dimension +- `output[i] = sum_j(attn[i][j] * V[j])` + +Block-per-sample, threads handle different timesteps. Shared memory for K/V caching. + +- [ ] **Step 3: Write the FFN kernel (embed_dim → 4×embed_dim → embed_dim)** + +```cuda +extern "C" __global__ void dt_ffn_kernel( + const float* input, // [B, T, embed_dim] + const float* W1, // [embed_dim, 4*embed_dim] + const float* b1, // [4*embed_dim] + const float* W2, // [4*embed_dim, embed_dim] + const float* b2, // [embed_dim] + float* output, // [B, T, embed_dim] + int B, int T, int D +); +``` + +GELU activation between layers. Residual connection: `output += input`. + +- [ ] **Step 4: Write the cross-entropy loss kernel** + +```cuda +extern "C" __global__ void dt_cross_entropy_kernel( + const float* logits, // [B, T, num_actions] + const int* target_actions, // [B, T] + float* per_sample_loss, // [B*T] + float* total_loss, // [1] atomicAdd + int B, int T, int num_actions +); +``` + +Per-token cross-entropy: `loss = -log(softmax(logits)[target_action])`. + +- [ ] **Step 5: Implement `pretrain_step()` in decision_transformer.rs** + +Wire the 5 forward kernels into a training loop. Then: + +- [ ] **Step 5b: Write 5 backward CUDA kernels** + +```cuda +// In dt_kernels.cu — add after forward kernels: +// dt_cross_entropy_backward_kernel: dL/d_logits = softmax - one_hot +// dt_ffn_backward_kernel: GELU backward + 2 linear backward +// dt_attention_backward_kernel: dQ,dK,dV from d_output (softmax Jacobian + causal mask) +// dt_layernorm_backward_kernel: chain through mean/variance +// dt_embed_backward_kernel: dW_embed from d_embedded +``` + +Each mirrors its forward kernel's thread/block layout. Total: 10 kernels. + +- [ ] **Step 5c: Implement `pretrain_step()` in decision_transformer.rs** + +1. **Forward:** Embed → {QKV → CausalAttn → LN → FFN → LN} × N → ActionHead → CE loss +2. **Backward:** CE backward → reverse chain through all N layers → embed backward +3. **Optimizer:** Reuse `dqn_grad_norm_kernel` + `dqn_adam_update_kernel` on DT params + +DT action space = 9 exposure actions (branch_0 only, not full 81 factored). + +- [ ] **Step 6: Compile check** + +```bash +SQLX_OFFLINE=true cargo check -p ml 2>&1 | tail -5 +``` + +- [ ] **Step 7: Commit** + +```bash +git add crates/ml/src/cuda_pipeline/dt_kernels.cu crates/ml/src/cuda_pipeline/decision_transformer.rs +git commit -m "feat: Decision Transformer CUDA kernels (embed, causal attention, FFN, CE loss)" +``` + +--- + +## Task 4: Decision Transformer Pre-Training Loop + +**Files:** +- Modify: `crates/ml/src/trainers/dqn/trainer/training_loop.rs` +- Modify: `crates/ml/src/trainers/dqn/config.rs` +- Modify: `crates/ml/src/hyperopt/adapters/dqn.rs` + +- [ ] **Step 1: Add DT pre-training config fields** + +In `DQNHyperparameters`: +```rust +pub dt_pretrain_epochs: usize, // 0 = disabled, 10-50 typical +pub dt_context_len: usize, // 20 timesteps +pub dt_embed_dim: usize, // 128 +pub dt_num_layers: usize, // 3 +pub dt_target_return: f64, // 2.0 (target Sharpe-like return) +``` + +- [ ] **Step 2: Add DT pre-training phase at training start** + +In `training_loop.rs`, before the main training loop: +```rust +if self.hyperparams.dt_pretrain_epochs > 0 { + let dt = DecisionTransformer::new(stream, dt_config)?; + for epoch in 0..self.hyperparams.dt_pretrain_epochs { + // Build trajectory batches from training_data + // Run dt.pretrain_step(trajectories, target_actions, batch_size) + // Log DT loss + } + // Transfer DT's learned representations to DQN trunk (optional) +} +``` + +- [ ] **Step 3: Add DT params to hyperopt search space** + +In `dqn.rs`, add bounds for `dt_pretrain_epochs`, `dt_context_len`, `dt_embed_dim`. + +- [ ] **Step 4: Compile check + commit** + +```bash +SQLX_OFFLINE=true cargo check --workspace +git add crates/ml/src/cuda_pipeline/ crates/ml/src/trainers/dqn/ crates/ml/src/hyperopt/ && git commit -m "feat: Decision Transformer pre-training phase in training loop" +``` + +--- + +## Task 5: Ensemble Multi-Head Q-Network + +**Files:** +- Create: `crates/ml/src/cuda_pipeline/ensemble_kernels.cu` +- Modify: `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs` +- Modify: `crates/ml/src/trainers/dqn/fused_training.rs` +- Modify: `crates/ml/src/trainers/dqn/config.rs` + +- [ ] **Step 1: Write the ensemble aggregation kernel** + +```cuda +extern "C" __global__ void ensemble_aggregate_kernel( + const float* head_q_values, // [K, B, num_actions] Q-values from K heads + float* mean_q, // [B, num_actions] mean across heads + float* var_q, // [B, num_actions] variance across heads + int K, int B, int num_actions +); +``` + +Computes mean and variance of Q-values across K ensemble heads for uncertainty estimation. + +- [ ] **Step 2: Write the KL-divergence diversity loss kernel** + +```cuda +extern "C" __global__ void ensemble_diversity_kernel( + const float* head_logits, // [K, B, num_atoms] per-head C51 logits + float* diversity_loss, // [1] total KL divergence (atomicAdd) + int K, int B, int num_atoms +); +``` + +For each pair of heads (i, j): `D_KL(p_i || p_j)` averaged over the batch. Uses hierarchical reduction (warp reduce → block reduce → atomicAdd per block, same pattern as `dqn_grad_norm_kernel`) to avoid single-address atomicAdd serialization. The diversity loss ENCOURAGES disagreement (subtracted from total loss). + +- [ ] **Step 3: Implement multi-head architecture in GpuDqnTrainer** + +The ensemble shares the TRUNK (W_s1, W_s2) but has K SEPARATE value + advantage heads: +```rust +pub struct EnsembleHeads { + heads: Vec<(DuelingWeightSet, BranchingWeightSet)>, // K head weight sets +} +``` + +During training (heads run OUTSIDE the CUDA Graph, same pattern as IQN/attention): +1. CUDA Graph: forward through shared trunk → h_s2 (using head 0's weights in the graph) +2. Post-graph: for each head k=1..K-1, run cuBLAS forward through head_k's value/advantage layers on `save_h_s2` (outside graph, with EventTrackingGuard) +3. Aggregate Q-values: mean ± std across K heads +4. C51 loss per head (head 0 inside graph, heads 1..K-1 outside) +5. Diversity regularization across heads +6. Backward: each head contributes trunk gradients via the existing IQN-style SGD/Adam correction + +Note: The CUDA Graph remains captured for head 0 only. Heads 1..K-1 run as post-graph operations with their own cuBLAS forwards. This avoids re-capturing the graph for each head. + +- [ ] **Step 4: Add ensemble config and wiring** + +In `DQNHyperparameters`: +```rust +pub ensemble_count: usize, // K heads (default 1 = no ensemble) +pub ensemble_diversity_weight: f64, // 0.01 = mild diversity encouragement +``` + +In `FusedTrainingCtx::run_full_step()`: +- If `ensemble_count > 1`: run K forward passes through different heads, average for action selection, sum losses with diversity term. + +- [ ] **Step 5: Compile check + commit** + +```bash +SQLX_OFFLINE=true cargo check --workspace +git add crates/ml/src/cuda_pipeline/ crates/ml/src/trainers/dqn/ crates/ml/src/hyperopt/ && git commit -m "feat: ensemble multi-head Q-network with KL diversity loss" +``` + +--- + +## Task 6: HER Episode Boundary Tracking + +**Files:** +- Create: `crates/ml/src/cuda_pipeline/her_episode_kernel.cu` +- Modify: `crates/ml/src/cuda_pipeline/gpu_her.rs` +- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs` + +- [ ] **Step 1: Add episode index buffer to experience output** + +In `GpuExperienceBatch`, add: +```rust +pub episode_ids: CudaSlice, // [N * L] episode index per transition +``` + +Fill via a simple CUDA kernel `fill_episode_ids_kernel(int* ids, int L, int N)` where thread `i` writes `ids[i] = i / L`. Launched as `grid=(ceil(N*L/256)), block=256`. Zero CPU fill — pure GPU. + +- [ ] **Step 2: Track episode boundaries in GPU replay buffer** + +When inserting transitions, also store the episode ID: +```rust +pub episode_id_buf: CudaSlice, // [capacity] episode index per slot +``` + +This allows the HER kernel to find other transitions from the same episode. + +- [ ] **Step 3: Write the episode-aware donor sampling kernel** + +```cuda +extern "C" __global__ void her_sample_future_donors( + const int* episode_ids, // [capacity] episode index per slot + const int* source_indices, // [her_batch_size] which to relabel + int* donor_indices, // [her_batch_size] output: future donors + unsigned int* rng_states, // [her_batch_size] per-sample RNG + int capacity, + int her_batch_size +); +``` + +Uses a GPU-resident `episode_boundary_offsets` auxiliary buffer (updated at insertion time) for O(log(N)) binary search instead of O(capacity) linear scan. Each thread binary-searches the boundary buffer to find its episode's [start, end) range, then uniformly samples from `[source+1, end)`. + +- [ ] **Step 4: Write the episode-end detection kernel (Final strategy)** + +```cuda +extern "C" __global__ void her_find_episode_end( + const int* episode_ids, // [capacity] + const int* source_indices, // [her_batch_size] + int* end_indices, // [her_batch_size] output: last index in same episode + int capacity, + int her_batch_size +); +``` + +For each source: binary search or linear scan to find the last index with the same episode_id. + +- [ ] **Step 5: Wire Future/Final strategies in `GpuHer`** + +```rust +pub fn relabel_batch_gpu( + &mut self, + strategy: HerGpuStrategy, + episode_ids: &CudaSlice, + ... +) -> Result<&HerBatch, MLError> { + let donors = match strategy { + HerGpuStrategy::Random => self.random_donors(her_batch_size), + HerGpuStrategy::Future => self.sample_future_donors(episode_ids, source_indices)?, + HerGpuStrategy::Final => self.find_episode_ends(episode_ids, source_indices)?, + }; + self.relabel_kernel(source_indices, donors, ...)?; + Ok(&self.output) +} +``` + +- [ ] **Step 6: Compile check + commit** + +```bash +SQLX_OFFLINE=true cargo check --workspace +git add crates/ml/src/cuda_pipeline/ crates/ml/src/trainers/dqn/ crates/ml/src/hyperopt/ && git commit -m "feat: HER Future/Final strategies with GPU episode boundary tracking" +``` + +--- + +## Task 7: Integration Test — Full Pipeline + +**Files:** +- Modify: `crates/ml/src/trainers/dqn/trainer/training_loop.rs` + +- [ ] **Step 1: Wire all 4 features into the training loop conditionally** + +```rust +// Attention: after trunk forward, before heads +if hyperparams.use_attention { ... } + +// DT pre-training: before main training loop +if hyperparams.dt_pretrain_epochs > 0 { ... } + +// Ensemble: K heads with diversity loss +if hyperparams.ensemble_count > 1 { ... } + +// HER: Future strategy with episode boundaries +if hyperparams.her_ratio > 0.0 && hyperparams.her_strategy == "future" { ... } +``` + +- [ ] **Step 2: Local test — all features enabled** + +```bash +rm -rf /tmp/.cubin_cache/ +SQLX_OFFLINE=true cargo build --release -p ml --example hyperopt_baseline_rl +SQLX_OFFLINE=true timeout 120 ./target/release/examples/hyperopt_baseline_rl \ + --model dqn --phase fast --trials 1 --n-initial 1 --epochs 3 \ + --data-dir test_data/futures-baseline --symbol ES.FUT \ + --output /tmp/full_pipeline_test.json --seed 42 +``` + +Expected: No NaN, no CUDA errors, train_loss < 50. + +- [ ] **Step 3: Commit** + +```bash +git add crates/ml/src/cuda_pipeline/ crates/ml/src/trainers/dqn/ crates/ml/src/hyperopt/ && git commit -m "feat: full pipeline integration — attention + DT + ensemble + HER Future" +``` + +--- + +## Task 8: H100 Submission + +- [ ] **Step 1: Push to main** + +```bash +git push origin main +``` + +- [ ] **Step 2: Submit H100 hyperopt run** + +50 epochs, 20 trials, with all features enabled via config: +```toml +use_attention = true +dt_pretrain_epochs = 10 +ensemble_count = 3 +her_ratio = 0.3 +her_strategy = "future" +``` + +- [ ] **Step 3: Monitor first trial** + +Check: no NaN, no CUDA OOM, loss decreasing, Q-gap > 0. + +--- + +## Dependency Graph + +``` +Task 1 (Attention Forward) → Task 2 (Attention Backward) → Task 7 (Integration) +Task 3 (DT Kernels) → Task 4 (DT Loop) → Task 7 (Integration) +Task 5 (Ensemble) → Task 7 (Integration) +Task 6 (HER Episodes) → Task 7 (Integration) +Task 7 (Integration) → Task 8 (H100) +``` + +Tasks 1-2, 3-4, 5, and 6 are INDEPENDENT and can be implemented in parallel. diff --git a/ml/hyperopt_results/dqn_best_trial_2026-03-22T23-43-16.144446538+00-00_sharpe_-0.0177.json b/ml/hyperopt_results/dqn_best_trial_2026-03-22T23-43-16.144446538+00-00_sharpe_-0.0177.json new file mode 100644 index 000000000..7d265dbea --- /dev/null +++ b/ml/hyperopt_results/dqn_best_trial_2026-03-22T23-43-16.144446538+00-00_sharpe_-0.0177.json @@ -0,0 +1,78 @@ +{ + "trial_number": 0, + "sharpe": -0.017729128897190093, + "win_rate": 0.024923645216040312, + "max_drawdown": 100.0, + "total_return": -5736.653628349304, + "hyperparameters": { + "learning_rate": 0.00005994999354431849, + "batch_size": 307, + "gamma": 0.9572818589229506, + "buffer_size": 66246, + "max_position_absolute": 1.1030284538648683, + "huber_delta": 17.775790174607682, + "entropy_coefficient": 0.38184099247597697, + "transaction_cost_multiplier": 1.7738774066741243, + "use_per": true, + "per_alpha": 0.45251155566697393, + "per_beta_start": 0.20130083854118397, + "use_dueling": true, + "dueling_hidden_dim": 128, + "n_steps": 4, + "tau": 0.008986246204051671, + "use_distributional": true, + "num_atoms": 51, + "v_min": -70.22777500053176, + "v_max": 70.22777500053176, + "use_noisy_nets": true, + "noisy_sigma_init": 0.24583814763225417, + "minimum_profit_factor": 1.5028035348681215, + "weight_decay": 0.00010597074820791314, + "kelly_fractional": 0.5096243658522589, + "kelly_max_fraction": 0.12010947414322902, + "kelly_min_trades": 20, + "volatility_window": 23, + "use_ensemble_uncertainty": false, + "ensemble_size": 5.0, + "beta_variance": 0.5, + "beta_disagreement": 0.5, + "beta_entropy": 0.2, + "warmup_ratio": 0.0, + "curiosity_weight": 0.0, + "td_error_clamp_max": 10.0, + "batch_diversity_cooldown": 50.0, + "lr_decay_type": 0.0, + "sharpe_weight": 0.10176748614555509, + "gae_lambda": 0.95, + "noisy_sigma_initial": 0.5, + "noisy_sigma_final": 0.3, + "use_spectral_norm": false, + "use_attention": false, + "use_residual": false, + "norm_type": 1.0, + "activation_type": 1.0, + "use_qr_dqn": false, + "num_quantiles": 64, + "qr_kappa": 1.0, + "iqn_lambda": 0.8479734309322158, + "hidden_dim_base": 128, + "noisy_epsilon_floor": 0.1, + "count_bonus_coefficient": 0.23831546372779777, + "cql_alpha": 0.5059776371295834, + "eval_softmax_temp": 1.0, + "dsr_eta": 0.003947815597664894, + "branch_hidden_dim": 64, + "use_branching": true, + "gradient_accumulation_steps": 1, + "w_dsr": 1.0, + "w_pnl": 0.3, + "w_dd": 1.0, + "w_idle": 0.01, + "dd_threshold": 0.01, + "loss_aversion": 1.5, + "time_decay_rate": 0.0005, + "q_gap_threshold": 0.1 + }, + "timestamp": "2026-03-22T23:43:16.144446538+00:00", + "gradient_clip_norm": 10.0 +} \ No newline at end of file diff --git a/ml/hyperopt_results/dqn_best_trial_2026-03-22T23-44-00.953235416+00-00_sharpe_0.0182.json b/ml/hyperopt_results/dqn_best_trial_2026-03-22T23-44-00.953235416+00-00_sharpe_0.0182.json new file mode 100644 index 000000000..a59b45dc5 --- /dev/null +++ b/ml/hyperopt_results/dqn_best_trial_2026-03-22T23-44-00.953235416+00-00_sharpe_0.0182.json @@ -0,0 +1,78 @@ +{ + "trial_number": 1, + "sharpe": 0.018215437326580285, + "win_rate": 0.020554036577232183, + "max_drawdown": 100.0, + "total_return": -547.1131277084351, + "hyperparameters": { + "learning_rate": 0.00004291359460284525, + "batch_size": 506, + "gamma": 0.9546657016504223, + "buffer_size": 68426, + "max_position_absolute": 1.084699669559996, + "huber_delta": 12.383291331070463, + "entropy_coefficient": 0.22305679391580635, + "transaction_cost_multiplier": 1.237118944760189, + "use_per": true, + "per_alpha": 0.7874757212411532, + "per_beta_start": 0.5405183949041945, + "use_dueling": true, + "dueling_hidden_dim": 128, + "n_steps": 3, + "tau": 0.00696401394764961, + "use_distributional": true, + "num_atoms": 51, + "v_min": -66.17506191155037, + "v_max": 66.17506191155037, + "use_noisy_nets": true, + "noisy_sigma_init": 0.6469317434464105, + "minimum_profit_factor": 1.199756461327086, + "weight_decay": 0.0006131056727707305, + "kelly_fractional": 0.36451619035608973, + "kelly_max_fraction": 0.1982886181169854, + "kelly_min_trades": 20, + "volatility_window": 23, + "use_ensemble_uncertainty": false, + "ensemble_size": 5.0, + "beta_variance": 0.5, + "beta_disagreement": 0.5, + "beta_entropy": 0.2, + "warmup_ratio": 0.0, + "curiosity_weight": 0.0, + "td_error_clamp_max": 10.0, + "batch_diversity_cooldown": 50.0, + "lr_decay_type": 1.0, + "sharpe_weight": 0.12231802599991748, + "gae_lambda": 0.95, + "noisy_sigma_initial": 0.5, + "noisy_sigma_final": 0.3, + "use_spectral_norm": false, + "use_attention": false, + "use_residual": false, + "norm_type": 1.0, + "activation_type": 1.0, + "use_qr_dqn": false, + "num_quantiles": 64, + "qr_kappa": 1.0, + "iqn_lambda": 0.1186854947736955, + "hidden_dim_base": 128, + "noisy_epsilon_floor": 0.1, + "count_bonus_coefficient": 0.09860756605998412, + "cql_alpha": 0.16860966429860813, + "eval_softmax_temp": 1.0, + "dsr_eta": 0.008085442687796646, + "branch_hidden_dim": 64, + "use_branching": true, + "gradient_accumulation_steps": 1, + "w_dsr": 1.0, + "w_pnl": 0.3, + "w_dd": 1.0, + "w_idle": 0.01, + "dd_threshold": 0.01, + "loss_aversion": 1.5, + "time_decay_rate": 0.0005, + "q_gap_threshold": 0.1 + }, + "timestamp": "2026-03-22T23:44:00.953235416+00:00", + "gradient_clip_norm": 10.0 +} \ No newline at end of file diff --git a/ml/hyperopt_results/dqn_best_trial_2026-03-23T07-22-11.285931152+00-00_sharpe_0.0091.json b/ml/hyperopt_results/dqn_best_trial_2026-03-23T07-22-11.285931152+00-00_sharpe_0.0091.json new file mode 100644 index 000000000..114e060e7 --- /dev/null +++ b/ml/hyperopt_results/dqn_best_trial_2026-03-23T07-22-11.285931152+00-00_sharpe_0.0091.json @@ -0,0 +1,78 @@ +{ + "trial_number": 1, + "sharpe": 0.009133303724229335, + "win_rate": 0.02989337162580341, + "max_drawdown": 100.0, + "total_return": -381.4813995361328, + "hyperparameters": { + "learning_rate": 0.00004291359460284525, + "batch_size": 506, + "gamma": 0.9546657016504223, + "buffer_size": 68426, + "max_position_absolute": 1.084699669559996, + "huber_delta": 12.383291331070463, + "entropy_coefficient": 0.22305679391580635, + "transaction_cost_multiplier": 1.237118944760189, + "use_per": true, + "per_alpha": 0.7874757212411532, + "per_beta_start": 0.5405183949041945, + "use_dueling": true, + "dueling_hidden_dim": 128, + "n_steps": 3, + "tau": 0.00696401394764961, + "use_distributional": true, + "num_atoms": 51, + "v_min": -66.17506191155037, + "v_max": 66.17506191155037, + "use_noisy_nets": true, + "noisy_sigma_init": 0.6469317434464105, + "minimum_profit_factor": 1.199756461327086, + "weight_decay": 0.0006131056727707305, + "kelly_fractional": 0.36451619035608973, + "kelly_max_fraction": 0.1982886181169854, + "kelly_min_trades": 20, + "volatility_window": 23, + "use_ensemble_uncertainty": false, + "ensemble_size": 5.0, + "beta_variance": 0.5, + "beta_disagreement": 0.5, + "beta_entropy": 0.2, + "warmup_ratio": 0.0, + "curiosity_weight": 0.0, + "td_error_clamp_max": 10.0, + "batch_diversity_cooldown": 50.0, + "lr_decay_type": 1.0, + "sharpe_weight": 0.12231802599991748, + "gae_lambda": 0.95, + "noisy_sigma_initial": 0.5, + "noisy_sigma_final": 0.3, + "use_spectral_norm": false, + "use_attention": false, + "use_residual": false, + "norm_type": 1.0, + "activation_type": 1.0, + "use_qr_dqn": false, + "num_quantiles": 64, + "qr_kappa": 1.0, + "iqn_lambda": 0.1186854947736955, + "hidden_dim_base": 128, + "noisy_epsilon_floor": 0.1, + "count_bonus_coefficient": 0.09860756605998412, + "cql_alpha": 0.16860966429860813, + "eval_softmax_temp": 1.0, + "dsr_eta": 0.008085442687796646, + "branch_hidden_dim": 64, + "use_branching": true, + "gradient_accumulation_steps": 1, + "w_dsr": 1.0, + "w_pnl": 0.3, + "w_dd": 1.0, + "w_idle": 0.01, + "dd_threshold": 0.01, + "loss_aversion": 1.5, + "time_decay_rate": 0.0005, + "q_gap_threshold": 0.1 + }, + "timestamp": "2026-03-23T07:22:11.285931152+00:00", + "gradient_clip_norm": 10.0 +} \ No newline at end of file diff --git a/ml/hyperopt_results/dqn_best_trial_2026-03-24T09-21-47.273661543+00-00_sharpe_0.0047.json b/ml/hyperopt_results/dqn_best_trial_2026-03-24T09-21-47.273661543+00-00_sharpe_0.0047.json new file mode 100644 index 000000000..87a4bf0b0 --- /dev/null +++ b/ml/hyperopt_results/dqn_best_trial_2026-03-24T09-21-47.273661543+00-00_sharpe_0.0047.json @@ -0,0 +1,79 @@ +{ + "trial_number": 0, + "sharpe": 0.004663038626313209, + "win_rate": 0.33112575113773346, + "max_drawdown": 100.0, + "total_return": -8658.971099853516, + "hyperparameters": { + "learning_rate": 0.00005994999354431849, + "batch_size": 307, + "gamma": 0.9572818589229506, + "buffer_size": 66246, + "max_position_absolute": 1.1030284538648683, + "huber_delta": 17.775790174607682, + "entropy_coefficient": 0.38184099247597697, + "transaction_cost_multiplier": 1.7738774066741243, + "use_per": true, + "per_alpha": 0.45251155566697393, + "per_beta_start": 0.20130083854118397, + "use_dueling": true, + "dueling_hidden_dim": 128, + "n_steps": 4, + "tau": 0.008986246204051671, + "use_distributional": true, + "num_atoms": 101, + "v_min": -84.27333000063813, + "v_max": 84.27333000063813, + "use_noisy_nets": true, + "noisy_sigma_init": 0.24583814763225417, + "minimum_profit_factor": 1.5028035348681215, + "weight_decay": 0.00010597074820791314, + "kelly_fractional": 0.5096243658522589, + "kelly_max_fraction": 0.12010947414322902, + "kelly_min_trades": 20, + "volatility_window": 23, + "use_ensemble_uncertainty": false, + "ensemble_size": 5.0, + "beta_variance": 0.5, + "beta_disagreement": 0.5, + "beta_entropy": 0.2, + "warmup_ratio": 0.0, + "curiosity_weight": 0.0, + "td_error_clamp_max": 10.0, + "batch_diversity_cooldown": 50.0, + "lr_decay_type": 0.0, + "sharpe_weight": 0.10176748614555509, + "gae_lambda": 0.95, + "noisy_sigma_initial": 0.5, + "noisy_sigma_final": 0.3, + "use_spectral_norm": true, + "use_attention": true, + "use_residual": true, + "norm_type": 1.0, + "activation_type": 1.0, + "num_quantiles": 32, + "qr_kappa": 1.0, + "iqn_lambda": 0.8479734309322158, + "spectral_norm_sigma_max": 4.8543579475657985, + "hidden_dim_base": 128, + "noisy_epsilon_floor": 0.1, + "count_bonus_coefficient": 0.23831546372779777, + "cql_alpha": 0.5059776371295834, + "eval_softmax_temp": 1.0, + "dsr_eta": 0.003947815597664894, + "branch_hidden_dim": 64, + "use_branching": true, + "gradient_accumulation_steps": 1, + "w_dsr": 1.0, + "w_pnl": 0.3, + "w_dd": 1.0, + "w_idle": 0.01, + "dd_threshold": 0.01, + "loss_aversion": 1.5, + "time_decay_rate": 0.0005, + "q_gap_threshold": 0.1, + "dt_pretrain_epochs": 0 + }, + "timestamp": "2026-03-24T09:21:47.273661543+00:00", + "gradient_clip_norm": 10.0 +} \ No newline at end of file diff --git a/ml/hyperopt_results/dqn_best_trial_2026-03-24T10-01-01.349397710+00-00_sharpe_0.0419.json b/ml/hyperopt_results/dqn_best_trial_2026-03-24T10-01-01.349397710+00-00_sharpe_0.0419.json new file mode 100644 index 000000000..4e9face1a --- /dev/null +++ b/ml/hyperopt_results/dqn_best_trial_2026-03-24T10-01-01.349397710+00-00_sharpe_0.0419.json @@ -0,0 +1,79 @@ +{ + "trial_number": 0, + "sharpe": 0.04188940078020096, + "win_rate": 0.2995686545968056, + "max_drawdown": 100.0, + "total_return": 233471.36112213135, + "hyperparameters": { + "learning_rate": 0.00005994999354431849, + "batch_size": 307, + "gamma": 0.9572818589229506, + "buffer_size": 66246, + "max_position_absolute": 1.1030284538648683, + "huber_delta": 17.775790174607682, + "entropy_coefficient": 0.38184099247597697, + "transaction_cost_multiplier": 1.7738774066741243, + "use_per": true, + "per_alpha": 0.45251155566697393, + "per_beta_start": 0.20130083854118397, + "use_dueling": true, + "dueling_hidden_dim": 128, + "n_steps": 4, + "tau": 0.008986246204051671, + "use_distributional": true, + "num_atoms": 101, + "v_min": -84.27333000063813, + "v_max": 84.27333000063813, + "use_noisy_nets": true, + "noisy_sigma_init": 0.24583814763225417, + "minimum_profit_factor": 1.5028035348681215, + "weight_decay": 0.00010597074820791314, + "kelly_fractional": 0.5096243658522589, + "kelly_max_fraction": 0.12010947414322902, + "kelly_min_trades": 20, + "volatility_window": 23, + "use_ensemble_uncertainty": false, + "ensemble_size": 5.0, + "beta_variance": 0.5, + "beta_disagreement": 0.5, + "beta_entropy": 0.2, + "warmup_ratio": 0.0, + "curiosity_weight": 0.0, + "td_error_clamp_max": 10.0, + "batch_diversity_cooldown": 50.0, + "lr_decay_type": 0.0, + "sharpe_weight": 0.10176748614555509, + "gae_lambda": 0.95, + "noisy_sigma_initial": 0.5, + "noisy_sigma_final": 0.3, + "use_spectral_norm": true, + "use_attention": true, + "use_residual": true, + "norm_type": 1.0, + "activation_type": 1.0, + "num_quantiles": 32, + "qr_kappa": 1.0, + "iqn_lambda": 0.8479734309322158, + "spectral_norm_sigma_max": 4.8543579475657985, + "hidden_dim_base": 128, + "noisy_epsilon_floor": 0.1, + "count_bonus_coefficient": 0.23831546372779777, + "cql_alpha": 0.5059776371295834, + "eval_softmax_temp": 1.0, + "dsr_eta": 0.003947815597664894, + "branch_hidden_dim": 64, + "use_branching": true, + "gradient_accumulation_steps": 1, + "w_dsr": 1.0, + "w_pnl": 0.3, + "w_dd": 1.0, + "w_idle": 0.01, + "dd_threshold": 0.01, + "loss_aversion": 1.5, + "time_decay_rate": 0.0005, + "q_gap_threshold": 0.1, + "dt_pretrain_epochs": 0 + }, + "timestamp": "2026-03-24T10:01:01.349397710+00:00", + "gradient_clip_norm": 10.0 +} \ No newline at end of file diff --git a/ml/hyperopt_results/dqn_best_trial_2026-03-24T14-27-44.051295574+00-00_sharpe_-0.0068.json b/ml/hyperopt_results/dqn_best_trial_2026-03-24T14-27-44.051295574+00-00_sharpe_-0.0068.json new file mode 100644 index 000000000..fd1e7f0ca --- /dev/null +++ b/ml/hyperopt_results/dqn_best_trial_2026-03-24T14-27-44.051295574+00-00_sharpe_-0.0068.json @@ -0,0 +1,79 @@ +{ + "trial_number": 0, + "sharpe": -0.006844052672386169, + "win_rate": 0.10371590945869684, + "max_drawdown": 100.0, + "total_return": -19307.970275878906, + "hyperparameters": { + "learning_rate": 0.00005994999354431849, + "batch_size": 307, + "gamma": 0.9572818589229506, + "buffer_size": 66246, + "max_position_absolute": 1.1030284538648683, + "huber_delta": 17.775790174607682, + "entropy_coefficient": 0.38184099247597697, + "transaction_cost_multiplier": 1.7738774066741243, + "use_per": true, + "per_alpha": 0.45251155566697393, + "per_beta_start": 0.20130083854118397, + "use_dueling": true, + "dueling_hidden_dim": 128, + "n_steps": 4, + "tau": 0.008986246204051671, + "use_distributional": true, + "num_atoms": 101, + "v_min": -42.13666500031906, + "v_max": 42.13666500031906, + "use_noisy_nets": true, + "noisy_sigma_init": 0.24583814763225417, + "minimum_profit_factor": 1.5028035348681215, + "weight_decay": 0.00010597074820791314, + "kelly_fractional": 0.5096243658522589, + "kelly_max_fraction": 0.12010947414322902, + "kelly_min_trades": 20, + "volatility_window": 23, + "use_ensemble_uncertainty": false, + "ensemble_size": 5.0, + "beta_variance": 0.5, + "beta_disagreement": 0.5, + "beta_entropy": 0.2, + "warmup_ratio": 0.0, + "curiosity_weight": 0.0, + "td_error_clamp_max": 10.0, + "batch_diversity_cooldown": 50.0, + "lr_decay_type": 0.0, + "sharpe_weight": 0.10176748614555509, + "gae_lambda": 0.95, + "noisy_sigma_initial": 0.5, + "noisy_sigma_final": 0.3, + "use_spectral_norm": true, + "use_attention": true, + "use_residual": true, + "norm_type": 1.0, + "activation_type": 1.0, + "num_quantiles": 32, + "qr_kappa": 1.0, + "iqn_lambda": 0.8479734309322158, + "spectral_norm_sigma_max": 4.8543579475657985, + "hidden_dim_base": 128, + "noisy_epsilon_floor": 0.1, + "count_bonus_coefficient": 0.23831546372779777, + "cql_alpha": 0.5059776371295834, + "eval_softmax_temp": 1.0, + "dsr_eta": 0.003947815597664894, + "branch_hidden_dim": 64, + "use_branching": true, + "gradient_accumulation_steps": 1, + "w_dsr": 1.0, + "w_pnl": 0.3, + "w_dd": 1.0, + "w_idle": 0.01, + "dd_threshold": 0.01, + "loss_aversion": 1.5, + "time_decay_rate": 0.0005, + "q_gap_threshold": 0.1, + "dt_pretrain_epochs": 0 + }, + "timestamp": "2026-03-24T14:27:44.051295574+00:00", + "gradient_clip_norm": 10.0 +} \ No newline at end of file diff --git a/ml/hyperopt_results/dqn_best_trial_2026-03-24T14-53-49.539858557+00-00_sharpe_-0.0116.json b/ml/hyperopt_results/dqn_best_trial_2026-03-24T14-53-49.539858557+00-00_sharpe_-0.0116.json new file mode 100644 index 000000000..b8aae383c --- /dev/null +++ b/ml/hyperopt_results/dqn_best_trial_2026-03-24T14-53-49.539858557+00-00_sharpe_-0.0116.json @@ -0,0 +1,79 @@ +{ + "trial_number": 0, + "sharpe": -0.01161600649356842, + "win_rate": 0.14867919757962228, + "max_drawdown": 100.0, + "total_return": -10207.207107543945, + "hyperparameters": { + "learning_rate": 0.00005994999354431849, + "batch_size": 307, + "gamma": 0.9572818589229506, + "buffer_size": 66246, + "max_position_absolute": 1.1030284538648683, + "huber_delta": 17.775790174607682, + "entropy_coefficient": 0.38184099247597697, + "transaction_cost_multiplier": 1.7738774066741243, + "use_per": true, + "per_alpha": 0.45251155566697393, + "per_beta_start": 0.20130083854118397, + "use_dueling": true, + "dueling_hidden_dim": 128, + "n_steps": 4, + "tau": 0.008986246204051671, + "use_distributional": true, + "num_atoms": 101, + "v_min": -42.13666500031906, + "v_max": 42.13666500031906, + "use_noisy_nets": true, + "noisy_sigma_init": 0.24583814763225417, + "minimum_profit_factor": 1.5028035348681215, + "weight_decay": 0.00010597074820791314, + "kelly_fractional": 0.5096243658522589, + "kelly_max_fraction": 0.12010947414322902, + "kelly_min_trades": 20, + "volatility_window": 23, + "use_ensemble_uncertainty": false, + "ensemble_size": 5.0, + "beta_variance": 0.5, + "beta_disagreement": 0.5, + "beta_entropy": 0.2, + "warmup_ratio": 0.0, + "curiosity_weight": 0.0, + "td_error_clamp_max": 10.0, + "batch_diversity_cooldown": 50.0, + "lr_decay_type": 0.0, + "sharpe_weight": 0.10176748614555509, + "gae_lambda": 0.95, + "noisy_sigma_initial": 0.5, + "noisy_sigma_final": 0.3, + "use_spectral_norm": true, + "use_attention": true, + "use_residual": true, + "norm_type": 1.0, + "activation_type": 1.0, + "num_quantiles": 32, + "qr_kappa": 1.0, + "iqn_lambda": 0.8479734309322158, + "spectral_norm_sigma_max": 4.8543579475657985, + "hidden_dim_base": 128, + "noisy_epsilon_floor": 0.1, + "count_bonus_coefficient": 0.23831546372779777, + "cql_alpha": 0.5059776371295834, + "eval_softmax_temp": 1.0, + "dsr_eta": 0.003947815597664894, + "branch_hidden_dim": 64, + "use_branching": true, + "gradient_accumulation_steps": 1, + "w_dsr": 1.0, + "w_pnl": 0.3, + "w_dd": 1.0, + "w_idle": 0.01, + "dd_threshold": 0.01, + "loss_aversion": 1.5, + "time_decay_rate": 0.0005, + "q_gap_threshold": 0.1, + "dt_pretrain_epochs": 0 + }, + "timestamp": "2026-03-24T14:53:49.539858557+00:00", + "gradient_clip_norm": 10.0 +} \ No newline at end of file diff --git a/ml/hyperopt_results/dqn_best_trial_2026-03-24T15-12-06.755573679+00-00_sharpe_0.0838.json b/ml/hyperopt_results/dqn_best_trial_2026-03-24T15-12-06.755573679+00-00_sharpe_0.0838.json new file mode 100644 index 000000000..42ee6355d --- /dev/null +++ b/ml/hyperopt_results/dqn_best_trial_2026-03-24T15-12-06.755573679+00-00_sharpe_0.0838.json @@ -0,0 +1,79 @@ +{ + "trial_number": 0, + "sharpe": 0.0837665757164359, + "win_rate": 0.21893585734069348, + "max_drawdown": 100.0, + "total_return": 2530.810966491699, + "hyperparameters": { + "learning_rate": 0.00005994999354431849, + "batch_size": 307, + "gamma": 0.9572818589229506, + "buffer_size": 66246, + "max_position_absolute": 1.1030284538648683, + "huber_delta": 17.775790174607682, + "entropy_coefficient": 0.38184099247597697, + "transaction_cost_multiplier": 1.7738774066741243, + "use_per": true, + "per_alpha": 0.45251155566697393, + "per_beta_start": 0.20130083854118397, + "use_dueling": true, + "dueling_hidden_dim": 128, + "n_steps": 4, + "tau": 0.008986246204051671, + "use_distributional": true, + "num_atoms": 101, + "v_min": -42.13666500031906, + "v_max": 42.13666500031906, + "use_noisy_nets": true, + "noisy_sigma_init": 0.24583814763225417, + "minimum_profit_factor": 1.5028035348681215, + "weight_decay": 0.00010597074820791314, + "kelly_fractional": 0.5096243658522589, + "kelly_max_fraction": 0.12010947414322902, + "kelly_min_trades": 20, + "volatility_window": 23, + "use_ensemble_uncertainty": false, + "ensemble_size": 5.0, + "beta_variance": 0.5, + "beta_disagreement": 0.5, + "beta_entropy": 0.2, + "warmup_ratio": 0.0, + "curiosity_weight": 0.0, + "td_error_clamp_max": 10.0, + "batch_diversity_cooldown": 50.0, + "lr_decay_type": 0.0, + "sharpe_weight": 0.10176748614555509, + "gae_lambda": 0.95, + "noisy_sigma_initial": 0.5, + "noisy_sigma_final": 0.3, + "use_spectral_norm": true, + "use_attention": true, + "use_residual": true, + "norm_type": 1.0, + "activation_type": 1.0, + "num_quantiles": 32, + "qr_kappa": 1.0, + "iqn_lambda": 0.8479734309322158, + "spectral_norm_sigma_max": 4.8543579475657985, + "hidden_dim_base": 128, + "noisy_epsilon_floor": 0.1, + "count_bonus_coefficient": 0.23831546372779777, + "cql_alpha": 0.5059776371295834, + "eval_softmax_temp": 1.0, + "dsr_eta": 0.003947815597664894, + "branch_hidden_dim": 64, + "use_branching": true, + "gradient_accumulation_steps": 1, + "w_dsr": 1.0, + "w_pnl": 0.3, + "w_dd": 1.0, + "w_idle": 0.01, + "dd_threshold": 0.01, + "loss_aversion": 1.5, + "time_decay_rate": 0.0005, + "q_gap_threshold": 0.1, + "dt_pretrain_epochs": 0 + }, + "timestamp": "2026-03-24T15:12:06.755573679+00:00", + "gradient_clip_norm": 10.0 +} \ No newline at end of file diff --git a/ml/hyperopt_results/dqn_best_trial_2026-03-24T15-19-27.798184419+00-00_sharpe_0.0355.json b/ml/hyperopt_results/dqn_best_trial_2026-03-24T15-19-27.798184419+00-00_sharpe_0.0355.json new file mode 100644 index 000000000..7c7285b59 --- /dev/null +++ b/ml/hyperopt_results/dqn_best_trial_2026-03-24T15-19-27.798184419+00-00_sharpe_0.0355.json @@ -0,0 +1,79 @@ +{ + "trial_number": 0, + "sharpe": 0.035534610599279405, + "win_rate": 0.12874939665198326, + "max_drawdown": 100.0, + "total_return": 18805.362014770508, + "hyperparameters": { + "learning_rate": 0.00008560720319413385, + "batch_size": 377, + "gamma": 0.9203133856027285, + "buffer_size": 89884, + "max_position_absolute": 3.9089452897651125, + "huber_delta": 11.90383664523535, + "entropy_coefficient": 0.19524300296372593, + "transaction_cost_multiplier": 1.6535717241939198, + "use_per": true, + "per_alpha": 0.40739064279321924, + "per_beta_start": 0.32217756667156455, + "use_dueling": true, + "dueling_hidden_dim": 128, + "n_steps": 3, + "tau": 0.005797449214565016, + "use_distributional": true, + "num_atoms": 101, + "v_min": -22.588486330040794, + "v_max": 22.588486330040794, + "use_noisy_nets": true, + "noisy_sigma_init": 0.13482975038367206, + "minimum_profit_factor": 1.776393501569344, + "weight_decay": 0.00012196287159107966, + "kelly_fractional": 0.41178193485773484, + "kelly_max_fraction": 0.26835742172433563, + "kelly_min_trades": 20, + "volatility_window": 23, + "use_ensemble_uncertainty": false, + "ensemble_size": 5.0, + "beta_variance": 0.5, + "beta_disagreement": 0.5, + "beta_entropy": 0.2, + "warmup_ratio": 0.0, + "curiosity_weight": 0.0, + "td_error_clamp_max": 10.0, + "batch_diversity_cooldown": 50.0, + "lr_decay_type": 1.0, + "sharpe_weight": 0.35540783507315155, + "gae_lambda": 0.95, + "noisy_sigma_initial": 0.5, + "noisy_sigma_final": 0.3, + "use_spectral_norm": true, + "use_attention": true, + "use_residual": true, + "norm_type": 1.0, + "activation_type": 1.0, + "num_quantiles": 32, + "qr_kappa": 1.0, + "iqn_lambda": 1.5521869540312672, + "spectral_norm_sigma_max": 7.971468117287978, + "hidden_dim_base": 128, + "noisy_epsilon_floor": 0.1, + "count_bonus_coefficient": 0.27486235230011086, + "cql_alpha": 0.9472537641839289, + "eval_softmax_temp": 1.0, + "dsr_eta": 0.0013662640295865858, + "branch_hidden_dim": 64, + "use_branching": true, + "gradient_accumulation_steps": 1, + "w_dsr": 1.0, + "w_pnl": 0.3, + "w_dd": 1.0, + "w_idle": 0.01, + "dd_threshold": 0.01, + "loss_aversion": 1.5, + "time_decay_rate": 0.0005, + "q_gap_threshold": 0.1, + "dt_pretrain_epochs": 0 + }, + "timestamp": "2026-03-24T15:19:27.798184419+00:00", + "gradient_clip_norm": 10.0 +} \ No newline at end of file diff --git a/ml/hyperopt_results/dqn_best_trial_2026-03-24T15-22-41.871557162+00-00_sharpe_0.0075.json b/ml/hyperopt_results/dqn_best_trial_2026-03-24T15-22-41.871557162+00-00_sharpe_0.0075.json new file mode 100644 index 000000000..78a273246 --- /dev/null +++ b/ml/hyperopt_results/dqn_best_trial_2026-03-24T15-22-41.871557162+00-00_sharpe_0.0075.json @@ -0,0 +1,79 @@ +{ + "trial_number": 0, + "sharpe": 0.007470622658729553, + "win_rate": 0.295475004427135, + "max_drawdown": 100.0, + "total_return": 3395.2147531509395, + "hyperparameters": { + "learning_rate": 0.00008560720319413385, + "batch_size": 377, + "gamma": 0.9203133856027285, + "buffer_size": 89884, + "max_position_absolute": 3.9089452897651125, + "huber_delta": 11.90383664523535, + "entropy_coefficient": 0.19524300296372593, + "transaction_cost_multiplier": 1.6535717241939198, + "use_per": true, + "per_alpha": 0.40739064279321924, + "per_beta_start": 0.32217756667156455, + "use_dueling": true, + "dueling_hidden_dim": 128, + "n_steps": 3, + "tau": 0.005797449214565016, + "use_distributional": true, + "num_atoms": 101, + "v_min": -22.588486330040794, + "v_max": 22.588486330040794, + "use_noisy_nets": true, + "noisy_sigma_init": 0.13482975038367206, + "minimum_profit_factor": 1.776393501569344, + "weight_decay": 0.00012196287159107966, + "kelly_fractional": 0.41178193485773484, + "kelly_max_fraction": 0.26835742172433563, + "kelly_min_trades": 20, + "volatility_window": 23, + "use_ensemble_uncertainty": false, + "ensemble_size": 5.0, + "beta_variance": 0.5, + "beta_disagreement": 0.5, + "beta_entropy": 0.2, + "warmup_ratio": 0.0, + "curiosity_weight": 0.0, + "td_error_clamp_max": 10.0, + "batch_diversity_cooldown": 50.0, + "lr_decay_type": 1.0, + "sharpe_weight": 0.35540783507315155, + "gae_lambda": 0.95, + "noisy_sigma_initial": 0.5, + "noisy_sigma_final": 0.3, + "use_spectral_norm": true, + "use_attention": true, + "use_residual": true, + "norm_type": 1.0, + "activation_type": 1.0, + "num_quantiles": 32, + "qr_kappa": 1.0, + "iqn_lambda": 1.5521869540312672, + "spectral_norm_sigma_max": 7.971468117287978, + "hidden_dim_base": 128, + "noisy_epsilon_floor": 0.1, + "count_bonus_coefficient": 0.27486235230011086, + "cql_alpha": 0.9472537641839289, + "eval_softmax_temp": 1.0, + "dsr_eta": 0.0013662640295865858, + "branch_hidden_dim": 64, + "use_branching": true, + "gradient_accumulation_steps": 1, + "w_dsr": 1.0, + "w_pnl": 0.3, + "w_dd": 1.0, + "w_idle": 0.01, + "dd_threshold": 0.01, + "loss_aversion": 1.5, + "time_decay_rate": 0.0005, + "q_gap_threshold": 0.1, + "dt_pretrain_epochs": 0 + }, + "timestamp": "2026-03-24T15:22:41.871557162+00:00", + "gradient_clip_norm": 10.0 +} \ No newline at end of file