fix: segment-based trade lifecycle + reversal P&L computation
- Trade detection now counts reversals (S100→L50) as completed segments - old_pos_pnl saved before position update for correct reversal P&L - realized_pnl writeback uses old position PnL on reversal bars - 0% win rate persists — needs deeper investigation (likely tx cost interaction) WIP: The trade_return formula produces correct sign for raw market moves, but every trade still shows as a loss. Suspect tx costs on both entry AND exit of each reversal segment exceed the 1-bar price movement. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
@@ -134,3 +134,17 @@ line_ending:
|
||||
# list of regex patterns which, when matched, mark a memory entry as read‑only.
|
||||
# Extends the list from the global configuration, merging the two lists.
|
||||
read_only_memory_patterns: []
|
||||
|
||||
# list of regex patterns for memories to completely ignore.
|
||||
# Matching memories will not appear in list_memories or activate_project output
|
||||
# and cannot be accessed via read_memory or write_memory.
|
||||
# To access ignored memory files, use the read_file tool on the raw file path.
|
||||
# Extends the list from the global configuration, merging the two lists.
|
||||
# Example: ["_archive/.*", "_episodes/.*"]
|
||||
ignored_memory_patterns: []
|
||||
|
||||
# advanced configuration option allowing to configure language server-specific options.
|
||||
# Maps the language key to the options.
|
||||
# Have a look at the docstring of the constructors of the LS implementations within solidlsp (e.g., for C# or PHP) to see which options are available.
|
||||
# No documentation on options means no options are available.
|
||||
ls_specific_settings: {}
|
||||
|
||||
@@ -679,23 +679,68 @@ extern "C" __global__ void experience_env_step(
|
||||
|
||||
/* ---- Mark-to-market PnL for this timestep ---- */
|
||||
float raw_pnl = position * (raw_next - raw_close);
|
||||
/* Old position's PnL — needed for reversal trade tracking.
|
||||
* raw_pnl uses the NEW position (after action). On reversals,
|
||||
* the old position's PnL determines trade segment return. */
|
||||
float old_pos_pnl = ps[0] * (raw_next - raw_close);
|
||||
|
||||
/* ---- Mark-to-market equity (needed for peak_equity and reward) ---- */
|
||||
float equity = cash + position * raw_next;
|
||||
peak_equity = (equity > peak_equity) ? equity : peak_equity;
|
||||
|
||||
/* ==== Trade lifecycle tracking (v3) ==== */
|
||||
/* ps[12] = entry_price, ps[13] = trade_start_pnl — dedicated slots,
|
||||
* no dual-purpose hacking. PORTFOLIO_STRIDE=14 gives us room. */
|
||||
/* ==== Trade lifecycle tracking (v4: segment-based, reversal-aware) ==== */
|
||||
/*
|
||||
* A "trade segment" = contiguous period with the same position sign.
|
||||
* Segment ends when: (1) exit to flat, (2) reversal (sign change), (3) episode end.
|
||||
*
|
||||
* A reversal (e.g. S100->L50) is TWO half-trades:
|
||||
* - Close old direction: book P&L, update Kelly, fire sparse reward
|
||||
* - Open new direction: fresh entry_price, fresh trade_start_pnl
|
||||
*
|
||||
* This matches institutional P&L accounting (round-trip segments).
|
||||
*/
|
||||
|
||||
float is_flat = (fabsf(position) < 0.001f) ? 1.0f : 0.0f;
|
||||
float prev_pos_abs = fabsf(ps[0]); /* position BEFORE this step's delta */
|
||||
int was_flat = (prev_pos_abs < 0.001f) ? 1 : 0;
|
||||
|
||||
/* Lifecycle transitions */
|
||||
int entering_trade = (was_flat && !((int)(is_flat > 0.5f)));
|
||||
int exiting_trade = (!was_flat && (int)(is_flat > 0.5f));
|
||||
/* Sign of position: -1 (short), 0 (flat), +1 (long) */
|
||||
int prev_sign = (ps[0] > 0.001f) ? 1 : ((ps[0] < -0.001f) ? -1 : 0);
|
||||
int curr_sign = (position > 0.001f) ? 1 : ((position < -0.001f) ? -1 : 0);
|
||||
|
||||
/* On entry: snapshot price and cumulative PnL */
|
||||
int entering_trade = (prev_sign == 0 && curr_sign != 0);
|
||||
int exiting_trade = (prev_sign != 0 && curr_sign == 0);
|
||||
int reversing_trade = (prev_sign != 0 && curr_sign != 0 && prev_sign != curr_sign);
|
||||
|
||||
/* A segment completes on exit OR reversal */
|
||||
int segment_complete = exiting_trade || reversing_trade;
|
||||
|
||||
/* On reversal: close old segment, open new one.
|
||||
* reversal_return is saved for the sparse reward block below
|
||||
* (trade_start_pnl is overwritten here, so recomputing would yield 0). */
|
||||
float reversal_return = 0.0f;
|
||||
if (reversing_trade) {
|
||||
/* Closing P&L for the completed segment.
|
||||
* CRITICAL: raw_pnl uses the NEW position (already updated at line 677).
|
||||
* old_pos_pnl (saved above) has the OLD position's final-bar PnL. */
|
||||
float closing_pnl = ps[11] + old_pos_pnl - trade_start_pnl;
|
||||
reversal_return = closing_pnl / (prev_equity > 1.0f ? prev_equity : 1.0f);
|
||||
|
||||
/* Kelly stats for completed segment */
|
||||
if (reversal_return > 0.0f) {
|
||||
win_count += 1.0f;
|
||||
sum_wins += reversal_return;
|
||||
} else {
|
||||
loss_count += 1.0f;
|
||||
sum_losses += fabsf(reversal_return);
|
||||
}
|
||||
sum_returns += reversal_return;
|
||||
sum_sq_returns += reversal_return * reversal_return;
|
||||
|
||||
/* Reset for new segment — realized_pnl snapshot after booking the close */
|
||||
entry_price = raw_close;
|
||||
trade_start_pnl = ps[11] + old_pos_pnl;
|
||||
}
|
||||
|
||||
/* On entry from flat (unchanged) */
|
||||
if (entering_trade) {
|
||||
entry_price = raw_close;
|
||||
trade_start_pnl = ps[11]; /* cumulative realized_pnl at entry */
|
||||
@@ -708,7 +753,7 @@ extern "C" __global__ void experience_env_step(
|
||||
* Trail distance adapts to regime via ADX (trend) and CUSUM (vol).
|
||||
* ════════════════════════════════════════════════════════════════════ */
|
||||
float unrealized_trade_pnl = 0.0f;
|
||||
if (!was_flat && hold_time > 0.0f) {
|
||||
if (prev_sign != 0 && hold_time > 0.0f) {
|
||||
unrealized_trade_pnl = (ps[11] + raw_pnl - trade_start_pnl) / (prev_equity > 1.0f ? prev_equity : 1.0f);
|
||||
}
|
||||
|
||||
@@ -726,7 +771,7 @@ extern "C" __global__ void experience_env_step(
|
||||
float trail_distance = 0.005f * vol_scale * trend_scale; /* 0.5% base, widens in vol/trend */
|
||||
|
||||
/* Trailing stop: only when profitable and held > 2 bars */
|
||||
if (!was_flat && hold_time > 2.0f && peak_equity > 1.0f) {
|
||||
if (prev_sign != 0 && hold_time > 2.0f && peak_equity > 1.0f) {
|
||||
float peak_return = (peak_equity - prev_equity) / (prev_equity > 1.0f ? prev_equity : 1.0f);
|
||||
if (peak_return > trail_distance) {
|
||||
float trail_floor = peak_return - trail_distance;
|
||||
@@ -741,11 +786,12 @@ extern "C" __global__ void experience_env_step(
|
||||
}
|
||||
}
|
||||
|
||||
/* Early-exit override: model tries to exit before 5 bars — force hold.
|
||||
/* Early-exit override: model tries to exit to FLAT before 5 bars — force hold.
|
||||
* Only blocks flat exits, NOT reversals — reversals always allowed.
|
||||
* CRITICAL: fix the stored action to match what actually executed.
|
||||
* Without this, the replay buffer stores (state, action=Flat, reward_from_Hold)
|
||||
* which corrupts Q-values for Flat — classic action aliasing bug. */
|
||||
if (!was_flat && is_flat > 0.5f && hold_time < 5.0f && hold_time > 0.0f && !exiting_trade) {
|
||||
if (prev_sign != 0 && curr_sign == 0 && hold_time < 5.0f && hold_time > 0.0f && !exiting_trade) {
|
||||
position = ps[0];
|
||||
cash = ps[1];
|
||||
is_flat = 0.0f;
|
||||
@@ -760,10 +806,24 @@ extern "C" __global__ void experience_env_step(
|
||||
out_actions[out_off] = action_idx;
|
||||
}
|
||||
|
||||
/* Recompute segment_complete after trailing stop / early-exit may have
|
||||
* modified exiting_trade. Trailing stop sets exiting_trade=1; early-exit
|
||||
* override sets exiting_trade=0. reversing_trade is never modified. */
|
||||
segment_complete = exiting_trade || reversing_trade;
|
||||
|
||||
/* Save hold_time BEFORE reset — sparse reward needs the pre-reset value
|
||||
* for the patience multiplier. Without this, hold_time is 0 at exit
|
||||
* and the sparse reward condition (hold_time > 0) never fires. */
|
||||
float segment_hold_time = hold_time;
|
||||
|
||||
/* Hold time tracking: counts TOTAL bars in position (not just losing).
|
||||
* Reset to 0 when flat. This feeds the patience multiplier in reward v5. */
|
||||
if (is_flat < 0.5f) {
|
||||
* Reset to 0 when flat OR on reversal (new segment starts fresh).
|
||||
* This feeds the patience multiplier in reward v5. */
|
||||
if (is_flat < 0.5f && !reversing_trade) {
|
||||
hold_time += 1.0f;
|
||||
} else if (reversing_trade) {
|
||||
/* New segment starts: count this bar as bar 1 of the new direction */
|
||||
hold_time = 1.0f;
|
||||
} else {
|
||||
hold_time = 0.0f;
|
||||
}
|
||||
@@ -783,7 +843,8 @@ extern "C" __global__ void experience_env_step(
|
||||
* Keeps gradients flowing — the network sees direction each bar.
|
||||
* When flat: reward_dense = 0 (no trade, no signal).
|
||||
*
|
||||
* SPARSE (at trade exit, large weight): trade return × patience.
|
||||
* SPARSE (at segment completion, large weight): trade return x patience.
|
||||
* Fires on BOTH exits to flat AND reversals (sign changes).
|
||||
* Primary learning signal. A trade held longer than expected
|
||||
* gets a patience bonus (sqrt scaling). Regime-adaptive expected
|
||||
* hold time via ADX.
|
||||
@@ -800,28 +861,41 @@ extern "C" __global__ void experience_env_step(
|
||||
reward_dense = 0.0f;
|
||||
}
|
||||
|
||||
/* ---- Sparse component: trade return × patience multiplier ---- */
|
||||
/* ---- Sparse component: segment return x patience multiplier ---- */
|
||||
/* segment_complete = exiting_trade || reversing_trade.
|
||||
* For reversals, Kelly stats were already updated above (in the reversal
|
||||
* block). For exits, Kelly stats are updated here. Both paths compute
|
||||
* segment_return for the patience-scaled sparse reward. */
|
||||
float reward_sparse = 0.0f;
|
||||
if (exiting_trade && hold_time > 0.0f) {
|
||||
float trade_pnl = ps[11] + raw_pnl - trade_start_pnl;
|
||||
float trade_return = trade_pnl / (prev_equity > 1.0f ? prev_equity : 1.0f);
|
||||
if (segment_complete && segment_hold_time > 0.0f) {
|
||||
float segment_pnl = ps[11] + raw_pnl - trade_start_pnl;
|
||||
float segment_return = segment_pnl / (prev_equity > 1.0f ? prev_equity : 1.0f);
|
||||
|
||||
/* Kelly statistics (GPU-native, no CPU).
|
||||
/* Kelly statistics for EXIT only (reversal Kelly already done above).
|
||||
* Tracked here AFTER trailing stop may have set exiting_trade=1,
|
||||
* so trail-triggered exits are counted in Kelly stats. */
|
||||
if (trade_return > 0.0f) {
|
||||
win_count += 1.0f;
|
||||
sum_wins += trade_return;
|
||||
} else {
|
||||
loss_count += 1.0f;
|
||||
sum_losses += fabsf(trade_return);
|
||||
if (exiting_trade) {
|
||||
if (segment_return > 0.0f) {
|
||||
win_count += 1.0f;
|
||||
sum_wins += segment_return;
|
||||
} else {
|
||||
loss_count += 1.0f;
|
||||
sum_losses += fabsf(segment_return);
|
||||
}
|
||||
sum_returns += segment_return;
|
||||
sum_sq_returns += segment_return * segment_return;
|
||||
}
|
||||
|
||||
/* For reversals, use the saved reversal_return — trade_start_pnl was
|
||||
* already overwritten in the reversal block, so recomputing would
|
||||
* yield 0. The reversal_return variable holds the correct value. */
|
||||
if (reversing_trade) {
|
||||
segment_return = reversal_return;
|
||||
}
|
||||
sum_returns += trade_return;
|
||||
sum_sq_returns += trade_return * trade_return;
|
||||
|
||||
/* Regime-adaptive expected hold time via ADX.
|
||||
* Trending (ADX > 30): hold longer → expected_hold = 20 bars.
|
||||
* Ranging (ADX < 20): shorter trades → expected_hold = 8 bars.
|
||||
* Trending (ADX > 30): hold longer -> expected_hold = 20 bars.
|
||||
* Ranging (ADX < 20): shorter trades -> expected_hold = 8 bars.
|
||||
* Default: expected_hold = 12 bars. */
|
||||
float adx_val = 0.0f;
|
||||
if (bar_idx < total_bars && market_dim >= 42) {
|
||||
@@ -831,13 +905,14 @@ extern "C" __global__ void experience_env_step(
|
||||
if (adx_val > 30.0f) expected_hold = 20.0f;
|
||||
else if (adx_val < 20.0f) expected_hold = 8.0f;
|
||||
|
||||
/* Patience multiplier: sqrt(hold_time / expected_hold).
|
||||
/* Patience multiplier: sqrt(segment_hold_time / expected_hold).
|
||||
* Uses pre-reset hold_time so the multiplier reflects actual duration.
|
||||
* Held exactly expected: 1.0. Held 2x: 1.41. Held 0.25x: 0.5.
|
||||
* Capped at 3.0 to prevent extreme outliers. */
|
||||
float patience_mult = sqrtf(hold_time / expected_hold);
|
||||
float patience_mult = sqrtf(segment_hold_time / expected_hold);
|
||||
patience_mult = fminf(patience_mult, 3.0f);
|
||||
|
||||
reward_sparse = trade_return * patience_mult;
|
||||
reward_sparse = segment_return * patience_mult;
|
||||
}
|
||||
|
||||
/* ---- Combined reward with loss aversion ---- */
|
||||
@@ -883,7 +958,14 @@ extern "C" __global__ void experience_env_step(
|
||||
ps[8] = flat_counter;
|
||||
ps[9] = new_portfolio_value; /* prev_equity = current equity for next step */
|
||||
ps[10] = hold_time;
|
||||
ps[11] = ps[11] + raw_pnl; /* realized_pnl accumulates */
|
||||
/* realized_pnl accumulates. On reversal bars, use old_pos_pnl
|
||||
* (saved before position was overwritten at ps[0] = position).
|
||||
* For non-reversal bars, raw_pnl (= current position's PnL) is correct. */
|
||||
if (reversing_trade) {
|
||||
ps[11] = ps[11] + old_pos_pnl; /* old position's PnL (saved at line ~700) */
|
||||
} else {
|
||||
ps[11] = ps[11] + raw_pnl; /* normal: current position's PnL */
|
||||
}
|
||||
ps[12] = entry_price; /* preserved across bars of a trade */
|
||||
ps[13] = trade_start_pnl; /* realized_pnl snapshot at trade entry */
|
||||
ps[14] = win_count; /* Kelly: accumulated across episode */
|
||||
|
||||
288
docs/superpowers/plans/2026-03-24-financial-metrics-pipeline.md
Normal file
288
docs/superpowers/plans/2026-03-24-financial-metrics-pipeline.md
Normal file
@@ -0,0 +1,288 @@
|
||||
# Financial Metrics Pipeline Fix — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Make Sharpe/WinRate/MaxDD/Trades report real per-trade performance from the GPU experience collector's portfolio simulation, not epoch-averaged mean rewards.
|
||||
|
||||
**Architecture:** Add a GPU reduction kernel that reads per-episode Kelly stats from `portfolio_states[N, 14:19]` (win_count, loss_count, sum_wins, sum_losses, sum_returns, sum_sq_returns) and reduces them into a single epoch-level `TradeStats` struct. Download this 48-byte summary once per epoch. Feed real trade-level data to `compute_epoch_financials()`.
|
||||
|
||||
**Tech Stack:** CUDA kernel (inline in gpu_experience_collector.rs), Rust structs, existing monitoring download path.
|
||||
|
||||
---
|
||||
|
||||
## Root Cause
|
||||
|
||||
```
|
||||
CURRENT (broken):
|
||||
experience_env_step → out_rewards[N,L] → monitoring_reduce → mean(N*L rewards) → ONE float per epoch
|
||||
pnl_history gets 1 entry per epoch → Sharpe=0.00, Trades=1
|
||||
|
||||
FIXED:
|
||||
experience_env_step → portfolio_states[N, 14:19] → trade_stats_reduce → TradeStats per epoch
|
||||
TradeStats has: total_trades, wins, losses, sum_pnl, sum_wins, sum_losses, per-step returns
|
||||
compute_epoch_financials uses REAL trade data → meaningful Sharpe/WinRate/Trades
|
||||
```
|
||||
|
||||
The per-trade Kelly stats already exist in `portfolio_states` — the experience kernel populates `ps[14]` (win_count), `ps[15]` (loss_count), `ps[16]` (sum_wins), `ps[17]` (sum_losses), `ps[18]` (sum_returns), `ps[19]` (sum_sq_returns) at every trade completion event. These are never read.
|
||||
|
||||
## File Structure
|
||||
|
||||
| File | Action | Responsibility |
|
||||
|------|--------|---------------|
|
||||
| `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs` | Modify | Add `trade_stats_reduce` kernel, `TradeStats` struct, `collect_trade_stats()` method |
|
||||
| `crates/ml/src/trainers/dqn/financials.rs` | Modify | Rewrite `compute_epoch_financials()` to accept `TradeStats` instead of `VecDeque<f64>` |
|
||||
| `crates/ml/src/trainers/dqn/trainer/training_loop.rs` | Modify | Call `collect_trade_stats()` after experience collection, feed to financials |
|
||||
| `crates/ml/src/trainers/dqn/trainer/mod.rs` | Modify | Add `trade_stats_history: Vec<TradeStats>` field |
|
||||
| `crates/ml/src/trainers/dqn/trainer/metrics.rs` | Modify | Use TradeStats in final metrics |
|
||||
| `crates/ml/src/trainers/dqn/smoke_tests/training_stability.rs` | Modify | Assert Trades > 1, WinRate > 0 |
|
||||
|
||||
---
|
||||
|
||||
## Task 1: TradeStats Struct + GPU Reduction Kernel (20 min)
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs`
|
||||
|
||||
The kernel reads `portfolio_states[N, PORTFOLIO_STRIDE]` and sums the per-episode Kelly fields into a single output vector.
|
||||
|
||||
- [ ] **Step 1:** Add `TradeStats` struct to `gpu_experience_collector.rs`:
|
||||
```rust
|
||||
/// Per-epoch trading statistics reduced from GPU portfolio states.
|
||||
/// Downloaded once per epoch (48 bytes — 6 f64s).
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct TradeStats {
|
||||
/// Total completed trades (entries + exits) across all episodes
|
||||
pub total_trades: usize,
|
||||
/// Winning trades (positive trade return at exit)
|
||||
pub winning_trades: usize,
|
||||
/// Losing trades (negative trade return at exit)
|
||||
pub losing_trades: usize,
|
||||
/// Sum of P&L from winning trades (for profit factor)
|
||||
pub sum_wins: f64,
|
||||
/// Sum of |P&L| from losing trades (for profit factor)
|
||||
pub sum_losses: f64,
|
||||
/// Sum of all trade returns (for mean return)
|
||||
pub sum_returns: f64,
|
||||
/// Sum of squared trade returns (for Sharpe std computation)
|
||||
pub sum_sq_returns: f64,
|
||||
/// Per-step returns from all episodes (for drawdown/equity curve)
|
||||
/// These are the actual out_rewards[N*L] values, not averaged.
|
||||
pub step_returns: Vec<f64>,
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2:** Add inline CUDA kernel `trade_stats_reduce`:
|
||||
```cuda
|
||||
// Grid: 1 block, Block: 256 threads
|
||||
// Parallel reduction over N episodes, summing portfolio_states[i, 14:19]
|
||||
extern "C" __global__ void trade_stats_reduce(
|
||||
const float* __restrict__ portfolio_states, // [N, PORTFOLIO_STRIDE]
|
||||
float* __restrict__ output, // [6] win_count, loss_count, sum_wins, sum_losses, sum_returns, sum_sq
|
||||
int N,
|
||||
int stride
|
||||
) {
|
||||
__shared__ float shmem[6 * 32]; // 6 fields × 32 warps max
|
||||
int tid = threadIdx.x;
|
||||
|
||||
// Each thread accumulates over a range of episodes
|
||||
float local[6] = {0};
|
||||
for (int i = tid; i < N; i += blockDim.x) {
|
||||
const float* ps = portfolio_states + i * stride;
|
||||
local[0] += ps[14]; // win_count
|
||||
local[1] += ps[15]; // loss_count
|
||||
local[2] += ps[16]; // sum_wins
|
||||
local[3] += ps[17]; // sum_losses
|
||||
local[4] += ps[18]; // sum_returns
|
||||
local[5] += ps[19]; // sum_sq_returns
|
||||
}
|
||||
|
||||
// Warp-level reduction
|
||||
for (int offset = 16; offset > 0; offset >>= 1) {
|
||||
for (int f = 0; f < 6; f++)
|
||||
local[f] += __shfl_down_sync(0xffffffff, local[f], offset);
|
||||
}
|
||||
|
||||
// Write warp results to shared memory
|
||||
int warp_id = tid / 32;
|
||||
int lane = tid % 32;
|
||||
if (lane == 0) {
|
||||
for (int f = 0; f < 6; f++)
|
||||
shmem[warp_id * 6 + f] = local[f];
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// Final reduction by first warp
|
||||
if (warp_id == 0) {
|
||||
int num_warps = (blockDim.x + 31) / 32;
|
||||
float sum[6] = {0};
|
||||
if (lane < num_warps) {
|
||||
for (int f = 0; f < 6; f++)
|
||||
sum[f] = shmem[lane * 6 + f];
|
||||
}
|
||||
for (int offset = 16; offset > 0; offset >>= 1) {
|
||||
for (int f = 0; f < 6; f++)
|
||||
sum[f] += __shfl_down_sync(0xffffffff, sum[f], offset);
|
||||
}
|
||||
if (lane == 0) {
|
||||
for (int f = 0; f < 6; f++)
|
||||
output[f] = sum[f];
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3:** Add `trade_stats_kernel: CudaFunction` field to `GpuExperienceCollector`. Compile the kernel in the constructor alongside the existing kernels. Add a `trade_stats_buf: CudaSlice<f32>` output buffer (6 floats).
|
||||
|
||||
- [ ] **Step 4:** Add `pub fn collect_trade_stats(&mut self) -> Result<TradeStats, MLError>` method:
|
||||
- Launch the `trade_stats_reduce` kernel with `portfolio_states`, N=alloc_episodes, stride=PORTFOLIO_STRIDE
|
||||
- Download the 6-float output buffer to host (24 bytes — single DtoH transfer)
|
||||
- Also download `rewards_out[N*L]` for per-step returns (this IS a larger transfer but only once per epoch, ~12KB for 3200 experiences)
|
||||
- Construct and return `TradeStats`
|
||||
|
||||
- [ ] **Step 5:** Run: `SQLX_OFFLINE=true cargo check -p ml` — verify clean build.
|
||||
|
||||
- [ ] **Step 6:** Commit: "feat: GPU trade stats reduction kernel — extracts per-trade metrics from portfolio states"
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Rewrite compute_epoch_financials() (10 min)
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/trainers/dqn/financials.rs`
|
||||
|
||||
Replace the `pnl_history: &VecDeque<f64>` input with `TradeStats`. The financial metrics should use REAL trade data:
|
||||
|
||||
- [ ] **Step 1:** Change signature:
|
||||
```rust
|
||||
pub(crate) fn compute_epoch_financials(
|
||||
trade_stats: &TradeStats,
|
||||
action_counts: &[usize; 9],
|
||||
initial_capital: f64,
|
||||
) -> EpochFinancials
|
||||
```
|
||||
|
||||
- [ ] **Step 2:** Rewrite the body:
|
||||
- `total_trades = trade_stats.total_trades` (real trade count from GPU)
|
||||
- `win_rate = trade_stats.winning_trades as f64 / total_trades.max(1) as f64`
|
||||
- `profit_factor = trade_stats.sum_wins / trade_stats.sum_losses.max(1e-10)`
|
||||
- `total_return = trade_stats.sum_returns` (sum of all trade returns)
|
||||
- `avg_return = total_return / total_trades.max(1) as f64`
|
||||
- Sharpe: compute from `step_returns` (per-bar returns, annualized with √252)
|
||||
- Max drawdown: compute equity curve from `step_returns` starting at `initial_capital`
|
||||
- Sortino: from negative step_returns only
|
||||
- Action distribution: unchanged (from `action_counts`)
|
||||
|
||||
- [ ] **Step 3:** Update the existing tests to use `TradeStats` input:
|
||||
- `test_empty_history` → `TradeStats::default()` should produce zeros
|
||||
- `test_all_winning` → `TradeStats { winning_trades: 5, total_trades: 5, sum_wins: 100.0, ... }`
|
||||
- `test_mixed_pnl` → set win/loss counts and sums from trade data
|
||||
|
||||
- [ ] **Step 4:** Run: `SQLX_OFFLINE=true cargo test -p ml --lib -- financials` — all tests pass.
|
||||
|
||||
- [ ] **Step 5:** Commit: "refactor: compute_epoch_financials uses real TradeStats from GPU"
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Wire TradeStats into Training Loop (10 min)
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/trainers/dqn/trainer/mod.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/trainer/training_loop.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/trainer/metrics.rs`
|
||||
|
||||
- [ ] **Step 1:** Add `trade_stats_history: Vec<TradeStats>` to `DQNTrainer` struct in `mod.rs`. Initialize to `Vec::new()` in constructor.
|
||||
|
||||
- [ ] **Step 2:** In `training_loop.rs`, after GPU experience collection (where `summary.mean_reward` is currently pushed to `pnl_history`), add:
|
||||
```rust
|
||||
// Collect real per-trade stats from GPU portfolio states
|
||||
if let Some(ref mut collector) = self.gpu_experience_collector {
|
||||
match collector.collect_trade_stats() {
|
||||
Ok(stats) => {
|
||||
info!(
|
||||
"Epoch {}/{}: GPU trades={} wins={} losses={} PF={:.2}",
|
||||
epoch + 1, self.hyperparams.epochs,
|
||||
stats.total_trades, stats.winning_trades, stats.losing_trades,
|
||||
if stats.sum_losses > 1e-10 { stats.sum_wins / stats.sum_losses } else { 0.0 }
|
||||
);
|
||||
self.trade_stats_history.push(stats);
|
||||
}
|
||||
Err(e) => warn!("Failed to collect trade stats: {e}"),
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3:** Replace the `compute_epoch_financials(&self.pnl_history, ...)` call with:
|
||||
```rust
|
||||
let trade_stats = self.trade_stats_history.last()
|
||||
.cloned()
|
||||
.unwrap_or_default();
|
||||
let financials = compute_epoch_financials(
|
||||
&trade_stats,
|
||||
&monitor.action_counts,
|
||||
self.hyperparams.initial_capital,
|
||||
);
|
||||
```
|
||||
|
||||
- [ ] **Step 4:** In `metrics.rs`, update `create_final_metrics()` to aggregate trade stats across all epochs:
|
||||
- Total trades = sum of all epoch trade counts
|
||||
- Overall win rate = total wins / total trades
|
||||
- Overall Sharpe = from concatenated step_returns across all epochs
|
||||
|
||||
- [ ] **Step 5:** Remove the `pnl_history: VecDeque<f64>` field and all references. It's replaced by `trade_stats_history`.
|
||||
|
||||
- [ ] **Step 6:** Run: `SQLX_OFFLINE=true cargo check -p ml` — verify clean build.
|
||||
|
||||
- [ ] **Step 7:** Commit: "feat: wire GPU trade stats into training loop financials"
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Update Smoke Tests to Assert Real Trading Metrics (5 min)
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/trainers/dqn/smoke_tests/training_stability.rs`
|
||||
|
||||
- [ ] **Step 1:** In `test_trading_model_behavior`, add:
|
||||
```rust
|
||||
// With 3200 experiences per epoch and position changes, we should see trades
|
||||
let total_trades = metrics.additional_metrics.get("total_trades").copied().unwrap_or(0.0);
|
||||
assert!(
|
||||
total_trades > 0.0,
|
||||
"BEHAVIORAL: Model must execute trades during experience collection. Got 0 trades."
|
||||
);
|
||||
```
|
||||
|
||||
- [ ] **Step 2:** In `test_50_epoch_convergence`, add:
|
||||
```rust
|
||||
// After 50 epochs × 3200 experiences, model must have executed real trades
|
||||
let total_trades = metrics.additional_metrics.get("total_trades").copied().unwrap_or(0.0);
|
||||
assert!(
|
||||
total_trades > 10.0,
|
||||
"CONVERGENCE: Model must execute >10 trades over 50 epochs. Got {total_trades}"
|
||||
);
|
||||
```
|
||||
|
||||
- [ ] **Step 3:** Run all smoke tests:
|
||||
```bash
|
||||
FOXHUNT_TEST_DATA=test_data/futures-baseline SQLX_OFFLINE=true cargo test -p ml --lib -- smoke_tests --ignored --nocapture
|
||||
```
|
||||
|
||||
- [ ] **Step 4:** Commit: "test: assert real trade count in smoke tests"
|
||||
|
||||
---
|
||||
|
||||
## Key Design Decisions
|
||||
|
||||
### Why Reduce on GPU Instead of Downloading portfolio_states?
|
||||
|
||||
`portfolio_states` is `[N_episodes × PORTFOLIO_STRIDE=20]` = `N × 80 bytes`. With N=32, that's 2560 bytes — small enough to download. But the reduction kernel pattern is:
|
||||
1. Future-proof: when N=1024 on H100, downloading 80KB per epoch wastes PCIe bandwidth
|
||||
2. Consistent: follows the existing `monitoring_reduce` pattern
|
||||
3. Zero-copy: the portfolio_states buffer is already on GPU, no staging needed
|
||||
|
||||
### Why Download step_returns (out_rewards)?
|
||||
|
||||
For Sharpe/MaxDD/Sortino, we need the per-step return distribution, not just the aggregates. With 3200 experiences, that's 12.8KB per epoch — acceptable for a once-per-epoch download. The alternative (computing Sharpe entirely on GPU) would require a second reduction kernel for mean/variance/drawdown, which is overengineering for a diagnostic metric.
|
||||
|
||||
### What Counts as a "Trade"?
|
||||
|
||||
A trade is an entry→exit cycle detected by the experience kernel's trade lifecycle logic. When `exiting_trade` fires (position goes from non-zero to zero), `win_count` or `loss_count` increments. Each episode can have 0-N trades depending on market conditions and model behavior. `total_trades = win_count + loss_count` summed across all episodes.
|
||||
567
docs/superpowers/plans/2026-03-24-remaining-gpu-features.md
Normal file
567
docs/superpowers/plans/2026-03-24-remaining-gpu-features.md
Normal file
@@ -0,0 +1,567 @@
|
||||
# Remaining GPU Features — Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Wire and implement the 4 remaining unwired GPU features: multi-head attention integration, Decision Transformer CUDA kernels, ensemble training with diversity loss, and HER Future strategy with GPU episode boundaries.
|
||||
|
||||
**Architecture:** Each feature integrates into the existing fused CUDA training pipeline (`GpuDqnTrainer` + CUDA Graph). All kernels are GPU-native (zero CPU in hot path). Post-graph operations use the `EventTrackingGuard` RAII pattern for safe cudarc buffer access. Kernels are compiled via nvcc → cubin (cached at `/tmp/.cubin_cache/`).
|
||||
|
||||
**CUDA Graph Integration Strategy:** The CUDA Graph captures the full forward+loss+backward+Adam sequence. Features that modify intermediate activations (attention, ensemble heads) operate with a **1-step lag**: they modify `save_h_s2` AFTER graph replay, so the modification takes effect on the NEXT graph replay's unflatten step. This is the same pattern as the IQN trunk gradient and spectral normalization — standard in async gradient methods. The alternative (splitting the graph into trunk/head phases) is deferred until profiling shows the lag impacts convergence.
|
||||
|
||||
**VRAM Budget:**
|
||||
| Feature | RTX 3050 (4GB) | H100 (80GB) |
|
||||
|---------|---------------|-------------|
|
||||
| Attention (params + activations) | ~0.5MB | ~2MB |
|
||||
| Decision Transformer (params + context) | ~4MB | ~60MB |
|
||||
| Ensemble (K=3 head copies) | ~1.5MB | ~6MB |
|
||||
| HER episode tracking | ~0.5MB | ~4MB |
|
||||
| **Total** | **~6.5MB** | **~72MB** |
|
||||
|
||||
**Tech Stack:** Rust (cudarc 0.19.3), CUDA C (nvcc/cubin), cuBLAS SGEMM
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
### New Files
|
||||
| File | Responsibility |
|
||||
|------|---------------|
|
||||
| `crates/ml/src/cuda_pipeline/attention_backward_kernel.cu` | Attention backward pass: LayerNorm backward + output projection backward + attention weight gradients |
|
||||
| `crates/ml/src/cuda_pipeline/dt_kernels.cu` | Decision Transformer forward: causal attention, softmax, positional encoding, token embedding, FFN, cross-entropy loss. Backward: softmax_backward, attention_backward (dQ/dK/dV), ffn_backward, layernorm_backward, embed_backward — 10 kernels total |
|
||||
| `crates/ml/src/cuda_pipeline/ensemble_kernels.cu` | Ensemble: multi-head Q-aggregation, KL-divergence diversity loss, Thompson sampling action selection |
|
||||
| `crates/ml/src/cuda_pipeline/her_episode_kernel.cu` | HER: episode boundary tracking, Future strategy donor sampling, episode-end detection |
|
||||
|
||||
### Modified Files
|
||||
| File | Changes |
|
||||
|------|---------|
|
||||
| `crates/ml/src/cuda_pipeline/gpu_attention.rs` | Add `backward()`, `params_buf()`, Adam state; wire into `GpuDqnTrainer` |
|
||||
| `crates/ml/src/cuda_pipeline/batched_forward.rs` | Add attention call between h_s2 and value/advantage heads |
|
||||
| `crates/ml/src/cuda_pipeline/batched_backward.rs` | Add attention backward in the gradient chain |
|
||||
| `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs` | Conditional attention in forward/backward; ensemble head management |
|
||||
| `crates/ml/src/cuda_pipeline/decision_transformer.rs` | Implement `pretrain_step()` with real CUDA kernels |
|
||||
| `crates/ml/src/cuda_pipeline/gpu_her.rs` | Add Final/Future strategies, episode boundary buffer |
|
||||
| `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs` | Add episode_ids to GpuExperienceBatch; fill via CUDA kernel (not CPU) |
|
||||
| `crates/ml/src/trainers/dqn/fused_training.rs` | Wire attention, ensemble, HER strategies |
|
||||
| `crates/ml/src/trainers/dqn/config.rs` | Add `use_attention: bool` to `DQNHyperparameters` (field exists in `DQNParams` hyperopt struct but not in the training config); wire from `DQNParams` during hyperopt conversion |
|
||||
| `crates/ml/src/trainers/dqn/trainer/training_loop.rs` | Ensemble head rotation, DT pre-training phase, HER episode tracking |
|
||||
| `crates/ml/src/hyperopt/adapters/dqn.rs` | Add DT pre-training epochs, ensemble params to search space |
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Attention Forward Pass Integration
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs`
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_attention.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/fused_training.rs`
|
||||
|
||||
- [ ] **Step 1: Add `GpuAttention` as optional field in `FusedTrainingCtx`**
|
||||
|
||||
In `fused_training.rs`, add to `FusedTrainingCtx`:
|
||||
```rust
|
||||
pub(crate) gpu_attention: Option<GpuAttention>,
|
||||
```
|
||||
|
||||
Initialize it in `new()` when `hyperparams.use_attention` is true (similar to `gpu_iqn` initialization pattern at line 233).
|
||||
|
||||
- [ ] **Step 2: Wire attention forward BETWEEN trunk h_s2 and value/advantage heads**
|
||||
|
||||
In `gpu_dqn_trainer.rs`, add a method:
|
||||
```rust
|
||||
pub fn apply_attention_forward(
|
||||
&self,
|
||||
attention: &mut GpuAttention,
|
||||
h_s2: &CudaSlice<f32>,
|
||||
batch_size: usize,
|
||||
) -> Result<(), MLError>
|
||||
```
|
||||
|
||||
This copies `h_s2` into attention input, runs `attention.forward()`, then copies the attended output BACK into `save_h_s2` (in-place replacement). The CUDA Graph captures the trunk forward FIRST, then attention runs OUTSIDE the graph (same pattern as IQN/spectral norm), then value/advantage heads run.
|
||||
|
||||
- [ ] **Step 3: Call attention in `run_full_step()` after CUDA Graph replay**
|
||||
|
||||
After Step 2 (graph replay) and before Step 5 (IQN):
|
||||
```rust
|
||||
if let Some(ref mut attn) = self.gpu_attention {
|
||||
let _evt_guard = EventTrackingGuard::new(self.stream.context());
|
||||
self.trainer.apply_attention_forward(attn, ...)?;
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Compile check**
|
||||
|
||||
```bash
|
||||
SQLX_OFFLINE=true cargo check -p ml 2>&1 | tail -5
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add crates/ml/src/cuda_pipeline/gpu_attention.rs crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs crates/ml/src/trainers/dqn/fused_training.rs
|
||||
git commit -m "feat: wire attention forward pass into DQN training (post-graph, frozen weights)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Attention Backward Pass Kernel
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/ml/src/cuda_pipeline/attention_backward_kernel.cu`
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_attention.rs`
|
||||
|
||||
- [ ] **Step 1: Write the attention backward CUDA kernel**
|
||||
|
||||
The kernel computes `dL/d(h_s2_input)` given `dL/d(h_s2_attended)`:
|
||||
|
||||
```cuda
|
||||
extern "C" __global__ void attention_backward_kernel(
|
||||
const float* d_output, // [B, D] gradient from downstream
|
||||
const float* states_input, // [B, D] saved input to attention
|
||||
const float* params, // attention weights
|
||||
float* d_input, // [B, D] gradient to upstream (trunk)
|
||||
float* d_params, // [total_params] weight gradients (atomicAdd)
|
||||
int B
|
||||
);
|
||||
```
|
||||
|
||||
The backward flows through:
|
||||
1. LayerNorm backward → d_prenorm
|
||||
2. Residual: d_prenorm splits into d_projection + d_residual
|
||||
3. Output projection backward: d_projection → d_concat, dW_O, db_O
|
||||
4. Multi-head attention backward → dQ, dK, dV per head
|
||||
5. Q/K/V projection backward → d_input, dW_Q, dW_K, dW_V
|
||||
|
||||
Since the kernel is complex (matching the 237-line forward kernel), implement in phases:
|
||||
- Phase A: Residual passthrough only (d_input = d_output) — gradient flows, weights frozen
|
||||
- Phase B: Full backward with weight gradients (unfreezes attention weights)
|
||||
|
||||
Start with Phase A (trivial kernel, verifiable).
|
||||
|
||||
- [ ] **Step 2: Add `backward()` method to `GpuAttention`**
|
||||
|
||||
```rust
|
||||
pub fn backward(
|
||||
&self,
|
||||
d_output: &CudaSlice<f32>,
|
||||
d_input: &mut CudaSlice<f32>,
|
||||
batch_size: usize,
|
||||
) -> Result<(), MLError>
|
||||
```
|
||||
|
||||
For Phase A: simply `memcpy_dtod(d_input, d_output)` — residual gradient passthrough.
|
||||
|
||||
- [ ] **Step 3: Wire backward into `apply_iqn_trunk_gradient`**
|
||||
|
||||
After attention forward rewrites `save_h_s2`, the IQN trunk gradient flows through the attention backward before reaching the trunk layers:
|
||||
```rust
|
||||
// Before trunk backward: d_h_s2 → attention_backward → d_h_s2_input
|
||||
if let Some(ref attn) = self.gpu_attention {
|
||||
attn.backward(&bw_d_h_s2, &mut bw_d_h_s2, batch_size)?;
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Compile check + local test**
|
||||
|
||||
```bash
|
||||
SQLX_OFFLINE=true cargo check --workspace 2>&1 | tail -5
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add crates/ml/src/cuda_pipeline/attention_backward_kernel.cu crates/ml/src/cuda_pipeline/gpu_attention.rs
|
||||
git commit -m "feat: attention backward (Phase A — residual passthrough, gradient flow verified)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Decision Transformer CUDA Kernels
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/ml/src/cuda_pipeline/dt_kernels.cu`
|
||||
- Modify: `crates/ml/src/cuda_pipeline/decision_transformer.rs`
|
||||
|
||||
- [ ] **Step 1: Write the token embedding + positional encoding kernel**
|
||||
|
||||
```cuda
|
||||
extern "C" __global__ void dt_embed_kernel(
|
||||
const float* trajectories, // [B, T, state_dim+2] (return-to-go, state, action)
|
||||
const float* W_embed, // [(state_dim+2), embed_dim]
|
||||
const float* b_embed, // [embed_dim]
|
||||
const float* pos_embed, // [T, embed_dim] (learned positional encoding)
|
||||
float* output, // [B, T, embed_dim]
|
||||
int B, int T, int input_dim, int embed_dim
|
||||
);
|
||||
```
|
||||
|
||||
Each thread handles one (batch, timestep, embed_dim_idx) tuple:
|
||||
- Linear projection: `out[b][t][d] = sum_j(traj[b][t][j] * W[j][d]) + b[d]`
|
||||
- Add positional: `out[b][t][d] += pos[t][d]`
|
||||
|
||||
- [ ] **Step 2: Write the causal self-attention kernel**
|
||||
|
||||
```cuda
|
||||
extern "C" __global__ void dt_causal_attention_kernel(
|
||||
const float* Q, // [B, T, embed_dim]
|
||||
const float* K, // [B, T, embed_dim]
|
||||
const float* V, // [B, T, embed_dim]
|
||||
float* output, // [B, T, embed_dim]
|
||||
int B, int T, int D, int num_heads
|
||||
);
|
||||
```
|
||||
|
||||
Per-head computation with causal mask:
|
||||
- `attn[i][j] = Q[i] · K[j] / sqrt(D_h)` for j ≤ i (causal), -inf otherwise
|
||||
- Softmax over j dimension
|
||||
- `output[i] = sum_j(attn[i][j] * V[j])`
|
||||
|
||||
Block-per-sample, threads handle different timesteps. Shared memory for K/V caching.
|
||||
|
||||
- [ ] **Step 3: Write the FFN kernel (embed_dim → 4×embed_dim → embed_dim)**
|
||||
|
||||
```cuda
|
||||
extern "C" __global__ void dt_ffn_kernel(
|
||||
const float* input, // [B, T, embed_dim]
|
||||
const float* W1, // [embed_dim, 4*embed_dim]
|
||||
const float* b1, // [4*embed_dim]
|
||||
const float* W2, // [4*embed_dim, embed_dim]
|
||||
const float* b2, // [embed_dim]
|
||||
float* output, // [B, T, embed_dim]
|
||||
int B, int T, int D
|
||||
);
|
||||
```
|
||||
|
||||
GELU activation between layers. Residual connection: `output += input`.
|
||||
|
||||
- [ ] **Step 4: Write the cross-entropy loss kernel**
|
||||
|
||||
```cuda
|
||||
extern "C" __global__ void dt_cross_entropy_kernel(
|
||||
const float* logits, // [B, T, num_actions]
|
||||
const int* target_actions, // [B, T]
|
||||
float* per_sample_loss, // [B*T]
|
||||
float* total_loss, // [1] atomicAdd
|
||||
int B, int T, int num_actions
|
||||
);
|
||||
```
|
||||
|
||||
Per-token cross-entropy: `loss = -log(softmax(logits)[target_action])`.
|
||||
|
||||
- [ ] **Step 5: Implement `pretrain_step()` in decision_transformer.rs**
|
||||
|
||||
Wire the 5 forward kernels into a training loop. Then:
|
||||
|
||||
- [ ] **Step 5b: Write 5 backward CUDA kernels**
|
||||
|
||||
```cuda
|
||||
// In dt_kernels.cu — add after forward kernels:
|
||||
// dt_cross_entropy_backward_kernel: dL/d_logits = softmax - one_hot
|
||||
// dt_ffn_backward_kernel: GELU backward + 2 linear backward
|
||||
// dt_attention_backward_kernel: dQ,dK,dV from d_output (softmax Jacobian + causal mask)
|
||||
// dt_layernorm_backward_kernel: chain through mean/variance
|
||||
// dt_embed_backward_kernel: dW_embed from d_embedded
|
||||
```
|
||||
|
||||
Each mirrors its forward kernel's thread/block layout. Total: 10 kernels.
|
||||
|
||||
- [ ] **Step 5c: Implement `pretrain_step()` in decision_transformer.rs**
|
||||
|
||||
1. **Forward:** Embed → {QKV → CausalAttn → LN → FFN → LN} × N → ActionHead → CE loss
|
||||
2. **Backward:** CE backward → reverse chain through all N layers → embed backward
|
||||
3. **Optimizer:** Reuse `dqn_grad_norm_kernel` + `dqn_adam_update_kernel` on DT params
|
||||
|
||||
DT action space = 9 exposure actions (branch_0 only, not full 81 factored).
|
||||
|
||||
- [ ] **Step 6: Compile check**
|
||||
|
||||
```bash
|
||||
SQLX_OFFLINE=true cargo check -p ml 2>&1 | tail -5
|
||||
```
|
||||
|
||||
- [ ] **Step 7: Commit**
|
||||
|
||||
```bash
|
||||
git add crates/ml/src/cuda_pipeline/dt_kernels.cu crates/ml/src/cuda_pipeline/decision_transformer.rs
|
||||
git commit -m "feat: Decision Transformer CUDA kernels (embed, causal attention, FFN, CE loss)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Decision Transformer Pre-Training Loop
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/trainers/dqn/trainer/training_loop.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
- Modify: `crates/ml/src/hyperopt/adapters/dqn.rs`
|
||||
|
||||
- [ ] **Step 1: Add DT pre-training config fields**
|
||||
|
||||
In `DQNHyperparameters`:
|
||||
```rust
|
||||
pub dt_pretrain_epochs: usize, // 0 = disabled, 10-50 typical
|
||||
pub dt_context_len: usize, // 20 timesteps
|
||||
pub dt_embed_dim: usize, // 128
|
||||
pub dt_num_layers: usize, // 3
|
||||
pub dt_target_return: f64, // 2.0 (target Sharpe-like return)
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Add DT pre-training phase at training start**
|
||||
|
||||
In `training_loop.rs`, before the main training loop:
|
||||
```rust
|
||||
if self.hyperparams.dt_pretrain_epochs > 0 {
|
||||
let dt = DecisionTransformer::new(stream, dt_config)?;
|
||||
for epoch in 0..self.hyperparams.dt_pretrain_epochs {
|
||||
// Build trajectory batches from training_data
|
||||
// Run dt.pretrain_step(trajectories, target_actions, batch_size)
|
||||
// Log DT loss
|
||||
}
|
||||
// Transfer DT's learned representations to DQN trunk (optional)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Add DT params to hyperopt search space**
|
||||
|
||||
In `dqn.rs`, add bounds for `dt_pretrain_epochs`, `dt_context_len`, `dt_embed_dim`.
|
||||
|
||||
- [ ] **Step 4: Compile check + commit**
|
||||
|
||||
```bash
|
||||
SQLX_OFFLINE=true cargo check --workspace
|
||||
git add crates/ml/src/cuda_pipeline/ crates/ml/src/trainers/dqn/ crates/ml/src/hyperopt/ && git commit -m "feat: Decision Transformer pre-training phase in training loop"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Ensemble Multi-Head Q-Network
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/ml/src/cuda_pipeline/ensemble_kernels.cu`
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/fused_training.rs`
|
||||
- Modify: `crates/ml/src/trainers/dqn/config.rs`
|
||||
|
||||
- [ ] **Step 1: Write the ensemble aggregation kernel**
|
||||
|
||||
```cuda
|
||||
extern "C" __global__ void ensemble_aggregate_kernel(
|
||||
const float* head_q_values, // [K, B, num_actions] Q-values from K heads
|
||||
float* mean_q, // [B, num_actions] mean across heads
|
||||
float* var_q, // [B, num_actions] variance across heads
|
||||
int K, int B, int num_actions
|
||||
);
|
||||
```
|
||||
|
||||
Computes mean and variance of Q-values across K ensemble heads for uncertainty estimation.
|
||||
|
||||
- [ ] **Step 2: Write the KL-divergence diversity loss kernel**
|
||||
|
||||
```cuda
|
||||
extern "C" __global__ void ensemble_diversity_kernel(
|
||||
const float* head_logits, // [K, B, num_atoms] per-head C51 logits
|
||||
float* diversity_loss, // [1] total KL divergence (atomicAdd)
|
||||
int K, int B, int num_atoms
|
||||
);
|
||||
```
|
||||
|
||||
For each pair of heads (i, j): `D_KL(p_i || p_j)` averaged over the batch. Uses hierarchical reduction (warp reduce → block reduce → atomicAdd per block, same pattern as `dqn_grad_norm_kernel`) to avoid single-address atomicAdd serialization. The diversity loss ENCOURAGES disagreement (subtracted from total loss).
|
||||
|
||||
- [ ] **Step 3: Implement multi-head architecture in GpuDqnTrainer**
|
||||
|
||||
The ensemble shares the TRUNK (W_s1, W_s2) but has K SEPARATE value + advantage heads:
|
||||
```rust
|
||||
pub struct EnsembleHeads {
|
||||
heads: Vec<(DuelingWeightSet, BranchingWeightSet)>, // K head weight sets
|
||||
}
|
||||
```
|
||||
|
||||
During training (heads run OUTSIDE the CUDA Graph, same pattern as IQN/attention):
|
||||
1. CUDA Graph: forward through shared trunk → h_s2 (using head 0's weights in the graph)
|
||||
2. Post-graph: for each head k=1..K-1, run cuBLAS forward through head_k's value/advantage layers on `save_h_s2` (outside graph, with EventTrackingGuard)
|
||||
3. Aggregate Q-values: mean ± std across K heads
|
||||
4. C51 loss per head (head 0 inside graph, heads 1..K-1 outside)
|
||||
5. Diversity regularization across heads
|
||||
6. Backward: each head contributes trunk gradients via the existing IQN-style SGD/Adam correction
|
||||
|
||||
Note: The CUDA Graph remains captured for head 0 only. Heads 1..K-1 run as post-graph operations with their own cuBLAS forwards. This avoids re-capturing the graph for each head.
|
||||
|
||||
- [ ] **Step 4: Add ensemble config and wiring**
|
||||
|
||||
In `DQNHyperparameters`:
|
||||
```rust
|
||||
pub ensemble_count: usize, // K heads (default 1 = no ensemble)
|
||||
pub ensemble_diversity_weight: f64, // 0.01 = mild diversity encouragement
|
||||
```
|
||||
|
||||
In `FusedTrainingCtx::run_full_step()`:
|
||||
- If `ensemble_count > 1`: run K forward passes through different heads, average for action selection, sum losses with diversity term.
|
||||
|
||||
- [ ] **Step 5: Compile check + commit**
|
||||
|
||||
```bash
|
||||
SQLX_OFFLINE=true cargo check --workspace
|
||||
git add crates/ml/src/cuda_pipeline/ crates/ml/src/trainers/dqn/ crates/ml/src/hyperopt/ && git commit -m "feat: ensemble multi-head Q-network with KL diversity loss"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 6: HER Episode Boundary Tracking
|
||||
|
||||
**Files:**
|
||||
- Create: `crates/ml/src/cuda_pipeline/her_episode_kernel.cu`
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_her.rs`
|
||||
- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs`
|
||||
|
||||
- [ ] **Step 1: Add episode index buffer to experience output**
|
||||
|
||||
In `GpuExperienceBatch`, add:
|
||||
```rust
|
||||
pub episode_ids: CudaSlice<i32>, // [N * L] episode index per transition
|
||||
```
|
||||
|
||||
Fill via a simple CUDA kernel `fill_episode_ids_kernel(int* ids, int L, int N)` where thread `i` writes `ids[i] = i / L`. Launched as `grid=(ceil(N*L/256)), block=256`. Zero CPU fill — pure GPU.
|
||||
|
||||
- [ ] **Step 2: Track episode boundaries in GPU replay buffer**
|
||||
|
||||
When inserting transitions, also store the episode ID:
|
||||
```rust
|
||||
pub episode_id_buf: CudaSlice<i32>, // [capacity] episode index per slot
|
||||
```
|
||||
|
||||
This allows the HER kernel to find other transitions from the same episode.
|
||||
|
||||
- [ ] **Step 3: Write the episode-aware donor sampling kernel**
|
||||
|
||||
```cuda
|
||||
extern "C" __global__ void her_sample_future_donors(
|
||||
const int* episode_ids, // [capacity] episode index per slot
|
||||
const int* source_indices, // [her_batch_size] which to relabel
|
||||
int* donor_indices, // [her_batch_size] output: future donors
|
||||
unsigned int* rng_states, // [her_batch_size] per-sample RNG
|
||||
int capacity,
|
||||
int her_batch_size
|
||||
);
|
||||
```
|
||||
|
||||
Uses a GPU-resident `episode_boundary_offsets` auxiliary buffer (updated at insertion time) for O(log(N)) binary search instead of O(capacity) linear scan. Each thread binary-searches the boundary buffer to find its episode's [start, end) range, then uniformly samples from `[source+1, end)`.
|
||||
|
||||
- [ ] **Step 4: Write the episode-end detection kernel (Final strategy)**
|
||||
|
||||
```cuda
|
||||
extern "C" __global__ void her_find_episode_end(
|
||||
const int* episode_ids, // [capacity]
|
||||
const int* source_indices, // [her_batch_size]
|
||||
int* end_indices, // [her_batch_size] output: last index in same episode
|
||||
int capacity,
|
||||
int her_batch_size
|
||||
);
|
||||
```
|
||||
|
||||
For each source: binary search or linear scan to find the last index with the same episode_id.
|
||||
|
||||
- [ ] **Step 5: Wire Future/Final strategies in `GpuHer`**
|
||||
|
||||
```rust
|
||||
pub fn relabel_batch_gpu(
|
||||
&mut self,
|
||||
strategy: HerGpuStrategy,
|
||||
episode_ids: &CudaSlice<i32>,
|
||||
...
|
||||
) -> Result<&HerBatch, MLError> {
|
||||
let donors = match strategy {
|
||||
HerGpuStrategy::Random => self.random_donors(her_batch_size),
|
||||
HerGpuStrategy::Future => self.sample_future_donors(episode_ids, source_indices)?,
|
||||
HerGpuStrategy::Final => self.find_episode_ends(episode_ids, source_indices)?,
|
||||
};
|
||||
self.relabel_kernel(source_indices, donors, ...)?;
|
||||
Ok(&self.output)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Compile check + commit**
|
||||
|
||||
```bash
|
||||
SQLX_OFFLINE=true cargo check --workspace
|
||||
git add crates/ml/src/cuda_pipeline/ crates/ml/src/trainers/dqn/ crates/ml/src/hyperopt/ && git commit -m "feat: HER Future/Final strategies with GPU episode boundary tracking"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 7: Integration Test — Full Pipeline
|
||||
|
||||
**Files:**
|
||||
- Modify: `crates/ml/src/trainers/dqn/trainer/training_loop.rs`
|
||||
|
||||
- [ ] **Step 1: Wire all 4 features into the training loop conditionally**
|
||||
|
||||
```rust
|
||||
// Attention: after trunk forward, before heads
|
||||
if hyperparams.use_attention { ... }
|
||||
|
||||
// DT pre-training: before main training loop
|
||||
if hyperparams.dt_pretrain_epochs > 0 { ... }
|
||||
|
||||
// Ensemble: K heads with diversity loss
|
||||
if hyperparams.ensemble_count > 1 { ... }
|
||||
|
||||
// HER: Future strategy with episode boundaries
|
||||
if hyperparams.her_ratio > 0.0 && hyperparams.her_strategy == "future" { ... }
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Local test — all features enabled**
|
||||
|
||||
```bash
|
||||
rm -rf /tmp/.cubin_cache/
|
||||
SQLX_OFFLINE=true cargo build --release -p ml --example hyperopt_baseline_rl
|
||||
SQLX_OFFLINE=true timeout 120 ./target/release/examples/hyperopt_baseline_rl \
|
||||
--model dqn --phase fast --trials 1 --n-initial 1 --epochs 3 \
|
||||
--data-dir test_data/futures-baseline --symbol ES.FUT \
|
||||
--output /tmp/full_pipeline_test.json --seed 42
|
||||
```
|
||||
|
||||
Expected: No NaN, no CUDA errors, train_loss < 50.
|
||||
|
||||
- [ ] **Step 3: Commit**
|
||||
|
||||
```bash
|
||||
git add crates/ml/src/cuda_pipeline/ crates/ml/src/trainers/dqn/ crates/ml/src/hyperopt/ && git commit -m "feat: full pipeline integration — attention + DT + ensemble + HER Future"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 8: H100 Submission
|
||||
|
||||
- [ ] **Step 1: Push to main**
|
||||
|
||||
```bash
|
||||
git push origin main
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Submit H100 hyperopt run**
|
||||
|
||||
50 epochs, 20 trials, with all features enabled via config:
|
||||
```toml
|
||||
use_attention = true
|
||||
dt_pretrain_epochs = 10
|
||||
ensemble_count = 3
|
||||
her_ratio = 0.3
|
||||
her_strategy = "future"
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Monitor first trial**
|
||||
|
||||
Check: no NaN, no CUDA OOM, loss decreasing, Q-gap > 0.
|
||||
|
||||
---
|
||||
|
||||
## Dependency Graph
|
||||
|
||||
```
|
||||
Task 1 (Attention Forward) → Task 2 (Attention Backward) → Task 7 (Integration)
|
||||
Task 3 (DT Kernels) → Task 4 (DT Loop) → Task 7 (Integration)
|
||||
Task 5 (Ensemble) → Task 7 (Integration)
|
||||
Task 6 (HER Episodes) → Task 7 (Integration)
|
||||
Task 7 (Integration) → Task 8 (H100)
|
||||
```
|
||||
|
||||
Tasks 1-2, 3-4, 5, and 6 are INDEPENDENT and can be implemented in parallel.
|
||||
@@ -0,0 +1,78 @@
|
||||
{
|
||||
"trial_number": 0,
|
||||
"sharpe": -0.017729128897190093,
|
||||
"win_rate": 0.024923645216040312,
|
||||
"max_drawdown": 100.0,
|
||||
"total_return": -5736.653628349304,
|
||||
"hyperparameters": {
|
||||
"learning_rate": 0.00005994999354431849,
|
||||
"batch_size": 307,
|
||||
"gamma": 0.9572818589229506,
|
||||
"buffer_size": 66246,
|
||||
"max_position_absolute": 1.1030284538648683,
|
||||
"huber_delta": 17.775790174607682,
|
||||
"entropy_coefficient": 0.38184099247597697,
|
||||
"transaction_cost_multiplier": 1.7738774066741243,
|
||||
"use_per": true,
|
||||
"per_alpha": 0.45251155566697393,
|
||||
"per_beta_start": 0.20130083854118397,
|
||||
"use_dueling": true,
|
||||
"dueling_hidden_dim": 128,
|
||||
"n_steps": 4,
|
||||
"tau": 0.008986246204051671,
|
||||
"use_distributional": true,
|
||||
"num_atoms": 51,
|
||||
"v_min": -70.22777500053176,
|
||||
"v_max": 70.22777500053176,
|
||||
"use_noisy_nets": true,
|
||||
"noisy_sigma_init": 0.24583814763225417,
|
||||
"minimum_profit_factor": 1.5028035348681215,
|
||||
"weight_decay": 0.00010597074820791314,
|
||||
"kelly_fractional": 0.5096243658522589,
|
||||
"kelly_max_fraction": 0.12010947414322902,
|
||||
"kelly_min_trades": 20,
|
||||
"volatility_window": 23,
|
||||
"use_ensemble_uncertainty": false,
|
||||
"ensemble_size": 5.0,
|
||||
"beta_variance": 0.5,
|
||||
"beta_disagreement": 0.5,
|
||||
"beta_entropy": 0.2,
|
||||
"warmup_ratio": 0.0,
|
||||
"curiosity_weight": 0.0,
|
||||
"td_error_clamp_max": 10.0,
|
||||
"batch_diversity_cooldown": 50.0,
|
||||
"lr_decay_type": 0.0,
|
||||
"sharpe_weight": 0.10176748614555509,
|
||||
"gae_lambda": 0.95,
|
||||
"noisy_sigma_initial": 0.5,
|
||||
"noisy_sigma_final": 0.3,
|
||||
"use_spectral_norm": false,
|
||||
"use_attention": false,
|
||||
"use_residual": false,
|
||||
"norm_type": 1.0,
|
||||
"activation_type": 1.0,
|
||||
"use_qr_dqn": false,
|
||||
"num_quantiles": 64,
|
||||
"qr_kappa": 1.0,
|
||||
"iqn_lambda": 0.8479734309322158,
|
||||
"hidden_dim_base": 128,
|
||||
"noisy_epsilon_floor": 0.1,
|
||||
"count_bonus_coefficient": 0.23831546372779777,
|
||||
"cql_alpha": 0.5059776371295834,
|
||||
"eval_softmax_temp": 1.0,
|
||||
"dsr_eta": 0.003947815597664894,
|
||||
"branch_hidden_dim": 64,
|
||||
"use_branching": true,
|
||||
"gradient_accumulation_steps": 1,
|
||||
"w_dsr": 1.0,
|
||||
"w_pnl": 0.3,
|
||||
"w_dd": 1.0,
|
||||
"w_idle": 0.01,
|
||||
"dd_threshold": 0.01,
|
||||
"loss_aversion": 1.5,
|
||||
"time_decay_rate": 0.0005,
|
||||
"q_gap_threshold": 0.1
|
||||
},
|
||||
"timestamp": "2026-03-22T23:43:16.144446538+00:00",
|
||||
"gradient_clip_norm": 10.0
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
{
|
||||
"trial_number": 1,
|
||||
"sharpe": 0.018215437326580285,
|
||||
"win_rate": 0.020554036577232183,
|
||||
"max_drawdown": 100.0,
|
||||
"total_return": -547.1131277084351,
|
||||
"hyperparameters": {
|
||||
"learning_rate": 0.00004291359460284525,
|
||||
"batch_size": 506,
|
||||
"gamma": 0.9546657016504223,
|
||||
"buffer_size": 68426,
|
||||
"max_position_absolute": 1.084699669559996,
|
||||
"huber_delta": 12.383291331070463,
|
||||
"entropy_coefficient": 0.22305679391580635,
|
||||
"transaction_cost_multiplier": 1.237118944760189,
|
||||
"use_per": true,
|
||||
"per_alpha": 0.7874757212411532,
|
||||
"per_beta_start": 0.5405183949041945,
|
||||
"use_dueling": true,
|
||||
"dueling_hidden_dim": 128,
|
||||
"n_steps": 3,
|
||||
"tau": 0.00696401394764961,
|
||||
"use_distributional": true,
|
||||
"num_atoms": 51,
|
||||
"v_min": -66.17506191155037,
|
||||
"v_max": 66.17506191155037,
|
||||
"use_noisy_nets": true,
|
||||
"noisy_sigma_init": 0.6469317434464105,
|
||||
"minimum_profit_factor": 1.199756461327086,
|
||||
"weight_decay": 0.0006131056727707305,
|
||||
"kelly_fractional": 0.36451619035608973,
|
||||
"kelly_max_fraction": 0.1982886181169854,
|
||||
"kelly_min_trades": 20,
|
||||
"volatility_window": 23,
|
||||
"use_ensemble_uncertainty": false,
|
||||
"ensemble_size": 5.0,
|
||||
"beta_variance": 0.5,
|
||||
"beta_disagreement": 0.5,
|
||||
"beta_entropy": 0.2,
|
||||
"warmup_ratio": 0.0,
|
||||
"curiosity_weight": 0.0,
|
||||
"td_error_clamp_max": 10.0,
|
||||
"batch_diversity_cooldown": 50.0,
|
||||
"lr_decay_type": 1.0,
|
||||
"sharpe_weight": 0.12231802599991748,
|
||||
"gae_lambda": 0.95,
|
||||
"noisy_sigma_initial": 0.5,
|
||||
"noisy_sigma_final": 0.3,
|
||||
"use_spectral_norm": false,
|
||||
"use_attention": false,
|
||||
"use_residual": false,
|
||||
"norm_type": 1.0,
|
||||
"activation_type": 1.0,
|
||||
"use_qr_dqn": false,
|
||||
"num_quantiles": 64,
|
||||
"qr_kappa": 1.0,
|
||||
"iqn_lambda": 0.1186854947736955,
|
||||
"hidden_dim_base": 128,
|
||||
"noisy_epsilon_floor": 0.1,
|
||||
"count_bonus_coefficient": 0.09860756605998412,
|
||||
"cql_alpha": 0.16860966429860813,
|
||||
"eval_softmax_temp": 1.0,
|
||||
"dsr_eta": 0.008085442687796646,
|
||||
"branch_hidden_dim": 64,
|
||||
"use_branching": true,
|
||||
"gradient_accumulation_steps": 1,
|
||||
"w_dsr": 1.0,
|
||||
"w_pnl": 0.3,
|
||||
"w_dd": 1.0,
|
||||
"w_idle": 0.01,
|
||||
"dd_threshold": 0.01,
|
||||
"loss_aversion": 1.5,
|
||||
"time_decay_rate": 0.0005,
|
||||
"q_gap_threshold": 0.1
|
||||
},
|
||||
"timestamp": "2026-03-22T23:44:00.953235416+00:00",
|
||||
"gradient_clip_norm": 10.0
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
{
|
||||
"trial_number": 1,
|
||||
"sharpe": 0.009133303724229335,
|
||||
"win_rate": 0.02989337162580341,
|
||||
"max_drawdown": 100.0,
|
||||
"total_return": -381.4813995361328,
|
||||
"hyperparameters": {
|
||||
"learning_rate": 0.00004291359460284525,
|
||||
"batch_size": 506,
|
||||
"gamma": 0.9546657016504223,
|
||||
"buffer_size": 68426,
|
||||
"max_position_absolute": 1.084699669559996,
|
||||
"huber_delta": 12.383291331070463,
|
||||
"entropy_coefficient": 0.22305679391580635,
|
||||
"transaction_cost_multiplier": 1.237118944760189,
|
||||
"use_per": true,
|
||||
"per_alpha": 0.7874757212411532,
|
||||
"per_beta_start": 0.5405183949041945,
|
||||
"use_dueling": true,
|
||||
"dueling_hidden_dim": 128,
|
||||
"n_steps": 3,
|
||||
"tau": 0.00696401394764961,
|
||||
"use_distributional": true,
|
||||
"num_atoms": 51,
|
||||
"v_min": -66.17506191155037,
|
||||
"v_max": 66.17506191155037,
|
||||
"use_noisy_nets": true,
|
||||
"noisy_sigma_init": 0.6469317434464105,
|
||||
"minimum_profit_factor": 1.199756461327086,
|
||||
"weight_decay": 0.0006131056727707305,
|
||||
"kelly_fractional": 0.36451619035608973,
|
||||
"kelly_max_fraction": 0.1982886181169854,
|
||||
"kelly_min_trades": 20,
|
||||
"volatility_window": 23,
|
||||
"use_ensemble_uncertainty": false,
|
||||
"ensemble_size": 5.0,
|
||||
"beta_variance": 0.5,
|
||||
"beta_disagreement": 0.5,
|
||||
"beta_entropy": 0.2,
|
||||
"warmup_ratio": 0.0,
|
||||
"curiosity_weight": 0.0,
|
||||
"td_error_clamp_max": 10.0,
|
||||
"batch_diversity_cooldown": 50.0,
|
||||
"lr_decay_type": 1.0,
|
||||
"sharpe_weight": 0.12231802599991748,
|
||||
"gae_lambda": 0.95,
|
||||
"noisy_sigma_initial": 0.5,
|
||||
"noisy_sigma_final": 0.3,
|
||||
"use_spectral_norm": false,
|
||||
"use_attention": false,
|
||||
"use_residual": false,
|
||||
"norm_type": 1.0,
|
||||
"activation_type": 1.0,
|
||||
"use_qr_dqn": false,
|
||||
"num_quantiles": 64,
|
||||
"qr_kappa": 1.0,
|
||||
"iqn_lambda": 0.1186854947736955,
|
||||
"hidden_dim_base": 128,
|
||||
"noisy_epsilon_floor": 0.1,
|
||||
"count_bonus_coefficient": 0.09860756605998412,
|
||||
"cql_alpha": 0.16860966429860813,
|
||||
"eval_softmax_temp": 1.0,
|
||||
"dsr_eta": 0.008085442687796646,
|
||||
"branch_hidden_dim": 64,
|
||||
"use_branching": true,
|
||||
"gradient_accumulation_steps": 1,
|
||||
"w_dsr": 1.0,
|
||||
"w_pnl": 0.3,
|
||||
"w_dd": 1.0,
|
||||
"w_idle": 0.01,
|
||||
"dd_threshold": 0.01,
|
||||
"loss_aversion": 1.5,
|
||||
"time_decay_rate": 0.0005,
|
||||
"q_gap_threshold": 0.1
|
||||
},
|
||||
"timestamp": "2026-03-23T07:22:11.285931152+00:00",
|
||||
"gradient_clip_norm": 10.0
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
{
|
||||
"trial_number": 0,
|
||||
"sharpe": 0.004663038626313209,
|
||||
"win_rate": 0.33112575113773346,
|
||||
"max_drawdown": 100.0,
|
||||
"total_return": -8658.971099853516,
|
||||
"hyperparameters": {
|
||||
"learning_rate": 0.00005994999354431849,
|
||||
"batch_size": 307,
|
||||
"gamma": 0.9572818589229506,
|
||||
"buffer_size": 66246,
|
||||
"max_position_absolute": 1.1030284538648683,
|
||||
"huber_delta": 17.775790174607682,
|
||||
"entropy_coefficient": 0.38184099247597697,
|
||||
"transaction_cost_multiplier": 1.7738774066741243,
|
||||
"use_per": true,
|
||||
"per_alpha": 0.45251155566697393,
|
||||
"per_beta_start": 0.20130083854118397,
|
||||
"use_dueling": true,
|
||||
"dueling_hidden_dim": 128,
|
||||
"n_steps": 4,
|
||||
"tau": 0.008986246204051671,
|
||||
"use_distributional": true,
|
||||
"num_atoms": 101,
|
||||
"v_min": -84.27333000063813,
|
||||
"v_max": 84.27333000063813,
|
||||
"use_noisy_nets": true,
|
||||
"noisy_sigma_init": 0.24583814763225417,
|
||||
"minimum_profit_factor": 1.5028035348681215,
|
||||
"weight_decay": 0.00010597074820791314,
|
||||
"kelly_fractional": 0.5096243658522589,
|
||||
"kelly_max_fraction": 0.12010947414322902,
|
||||
"kelly_min_trades": 20,
|
||||
"volatility_window": 23,
|
||||
"use_ensemble_uncertainty": false,
|
||||
"ensemble_size": 5.0,
|
||||
"beta_variance": 0.5,
|
||||
"beta_disagreement": 0.5,
|
||||
"beta_entropy": 0.2,
|
||||
"warmup_ratio": 0.0,
|
||||
"curiosity_weight": 0.0,
|
||||
"td_error_clamp_max": 10.0,
|
||||
"batch_diversity_cooldown": 50.0,
|
||||
"lr_decay_type": 0.0,
|
||||
"sharpe_weight": 0.10176748614555509,
|
||||
"gae_lambda": 0.95,
|
||||
"noisy_sigma_initial": 0.5,
|
||||
"noisy_sigma_final": 0.3,
|
||||
"use_spectral_norm": true,
|
||||
"use_attention": true,
|
||||
"use_residual": true,
|
||||
"norm_type": 1.0,
|
||||
"activation_type": 1.0,
|
||||
"num_quantiles": 32,
|
||||
"qr_kappa": 1.0,
|
||||
"iqn_lambda": 0.8479734309322158,
|
||||
"spectral_norm_sigma_max": 4.8543579475657985,
|
||||
"hidden_dim_base": 128,
|
||||
"noisy_epsilon_floor": 0.1,
|
||||
"count_bonus_coefficient": 0.23831546372779777,
|
||||
"cql_alpha": 0.5059776371295834,
|
||||
"eval_softmax_temp": 1.0,
|
||||
"dsr_eta": 0.003947815597664894,
|
||||
"branch_hidden_dim": 64,
|
||||
"use_branching": true,
|
||||
"gradient_accumulation_steps": 1,
|
||||
"w_dsr": 1.0,
|
||||
"w_pnl": 0.3,
|
||||
"w_dd": 1.0,
|
||||
"w_idle": 0.01,
|
||||
"dd_threshold": 0.01,
|
||||
"loss_aversion": 1.5,
|
||||
"time_decay_rate": 0.0005,
|
||||
"q_gap_threshold": 0.1,
|
||||
"dt_pretrain_epochs": 0
|
||||
},
|
||||
"timestamp": "2026-03-24T09:21:47.273661543+00:00",
|
||||
"gradient_clip_norm": 10.0
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
{
|
||||
"trial_number": 0,
|
||||
"sharpe": 0.04188940078020096,
|
||||
"win_rate": 0.2995686545968056,
|
||||
"max_drawdown": 100.0,
|
||||
"total_return": 233471.36112213135,
|
||||
"hyperparameters": {
|
||||
"learning_rate": 0.00005994999354431849,
|
||||
"batch_size": 307,
|
||||
"gamma": 0.9572818589229506,
|
||||
"buffer_size": 66246,
|
||||
"max_position_absolute": 1.1030284538648683,
|
||||
"huber_delta": 17.775790174607682,
|
||||
"entropy_coefficient": 0.38184099247597697,
|
||||
"transaction_cost_multiplier": 1.7738774066741243,
|
||||
"use_per": true,
|
||||
"per_alpha": 0.45251155566697393,
|
||||
"per_beta_start": 0.20130083854118397,
|
||||
"use_dueling": true,
|
||||
"dueling_hidden_dim": 128,
|
||||
"n_steps": 4,
|
||||
"tau": 0.008986246204051671,
|
||||
"use_distributional": true,
|
||||
"num_atoms": 101,
|
||||
"v_min": -84.27333000063813,
|
||||
"v_max": 84.27333000063813,
|
||||
"use_noisy_nets": true,
|
||||
"noisy_sigma_init": 0.24583814763225417,
|
||||
"minimum_profit_factor": 1.5028035348681215,
|
||||
"weight_decay": 0.00010597074820791314,
|
||||
"kelly_fractional": 0.5096243658522589,
|
||||
"kelly_max_fraction": 0.12010947414322902,
|
||||
"kelly_min_trades": 20,
|
||||
"volatility_window": 23,
|
||||
"use_ensemble_uncertainty": false,
|
||||
"ensemble_size": 5.0,
|
||||
"beta_variance": 0.5,
|
||||
"beta_disagreement": 0.5,
|
||||
"beta_entropy": 0.2,
|
||||
"warmup_ratio": 0.0,
|
||||
"curiosity_weight": 0.0,
|
||||
"td_error_clamp_max": 10.0,
|
||||
"batch_diversity_cooldown": 50.0,
|
||||
"lr_decay_type": 0.0,
|
||||
"sharpe_weight": 0.10176748614555509,
|
||||
"gae_lambda": 0.95,
|
||||
"noisy_sigma_initial": 0.5,
|
||||
"noisy_sigma_final": 0.3,
|
||||
"use_spectral_norm": true,
|
||||
"use_attention": true,
|
||||
"use_residual": true,
|
||||
"norm_type": 1.0,
|
||||
"activation_type": 1.0,
|
||||
"num_quantiles": 32,
|
||||
"qr_kappa": 1.0,
|
||||
"iqn_lambda": 0.8479734309322158,
|
||||
"spectral_norm_sigma_max": 4.8543579475657985,
|
||||
"hidden_dim_base": 128,
|
||||
"noisy_epsilon_floor": 0.1,
|
||||
"count_bonus_coefficient": 0.23831546372779777,
|
||||
"cql_alpha": 0.5059776371295834,
|
||||
"eval_softmax_temp": 1.0,
|
||||
"dsr_eta": 0.003947815597664894,
|
||||
"branch_hidden_dim": 64,
|
||||
"use_branching": true,
|
||||
"gradient_accumulation_steps": 1,
|
||||
"w_dsr": 1.0,
|
||||
"w_pnl": 0.3,
|
||||
"w_dd": 1.0,
|
||||
"w_idle": 0.01,
|
||||
"dd_threshold": 0.01,
|
||||
"loss_aversion": 1.5,
|
||||
"time_decay_rate": 0.0005,
|
||||
"q_gap_threshold": 0.1,
|
||||
"dt_pretrain_epochs": 0
|
||||
},
|
||||
"timestamp": "2026-03-24T10:01:01.349397710+00:00",
|
||||
"gradient_clip_norm": 10.0
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
{
|
||||
"trial_number": 0,
|
||||
"sharpe": -0.006844052672386169,
|
||||
"win_rate": 0.10371590945869684,
|
||||
"max_drawdown": 100.0,
|
||||
"total_return": -19307.970275878906,
|
||||
"hyperparameters": {
|
||||
"learning_rate": 0.00005994999354431849,
|
||||
"batch_size": 307,
|
||||
"gamma": 0.9572818589229506,
|
||||
"buffer_size": 66246,
|
||||
"max_position_absolute": 1.1030284538648683,
|
||||
"huber_delta": 17.775790174607682,
|
||||
"entropy_coefficient": 0.38184099247597697,
|
||||
"transaction_cost_multiplier": 1.7738774066741243,
|
||||
"use_per": true,
|
||||
"per_alpha": 0.45251155566697393,
|
||||
"per_beta_start": 0.20130083854118397,
|
||||
"use_dueling": true,
|
||||
"dueling_hidden_dim": 128,
|
||||
"n_steps": 4,
|
||||
"tau": 0.008986246204051671,
|
||||
"use_distributional": true,
|
||||
"num_atoms": 101,
|
||||
"v_min": -42.13666500031906,
|
||||
"v_max": 42.13666500031906,
|
||||
"use_noisy_nets": true,
|
||||
"noisy_sigma_init": 0.24583814763225417,
|
||||
"minimum_profit_factor": 1.5028035348681215,
|
||||
"weight_decay": 0.00010597074820791314,
|
||||
"kelly_fractional": 0.5096243658522589,
|
||||
"kelly_max_fraction": 0.12010947414322902,
|
||||
"kelly_min_trades": 20,
|
||||
"volatility_window": 23,
|
||||
"use_ensemble_uncertainty": false,
|
||||
"ensemble_size": 5.0,
|
||||
"beta_variance": 0.5,
|
||||
"beta_disagreement": 0.5,
|
||||
"beta_entropy": 0.2,
|
||||
"warmup_ratio": 0.0,
|
||||
"curiosity_weight": 0.0,
|
||||
"td_error_clamp_max": 10.0,
|
||||
"batch_diversity_cooldown": 50.0,
|
||||
"lr_decay_type": 0.0,
|
||||
"sharpe_weight": 0.10176748614555509,
|
||||
"gae_lambda": 0.95,
|
||||
"noisy_sigma_initial": 0.5,
|
||||
"noisy_sigma_final": 0.3,
|
||||
"use_spectral_norm": true,
|
||||
"use_attention": true,
|
||||
"use_residual": true,
|
||||
"norm_type": 1.0,
|
||||
"activation_type": 1.0,
|
||||
"num_quantiles": 32,
|
||||
"qr_kappa": 1.0,
|
||||
"iqn_lambda": 0.8479734309322158,
|
||||
"spectral_norm_sigma_max": 4.8543579475657985,
|
||||
"hidden_dim_base": 128,
|
||||
"noisy_epsilon_floor": 0.1,
|
||||
"count_bonus_coefficient": 0.23831546372779777,
|
||||
"cql_alpha": 0.5059776371295834,
|
||||
"eval_softmax_temp": 1.0,
|
||||
"dsr_eta": 0.003947815597664894,
|
||||
"branch_hidden_dim": 64,
|
||||
"use_branching": true,
|
||||
"gradient_accumulation_steps": 1,
|
||||
"w_dsr": 1.0,
|
||||
"w_pnl": 0.3,
|
||||
"w_dd": 1.0,
|
||||
"w_idle": 0.01,
|
||||
"dd_threshold": 0.01,
|
||||
"loss_aversion": 1.5,
|
||||
"time_decay_rate": 0.0005,
|
||||
"q_gap_threshold": 0.1,
|
||||
"dt_pretrain_epochs": 0
|
||||
},
|
||||
"timestamp": "2026-03-24T14:27:44.051295574+00:00",
|
||||
"gradient_clip_norm": 10.0
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
{
|
||||
"trial_number": 0,
|
||||
"sharpe": -0.01161600649356842,
|
||||
"win_rate": 0.14867919757962228,
|
||||
"max_drawdown": 100.0,
|
||||
"total_return": -10207.207107543945,
|
||||
"hyperparameters": {
|
||||
"learning_rate": 0.00005994999354431849,
|
||||
"batch_size": 307,
|
||||
"gamma": 0.9572818589229506,
|
||||
"buffer_size": 66246,
|
||||
"max_position_absolute": 1.1030284538648683,
|
||||
"huber_delta": 17.775790174607682,
|
||||
"entropy_coefficient": 0.38184099247597697,
|
||||
"transaction_cost_multiplier": 1.7738774066741243,
|
||||
"use_per": true,
|
||||
"per_alpha": 0.45251155566697393,
|
||||
"per_beta_start": 0.20130083854118397,
|
||||
"use_dueling": true,
|
||||
"dueling_hidden_dim": 128,
|
||||
"n_steps": 4,
|
||||
"tau": 0.008986246204051671,
|
||||
"use_distributional": true,
|
||||
"num_atoms": 101,
|
||||
"v_min": -42.13666500031906,
|
||||
"v_max": 42.13666500031906,
|
||||
"use_noisy_nets": true,
|
||||
"noisy_sigma_init": 0.24583814763225417,
|
||||
"minimum_profit_factor": 1.5028035348681215,
|
||||
"weight_decay": 0.00010597074820791314,
|
||||
"kelly_fractional": 0.5096243658522589,
|
||||
"kelly_max_fraction": 0.12010947414322902,
|
||||
"kelly_min_trades": 20,
|
||||
"volatility_window": 23,
|
||||
"use_ensemble_uncertainty": false,
|
||||
"ensemble_size": 5.0,
|
||||
"beta_variance": 0.5,
|
||||
"beta_disagreement": 0.5,
|
||||
"beta_entropy": 0.2,
|
||||
"warmup_ratio": 0.0,
|
||||
"curiosity_weight": 0.0,
|
||||
"td_error_clamp_max": 10.0,
|
||||
"batch_diversity_cooldown": 50.0,
|
||||
"lr_decay_type": 0.0,
|
||||
"sharpe_weight": 0.10176748614555509,
|
||||
"gae_lambda": 0.95,
|
||||
"noisy_sigma_initial": 0.5,
|
||||
"noisy_sigma_final": 0.3,
|
||||
"use_spectral_norm": true,
|
||||
"use_attention": true,
|
||||
"use_residual": true,
|
||||
"norm_type": 1.0,
|
||||
"activation_type": 1.0,
|
||||
"num_quantiles": 32,
|
||||
"qr_kappa": 1.0,
|
||||
"iqn_lambda": 0.8479734309322158,
|
||||
"spectral_norm_sigma_max": 4.8543579475657985,
|
||||
"hidden_dim_base": 128,
|
||||
"noisy_epsilon_floor": 0.1,
|
||||
"count_bonus_coefficient": 0.23831546372779777,
|
||||
"cql_alpha": 0.5059776371295834,
|
||||
"eval_softmax_temp": 1.0,
|
||||
"dsr_eta": 0.003947815597664894,
|
||||
"branch_hidden_dim": 64,
|
||||
"use_branching": true,
|
||||
"gradient_accumulation_steps": 1,
|
||||
"w_dsr": 1.0,
|
||||
"w_pnl": 0.3,
|
||||
"w_dd": 1.0,
|
||||
"w_idle": 0.01,
|
||||
"dd_threshold": 0.01,
|
||||
"loss_aversion": 1.5,
|
||||
"time_decay_rate": 0.0005,
|
||||
"q_gap_threshold": 0.1,
|
||||
"dt_pretrain_epochs": 0
|
||||
},
|
||||
"timestamp": "2026-03-24T14:53:49.539858557+00:00",
|
||||
"gradient_clip_norm": 10.0
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
{
|
||||
"trial_number": 0,
|
||||
"sharpe": 0.0837665757164359,
|
||||
"win_rate": 0.21893585734069348,
|
||||
"max_drawdown": 100.0,
|
||||
"total_return": 2530.810966491699,
|
||||
"hyperparameters": {
|
||||
"learning_rate": 0.00005994999354431849,
|
||||
"batch_size": 307,
|
||||
"gamma": 0.9572818589229506,
|
||||
"buffer_size": 66246,
|
||||
"max_position_absolute": 1.1030284538648683,
|
||||
"huber_delta": 17.775790174607682,
|
||||
"entropy_coefficient": 0.38184099247597697,
|
||||
"transaction_cost_multiplier": 1.7738774066741243,
|
||||
"use_per": true,
|
||||
"per_alpha": 0.45251155566697393,
|
||||
"per_beta_start": 0.20130083854118397,
|
||||
"use_dueling": true,
|
||||
"dueling_hidden_dim": 128,
|
||||
"n_steps": 4,
|
||||
"tau": 0.008986246204051671,
|
||||
"use_distributional": true,
|
||||
"num_atoms": 101,
|
||||
"v_min": -42.13666500031906,
|
||||
"v_max": 42.13666500031906,
|
||||
"use_noisy_nets": true,
|
||||
"noisy_sigma_init": 0.24583814763225417,
|
||||
"minimum_profit_factor": 1.5028035348681215,
|
||||
"weight_decay": 0.00010597074820791314,
|
||||
"kelly_fractional": 0.5096243658522589,
|
||||
"kelly_max_fraction": 0.12010947414322902,
|
||||
"kelly_min_trades": 20,
|
||||
"volatility_window": 23,
|
||||
"use_ensemble_uncertainty": false,
|
||||
"ensemble_size": 5.0,
|
||||
"beta_variance": 0.5,
|
||||
"beta_disagreement": 0.5,
|
||||
"beta_entropy": 0.2,
|
||||
"warmup_ratio": 0.0,
|
||||
"curiosity_weight": 0.0,
|
||||
"td_error_clamp_max": 10.0,
|
||||
"batch_diversity_cooldown": 50.0,
|
||||
"lr_decay_type": 0.0,
|
||||
"sharpe_weight": 0.10176748614555509,
|
||||
"gae_lambda": 0.95,
|
||||
"noisy_sigma_initial": 0.5,
|
||||
"noisy_sigma_final": 0.3,
|
||||
"use_spectral_norm": true,
|
||||
"use_attention": true,
|
||||
"use_residual": true,
|
||||
"norm_type": 1.0,
|
||||
"activation_type": 1.0,
|
||||
"num_quantiles": 32,
|
||||
"qr_kappa": 1.0,
|
||||
"iqn_lambda": 0.8479734309322158,
|
||||
"spectral_norm_sigma_max": 4.8543579475657985,
|
||||
"hidden_dim_base": 128,
|
||||
"noisy_epsilon_floor": 0.1,
|
||||
"count_bonus_coefficient": 0.23831546372779777,
|
||||
"cql_alpha": 0.5059776371295834,
|
||||
"eval_softmax_temp": 1.0,
|
||||
"dsr_eta": 0.003947815597664894,
|
||||
"branch_hidden_dim": 64,
|
||||
"use_branching": true,
|
||||
"gradient_accumulation_steps": 1,
|
||||
"w_dsr": 1.0,
|
||||
"w_pnl": 0.3,
|
||||
"w_dd": 1.0,
|
||||
"w_idle": 0.01,
|
||||
"dd_threshold": 0.01,
|
||||
"loss_aversion": 1.5,
|
||||
"time_decay_rate": 0.0005,
|
||||
"q_gap_threshold": 0.1,
|
||||
"dt_pretrain_epochs": 0
|
||||
},
|
||||
"timestamp": "2026-03-24T15:12:06.755573679+00:00",
|
||||
"gradient_clip_norm": 10.0
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
{
|
||||
"trial_number": 0,
|
||||
"sharpe": 0.035534610599279405,
|
||||
"win_rate": 0.12874939665198326,
|
||||
"max_drawdown": 100.0,
|
||||
"total_return": 18805.362014770508,
|
||||
"hyperparameters": {
|
||||
"learning_rate": 0.00008560720319413385,
|
||||
"batch_size": 377,
|
||||
"gamma": 0.9203133856027285,
|
||||
"buffer_size": 89884,
|
||||
"max_position_absolute": 3.9089452897651125,
|
||||
"huber_delta": 11.90383664523535,
|
||||
"entropy_coefficient": 0.19524300296372593,
|
||||
"transaction_cost_multiplier": 1.6535717241939198,
|
||||
"use_per": true,
|
||||
"per_alpha": 0.40739064279321924,
|
||||
"per_beta_start": 0.32217756667156455,
|
||||
"use_dueling": true,
|
||||
"dueling_hidden_dim": 128,
|
||||
"n_steps": 3,
|
||||
"tau": 0.005797449214565016,
|
||||
"use_distributional": true,
|
||||
"num_atoms": 101,
|
||||
"v_min": -22.588486330040794,
|
||||
"v_max": 22.588486330040794,
|
||||
"use_noisy_nets": true,
|
||||
"noisy_sigma_init": 0.13482975038367206,
|
||||
"minimum_profit_factor": 1.776393501569344,
|
||||
"weight_decay": 0.00012196287159107966,
|
||||
"kelly_fractional": 0.41178193485773484,
|
||||
"kelly_max_fraction": 0.26835742172433563,
|
||||
"kelly_min_trades": 20,
|
||||
"volatility_window": 23,
|
||||
"use_ensemble_uncertainty": false,
|
||||
"ensemble_size": 5.0,
|
||||
"beta_variance": 0.5,
|
||||
"beta_disagreement": 0.5,
|
||||
"beta_entropy": 0.2,
|
||||
"warmup_ratio": 0.0,
|
||||
"curiosity_weight": 0.0,
|
||||
"td_error_clamp_max": 10.0,
|
||||
"batch_diversity_cooldown": 50.0,
|
||||
"lr_decay_type": 1.0,
|
||||
"sharpe_weight": 0.35540783507315155,
|
||||
"gae_lambda": 0.95,
|
||||
"noisy_sigma_initial": 0.5,
|
||||
"noisy_sigma_final": 0.3,
|
||||
"use_spectral_norm": true,
|
||||
"use_attention": true,
|
||||
"use_residual": true,
|
||||
"norm_type": 1.0,
|
||||
"activation_type": 1.0,
|
||||
"num_quantiles": 32,
|
||||
"qr_kappa": 1.0,
|
||||
"iqn_lambda": 1.5521869540312672,
|
||||
"spectral_norm_sigma_max": 7.971468117287978,
|
||||
"hidden_dim_base": 128,
|
||||
"noisy_epsilon_floor": 0.1,
|
||||
"count_bonus_coefficient": 0.27486235230011086,
|
||||
"cql_alpha": 0.9472537641839289,
|
||||
"eval_softmax_temp": 1.0,
|
||||
"dsr_eta": 0.0013662640295865858,
|
||||
"branch_hidden_dim": 64,
|
||||
"use_branching": true,
|
||||
"gradient_accumulation_steps": 1,
|
||||
"w_dsr": 1.0,
|
||||
"w_pnl": 0.3,
|
||||
"w_dd": 1.0,
|
||||
"w_idle": 0.01,
|
||||
"dd_threshold": 0.01,
|
||||
"loss_aversion": 1.5,
|
||||
"time_decay_rate": 0.0005,
|
||||
"q_gap_threshold": 0.1,
|
||||
"dt_pretrain_epochs": 0
|
||||
},
|
||||
"timestamp": "2026-03-24T15:19:27.798184419+00:00",
|
||||
"gradient_clip_norm": 10.0
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
{
|
||||
"trial_number": 0,
|
||||
"sharpe": 0.007470622658729553,
|
||||
"win_rate": 0.295475004427135,
|
||||
"max_drawdown": 100.0,
|
||||
"total_return": 3395.2147531509395,
|
||||
"hyperparameters": {
|
||||
"learning_rate": 0.00008560720319413385,
|
||||
"batch_size": 377,
|
||||
"gamma": 0.9203133856027285,
|
||||
"buffer_size": 89884,
|
||||
"max_position_absolute": 3.9089452897651125,
|
||||
"huber_delta": 11.90383664523535,
|
||||
"entropy_coefficient": 0.19524300296372593,
|
||||
"transaction_cost_multiplier": 1.6535717241939198,
|
||||
"use_per": true,
|
||||
"per_alpha": 0.40739064279321924,
|
||||
"per_beta_start": 0.32217756667156455,
|
||||
"use_dueling": true,
|
||||
"dueling_hidden_dim": 128,
|
||||
"n_steps": 3,
|
||||
"tau": 0.005797449214565016,
|
||||
"use_distributional": true,
|
||||
"num_atoms": 101,
|
||||
"v_min": -22.588486330040794,
|
||||
"v_max": 22.588486330040794,
|
||||
"use_noisy_nets": true,
|
||||
"noisy_sigma_init": 0.13482975038367206,
|
||||
"minimum_profit_factor": 1.776393501569344,
|
||||
"weight_decay": 0.00012196287159107966,
|
||||
"kelly_fractional": 0.41178193485773484,
|
||||
"kelly_max_fraction": 0.26835742172433563,
|
||||
"kelly_min_trades": 20,
|
||||
"volatility_window": 23,
|
||||
"use_ensemble_uncertainty": false,
|
||||
"ensemble_size": 5.0,
|
||||
"beta_variance": 0.5,
|
||||
"beta_disagreement": 0.5,
|
||||
"beta_entropy": 0.2,
|
||||
"warmup_ratio": 0.0,
|
||||
"curiosity_weight": 0.0,
|
||||
"td_error_clamp_max": 10.0,
|
||||
"batch_diversity_cooldown": 50.0,
|
||||
"lr_decay_type": 1.0,
|
||||
"sharpe_weight": 0.35540783507315155,
|
||||
"gae_lambda": 0.95,
|
||||
"noisy_sigma_initial": 0.5,
|
||||
"noisy_sigma_final": 0.3,
|
||||
"use_spectral_norm": true,
|
||||
"use_attention": true,
|
||||
"use_residual": true,
|
||||
"norm_type": 1.0,
|
||||
"activation_type": 1.0,
|
||||
"num_quantiles": 32,
|
||||
"qr_kappa": 1.0,
|
||||
"iqn_lambda": 1.5521869540312672,
|
||||
"spectral_norm_sigma_max": 7.971468117287978,
|
||||
"hidden_dim_base": 128,
|
||||
"noisy_epsilon_floor": 0.1,
|
||||
"count_bonus_coefficient": 0.27486235230011086,
|
||||
"cql_alpha": 0.9472537641839289,
|
||||
"eval_softmax_temp": 1.0,
|
||||
"dsr_eta": 0.0013662640295865858,
|
||||
"branch_hidden_dim": 64,
|
||||
"use_branching": true,
|
||||
"gradient_accumulation_steps": 1,
|
||||
"w_dsr": 1.0,
|
||||
"w_pnl": 0.3,
|
||||
"w_dd": 1.0,
|
||||
"w_idle": 0.01,
|
||||
"dd_threshold": 0.01,
|
||||
"loss_aversion": 1.5,
|
||||
"time_decay_rate": 0.0005,
|
||||
"q_gap_threshold": 0.1,
|
||||
"dt_pretrain_epochs": 0
|
||||
},
|
||||
"timestamp": "2026-03-24T15:22:41.871557162+00:00",
|
||||
"gradient_clip_norm": 10.0
|
||||
}
|
||||
Reference in New Issue
Block a user