diff --git a/config/training/dqn-hyperopt.toml b/config/training/dqn-hyperopt.toml index 0247bf045..e7cb50794 100644 --- a/config/training/dqn-hyperopt.toml +++ b/config/training/dqn-hyperopt.toml @@ -92,7 +92,7 @@ exit_timing_weight = [0.0, 0.2] ofi_reward_weight = [0.0, 0.5] kelly_sizing_weight = [0.0, 0.5] reward_noise_scale = [0.01, 0.1] -exposure_aux_weight = [0.1, 1.0] +exposure_aux_weight = [0.001, 0.1] # v8 search ranges micro_reward_scale = [0.0, 0.05] @@ -139,7 +139,7 @@ exit_timing_weight = 0.05 ofi_reward_weight = 0.2 kelly_sizing_weight = 0.1 reward_noise_scale = 0.05 -exposure_aux_weight = 0.5 +exposure_aux_weight = 0.01 exposure_aux_warmup_epochs = 5 [fixed] @@ -182,5 +182,5 @@ exit_timing_weight = 0.05 ofi_reward_weight = 0.2 kelly_sizing_weight = 0.1 reward_noise_scale = 0.05 -exposure_aux_weight = 0.5 +exposure_aux_weight = 0.01 exposure_aux_warmup_epochs = 5 diff --git a/config/training/dqn-localdev.toml b/config/training/dqn-localdev.toml index 62a98524b..dacb6f82f 100644 --- a/config/training/dqn-localdev.toml +++ b/config/training/dqn-localdev.toml @@ -131,5 +131,5 @@ exit_timing_weight = 0.05 ofi_reward_weight = 0.2 kelly_sizing_weight = 0.1 reward_noise_scale = 0.05 -exposure_aux_weight = 0.5 +exposure_aux_weight = 0.01 exposure_aux_warmup_epochs = 5 diff --git a/config/training/dqn-production.toml b/config/training/dqn-production.toml index 4259fc8cf..f10e07db7 100644 --- a/config/training/dqn-production.toml +++ b/config/training/dqn-production.toml @@ -142,5 +142,5 @@ exit_timing_weight = 0.05 ofi_reward_weight = 0.2 kelly_sizing_weight = 0.1 reward_noise_scale = 0.05 -exposure_aux_weight = 0.5 +exposure_aux_weight = 0.01 exposure_aux_warmup_epochs = 5 diff --git a/config/training/dqn-smoketest.toml b/config/training/dqn-smoketest.toml index 68931a4f9..7ad05e31e 100644 --- a/config/training/dqn-smoketest.toml +++ b/config/training/dqn-smoketest.toml @@ -138,7 +138,7 @@ exit_timing_weight = 0.05 ofi_reward_weight = 0.2 kelly_sizing_weight = 0.1 reward_noise_scale = 0.05 -exposure_aux_weight = 0.5 +exposure_aux_weight = 0.01 exposure_aux_warmup_epochs = 5 [walk_forward] diff --git a/crates/ml/src/trainers/dqn/config.rs b/crates/ml/src/trainers/dqn/config.rs index 4791e8b0c..247ed1851 100644 --- a/crates/ml/src/trainers/dqn/config.rs +++ b/crates/ml/src/trainers/dqn/config.rs @@ -1653,7 +1653,7 @@ impl DQNHyperparameters { ofi_reward_weight: 0.2, kelly_sizing_weight: 0.1, reward_noise_scale: 0.05, - exposure_aux_weight: 0.5, + exposure_aux_weight: 0.01, exposure_aux_warmup_epochs: 5, // v8: Dense micro-reward, TD(λ), PopArt, curriculum, hindsight micro_reward_scale: 0.01, diff --git a/crates/ml/src/trainers/dqn/fused_training.rs b/crates/ml/src/trainers/dqn/fused_training.rs index 4e35d6e9d..63bbf618a 100644 --- a/crates/ml/src/trainers/dqn/fused_training.rs +++ b/crates/ml/src/trainers/dqn/fused_training.rs @@ -1061,6 +1061,10 @@ impl FusedTrainingCtx { ) { tracing::warn!("Exposure aux grad failed (non-fatal): {e}"); } + // Clip combined gradient to prevent aux GEMM backward from dominating. + // Same pattern as CQL gradient budget — keeps total norm within max_grad_norm. + self.trainer.clip_grad_buf_inplace(self.trainer.config().max_grad_norm) + .map_err(|e| anyhow::anyhow!("Aux gradient clip: {e}"))?; } Ok(()) diff --git a/docs/superpowers/plans/2026-04-08-h100-graph-optimizations.md b/docs/superpowers/plans/2026-04-08-h100-graph-optimizations.md new file mode 100644 index 000000000..44b642090 --- /dev/null +++ b/docs/superpowers/plans/2026-04-08-h100-graph-optimizations.md @@ -0,0 +1,486 @@ +# H100 Graph-Level Optimizations — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Implement 4 remaining H100 performance optimizations saving ~13s/epoch. Eliminates validation blocking, fuses training graphs, captures aux GEMM. + +**Architecture:** Fix 2 removes unnecessary synchronization. Fix 3 adds async validation on a separate stream. Fix 4 captures the aux GEMM in graph_aux via GPU-resident scalar. Fix 7 merges spectral+forward+aux into one mega-graph. All GPU-side, no CPU path. + +**Tech Stack:** CUDA driver API (cuEventRecord, cuStreamWaitEvent), cudarc, cuBLAS, CUDA Graphs + +**IMPORTANT CORRECTION:** The backtest evaluator does NOT use CUDA graphs for its step loop (500K+ launches would SIGSEGV — see comment at gpu_backtest_evaluator.rs:904). `invalidate_dqn_graph()` is already a no-op. Fix 2 focuses on removing the blocking `stream.synchronize()` call instead. + +--- + +### Task 1: Fix 2 — Remove Validation Sync Stall + Unnecessary Invalidation + +**Files:** +- Modify: `crates/ml/src/trainers/dqn/trainer/metrics.rs` +- Modify: `crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs` + +- [ ] **Step 1: Remove `invalidate_dqn_graph()` call in metrics.rs** + +Find `evaluator.invalidate_dqn_graph()` in `compute_validation_loss()` (around line 510). Delete the line — the evaluator doesn't use graphs for its step loop, so this is a no-op. + +- [ ] **Step 2: Replace `stream.synchronize()` with event-based wait in evaluator** + +In `gpu_backtest_evaluator.rs`, find `evaluate_dqn_graphed()` (line ~874). The method calls `self.stream.synchronize()` at line 882. This is a CPU stall that blocks until ALL GPU work completes. + +Replace: +```rust + // Sync stream and drain any CUDA errors before evaluator starts. + if let Err(sync_err) = self.stream.synchronize() { + eprintln!("EVALUATOR PRE-SYNC: training left stale CUDA error: {sync_err}"); + } +``` + +With: +```rust + // Drain stale CUDA errors without blocking (no CPU sync needed — + // the caller ensures training is complete via event wait before calling) + let _ = self.stream.context().check_err(); +``` + +The caller (Fix 3's async validation) will use `cuStreamWaitEvent` to ensure training completes before validation starts — no CPU sync needed. + +- [ ] **Step 3: Verify build + test** + +```bash +SQLX_OFFLINE=true cargo check -p ml 2>&1 | tail -5 +SQLX_OFFLINE=true cargo test -p ml --lib 2>&1 | grep "^test result:" +``` + +- [ ] **Step 4: Commit** + +```bash +git add crates/ml/src/trainers/dqn/trainer/metrics.rs \ + crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs +git commit -m "perf: remove validation sync stall + no-op invalidate_dqn_graph call" +``` + +--- + +### Task 2: Fix 4 — Aux GEMM Kernel Sig Change + GPU Scalar Buffer + +**Files:** +- Modify: `crates/ml/src/cuda_pipeline/dqn_utility_kernels.cu` +- Modify: `crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs` + +- [ ] **Step 1: Change kernel signature from scalar to pointer** + +In `dqn_utility_kernels.cu`, find `exposure_aux_grad_kernel`. Change the `aux_weight` parameter from scalar to pointer: + +```cuda +// BEFORE: + float aux_weight, + +// AFTER: + const float* __restrict__ aux_weight_ptr, +``` + +Inside the kernel body, add as the first line after the bounds check: +```cuda + float aux_weight = *aux_weight_ptr; +``` + +- [ ] **Step 2: Add GPU scalar buffer in GpuDqnTrainer** + +In `gpu_dqn_trainer.rs`, add field: +```rust + pub(crate) exposure_aux_weight_gpu: CudaSlice, +``` + +Allocate in constructor (near other buffer allocations): +```rust + let exposure_aux_weight_gpu = stream.alloc_zeros::(1) + .map_err(|e| MLError::ModelError(format!("alloc exposure_aux_weight_gpu: {e}")))?; +``` + +Store in struct literal: +```rust + exposure_aux_weight_gpu, +``` + +- [ ] **Step 3: Update launch method to pass pointer** + +In `launch_exposure_aux_grad()`, find `.arg(&aux_weight)`. Replace: + +```rust +// BEFORE: + .arg(&aux_weight) + +// AFTER: + .arg(&self.exposure_aux_weight_gpu.raw_ptr()) +``` + +Add a method to update the GPU scalar: +```rust + /// Update the GPU-resident aux weight scalar (call before graph replay). + pub fn set_exposure_aux_weight(&mut self, weight: f32) -> Result<(), MLError> { + let val = [weight]; + unsafe { + cudarc::driver::sys::cuMemcpyHtoDAsync_v2( + self.exposure_aux_weight_gpu.raw_ptr(), + val.as_ptr().cast(), + std::mem::size_of::(), + self.stream.cu_stream(), + ); + } + Ok(()) + } +``` + +- [ ] **Step 4: Verify build** + +```bash +SQLX_OFFLINE=true cargo check -p ml 2>&1 | tail -5 +``` + +- [ ] **Step 5: Commit** + +```bash +git add crates/ml/src/cuda_pipeline/dqn_utility_kernels.cu \ + crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs +git commit -m "perf: aux GEMM kernel takes GPU scalar pointer (enables graph capture)" +``` + +--- + +### Task 3: Fix 4 continued — Move Aux GEMM into submit_aux_ops + Fix 7 Mega-Graph + +**Files:** +- Modify: `crates/ml/src/trainers/dqn/fused_training.rs` + +This task combines Fix 4 (move aux into submit_aux_ops) and Fix 7 (mega-graph fusion) since they both modify `run_full_step()` and `submit_aux_ops()`. + +- [ ] **Step 1: Move aux GEMM from run_full_step into submit_aux_ops** + +In `fused_training.rs`, find the exposure aux GEMM block in `run_full_step()` (line ~826-835): +```rust + // v7.1: Exposure auxiliary gradient (breaks i%3 degeneracy) + if self.exposure_aux_weight > 1e-6 { + ... + } +``` + +CUT this block. PASTE it at the END of `submit_aux_ops()`, before `Ok(())`. + +- [ ] **Step 2: Update aux weight via GPU scalar before graph replay** + +In `run_full_step()`, find where `graph_aux` is replayed (line ~792-794): +```rust + if self.graph_aux.is_some() { + self.pre_replay_state_update(agent); +``` + +Add the GPU scalar update BEFORE the graph replay: +```rust + if self.graph_aux.is_some() { + self.pre_replay_state_update(agent); + // Update GPU-resident aux weight scalar for graphed kernel + self.trainer.set_exposure_aux_weight(self.exposure_aux_weight as f32)?; + let graph_aux = self.graph_aux.as_ref().unwrap(); + graph_aux.launch(self.stream.cu_stream())?; +``` + +- [ ] **Step 3: Add graph_mega field and capture method** + +Add field: +```rust + graph_mega: Option, +``` + +Initialize as `None` in constructor. + +Add `capture_graph_mega`: +```rust + fn capture_graph_mega( + &mut self, + agent: &mut DQNAgentType, + gpu_batch: &crate::dqn::replay_buffer_type::GpuBatch, + ) -> Result<()> { + self.trainer.sync_all_streams() + .map_err(|e| anyhow::anyhow!("pre-mega-capture sync: {e}"))?; + let _ = self.stream.context().check_err(); + + let cu_stream = self.stream.cu_stream(); + let mut graph: cuda_sys::CUgraph = std::ptr::null_mut(); + let mut exec: cuda_sys::CUgraphExec = std::ptr::null_mut(); + + let begin_result = unsafe { + cuda_sys::cuStreamBeginCapture_v2( + cu_stream, + cuda_sys::CUstreamCaptureMode::CU_STREAM_CAPTURE_MODE_THREAD_LOCAL, + ) + }; + if begin_result != cuda_sys::cudaError_enum::CUDA_SUCCESS { + anyhow::bail!("cuStreamBeginCapture_v2 (mega) failed: {begin_result:?}"); + } + + // Phase 1: Spectral norm + self.trainer.apply_spectral_norm(&self.online_dueling, &self.online_branching) + .map_err(|e| anyhow::anyhow!("mega capture spectral: {e}"))?; + + // Phase 2: Forward + backward + self.trainer.submit_forward_ops_main() + .map_err(|e| anyhow::anyhow!("mega capture forward: {e}"))?; + + // Phase 3: Aux ops (includes aux GEMM from Fix 4) + self.submit_aux_ops(agent, gpu_batch)?; + + let end_result = unsafe { cuda_sys::cuStreamEndCapture(cu_stream, &mut graph) }; + if end_result != cuda_sys::cudaError_enum::CUDA_SUCCESS { + anyhow::bail!("cuStreamEndCapture (mega) failed: {end_result:?}"); + } + if graph.is_null() { + anyhow::bail!("cuStreamEndCapture (mega) returned null graph"); + } + + let inst_result = unsafe { cuda_sys::cuGraphInstantiateWithFlags(&mut exec, graph, 0) }; + if inst_result != cuda_sys::cudaError_enum::CUDA_SUCCESS { + unsafe { cuda_sys::cuGraphDestroy(graph); } + anyhow::bail!("cuGraphInstantiateWithFlags (mega) failed: {inst_result:?}"); + } + + info!("graph_mega captured: spectral + forward + backward + aux ops (single replay)"); + self.graph_mega = Some(RawCudaGraph { exec, graph }); + // Supersede individual graphs + self.graph_spectral = None; + self.graph_forward = None; + self.graph_forward_mse = None; + self.graph_forward_ddqn = None; + self.graph_aux = None; + Ok(()) + } +``` + +- [ ] **Step 4: Update run_full_step to use graph_mega** + +Replace the current step sequence (spectral → forward → aux) with: + +```rust + // ── Mega-graph: spectral + forward + backward + aux in ONE replay ── + if let Some(ref graph) = self.graph_mega { + self.pre_replay_state_update(agent); + self.trainer.set_exposure_aux_weight(self.exposure_aux_weight as f32)?; + self.update_stochastic_depth_mask()?; + // Adam step counter update (outside graph — value captured by address) + self.trainer.adam_step += 1; + unsafe { + cudarc::driver::sys::cuMemcpyHtoDAsync_v2( + self.trainer.t_buf.raw_ptr(), + (&self.trainer.adam_step as *const i32).cast(), + std::mem::size_of::(), + self.stream.cu_stream(), + ); + } + graph.launch(self.stream.cu_stream())?; + } else { + // Steps 0-1: ungraphed execution (original path) + // Step 1: Spectral norm + if let Some(ref graph) = self.graph_spectral { + graph.launch(self.stream.cu_stream())?; + } else { + self.trainer.apply_spectral_norm(&self.online_dueling, &self.online_branching)?; + if step == 1 { + if let Err(e) = self.capture_graph_spectral() { + tracing::warn!("graph_spectral capture failed: {e}"); + } + } + } + + // Step 2: Forward + let _fused_placeholder = self.trainer.train_step_gpu( + gpu_batch, &self.online_dueling, &self.online_branching, + &self.target_dueling, &self.target_branching, + )?; + self.trainer.run_nan_checks_post_forward(self.batch_size)?; + + // Step 2b: HER donors + // ... (existing HER code stays) ... + + // Step 3: Aux ops + if self.graph_aux.is_some() { + self.pre_replay_state_update(agent); + self.trainer.set_exposure_aux_weight(self.exposure_aux_weight as f32)?; + self.graph_aux.as_ref().unwrap().launch(self.stream.cu_stream())?; + } else { + self.submit_aux_ops(agent, gpu_batch)?; + } + + // Capture mega-graph at step 2 + if self.steps_since_varmap_sync == 2 { + if let Err(e) = self.capture_graph_mega(agent, gpu_batch) { + tracing::warn!("graph_mega capture failed (non-fatal, continuing with individual graphs): {e}"); + } + } + } +``` + +Keep everything AFTER this block unchanged (conditional ops, grad_norm, adam, PER update). + +- [ ] **Step 5: Add graph_mega to reset_for_fold** + +Find `reset_for_fold()`. Add: +```rust + self.graph_mega = None; +``` + +- [ ] **Step 6: Verify build** + +```bash +SQLX_OFFLINE=true cargo check -p ml 2>&1 | tail -5 +``` + +- [ ] **Step 7: Commit** + +```bash +git add crates/ml/src/trainers/dqn/fused_training.rs +git commit -m "perf: mega-graph fusion (spectral+forward+aux → 1 launch) + aux GEMM in graph" +``` + +--- + +### Task 4: Fix 3 — Async Validation on Separate Stream + +**Files:** +- Modify: `crates/ml/src/trainers/dqn/trainer/mod.rs` +- Modify: `crates/ml/src/trainers/dqn/trainer/training_loop.rs` +- Modify: `crates/ml/src/trainers/dqn/trainer/metrics.rs` +- Modify: `crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs` + +- [ ] **Step 1: Add validation stream + event fields to DQNTrainer** + +In `mod.rs`, find the struct fields. Add: + +```rust + /// Dedicated CUDA stream for async validation (overlaps with next epoch). + pub(crate) validation_stream: Option>, + /// Event recorded after training — validation waits on this. + pub(crate) validation_event: Option, + /// Pending validation result from previous epoch (computed asynchronously). + pub(crate) pending_val_loss: Option, +``` + +Initialize in constructor (find where `cuda_stream` is stored): +```rust + validation_stream: if let Some(ref s) = cuda_stream { + match s.context().new_stream() { + Ok(vs) => Some(Arc::new(vs)), + Err(e) => { tracing::warn!("validation stream alloc failed: {e}"); None } + } + } else { None }, + validation_event: if cuda_stream.is_some() { + let mut event: cudarc::driver::sys::CUevent = std::ptr::null_mut(); + let result = unsafe { cudarc::driver::sys::cuEventCreate(&mut event, 0) }; + if result == cudarc::driver::sys::CUresult::CUDA_SUCCESS { + Some(event) + } else { None } + } else { None }, + pending_val_loss: None, +``` + +- [ ] **Step 2: Make validation async in epoch loop** + +In `training_loop.rs`, find where `compute_validation_loss()` is called in the epoch loop. It's inside `process_epoch_boundary()` or `log_epoch_metrics_and_financials()`. + +Find the validation call. Wrap it with async launch: + +At the START of each epoch iteration (after `self.current_epoch = epoch`), add: +```rust + // Collect pending async validation from PREVIOUS epoch + if let Some(val) = self.pending_val_loss.take() { + // Store for this epoch's logging (one-epoch delay is acceptable) + self.cached_async_val_loss = Some(val); + } +``` + +At the END of the epoch (after training completes, before the next loop iteration), add: +```rust + // Launch validation asynchronously on separate stream + if let (Some(ref val_stream), Some(event)) = (&self.validation_stream, self.validation_event) { + if let Some(ref main_stream) = self.cuda_stream { + // Record training completion on main stream + unsafe { + cudarc::driver::sys::cuEventRecord(event, main_stream.cu_stream()); + // Validation stream waits for training to finish + cudarc::driver::sys::cuStreamWaitEvent(val_stream.cu_stream(), event, 0); + } + // Validation runs on stream 2 — main stream continues to next epoch + match self.compute_validation_loss().await { + Ok(val) => { self.pending_val_loss = Some(val); } + Err(e) => { tracing::warn!("Async validation failed: {e}"); } + } + } + } +``` + +Add field: +```rust + pub(crate) cached_async_val_loss: Option, +``` + +- [ ] **Step 3: Pass validation stream to evaluator** + +In `metrics.rs`, find `compute_validation_loss()`. The evaluator is created with `stream` from `self.cuda_stream`. Change to use `self.validation_stream` if available: + +```rust + let eval_stream = self.validation_stream.as_ref() + .unwrap_or_else(|| self.cuda_stream.as_ref().unwrap()); +``` + +Pass `eval_stream` when creating the evaluator: +```rust + let evaluator = GpuBacktestEvaluator::new( + &window_prices, &window_features, feature_dim, config, eval_stream, + )?; +``` + +- [ ] **Step 4: Verify build + GPU smoke test** + +```bash +SQLX_OFFLINE=true cargo check -p ml 2>&1 | tail -5 +FOXHUNT_TEST_DATA=test_data/futures-baseline SQLX_OFFLINE=true cargo test -p ml --lib -- smoke_tests::walk_forward --ignored --nocapture 2>&1 | tail -5 +``` + +- [ ] **Step 5: Commit** + +```bash +git add crates/ml/src/trainers/dqn/trainer/mod.rs \ + crates/ml/src/trainers/dqn/trainer/training_loop.rs \ + crates/ml/src/trainers/dqn/trainer/metrics.rs \ + crates/ml/src/cuda_pipeline/gpu_backtest_evaluator.rs +git commit -m "perf: async validation on separate CUDA stream — overlaps with experience collection" +``` + +--- + +### Task 5: Final Build + GPU Smoke Test + +- [ ] **Step 1: Full workspace build** + +```bash +SQLX_OFFLINE=true cargo check --workspace 2>&1 | tail -5 +``` + +- [ ] **Step 2: Full test suite** + +```bash +SQLX_OFFLINE=true cargo test -p ml --lib 2>&1 | grep "^test result:" +``` +Expected: 900+ passed, 0 failed + +- [ ] **Step 3: GPU smoke test on RTX 3050** + +```bash +FOXHUNT_TEST_DATA=test_data/futures-baseline SQLX_OFFLINE=true cargo test -p ml --lib -- smoke_tests --ignored --nocapture 2>&1 | grep -E "^test result:|FAILED|SIGSEGV" +``` +Expected: all pass, no SIGSEGV + +- [ ] **Step 4: Commit + push** + +```bash +git add -A +git commit -m "fix: final test pass for H100 graph optimizations" +git push origin main +```