diff --git a/crates/ml/src/cuda_pipeline/mod.rs b/crates/ml/src/cuda_pipeline/mod.rs index 67968a674..b642e8605 100644 --- a/crates/ml/src/cuda_pipeline/mod.rs +++ b/crates/ml/src/cuda_pipeline/mod.rs @@ -132,8 +132,20 @@ pub use gpu_dqn_trainer::{ }; // gpu_replay_buffer moved to ml-dqn crate -/// Maximum bytes allowed for a single GPU upload (2 GB safety limit). -const MAX_UPLOAD_BYTES: usize = 2 * 1024 * 1024 * 1024; +/// Maximum bytes allowed for a single GPU upload (8 GB safety limit). +/// +/// 8 GB is a deliberate budget for both L40S (48 GB total) and H100 (80 GB): +/// - Model weights + Adam moments at hidden_dim=256: ~2 GB +/// - Activations during forward/backward at batch_size=16384: ~4-8 GB +/// - Replay buffer GPU side: <1 GB +/// - CUDA workspace, cuBLAS, etc.: ~4 GB +/// - Training data slabs (this limit): up to 8 GB +/// +/// Total: ~20 GB — well within L40S 48 GB. Pre-2026-05-10 the limit was 2 GB, +/// hardcoded conservatively when the project assumed smaller GPUs. Bumped to +/// accommodate 17.8M+ imbalance bars (3.2 GB) on workflow zgjgc, plus future +/// scale headroom for richer bar densities. +const MAX_UPLOAD_BYTES: usize = 8 * 1024 * 1024 * 1024; /// Round dimension up to nearest multiple of 8 for tensor core alignment. /// @@ -392,7 +404,7 @@ impl DqnGpuData { let estimated_bytes = estimate_vram_bytes(num_bars * (feature_dim + target_dim)); if estimated_bytes > MAX_UPLOAD_BYTES { return Err(MLError::ModelError(format!( - "Training data too large for GPU upload: {:.1} GB > 2.0 GB limit ({} bars)", + "Training data too large for GPU upload: {:.1} GB > 8.0 GB limit ({} bars)", estimated_bytes as f64 / 1_073_741_824.0, num_bars, ))); @@ -838,7 +850,7 @@ impl PpoGpuData { let estimated_bytes = estimate_vram_bytes(num_steps * state_dim); if estimated_bytes > MAX_UPLOAD_BYTES { return Err(MLError::ModelError(format!( - "Market data too large for GPU upload: {:.1} GB > 2.0 GB limit ({} steps × {} dims)", + "Market data too large for GPU upload: {:.1} GB > 8.0 GB limit ({} steps × {} dims)", estimated_bytes as f64 / 1_073_741_824.0, num_steps, state_dim, diff --git a/docs/dqn-wire-up-audit.md b/docs/dqn-wire-up-audit.md index 4b74074a6..07121479c 100644 --- a/docs/dqn-wire-up-audit.md +++ b/docs/dqn-wire-up-audit.md @@ -2,6 +2,10 @@ **Status:** Populated during Plan 1 Task 6 (A.5 orphan audit). Updated on every commit per Invariant 7. +## 2026-05-10 — MAX_UPLOAD_BYTES 2GB → 8GB + +`crates/ml/src/cuda_pipeline/mod.rs:136`. L40S (48GB) and H100 (80GB) have plenty of headroom; the 2GB cap was a conservative leftover that tripped on workflow `zgjgc` with 17.8M imbalance bars (3.2GB needed). 8GB accommodates richer bar densities while leaving ~28GB free on L40S after model + activations + workspace. Updates both `DqnGpuData::upload_slices` (training data) and `PpoGpuData::upload` (market data) — same constant, same error message format. + ## 2026-05-10 — SP20 Phase 2 fix + Phase 3 merge: restore is_win_per_env Cherry-picking Phase 3 commits onto Phase 2 fix tip with `--strategy-option=theirs` dropped the `is_win_per_env` kernel arg + struct field + reset registration that the wr_ema fix branch added (Phase 3 had its own version of the `experience_env_step` signature without that arg). Manual restoration: kernel signature, struct field, alloc, launcher arg, registry entry, reset dispatch arm. Test `sp20_is_win_per_env_registered_fold_reset` passes.