From 0beccd5e825a1f7542bc4aeef2dbdba3cdb85121 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 22 Apr 2026 02:01:47 +0200 Subject: [PATCH] =?UTF-8?q?fix(smoke):=20multi=5Ffold=5Fconvergence=20?= =?UTF-8?q?=E2=80=94=20laptop-sized=20config?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two wrong-scale assumptions in the test made it unachievable on the RTX 3050 Ti / 4 GB laptop this smoke is meant to run on: 1. `train_baseline_rl` was invoked without `--training-profile`, so it defaulted to `dqn-production`: batch_size=16384, buffer=500k, num_atoms=52, hidden_dim_base=256. Fused-CUDA init OOMs at `kan_d_coeff_per_elem alloc` on 4 GB, leaving `fused_ctx = None` and every subsequent fold failing with "GPU experience collector MUST be active for CUDA training". Fix: pass `--training-profile=dqn-smoketest`. 2. Default walk-forward windows (12 train / 3 val / 3 test / 3 step) only yield 2 folds in the 24-month baseline dataset — fold 2's test-end lands one month past `data_end`. The test's pass-gate is "≥2/3 folds produce a checkpoint", so a test that can only ever generate 2 folds is degenerate. Fix: explicit shorter windows (6 / 2 / 2, step 2) that yield all 3 folds (`6 + 2*2 + 2 + 2 = 14 ≤ 24`, comfortable margin). Also drops `--epochs 20` → `--epochs 5`. Each fold runs ~5500 batches at ~33 s/epoch on this GPU; 20 × 3 folds ≈ 33 min was exceeding the smoke budget (kill observed around the 10-minute mark). 5 epochs is ample for the checkpoint gate — `best_sharpe` saves on the first improving epoch (epoch 1 in practice), so more epochs add no pass/fail signal, only wall-clock. Verified locally: 3/3 folds produce `dqn_fold{N}_best.safetensors`, total wall-clock ~7 min. [MULTI_FOLD] fold 0 checkpoint OK [MULTI_FOLD] fold 1 checkpoint OK [MULTI_FOLD] fold 2 checkpoint OK test result: ok. 1 passed; 0 failed ... finished in 416.36s Docstring updated to reflect new sizing and call out the 4 GB / 24-month constraints explicitly so the next person reading this can see why the numbers are what they are. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../dqn/smoke_tests/multi_fold_convergence.rs | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/crates/ml/src/trainers/dqn/smoke_tests/multi_fold_convergence.rs b/crates/ml/src/trainers/dqn/smoke_tests/multi_fold_convergence.rs index b4edb3967..3711d424f 100644 --- a/crates/ml/src/trainers/dqn/smoke_tests/multi_fold_convergence.rs +++ b/crates/ml/src/trainers/dqn/smoke_tests/multi_fold_convergence.rs @@ -1,15 +1,30 @@ //! Smoke test: multi-fold convergence (fast local variant of Phase 3 L40S gate). //! //! The full Phase 3 gate runs 6 folds × 50 epochs on L40S (~1 hour). This -//! smoke runs 3 folds × 20 epochs locally via the `train_baseline_rl` +//! smoke runs 3 folds × 5 epochs locally via the `train_baseline_rl` //! example binary with `--max-folds 3`. Serves as an early-warning gate //! before spending GPU time on the full L40S run. //! //! ## Pass criteria //! //! - train_baseline_rl exits 0 (no NaN/Inf crash) -//! - At least 2/3 folds have Best val Sharpe > 0 (policy actually learned -//! something on the majority of windows) +//! - At least 2/3 folds produce `dqn_fold{N}_best.safetensors` (policy +//! actually reached a "best val sharpe" epoch on the majority of windows). +//! +//! ## Local-laptop sizing (RTX 3050 Ti, 4 GB VRAM, ~24 months of baseline data) +//! +//! - `--training-profile=dqn-smoketest`: batch_size=64, buffer=256, hidden_dim_base=64. +//! Production defaults (batch 16384, buffer 500k, num_atoms 52) OOM the 4 GB GPU +//! at the `kan_d_coeff_per_elem` allocation during fused-ctx init. +//! - `--train-months=6 --val-months=2 --test-months=2 --step-months=2`: +//! production defaults (12/3/3/3) only fit 2 folds in the 24-month baseline +//! dataset — this is the test's "3 folds" promise, so shrink windows enough +//! that fold 2's test-end lands inside the data (need +//! `train + 2*step + val + test <= 24`; 6+4+2+2 = 14, leaves headroom). +//! - `--epochs=5`: each fold runs ~5500 batches/epoch at ~33s/epoch on this +//! GPU. 20 epochs × 3 folds = ~33 min; 5 × 3 ≈ 8 min fits the smoke budget +//! and is more than enough for the checkpoint gate (best-sharpe saves on +//! the first improving epoch, which is typically epoch 1). //! //! Run: `FOXHUNT_TEST_DATA=test_data/futures-baseline \ //! cargo test -p ml --release --lib -- multi_fold_convergence --ignored --nocapture` @@ -19,7 +34,7 @@ use anyhow::{anyhow, Context, Result}; use std::path::PathBuf; #[test] -#[ignore] // Requires fxcache + substantial runtime (~5 min on RTX 3050) +#[ignore] // Requires fxcache + substantial runtime (~8 min on RTX 3050 Ti) fn test_multi_fold_convergence() -> Result<()> { let data_dir = test_data_dir().expect("FOXHUNT_TEST_DATA or test_data/ must exist"); let output_dir: PathBuf = workspace_root().join("target").join("smoke_multi_fold_output"); @@ -30,6 +45,11 @@ fn test_multi_fold_convergence() -> Result<()> { std::fs::create_dir_all(&output_dir)?; let workspace = workspace_root(); + // See module-level docstring for sizing rationale (profile + walk-forward + // windows + epoch count). In short: dqn-smoketest avoids 4 GB VRAM OOM, + // the 6/2/2 (step 2) walk-forward actually yields 3 folds on the baseline + // dataset (vs production 12/3/3 step 3 which only fits 2), and 5 epochs + // is ample to save a best-sharpe checkpoint per fold. let status = std::process::Command::new("cargo") .current_dir(&workspace) .env("SQLX_OFFLINE", "true") @@ -43,8 +63,13 @@ fn test_multi_fold_convergence() -> Result<()> { "--model", "dqn", "--data-dir", &data_dir, "--symbol", "ES.FUT", - "--epochs", "20", + "--epochs", "5", "--max-folds", "3", + "--training-profile", "dqn-smoketest", + "--train-months", "6", + "--val-months", "2", + "--test-months", "2", + "--step-months", "2", "--output-dir", output_dir.to_str().unwrap(), ]) .status()