diff --git a/crates/ml/src/trainers/dqn/config.rs b/crates/ml/src/trainers/dqn/config.rs index 6245ebe93..87127813f 100644 --- a/crates/ml/src/trainers/dqn/config.rs +++ b/crates/ml/src/trainers/dqn/config.rs @@ -1119,10 +1119,6 @@ pub struct DQNHyperparameters { /// Production: leave at 0 for full-dataset training. pub max_training_steps_per_epoch: usize, - /// Maximum number of bars to load from training data. - /// 0 = unlimited (load all). CI/smoke: set to 2000-5000 for fast I/O. - pub max_bars: usize, - /// Hidden dimension base for GPU-dynamic network sizing. /// None = use default [256, 128, 64]. Some(base) = [base, base/2, base/4]. pub hidden_dim_base: Option, @@ -1384,7 +1380,6 @@ impl DQNHyperparameters { gpu_timesteps_per_episode: 500, // Default: 500 timesteps per episode avg_spread: 0.0001, // Default: 1bp (ES/NQ futures) max_training_steps_per_epoch: 0, // Default: unlimited (full dataset training) - max_bars: 0, // Default: unlimited (load all bars) // GPU-dynamic network sizing — None means auto-detect from hardware. // H100: optimal_n_episodes fills 132 SMs; hidden_dim_base expanded by hyperopt bounds. diff --git a/crates/ml/src/trainers/dqn/data_loading.rs b/crates/ml/src/trainers/dqn/data_loading.rs index fd57d0c80..870e99f00 100644 --- a/crates/ml/src/trainers/dqn/data_loading.rs +++ b/crates/ml/src/trainers/dqn/data_loading.rs @@ -634,13 +634,6 @@ impl DQNTrainer { ); all_ohlcv_bars.extend(file_bars); - - // CI/smoke: cap total bars for fast I/O - if self.hyperparams.max_bars > 0 && all_ohlcv_bars.len() >= self.hyperparams.max_bars { - all_ohlcv_bars.truncate(self.hyperparams.max_bars); - info!("max_bars={}: truncated to {} bars", self.hyperparams.max_bars, all_ohlcv_bars.len()); - break; - } } if all_ohlcv_bars.is_empty() { diff --git a/crates/ml/tests/smoke_test_real_data.rs b/crates/ml/tests/smoke_test_real_data.rs index b0f4404e1..d8a6f4b1d 100644 --- a/crates/ml/tests/smoke_test_real_data.rs +++ b/crates/ml/tests/smoke_test_real_data.rs @@ -138,13 +138,11 @@ fn try_data(relative: &str) -> Option { #[test] fn smoke_ohlcv_parse_and_extract_features() { - // Single symbol for CI speed - let Some(ohlcv_dir) = try_data("ohlcv/ES.FUT").or_else(|| try_data("ohlcv")) else { return }; + let Some(ohlcv_dir) = try_data("ohlcv") else { return }; - // Load OHLCV bars from .dbn.zst (capped for CI speed) - let mut bars = ml::hyperopt::adapters::dbn_loader::load_bars_from_dbn_dir(&ohlcv_dir) + // Load OHLCV bars from .dbn.zst + let bars = ml::hyperopt::adapters::dbn_loader::load_bars_from_dbn_dir(&ohlcv_dir) .expect("Failed to load OHLCV bars from DBN"); - if bars.len() > 2000 { bars.truncate(2000); } info!(count = bars.len(), dir = %ohlcv_dir.display(), "Loaded OHLCV bars"); assert!(bars.len() > 100, "Expected >100 bars, got {}", bars.len()); @@ -430,14 +428,10 @@ mod gpu_smoke { mbp10_dir: Option<&Path>, stream: &Arc, ) -> Result<(CudaSlice, CudaSlice, usize), anyhow::Error> { - // 1. Load OHLCV bars (capped for CI speed) - const MAX_BARS_CI: usize = 2000; - let mut bars = ml::hyperopt::adapters::dbn_loader::load_bars_from_dbn_dir(ohlcv_dir) + // 1. Load OHLCV bars + let bars = ml::hyperopt::adapters::dbn_loader::load_bars_from_dbn_dir(ohlcv_dir) .expect("Failed to load OHLCV bars"); - if bars.len() > MAX_BARS_CI { - bars.truncate(MAX_BARS_CI); - } - info!(count = bars.len(), "Loaded OHLCV bars (capped to {})", MAX_BARS_CI); + info!(count = bars.len(), "Loaded OHLCV bars"); // 2. Extract 42-dim features let features = ml::features::extract_ml_features(&bars) @@ -521,8 +515,7 @@ mod gpu_smoke { #[test] fn smoke_gpu_real_ohlcv_forward() -> Result<(), anyhow::Error> { - // Use single symbol for CI speed — full dataset is tested via nightly - let Some(ohlcv_dir) = try_data("ohlcv/ES.FUT").or_else(|| try_data("ohlcv")) else { return Ok(()) }; + let Some(ohlcv_dir) = try_data("ohlcv") else { return Ok(()) }; let Some((_device, stream)) = try_cuda() else { return Ok(()) }; let (market_buf, target_buf, num_bars) = @@ -603,7 +596,7 @@ mod gpu_smoke { #[test] fn smoke_gpu_real_ohlcv_plus_ofi_forward() -> Result<(), anyhow::Error> { if !has_enough_vram(2048) { return Ok(()) } - let Some(ohlcv_dir) = try_data("ohlcv/ES.FUT").or_else(|| try_data("ohlcv")) else { return Ok(()) }; + let Some(ohlcv_dir) = try_data("ohlcv") else { return Ok(()) }; let Some(mbp10_dir) = try_data("mbp10/ES.FUT") else { return Ok(()) }; let Some((_device, stream)) = try_cuda() else { return Ok(()) }; @@ -675,7 +668,7 @@ mod gpu_smoke { fn smoke_gpu_real_data_noisy_distributional() -> Result<(), anyhow::Error> { use ml::dqn::branching::{BranchingConfig, BranchingDuelingQNetwork}; - let Some(ohlcv_dir) = try_data("ohlcv/ES.FUT").or_else(|| try_data("ohlcv")) else { return Ok(()) }; + let Some(ohlcv_dir) = try_data("ohlcv") else { return Ok(()) }; let Some((_device, stream)) = try_cuda() else { return Ok(()) }; let (market_buf, target_buf, num_bars) = @@ -786,15 +779,7 @@ async fn smoke_e2e_dqn_training_loop() { return; } let Some(ohlcv_dir) = try_data("ohlcv") else { return }; - // Use single symbol subdir for CI speed — loading all 4 symbols × 9 quarters - // takes minutes just for I/O. One symbol (ES.FUT, ~600K bars) is enough to - // validate the full pipeline: data loading → GPU upload → training → checkpoint. - let es_dir = ohlcv_dir.join("ES.FUT"); - let data_dir = if es_dir.exists() { - es_dir.to_string_lossy().to_string() - } else { - ohlcv_dir.to_string_lossy().to_string() - }; + let data_dir = ohlcv_dir.to_string_lossy().to_string(); // Configure for a fast smoke run: few epochs, small batch, low warmup let mut hyperparams = DQNHyperparameters::conservative(); @@ -817,8 +802,6 @@ async fn smoke_e2e_dqn_training_loop() { // 64 steps x batch_size 32 = 2048 gradient updates — sufficient to validate // finite loss, gradient flow, and action diversity without 400s/epoch overhead. hyperparams.max_training_steps_per_epoch = 64; - // CI: load only 2000 bars (not 600K) — validates pipeline, not data coverage. - hyperparams.max_bars = 2000; let checkpoint_dir = tempfile::tempdir().expect("Failed to create temp dir"); let mut trainer = DQNTrainer::new(hyperparams).expect("Failed to create DQN trainer"); diff --git a/infra/k8s/argo/gpu-test-pipeline-template.yaml b/infra/k8s/argo/gpu-test-pipeline-template.yaml index c9f9c10e0..8653870ef 100644 --- a/infra/k8s/argo/gpu-test-pipeline-template.yaml +++ b/infra/k8s/argo/gpu-test-pipeline-template.yaml @@ -208,9 +208,6 @@ spec: export PATH="${CARGO_HOME}/bin:${PATH}" export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}} - # --- Clean stale test binaries (PVC incremental cache) --- - cargo clean -p ml -p ml-core -p ml-dqn -p ml-ppo 2>/dev/null || true - # --- PTX cache invalidation --- # Purge stale cached PTX if any CUDA kernel source changed since last run. bash scripts/ptx-cache-invalidate.sh "${CARGO_TARGET_DIR}/.ptx_cache"