From 282f3aff7c80228ba6707840b53d633bde6b0930 Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Thu, 19 Mar 2026 21:06:29 +0100
Subject: [PATCH] feat: add max_bars hyperparameter for CI-fast data loading
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

DQNHyperparameters.max_bars caps total bars loaded from .dbn files.
CI smoke test now loads 2000 bars (not 600K) — validates the full
pipeline in seconds instead of minutes of I/O.

Default: 0 (unlimited, for production training).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 crates/ml/src/trainers/dqn/config.rs       | 5 +++++
 crates/ml/src/trainers/dqn/data_loading.rs | 7 +++++++
 crates/ml/tests/smoke_test_real_data.rs    | 2 ++
 3 files changed, 14 insertions(+)

diff --git a/crates/ml/src/trainers/dqn/config.rs b/crates/ml/src/trainers/dqn/config.rs
index 87127813f..6245ebe93 100644
--- a/crates/ml/src/trainers/dqn/config.rs
+++ b/crates/ml/src/trainers/dqn/config.rs
@@ -1119,6 +1119,10 @@ pub struct DQNHyperparameters {
     /// Production: leave at 0 for full-dataset training.
     pub max_training_steps_per_epoch: usize,
 
+    /// Maximum number of bars to load from training data.
+    /// 0 = unlimited (load all). CI/smoke: set to 2000-5000 for fast I/O.
+    pub max_bars: usize,
+
     /// Hidden dimension base for GPU-dynamic network sizing.
     /// None = use default [256, 128, 64]. Some(base) = [base, base/2, base/4].
     pub hidden_dim_base: Option<usize>,
@@ -1380,6 +1384,7 @@ impl DQNHyperparameters {
             gpu_timesteps_per_episode: 500,       // Default: 500 timesteps per episode
             avg_spread: 0.0001,                   // Default: 1bp (ES/NQ futures)
             max_training_steps_per_epoch: 0,      // Default: unlimited (full dataset training)
+            max_bars: 0,                          // Default: unlimited (load all bars)
 
             // GPU-dynamic network sizing — None means auto-detect from hardware.
             // H100: optimal_n_episodes fills 132 SMs; hidden_dim_base expanded by hyperopt bounds.
diff --git a/crates/ml/src/trainers/dqn/data_loading.rs b/crates/ml/src/trainers/dqn/data_loading.rs
index 870e99f00..fd57d0c80 100644
--- a/crates/ml/src/trainers/dqn/data_loading.rs
+++ b/crates/ml/src/trainers/dqn/data_loading.rs
@@ -634,6 +634,13 @@ impl DQNTrainer {
             );
 
             all_ohlcv_bars.extend(file_bars);
+
+            // CI/smoke: cap total bars for fast I/O
+            if self.hyperparams.max_bars > 0 && all_ohlcv_bars.len() >= self.hyperparams.max_bars {
+                all_ohlcv_bars.truncate(self.hyperparams.max_bars);
+                info!("max_bars={}: truncated to {} bars", self.hyperparams.max_bars, all_ohlcv_bars.len());
+                break;
+            }
         }
 
         if all_ohlcv_bars.is_empty() {
diff --git a/crates/ml/tests/smoke_test_real_data.rs b/crates/ml/tests/smoke_test_real_data.rs
index 030e3cff8..4a6f8ba56 100644
--- a/crates/ml/tests/smoke_test_real_data.rs
+++ b/crates/ml/tests/smoke_test_real_data.rs
@@ -812,6 +812,8 @@ async fn smoke_e2e_dqn_training_loop() {
     // 64 steps x batch_size 32 = 2048 gradient updates — sufficient to validate
     // finite loss, gradient flow, and action diversity without 400s/epoch overhead.
     hyperparams.max_training_steps_per_epoch = 64;
+    // CI: load only 2000 bars (not 600K) — validates pipeline, not data coverage.
+    hyperparams.max_bars = 2000;
 
     let checkpoint_dir = tempfile::tempdir().expect("Failed to create temp dir");
     let mut trainer = DQNTrainer::new(hyperparams).expect("Failed to create DQN trainer");