feat: add max_bars hyperparameter for CI-fast data loading
DQNHyperparameters.max_bars caps total bars loaded from .dbn files. CI smoke test now loads 2000 bars (not 600K) — validates the full pipeline in seconds instead of minutes of I/O. Default: 0 (unlimited, for production training). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -1119,6 +1119,10 @@ pub struct DQNHyperparameters {
|
||||
/// Production: leave at 0 for full-dataset training.
|
||||
pub max_training_steps_per_epoch: usize,
|
||||
|
||||
/// Maximum number of bars to load from training data.
|
||||
/// 0 = unlimited (load all). CI/smoke: set to 2000-5000 for fast I/O.
|
||||
pub max_bars: usize,
|
||||
|
||||
/// Hidden dimension base for GPU-dynamic network sizing.
|
||||
/// None = use default [256, 128, 64]. Some(base) = [base, base/2, base/4].
|
||||
pub hidden_dim_base: Option<usize>,
|
||||
@@ -1380,6 +1384,7 @@ impl DQNHyperparameters {
|
||||
gpu_timesteps_per_episode: 500, // Default: 500 timesteps per episode
|
||||
avg_spread: 0.0001, // Default: 1bp (ES/NQ futures)
|
||||
max_training_steps_per_epoch: 0, // Default: unlimited (full dataset training)
|
||||
max_bars: 0, // Default: unlimited (load all bars)
|
||||
|
||||
// GPU-dynamic network sizing — None means auto-detect from hardware.
|
||||
// H100: optimal_n_episodes fills 132 SMs; hidden_dim_base expanded by hyperopt bounds.
|
||||
|
||||
@@ -634,6 +634,13 @@ impl DQNTrainer {
|
||||
);
|
||||
|
||||
all_ohlcv_bars.extend(file_bars);
|
||||
|
||||
// CI/smoke: cap total bars for fast I/O
|
||||
if self.hyperparams.max_bars > 0 && all_ohlcv_bars.len() >= self.hyperparams.max_bars {
|
||||
all_ohlcv_bars.truncate(self.hyperparams.max_bars);
|
||||
info!("max_bars={}: truncated to {} bars", self.hyperparams.max_bars, all_ohlcv_bars.len());
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if all_ohlcv_bars.is_empty() {
|
||||
|
||||
@@ -812,6 +812,8 @@ async fn smoke_e2e_dqn_training_loop() {
|
||||
// 64 steps x batch_size 32 = 2048 gradient updates — sufficient to validate
|
||||
// finite loss, gradient flow, and action diversity without 400s/epoch overhead.
|
||||
hyperparams.max_training_steps_per_epoch = 64;
|
||||
// CI: load only 2000 bars (not 600K) — validates pipeline, not data coverage.
|
||||
hyperparams.max_bars = 2000;
|
||||
|
||||
let checkpoint_dir = tempfile::tempdir().expect("Failed to create temp dir");
|
||||
let mut trainer = DQNTrainer::new(hyperparams).expect("Failed to create DQN trainer");
|
||||
|
||||
Reference in New Issue
Block a user