gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add max_bars hyperparameter for CI-fast data loading

Browse Source 
DQNHyperparameters.max_bars caps total bars loaded from .dbn files.
CI smoke test now loads 2000 bars (not 600K) — validates the full
pipeline in seconds instead of minutes of I/O.

Default: 0 (unlimited, for production training).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-19 21:06:29 +01:00
parent 8139911c3d
commit 282f3aff7c
3 changed files with 14 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
crates/ml/src/trainers/dqn/config.rs
View File

@@ -1119,6 +1119,10 @@ pub struct DQNHyperparameters {
/// Production: leave at 0 for full-dataset training. /// Production: leave at 0 for full-dataset training.
pub max_training_steps_per_epoch: usize, pub max_training_steps_per_epoch: usize,
/// Maximum number of bars to load from training data.
/// 0 = unlimited (load all). CI/smoke: set to 2000-5000 for fast I/O.
pub max_bars: usize,
/// Hidden dimension base for GPU-dynamic network sizing. /// Hidden dimension base for GPU-dynamic network sizing.
/// None = use default [256, 128, 64]. Some(base) = [base, base/2, base/4]. /// None = use default [256, 128, 64]. Some(base) = [base, base/2, base/4].
pub hidden_dim_base: Option<usize>, pub hidden_dim_base: Option<usize>,
@@ -1380,6 +1384,7 @@ impl DQNHyperparameters {
gpu_timesteps_per_episode: 500, // Default: 500 timesteps per episode gpu_timesteps_per_episode: 500, // Default: 500 timesteps per episode
avg_spread: 0.0001, // Default: 1bp (ES/NQ futures) avg_spread: 0.0001, // Default: 1bp (ES/NQ futures)
max_training_steps_per_epoch: 0, // Default: unlimited (full dataset training) max_training_steps_per_epoch: 0, // Default: unlimited (full dataset training)
max_bars: 0, // Default: unlimited (load all bars)
// GPU-dynamic network sizing — None means auto-detect from hardware. // GPU-dynamic network sizing — None means auto-detect from hardware.
// H100: optimal_n_episodes fills 132 SMs; hidden_dim_base expanded by hyperopt bounds. // H100: optimal_n_episodes fills 132 SMs; hidden_dim_base expanded by hyperopt bounds.

7
crates/ml/src/trainers/dqn/data_loading.rs
View File

@@ -634,6 +634,13 @@ impl DQNTrainer {
); );
all_ohlcv_bars.extend(file_bars); all_ohlcv_bars.extend(file_bars);
// CI/smoke: cap total bars for fast I/O
if self.hyperparams.max_bars > 0 && all_ohlcv_bars.len() >= self.hyperparams.max_bars {
all_ohlcv_bars.truncate(self.hyperparams.max_bars);
info!("max_bars={}: truncated to {} bars", self.hyperparams.max_bars, all_ohlcv_bars.len());
break;
}
} }
if all_ohlcv_bars.is_empty() { if all_ohlcv_bars.is_empty() {

2
crates/ml/tests/smoke_test_real_data.rs
View File

@@ -812,6 +812,8 @@ async fn smoke_e2e_dqn_training_loop() {
// 64 steps x batch_size 32 = 2048 gradient updates — sufficient to validate // 64 steps x batch_size 32 = 2048 gradient updates — sufficient to validate
// finite loss, gradient flow, and action diversity without 400s/epoch overhead. // finite loss, gradient flow, and action diversity without 400s/epoch overhead.
hyperparams.max_training_steps_per_epoch = 64; hyperparams.max_training_steps_per_epoch = 64;
// CI: load only 2000 bars (not 600K) — validates pipeline, not data coverage.
hyperparams.max_bars = 2000;
let checkpoint_dir = tempfile::tempdir().expect("Failed to create temp dir"); let checkpoint_dir = tempfile::tempdir().expect("Failed to create temp dir");
let mut trainer = DQNTrainer::new(hyperparams).expect("Failed to create DQN trainer"); let mut trainer = DQNTrainer::new(hyperparams).expect("Failed to create DQN trainer");