gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: remove max_training_steps_per_epoch — always train full dataset

Browse Source 
Epoch duration self-balances: bigger GPU → bigger auto-scaled batch →
fewer steps per epoch. The manual cap created 7 different values
(0, 8, 64, 100, 200, 300, 2000) across configs/tests/examples, making
behavior inconsistent between environments.

Removed from: DQNHyperparameters, training profiles (smoketest,
localdev, production), CLI args, Argo templates, hyperopt adapter,
all test overrides, supervised example.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-02 14:41:16 +02:00
parent 5fe6acc99b
commit 9aad6ff60e
14 changed files with 5 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
config/training/dqn-localdev.toml
View File

@@ -12,7 +12,6 @@ weight_decay = 0.0001
adam_epsilon = 1e-8
warmup_steps = 0
hidden_dim_base = 64
max_steps_per_epoch = 200
reward_scale = 1.0
huber_delta = 1.0
lr_decay_type = 2

1
config/training/dqn-production.toml
View File

@@ -15,7 +15,6 @@ weight_decay = 0.0001
adam_epsilon = 1e-8
warmup_steps = 0
hidden_dim_base = 256
max_steps_per_epoch = 2000
reward_scale = 1.0
huber_delta = 1.0
lr_decay_type = 2

1
config/training/dqn-smoketest.toml
View File

@@ -18,7 +18,6 @@ weight_decay = 0.0001
adam_epsilon = 1e-8
warmup_steps = 0
hidden_dim_base = 64
max_steps_per_epoch = 200
reward_scale = 1.0
huber_delta = 1.0
data_source = "mbp10"

11
crates/ml/examples/train_baseline_rl.rs
View File

@@ -179,11 +179,6 @@ struct Args {
#[arg(long, default_value_t = 1e-4)]
learning_rate: f64,
/// Max environment steps per epoch (caps trajectory length for OOM safety;
/// 0 = use all bars, but this can use >1GB RAM per fold on large datasets)
#[arg(long, default_value_t = 2000)]
max_steps_per_epoch: usize,
/// Symbol subdirectory to load (e.g. "ES.FUT", "NQ.FUT")
#[arg(long, default_value = "ES.FUT")]
symbol: String,
@@ -562,7 +557,6 @@ fn train_dqn_fold(
replay_buffer_vram_fraction: gpu_profile.training.replay_buffer_vram_fraction,
// GPU experience collection: n_episodes auto-scales from VRAM
gpu_timesteps_per_episode: gpu_profile.experience.gpu_timesteps_per_episode,
max_training_steps_per_epoch: args.max_steps_per_epoch,
..DQNHyperparameters::default()
};
@@ -573,7 +567,6 @@ fn train_dqn_fold(
// CLI args override profile: re-apply any arg that the user can set explicitly.
hyperparams.epochs = args.epochs;
hyperparams.learning_rate = hp_f64(hp, "learning_rate").unwrap_or(args.learning_rate);
hyperparams.max_training_steps_per_epoch = args.max_steps_per_epoch;
hyperparams.initial_capital = args.initial_capital as f32;
if let Some(mhb) = args.min_hold_bars {
hyperparams.min_hold_bars = mhb;
@@ -702,9 +695,7 @@ fn train_ppo_fold(
vf_coef: hp_f64(hp, "value_loss_coeff").unwrap_or(0.5) as f32,
ent_coef: hp_f64(hp, "entropy_coeff").unwrap_or(0.01) as f32,
gae_lambda: hp_f64(hp, "gae_lambda").unwrap_or(0.95) as f32,
rollout_steps: hp_usize(hp, "rollout_steps").unwrap_or(
if args.max_steps_per_epoch > 0 { args.max_steps_per_epoch } else { 2048 }
),
rollout_steps: hp_usize(hp, "rollout_steps").unwrap_or(2048),
minibatch_size: hp_usize(hp, "minibatch_size").unwrap_or(64),
epochs: args.epochs,
early_stopping_enabled: true,

11
crates/ml/examples/train_baseline_supervised.rs
View File

@@ -166,10 +166,6 @@ struct Args {
#[arg(long, default_value_t = 51)]
feature_dim: usize,
/// Max training steps per epoch (0 = use all bars)
#[arg(long, default_value_t = 2000)]
max_steps_per_epoch: usize,
/// Output directory for trained model checkpoints
#[arg(long, default_value = "ml/trained_models")]
output_dir: PathBuf,
@@ -489,12 +485,7 @@ fn build_vector_pairs(
) -> Result<Vec<(Vec<f32>, Vec<f32>)>> {
let mut pairs = Vec::new();
let n = norm_features.len();
let limit = n.saturating_sub(1);
let step_limit = if args.max_steps_per_epoch > 0 {
args.max_steps_per_epoch.min(limit)
} else {
limit
};
let step_limit = n.saturating_sub(1);
for i in 0..step_limit {
let Some(feat) = norm_features.get(i) else {

1
crates/ml/src/hyperopt/adapters/dqn.rs
View File

@@ -2042,7 +2042,6 @@ impl HyperparameterOptimizable for DQNTrainer {
hyperparams.mbp10_data_dir = self.mbp10_data_dir.clone().unwrap_or_default();
hyperparams.trades_data_dir = self.trades_data_dir.clone().unwrap_or_default();
hyperparams.max_training_steps_per_epoch = if budget.gpu_memory_mb >= 40960 { 2000 } else { 200 };
// AutoReplaySizer VRAM fraction
hyperparams.replay_buffer_vram_fraction = {
let raw = ((budget.gpu_memory_mb as f64 - 8192.0)

1
crates/ml/src/hyperopt/campaign.rs
View File

@@ -98,7 +98,6 @@ impl CampaignConfig {
}
/// DQN local dev campaign (10 trials × 20 epochs — RTX 3050 friendly).
/// ~15 min on RTX 3050 with max_training_steps_per_epoch=200.
pub fn dqn_localdev() -> Self {
Self {
model_type: ModelType::DQN,

7
crates/ml/src/trainers/dqn/config.rs
View File

@@ -1179,10 +1179,8 @@ pub struct DQNHyperparameters {
pub avg_spread: f64,
/// Maximum training steps per epoch (0 = unlimited, uses full dataset).
/// Caps `training_data.len() / batch_size` to reduce epoch walltime.
/// CI/smoke: set to 50-100 for fast validation without full dataset sweeps.
/// Production: leave at 0 for full-dataset training.
pub max_training_steps_per_epoch: usize,
// max_training_steps_per_epoch removed — always train on full dataset.
// Epoch duration self-balances: bigger GPU → bigger batch → fewer steps.
/// Hidden dimension base for GPU-dynamic network sizing.
/// None = use default [256, 128, 64]. Some(base) = [base, base/2, base/4].
@@ -1680,7 +1678,6 @@ impl DQNHyperparameters {
// Phase 3: GPU experience collection
gpu_timesteps_per_episode: 500, // Default: 500 timesteps per episode
avg_spread: 0.0001, // Default: 1bp (ES/NQ futures)
max_training_steps_per_epoch: 0, // Default: unlimited (full dataset training)
// GPU-dynamic network sizing — None means auto-detect from hardware.
// H100: optimal_n_episodes fills 132 SMs; hidden_dim_base expanded by hyperopt bounds.

4
crates/ml/src/trainers/dqn/trainer/training_loop.rs
View File

@@ -1285,9 +1285,7 @@ impl DQNTrainer {
) -> Result<usize> {
let batch_size = self.hyperparams.batch_size;
let num_training_steps = if self.can_train().await? {
let full_steps = (training_data.len() / batch_size).max(1);
let cap = self.hyperparams.max_training_steps_per_epoch;
if cap > 0 { full_steps.min(cap) } else { full_steps }
(training_data.len() / batch_size).max(1)
} else {
0
};

7
crates/ml/src/training_profile.rs
View File

@@ -60,8 +60,6 @@ pub struct TrainingSection {
pub warmup_steps: Option<usize>,
pub gradient_clip_norm: Option<f64>,
pub weight_decay: Option<f64>,
/// Maps to `DQNHyperparameters::max_training_steps_per_epoch`.
pub max_steps_per_epoch: Option<usize>,
pub hidden_dim_base: Option<usize>,
/// Reward scale factor for v_range computation.
/// v_range = (reward_scale / (1 - gamma) * 1.2).clamp(20.0, 300.0)
@@ -695,10 +693,6 @@ impl DqnTrainingProfile {
if let Some(v) = t.weight_decay {
hp.weight_decay = v;
}
// TOML: max_steps_per_epoch → hp: max_training_steps_per_epoch
if let Some(v) = t.max_steps_per_epoch {
hp.max_training_steps_per_epoch = v;
}
if let Some(v) = t.hidden_dim_base {
hp.hidden_dim_base = Some(v);
}
@@ -1099,7 +1093,6 @@ mod tests {
// smoketest enables early stopping with high min_epochs (won't trigger in 3 epochs)
assert!(hp.early_stopping_enabled);
// smoketest caps max steps per epoch
assert_eq!(hp.max_training_steps_per_epoch, 200);
// smoketest doesn't override mbp10_data_dir — default path is kept
// (the directory may not exist on the smoketest machine, but the field is always populated)
assert!(!hp.mbp10_data_dir.is_empty(), "mbp10_data_dir should have default value");

4
crates/ml/tests/dqn_early_stopping_termination_test.rs
View File

@@ -152,7 +152,6 @@ async fn test_early_stopping_terminates_with_error() {
hyperparams.buffer_size = 1024;
hyperparams.min_replay_size = 32;
hyperparams.warmup_steps = 0;
hyperparams.max_training_steps_per_epoch = 300;
hyperparams.replay_buffer_vram_fraction = 0.0;
hyperparams.gpu_timesteps_per_episode = 50;
@@ -213,7 +212,6 @@ async fn test_gradient_collapse_propagates_error() {
// With v_range ±240 (C51), grad norms can reach 100-10000. Threshold must be above
// the maximum expected norm to guarantee collapse detection fires.
// GPU PER requires buffer_size >= 1024. Warmup = 1024 × 0.2 = 204 steps.
// max_training_steps_per_epoch=300 clears warmup in epoch 1 (300 > 204).
hyperparams.learning_rate = 5e-9;
hyperparams.epochs = 10;
hyperparams.gradient_collapse_multiplier = 1e12;
@@ -224,7 +222,6 @@ async fn test_gradient_collapse_propagates_error() {
hyperparams.buffer_size = 1024;
hyperparams.min_replay_size = 32;
hyperparams.warmup_steps = 0;
hyperparams.max_training_steps_per_epoch = 300;
hyperparams.replay_buffer_vram_fraction = 0.0;
hyperparams.gpu_timesteps_per_episode = 50;
@@ -284,7 +281,6 @@ async fn test_healthy_training_completes_successfully() {
hyperparams.buffer_size = 1024; // Minimum for GPU PER (MIN_GPU_CAPACITY)
hyperparams.min_replay_size = 32;
hyperparams.warmup_steps = 0; // Skip train_step() warmup so training_steps increments immediately
hyperparams.max_training_steps_per_epoch = 300; // Fast epochs: ~3s vs ~370s
hyperparams.replay_buffer_vram_fraction = 0.0; // Disable AutoReplaySizer for test determinism
hyperparams.gpu_timesteps_per_episode = 50;

6
crates/ml/tests/dqn_training_pipeline_test.rs
View File

@@ -195,7 +195,6 @@ async fn test_dqn_trains_on_es_fut() -> Result<()> {
info!("ACT: Running DQN training...");
hyperparams.gpu_timesteps_per_episode = 50;
hyperparams.max_training_steps_per_epoch = 64;
scale_for_gpu(&mut hyperparams);
let mut trainer = DQNTrainer::new(hyperparams.clone())?;
@@ -312,7 +311,6 @@ async fn test_dqn_loss_decreases() -> Result<()> {
hyperparams.early_stopping_enabled = false;
hyperparams.gpu_timesteps_per_episode = 50;
hyperparams.max_training_steps_per_epoch = 64;
scale_for_gpu(&mut hyperparams);
let mut trainer = DQNTrainer::new(hyperparams)?;
@@ -386,7 +384,6 @@ async fn test_dqn_checkpoint_save_load() -> Result<()> {
hyperparams.checkpoint_frequency = 2;
hyperparams.gpu_timesteps_per_episode = 50;
hyperparams.max_training_steps_per_epoch = 64;
scale_for_gpu(&mut hyperparams);
let mut trainer = DQNTrainer::new(hyperparams)?;
@@ -455,7 +452,6 @@ async fn test_dqn_q_value_predictions() -> Result<()> {
hyperparams.batch_size = 32;
hyperparams.gpu_timesteps_per_episode = 50;
hyperparams.max_training_steps_per_epoch = 64;
scale_for_gpu(&mut hyperparams);
let mut trainer = DQNTrainer::new(hyperparams)?;
@@ -514,7 +510,6 @@ async fn test_dqn_epsilon_greedy() -> Result<()> {
hyperparams.epsilon_decay = 0.9; // Fast decay
hyperparams.batch_size = 64;
hyperparams.max_training_steps_per_epoch = 64;
hyperparams.gpu_timesteps_per_episode = 50;
hyperparams.min_replay_size = 50;
hyperparams.warmup_steps = 0;
@@ -597,7 +592,6 @@ async fn test_dqn_full_production_training() -> Result<()> {
hyperparams.early_stopping_enabled = true;
hyperparams.gpu_timesteps_per_episode = 50;
hyperparams.max_training_steps_per_epoch = 64;
scale_for_gpu(&mut hyperparams);
let mut trainer = DQNTrainer::new(hyperparams.clone())?;

1
crates/ml/tests/smoke_test_real_data.rs
View File

@@ -805,7 +805,6 @@ async fn smoke_e2e_dqn_training_loop() {
// CI: cap training steps to avoid full 204K-bar dataset sweep (6375->64 steps).
// 64 steps x batch_size 32 = 2048 gradient updates — sufficient to validate
// finite loss, gradient flow, and action diversity without 400s/epoch overhead.
hyperparams.max_training_steps_per_epoch = 8;
// Curiosity disabled: kernel crashes on RTX 3050 (needs investigation)
// C51 atom count from GPU profile (replaces hardcoded VRAM if/else)
let gpu_profile = ml_core::gpu::profile::GpuProfile::load();

1
infra/k8s/argo/gpu-test-pipeline-template.yaml
View File

@@ -432,7 +432,6 @@ spec:
--trades-data-dir /data/test-data/trades \
--symbol ES.FUT \
--epochs 5 \
--max-steps-per-epoch 100 \
--train-months 3 --val-months 1 --test-months 1 --step-months 3 \
2>&1)