a6b6f27cdd
- Remove Default trait implementations from DQN and PPO trainers - Add conservative() methods for testing/examples - Create canonical hyperparameter config files in ml/hyperparams/ - Update all examples and tests to use conservative() This prevents production failures from incorrect defaults (e.g., Pod 0hczpx9nj1ub88 failure where default LR was 1000x too high for PPO). Changes: - ml/src/trainers/dqn.rs: Remove Default, add conservative() + monitoring - ml/src/trainers/ppo.rs: Remove Default, add conservative() + dual LRs - ml/hyperparams/ppo_best.toml: Best params from hyperopt Trial #1 - ml/hyperparams/dqn_best.toml: Conservative DQN defaults - ml/hyperparams/README.md: Usage documentation - Updated 5 examples to use conservative() - Updated 7 test files (69 occurrences) Test Results: 24/24 trainer tests passing (15 DQN + 9 PPO) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
65 lines
2.6 KiB
TOML
65 lines
2.6 KiB
TOML
# DQN Best Hyperparameters
|
|
# Source: Historical training + manual tuning
|
|
# Date: 2025-11-02
|
|
# Status: PENDING - Awaiting hyperopt completion for optimal values
|
|
# Dataset: ES_FUT_180d.parquet
|
|
|
|
# NOTE: These are conservative defaults based on successful training runs.
|
|
# After completing DQN hyperopt (with action-dependent rewards fix), update
|
|
# this file with optimal parameters from the best trial.
|
|
|
|
# Learning Parameters
|
|
learning_rate = 0.0001 # Conservative LR for stable convergence
|
|
batch_size = 128 # Optimal for RTX 3050 Ti 4GB (max 230)
|
|
gamma = 0.99 # Discount factor (standard RL)
|
|
|
|
# Exploration Schedule (Epsilon-Greedy)
|
|
epsilon_start = 1.0 # Start with full exploration
|
|
epsilon_end = 0.01 # Minimum exploration (1%)
|
|
epsilon_decay = 0.995 # Decay rate per episode
|
|
|
|
# Replay Buffer Configuration
|
|
buffer_size = 100000 # Experience replay capacity
|
|
min_replay_size = 1000 # Minimum experiences before training
|
|
|
|
# Training Configuration
|
|
epochs = 100 # Training epochs (default)
|
|
checkpoint_frequency = 10 # Save every 10 epochs
|
|
|
|
# Early Stopping Configuration
|
|
early_stopping_enabled = true
|
|
q_value_floor = 0.5 # Minimum Q-value threshold
|
|
min_loss_improvement_pct = 2.0 # Minimum 2% improvement over window
|
|
plateau_window = 30 # Epochs to check for plateau
|
|
min_epochs_before_stopping = 50 # Minimum training duration
|
|
|
|
# GPU Configuration
|
|
# Max batch size for RTX 3050 Ti 4GB: 230
|
|
# Max batch size for RTX A4000 16GB: 512+
|
|
# Recommended: 128 (safe for all GPUs)
|
|
|
|
# Historical Performance Notes
|
|
# - Training time: ~15s for 50 epochs (RTX 3050 Ti)
|
|
# - GPU memory: ~6MB model size
|
|
# - Inference: ~200μs per action
|
|
# - Tests: 16/16 passing (100% coverage)
|
|
|
|
# Known Issues (FIXED as of 2025-11-02)
|
|
# ✅ Action-dependent rewards implemented (lines 722-743, trainers/dqn.rs)
|
|
# ✅ Hyperopt objective fixed to maximize episode rewards (not loss)
|
|
# ✅ 16 comprehensive tests added (dqn_action_dependent_reward_test.rs)
|
|
# ⏳ Awaiting hyperopt deployment to find optimal hyperparameters
|
|
|
|
# Deployment
|
|
# After hyperopt completes, update this file with:
|
|
# - Optimal batch_size (expected: 128-512)
|
|
# - Optimal learning_rate (expected: 1e-5 to 1e-3)
|
|
# - Optimal gamma (expected: 0.95-0.99)
|
|
# - Optimal epsilon_decay (expected: 0.99-0.999)
|
|
|
|
# Related Files
|
|
# - DQN_ACTION_DEPENDENT_REWARDS_FIX_SUMMARY.md: Fix documentation
|
|
# - DQN_HYPEROPT_CORRECTED_QUICKREF.md: Hyperopt guide
|
|
# - ml/src/trainers/dqn.rs: Trainer implementation
|
|
# - deploy_dqn_hyperopt_corrected.sh: Deployment script
|