a6b6f27cdd
- Remove Default trait implementations from DQN and PPO trainers - Add conservative() methods for testing/examples - Create canonical hyperparameter config files in ml/hyperparams/ - Update all examples and tests to use conservative() This prevents production failures from incorrect defaults (e.g., Pod 0hczpx9nj1ub88 failure where default LR was 1000x too high for PPO). Changes: - ml/src/trainers/dqn.rs: Remove Default, add conservative() + monitoring - ml/src/trainers/ppo.rs: Remove Default, add conservative() + dual LRs - ml/hyperparams/ppo_best.toml: Best params from hyperopt Trial #1 - ml/hyperparams/dqn_best.toml: Conservative DQN defaults - ml/hyperparams/README.md: Usage documentation - Updated 5 examples to use conservative() - Updated 7 test files (69 occurrences) Test Results: 24/24 trainer tests passing (15 DQN + 9 PPO) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
52 lines
2.1 KiB
TOML
52 lines
2.1 KiB
TOML
# PPO Best Hyperparameters
|
|
# Source: Hyperopt Trial #1 (Pod bpxgh10c5ocus5, EUR-IS-1, RTX A4000)
|
|
# Date: 2025-11-01
|
|
# Objective Score: 2.4023 (best of 63 trials)
|
|
# Duration: 14.3 minutes
|
|
# Cost: $0.06
|
|
# Dataset: ES_FUT_180d.parquet
|
|
|
|
# Learning Rates (ASYMMETRIC - CRITICAL)
|
|
# Policy network requires ultra-conservative LR to prevent catastrophic forgetting
|
|
# Value network can handle aggressive LR for fast convergence
|
|
learning_rate = 1.0e-4 # Deprecated: use actor/critic rates instead
|
|
actor_learning_rate = 1.0e-6 # Policy LR (ultra-conservative)
|
|
critic_learning_rate = 0.001 # Value LR (aggressive, 1000x higher)
|
|
|
|
# PPO Algorithm Parameters
|
|
clip_epsilon = 0.1126 # PPO clip range (conservative vs 0.2 default)
|
|
vf_coef = 0.5 # Value loss coefficient (balanced)
|
|
ent_coef = 0.006142 # Entropy coefficient (low exploration)
|
|
gae_lambda = 0.95 # GAE lambda (default)
|
|
gamma = 0.99 # Discount factor (default)
|
|
|
|
# Training Configuration
|
|
batch_size = 64 # Batch size (standard)
|
|
minibatch_size = 64 # Mini-batch size
|
|
rollout_steps = 2048 # Steps per rollout
|
|
epochs = 10000 # Training epochs (production)
|
|
|
|
# Early Stopping Configuration
|
|
early_stopping_enabled = true
|
|
min_value_loss_improvement_pct = 2.0
|
|
min_explained_variance = 0.4
|
|
plateau_window = 30
|
|
min_epochs_before_stopping = 50
|
|
|
|
# Performance Notes
|
|
# - Trial #1 achieved best objective (2.4023) among 63 trials
|
|
# - Policy LR cluster: 0.7e-6 to 2.5e-6 (all top 5 trials)
|
|
# - Value LR range: 0.0009 to 0.0012 (tight variance)
|
|
# - Using single learning_rate 0.001 caused loss stagnation at 1.158-1.159 (Pod 0hczpx9nj1ub88)
|
|
# - Asymmetric learning rates (33x ratio) are REQUIRED for PPO convergence
|
|
|
|
# Deployment
|
|
# - Production deployment: deploy_ppo_production_corrected.sh
|
|
# - Expected duration: 30-90 minutes on RTX A4000
|
|
# - Expected cost: $0.12-$0.38
|
|
|
|
# Related Files
|
|
# - CLAUDE.md: Recent Updates (lines 9-47)
|
|
# - PPO_PARAMETERS_QUICK_REF.md: Complete hyperopt analysis
|
|
# - ml/src/trainers/ppo.rs: Trainer implementation
|