3853988af7
- Fixed PSO budget calculation bug in ml/src/hyperopt/optimizer.rs - Root cause: Division by n_particles in sequential execution - Now correctly calculates max_iters = remaining_trials (no division) - Result: 50 trials complete instead of 23 (100% vs 46%) - Added comprehensive DQN hyperopt results analysis - 39/50 trials analyzed across 2 RunPod deployments - Best hyperparameters identified: LR 4.89e-5 (ultra-low) - Created DQN_HYPEROPT_RESULTS_SUMMARY.md with expert validation - GitLab CI/CD pipeline operational (48 lines fixed) - Fixed YAML syntax errors (unquoted colons) - All 7 jobs validated and working - Warning cleanup complete (136 → 0 warnings) - Removed 143 lines dead code - Fixed visibility, unused imports, Debug traits - Archived Wave D reports to docs/archive/ - 8 early stopping reports moved - Root directory cleaned up 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
228 lines
9.6 KiB
Bash
228 lines
9.6 KiB
Bash
#!/bin/bash
|
|
# PPO Deployment Examples - Correct Usage After Binary Update
|
|
# Reference: PPO_PARAMETERS_QUICK_REF.md
|
|
# Status: ⏳ Pending binary update to support --policy-lr and --value-lr
|
|
|
|
set -e
|
|
|
|
echo "========================================="
|
|
echo "PPO Deployment Examples (Post Binary Fix)"
|
|
echo "========================================="
|
|
echo ""
|
|
echo "⚠️ NOTE: These examples assume train_ppo_parquet.rs has been updated"
|
|
echo " to accept --policy-lr and --value-lr parameters."
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# Example 1: Local Development with Optimal Parameters
|
|
# ============================================================================
|
|
echo "Example 1: Local Development (Recommended Parameters)"
|
|
echo "========================================="
|
|
echo ""
|
|
echo "Command:"
|
|
echo 'cargo run -p ml --example train_ppo_parquet --release --features cuda -- \'
|
|
echo ' --parquet-file test_data/ES_FUT_180d.parquet \'
|
|
echo ' --policy-lr 0.000001 \'
|
|
echo ' --value-lr 0.001 \'
|
|
echo ' --epochs 500 \'
|
|
echo ' --batch-size 64 \'
|
|
echo ' --output-dir ml/trained_models/ppo_dev'
|
|
echo ""
|
|
echo "Expected results:"
|
|
echo " • Duration: ~5 minutes (500 epochs locally)"
|
|
echo " • GPU: RTX 3050 Ti 4GB"
|
|
echo " • Policy loss: decreasing (not stagnating)"
|
|
echo " • Value loss: < 0.5 (much better than 1.158 failure)"
|
|
echo " • Explained variance: > 0.6"
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# Example 2: Hyperopt Rerun (Verify Reproducibility)
|
|
# ============================================================================
|
|
echo "Example 2: Hyperopt Rerun (Verify Reproducibility)"
|
|
echo "========================================="
|
|
echo ""
|
|
echo "Command:"
|
|
echo 'python3 scripts/runpod_deploy.py \'
|
|
echo ' --gpu-type "RTX A4000" \'
|
|
echo ' --image "jgrusewski/foxhunt-hyperopt:latest" \'
|
|
echo ' --command "hyperopt_ppo_demo \'
|
|
echo ' --parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet \'
|
|
echo ' --trials 50 \'
|
|
echo ' --episodes 2000 \'
|
|
echo ' --base-dir /runpod-volume/ml_training/ppo_hyperopt_verify"'
|
|
echo ""
|
|
echo "Expected results:"
|
|
echo " • Duration: ~15 minutes (vs 18-24 hours original)"
|
|
echo " • Cost: ~\$0.06 (vs \$4.50-\$6.00 original)"
|
|
echo " • Best trial policy_lr: 1e-6 to 3e-6 range"
|
|
echo " • Best trial value_lr: 0.0009 to 0.0012 range"
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# Example 3: Production Training (Full 10k Epochs)
|
|
# ============================================================================
|
|
echo "Example 3: Production Training (10k Epochs, Full Dataset)"
|
|
echo "========================================="
|
|
echo ""
|
|
echo "Command:"
|
|
echo 'python3 scripts/runpod_deploy.py \'
|
|
echo ' --gpu-type "RTX A4000" \'
|
|
echo ' --image "jgrusewski/foxhunt-hyperopt:latest" \'
|
|
echo ' --command "train_ppo_parquet \'
|
|
echo ' --parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet \'
|
|
echo ' --policy-lr 0.000001 \'
|
|
echo ' --value-lr 0.001 \'
|
|
echo ' --epochs 10000 \'
|
|
echo ' --batch-size 64 \'
|
|
echo ' --output-dir /runpod-volume/ml_training/ppo_production_final \'
|
|
echo ' --no-early-stopping"'
|
|
echo ""
|
|
echo "Expected results:"
|
|
echo " • Duration: 30-90 minutes"
|
|
echo " • Cost: \$0.12-\$0.38 @ \$0.25/hr"
|
|
echo " • GPU Memory: ~145MB (plenty of headroom)"
|
|
echo " • Convergence: Policy loss stable, value loss < 0.3"
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# Example 4: Conservative Variant (Smaller Learning Rates)
|
|
# ============================================================================
|
|
echo "Example 4: Conservative Variant (Smaller Learning Rates)"
|
|
echo "========================================="
|
|
echo ""
|
|
echo "Command (Lower policy LR for extreme stability):"
|
|
echo 'cargo run -p ml --example train_ppo_parquet --release --features cuda -- \'
|
|
echo ' --parquet-file test_data/ES_FUT_180d.parquet \'
|
|
echo ' --policy-lr 0.0000005 \'
|
|
echo ' --value-lr 0.0008 \'
|
|
echo ' --epochs 500 \'
|
|
echo ' --batch-size 64'
|
|
echo ""
|
|
echo "Use cases:"
|
|
echo " • High-frequency sensitive markets"
|
|
echo " • Risk-averse deployments"
|
|
echo " • Extended convergence tolerance (slower but more stable)"
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# Example 5: Aggressive Variant (Higher Learning Rates)
|
|
# ============================================================================
|
|
echo "Example 5: Aggressive Variant (Higher Learning Rates)"
|
|
echo "========================================="
|
|
echo ""
|
|
echo "Command (Higher rates for faster convergence):"
|
|
echo 'cargo run -p ml --example train_ppo_parquet --release --features cuda -- \'
|
|
echo ' --parquet-file test_data/ES_FUT_180d.parquet \'
|
|
echo ' --policy-lr 0.000003 \'
|
|
echo ' --value-lr 0.0012 \'
|
|
echo ' --epochs 500 \'
|
|
echo ' --batch-size 64'
|
|
echo ""
|
|
echo "Use cases:"
|
|
echo " • Fast iteration (dev/test cycles)"
|
|
echo " • Non-critical experiments"
|
|
echo " • Baseline comparisons"
|
|
echo ""
|
|
echo "⚠️ WARNING: Slightly higher risk of instability"
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# Example 6: Early Stopping Enabled (Default)
|
|
# ============================================================================
|
|
echo "Example 6: With Early Stopping (Recommended for Dev)"
|
|
echo "========================================="
|
|
echo ""
|
|
echo "Command:"
|
|
echo 'cargo run -p ml --example train_ppo_parquet --release --features cuda -- \'
|
|
echo ' --parquet-file test_data/ES_FUT_180d.parquet \'
|
|
echo ' --policy-lr 0.000001 \'
|
|
echo ' --value-lr 0.001 \'
|
|
echo ' --epochs 10000 \'
|
|
echo ' --batch-size 64 \'
|
|
echo ' --min-value-loss-improvement 2.0 \'
|
|
echo ' --plateau-window 30'
|
|
echo ""
|
|
echo "Expected behavior:"
|
|
echo " • Stops if value loss plateaus (< 2% improvement for 30 epochs)"
|
|
echo " • Prevents wasted computation"
|
|
echo " • Duration: typically 500-2000 epochs (vs 10000 full)"
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# Comparison Table
|
|
# ============================================================================
|
|
echo "============================================================================"
|
|
echo "Parameter Comparison (Variants)"
|
|
echo "============================================================================"
|
|
cat <<'EOF'
|
|
|
|
Scenario | Policy LR | Value LR | Epochs | Est. Time | Est. Cost | Risk
|
|
--------- |-----------|---------| ------ | --------- | --------- | ----
|
|
Conservative (Prod) | 1.0e-6 | 0.0008 | 10000 | 60-90 min | $0.25-38 | Low
|
|
Optimal (Hyperopt) | 1.0e-6 | 0.001 | 10000 | 30-90 min | $0.12-38 | Low ⭐
|
|
Aggressive (Dev) | 3.0e-6 | 0.0012 | 500 | 5-10 min | $0.02-04 | Med
|
|
Ultra-Conservative | 5.0e-7 | 0.0008 | 10000 | 90-120 min| $0.37-50 | V.Low
|
|
Failed (Single LR) | 0.001 | 0.001 | 200+ | Stagnate | $0.10+ | FAIL ✗
|
|
|
|
EOF
|
|
|
|
# ============================================================================
|
|
# Validation Checklist
|
|
# ============================================================================
|
|
echo "============================================================================"
|
|
echo "Validation Checklist (After Binary Update)"
|
|
echo "============================================================================"
|
|
echo ""
|
|
echo "Before running any deployment:"
|
|
echo " [ ] Check help text: cargo run -p ml --example train_ppo_parquet -- --help"
|
|
echo " Should show both --policy-lr and --value-lr parameters"
|
|
echo ""
|
|
echo " [ ] Verify PpoHyperparameters struct in ml/src/trainers/ppo.rs"
|
|
echo " Should have separate policy_learning_rate and value_learning_rate fields"
|
|
echo ""
|
|
echo " [ ] Test locally with small dataset (100 epochs)"
|
|
echo " Confirm parameters are parsed correctly"
|
|
echo ""
|
|
echo "During training (first 10 epochs):"
|
|
echo " [ ] Policy loss should decrease (not stay flat)"
|
|
echo " [ ] Value loss should decrease (not stagnate at 1.158)"
|
|
echo " [ ] KL divergence > 0.01 (policy is updating)"
|
|
echo ""
|
|
echo "After training:"
|
|
echo " [ ] Checkpoint files created at output-dir"
|
|
echo " [ ] Policy loss < 0.5 (much better than failure)"
|
|
echo " [ ] Explained variance > 0.6 (good value fit)"
|
|
echo " [ ] Training log shows convergence trend"
|
|
echo ""
|
|
|
|
# ============================================================================
|
|
# Troubleshooting
|
|
# ============================================================================
|
|
echo "============================================================================"
|
|
echo "Troubleshooting"
|
|
echo "============================================================================"
|
|
echo ""
|
|
echo "Issue: Parameter not recognized"
|
|
echo " → Binary not rebuilt after source changes"
|
|
echo " → Solution: cargo clean && cargo build -p ml --example train_ppo_parquet --release"
|
|
echo ""
|
|
echo "Issue: Loss stagnates at 1.158-1.159"
|
|
echo " → Likely policy_lr is too high (should be 1e-6, not 0.001)"
|
|
echo " → Check arguments: train_ppo_parquet --policy-lr 0.000001"
|
|
echo ""
|
|
echo "Issue: Value loss increases (getting worse)"
|
|
echo " → Value network LR might be too high"
|
|
echo " → Try reducing: --value-lr 0.0008 (from 0.001)"
|
|
echo ""
|
|
echo "Issue: Convergence too slow"
|
|
echo " → Policy network might be too conservative (LR too low)"
|
|
echo " → Try: --policy-lr 0.000002 (slightly higher)"
|
|
echo " → Or increase batch size: --batch-size 128 (if VRAM allows)"
|
|
echo ""
|
|
|
|
echo ""
|
|
echo "========================================="
|
|
echo "For more details, see: PPO_PARAMETERS_QUICK_REF.md"
|
|
echo "========================================="
|