04d8802c94
Remove use_double_dqn, use_dueling, use_per, use_branching,
use_distributional, use_noisy_nets, use_huber_loss, and use_cql
from DQNConfig, DQNHyperparameters, and DqnParams structs.
These features are always enabled (Rainbow DQN standard). The boolean
flags were dead code — every constructor set them to true, and the
only code paths that set them to false were in tests that disabled
features for simplicity. With the fields removed, the features are
unconditionally active, eliminating ~490 lines of dead configuration.
Key changes:
- Struct field declarations removed from 3 core config structs
- Conditional branches (if use_X { ... } else { ... }) simplified:
dueling/branching/PER network creation is now unconditional
- Checkpoint metadata hardcodes "true" for backward compatibility
- Hyperopt search space index 11 (use_branching) fixed at 1.0
- TOML/YAML config files cleaned of removed fields
- Tests that toggled these flags updated or rewritten
45 files changed, -487 net lines. Zero new test failures.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
165 lines
8.5 KiB
TOML
165 lines
8.5 KiB
TOML
# DQN Production Configuration v2.0
|
|
# Generated: 2025-11-04
|
|
# Status: Wave 1/2 Complete + Trial #68 Hyperopt Optimized
|
|
#
|
|
# This configuration consolidates:
|
|
# - Wave 1 improvements: Huber loss, HOLD penalty, Double DQN, gradient clipping
|
|
# - Wave 2 improvements: Validation system, target network optimization, replay buffer
|
|
# - Trial #68 hyperopt results: Best of 116 trials (objective: 0.0006354887)
|
|
|
|
[model]
|
|
name = "DQN-Production-v2"
|
|
version = "2.0.0"
|
|
description = "Production DQN with all Wave 1/2 improvements + Trial #68 hyperopt optimization"
|
|
date_created = "2025-11-04"
|
|
hyperopt_source = "Trial #68 (116 trials, Pod nk5q3xxmb8x40i, 2025-11-03)"
|
|
|
|
[architecture]
|
|
# Network architecture
|
|
state_dim = 225 # Full feature vector (201 Wave C + 24 Wave D)
|
|
num_actions = 3 # BUY, SELL, HOLD
|
|
hidden_dims = [128, 64] # Standard 2-layer architecture
|
|
|
|
[hyperparameters]
|
|
# Trial #68 Optimal Parameters (5.5x-9.6x better than previous defaults)
|
|
learning_rate = 0.00055 # Was 0.0001 conservative (5.5x improvement from hyperopt)
|
|
batch_size = 230 # Was 32 old hyperopt (7.2x improvement, max GPU capacity RTX 3050 Ti)
|
|
gamma = 0.99 # Was 0.9626 old hyperopt (long-term reward focus, top 5 avg=0.9887)
|
|
epsilon_start = 1.0 # Full exploration at training start
|
|
epsilon_end = 0.01 # Minimal exploration at convergence
|
|
epsilon_decay = 0.99 # Was 0.995 (faster exploration→exploitation transition)
|
|
buffer_size = 1000000 # Was 104,346 old hyperopt (9.6x improvement, top 3 trials all used max)
|
|
min_replay_size = 2000 # 2x batch_size minimum for diversity
|
|
|
|
# Rationale for Trial #68 Parameters:
|
|
# - Learning Rate 5.5e-4: Optimal balance (too low <1e-4 = slow, too high >8e-4 = unstable)
|
|
# - Batch Size 230: Top 5 trials avg=206, larger batches = more stable gradients
|
|
# - Gamma 0.99: All top 5 trials used ≥0.98 (long-term trading strategy focus)
|
|
# - Epsilon Decay 0.99: Standard decay optimal (0.999 very slow decay underperformed)
|
|
# - Buffer Size 1M: Top 3 trials ALL used maximum (better sample diversity)
|
|
|
|
[wave1_features]
|
|
# Wave 1: Address pathological behaviors (99.4% HOLD, Q-value outliers, gradient explosions)
|
|
|
|
# Double DQN: Reduce Q-value overestimation bias
|
|
|
|
# Huber Loss: Robust outlier handling (Q-values ranged -87,610 to +142,892)
|
|
huber_delta = 1.0 # Standard threshold (quadratic→linear transition)
|
|
# Benefits: 20x-200x gradient reduction on outliers, 50% robustness improvement vs MSE
|
|
|
|
# Gradient Clipping: Prevent training divergence
|
|
gradient_clip_norm = 1.0 # Max gradient norm (tested range: 0.5-2.0, 1.0 optimal)
|
|
|
|
# HOLD Penalty: Address 99.4% HOLD action problem (Trial #35: -1.92% returns)
|
|
hold_penalty_weight = 0.01 # 1% penalty per 1% excess price movement
|
|
movement_threshold = 0.02 # 2% deadzone (no penalty below this threshold)
|
|
# Expected impact: HOLD rate 99.4% → 30-50%, Returns -1.92% → +5-15%
|
|
|
|
[wave2_features]
|
|
# Wave 2: Faster convergence + comprehensive validation + replay buffer optimization
|
|
|
|
# Target Network Updates
|
|
target_update_frequency = 500 # Was 1000 (2x faster convergence from Wave 2)
|
|
target_update_tau = null # Hard updates (could use 0.001 for soft updates)
|
|
|
|
# Prioritized Experience Replay (PER) - Phase 1 Complete, Phase 2 Integration Pending
|
|
use_prioritized_replay = false # Disabled pending DQN trainer integration (2-3 hours)
|
|
per_alpha = 0.6 # Priority exponent when enabled (literature standard)
|
|
per_beta_start = 0.4 # Importance sampling correction start
|
|
per_beta_end = 1.0 # Importance sampling correction end (anneal over training)
|
|
# Expected PER benefits: 20-40% faster convergence, +10-15% Sharpe improvement
|
|
|
|
# Extended Validation System (prevents Trial #35 loss explosion: 1.207 → 2,612)
|
|
validation_split = 0.2 # 20% holdout set
|
|
validation_log_frequency = 10 # Log metrics every 10 epochs
|
|
validation_patience = 5 # Early stop after 5 epochs val loss increase
|
|
|
|
# Early Stopping Criteria (6 failure modes monitored)
|
|
early_stopping_enabled = true
|
|
q_value_floor = 0.5 # Min Q-value before stopping
|
|
min_loss_improvement_pct = 0.1 # 0.1% improvement threshold (was 2.0%, more sensitive)
|
|
plateau_window = 5 # Window for plateau detection (was 30, faster detection)
|
|
min_epochs_before_stopping = 50 # Prevent premature stopping
|
|
|
|
# Validation Thresholds (extended system)
|
|
hold_collapse_threshold = 0.90 # Trigger if HOLD > 90% for 10 epochs
|
|
entropy_collapse_threshold = 0.10 # Trigger if policy entropy < 0.10
|
|
q_explosion_threshold = 10000.0 # Trigger if |Q| > 10,000
|
|
grad_explosion_threshold = 100.0 # Trigger if gradient norm > 100
|
|
|
|
[training]
|
|
epochs = 500 # Production training duration
|
|
checkpoint_frequency = 10 # Save checkpoint every 10 epochs
|
|
save_best_only = true # Only save checkpoints that improve validation loss
|
|
|
|
[production_criteria]
|
|
# Validation gates for production deployment (is_production_ready() checks)
|
|
min_sharpe_ratio = 1.5 # Minimum risk-adjusted returns
|
|
max_drawdown_pct = 20.0 # Maximum acceptable drawdown
|
|
min_win_rate = 0.50 # Minimum profitable action rate (50%)
|
|
max_hold_pct = 0.70 # Maximum HOLD action rate (ensure diversity)
|
|
min_q_value = -1000.0 # Q-value stability lower bound
|
|
max_q_value = 1000.0 # Q-value stability upper bound
|
|
min_policy_entropy = 0.1 # Minimum exploration (prevent deterministic collapse)
|
|
|
|
[comparison_to_trial35]
|
|
# Trial #35 Analysis (stopped epoch 311, loss 1.207 → exploded to 2,612 at epoch 500)
|
|
trial35_learning_rate = 0.00001 # 55x lower than production (too conservative)
|
|
trial35_batch_size = 110 # 2.1x lower than production
|
|
trial35_gamma = 0.9775 # 1.2% lower than production
|
|
trial35_epsilon_decay = 0.9394 # 5% lower (slower exploration decay)
|
|
trial35_buffer_size = 34000 # 29x lower than production
|
|
trial35_hold_penalty = false # Missing (99.4% HOLD issue not addressed)
|
|
trial35_validation_system = "minimal" # Only val loss plateau (no overfitting detection)
|
|
trial35_failure = "loss_explosion" # Root cause: insufficient validation, suboptimal hyperparams
|
|
|
|
# Production v2 Improvements:
|
|
# - 5.5x higher learning rate (faster convergence)
|
|
# - 2.1x larger batch size (more stable gradients)
|
|
# - 29x larger buffer (better sample diversity)
|
|
# - HOLD penalty enabled (address passivity)
|
|
# - Extended validation (6 failure modes vs 1)
|
|
# - All Wave 1/2 features enabled
|
|
|
|
[deployment]
|
|
# Runpod GPU Training Configuration
|
|
gpu_type = "RTX A4000" # 16GB VRAM (sufficient for batch=230)
|
|
container_disk_size = 50 # GB
|
|
volume_mount = "/runpod-volume"
|
|
estimated_duration_minutes = 60 # 500 epochs @ 7-8s/epoch ≈ 60 min
|
|
estimated_cost_usd = 0.25 # 60 min @ $0.25/hr
|
|
|
|
# Local Training Configuration (RTX 3050 Ti 4GB)
|
|
local_gpu_compatible = true # Batch=230 fits in 4GB VRAM
|
|
local_duration_minutes = 60 # Similar to Runpod
|
|
|
|
[logging]
|
|
log_level = "info"
|
|
metrics_frequency = 10 # Log metrics every 10 epochs
|
|
save_action_distribution = true # Track BUY/SELL/HOLD percentages
|
|
save_q_value_stats = true # Track Q-value mean/std/min/max
|
|
save_gradient_norms = true # Track gradient magnitudes (explosion detection)
|
|
save_epsilon_values = true # Track exploration decay over time
|
|
|
|
[expected_performance]
|
|
# Conservative estimates based on Trial #68 + Wave 1/2 improvements
|
|
sharpe_ratio = "1.5-2.5" # Target range
|
|
win_rate = "50-60%" # Profitable action rate
|
|
hold_percentage = "30-50%" # Reduced from 99.4% (Trial #35 baseline)
|
|
returns = "+5-15%" # Improved from -1.92% (Trial #35 baseline)
|
|
max_drawdown = "15-20%" # Within production criteria
|
|
convergence_epochs = "200-300" # Faster than Trial #35 (311 epochs to best)
|
|
|
|
[notes]
|
|
wave1_status = "✅ COMPLETE - All 4 improvements implemented and tested"
|
|
wave2_status = "✅ COMPLETE - Validation system + replay buffer + target network optimized"
|
|
hyperopt_status = "✅ COMPLETE - 116 trials, Trial #68 best (objective: 0.0006354887)"
|
|
integration_status = "⏳ PENDING - PER Phase 2 integration (2-3 hours)"
|
|
deployment_readiness = "✅ READY - Can deploy immediately with current config"
|
|
|
|
# Next Steps:
|
|
# 1. Deploy production training (IMMEDIATE - 60 min, $0.25)
|
|
# 2. Backtest on ES_FUT_unseen.parquet (verify +20-40% improvement)
|
|
# 3. (Optional) Integrate PER Phase 2 (2-3 hours dev, +20-40% convergence speed)
|
|
# 4. (Optional) Hyperopt PER alpha/beta (30-90 min GPU, 15 trials)
|