From 42634014b653bae4bc3a665349dddc9abf575eba Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 22 Feb 2026 20:49:22 +0100 Subject: [PATCH] refactor: consolidate DQNConfig to single canonical definition Removed duplicate DQNConfig from agent.rs (13 fields, pre-Rainbow with f64 gamma/epsilon) and adaptive-strategy stub (unit struct). Canonical definition in dqn/dqn.rs now has 51 fields covering full Rainbow DQN plus agent-level trading parameters (minimum_profit_factor, weight_decay). Key changes: - agent.rs imports DQNConfig from dqn.rs instead of defining its own - Fixed f32/f64 type mismatches (epsilon_start/end/decay cast to f64 where QNetworkConfig expects f64) - Renamed replay_buffer_size -> replay_buffer_capacity across all callers - Updated 13 files across ml, adaptive-strategy, and trading_service - All 2009 ml tests pass, 0 clippy warnings in modified files Co-Authored-By: Claude Opus 4.6 --- adaptive-strategy/src/models/deep_learning.rs | 3 +- ml/examples/evaluate_dqn_load_function.rs | 3 +- ml/src/benchmark/dqn_benchmark.rs | 3 + ml/src/checkpoint/model_implementations.rs | 15 +-- ml/src/dqn/agent.rs | 92 ++++++------------- ml/src/dqn/demo_dqn.rs | 3 +- ml/src/dqn/dqn.rs | 20 ++++ ml/src/examples.rs | 18 ++-- ml/src/integration/strategy_dqn_bridge.rs | 3 +- ml/src/trainers/dqn/config.rs | 3 + ml/src/trainers/dqn/trainer.rs | 3 + ml/tests/training_edge_cases.rs | 7 +- .../src/services/enhanced_ml.rs | 6 +- 13 files changed, 89 insertions(+), 90 deletions(-) diff --git a/adaptive-strategy/src/models/deep_learning.rs b/adaptive-strategy/src/models/deep_learning.rs index 82013e07c..46ac08024 100644 --- a/adaptive-strategy/src/models/deep_learning.rs +++ b/adaptive-strategy/src/models/deep_learning.rs @@ -24,8 +24,7 @@ use config::ml_config::Mamba2Config; pub type AgentMetrics = u64; /// Deep Q-Network agent for reinforcement learning pub struct DQNAgent; -/// `Configuration` for `DQN` agent training -pub struct DQNConfig; +// DQNConfig: removed stub — canonical definition is ml::dqn::DQNConfig /// Experience tuple for replay buffer pub struct Experience; /// Trading action space representation diff --git a/ml/examples/evaluate_dqn_load_function.rs b/ml/examples/evaluate_dqn_load_function.rs index 849113357..8fe091f16 100644 --- a/ml/examples/evaluate_dqn_load_function.rs +++ b/ml/examples/evaluate_dqn_load_function.rs @@ -244,12 +244,13 @@ fn load_dqn_model(model_path: &Path, device_str: &str) -> Result { hidden_dims: hidden_dims.clone(), // Clone to avoid move learning_rate: 0.001, // Default (not used for inference) gamma: 0.99, // Default (not used for inference) - replay_buffer_size: 100_000, // Default (not used for inference) + replay_buffer_capacity: 100_000, // Default (not used for inference) batch_size: 32, // Default (not used for inference) target_update_freq: 1000, // Default (not used for inference) epsilon_start: 0.0, // Disable exploration for evaluation epsilon_end: 0.0, // Disable exploration for evaluation epsilon_decay: 1.0, // No decay needed for evaluation + ..DQNConfig::default() }; info!("Creating DQNAgent with configuration:"); diff --git a/ml/src/benchmark/dqn_benchmark.rs b/ml/src/benchmark/dqn_benchmark.rs index 3c44496b3..211c1a83a 100644 --- a/ml/src/benchmark/dqn_benchmark.rs +++ b/ml/src/benchmark/dqn_benchmark.rs @@ -467,6 +467,9 @@ impl DqnBenchmarkRunner { iqn_embedding_dim: 64, use_cvar_action_selection: false, cvar_alpha: 0.05, + + minimum_profit_factor: 1.5, + weight_decay: 1e-4, } } diff --git a/ml/src/checkpoint/model_implementations.rs b/ml/src/checkpoint/model_implementations.rs index 871ecf0ce..2218eae1e 100644 --- a/ml/src/checkpoint/model_implementations.rs +++ b/ml/src/checkpoint/model_implementations.rs @@ -15,7 +15,8 @@ use crate::MLError; use rust_decimal::Decimal; // Import all model types -use crate::dqn::agent::{DQNAgent, DQNConfig}; +use crate::dqn::agent::DQNAgent; +use crate::dqn::DQNConfig; use crate::liquid::LiquidNetworkConfig; use crate::mamba::{Mamba2Config, Mamba2SSM}; use crate::tft::TFTConfig; @@ -77,9 +78,9 @@ impl Checkpointable for DQNAgent { .unwrap_or_default(), target_network_weights: vec![], replay_buffer_size: self.get_replay_buffer_size(), - replay_buffer_capacity: self.config.replay_buffer_size, + replay_buffer_capacity: self.config.replay_buffer_capacity, average_reward: self.get_average_reward(), - epsilon: self.config.epsilon_start, // Use epsilon_start instead of epsilon + epsilon: self.config.epsilon_start as f64, // Use epsilon_start instead of epsilon loss_history: vec![], total_inferences: 0, avg_inference_time_us: 0.0, @@ -155,7 +156,7 @@ impl Checkpointable for DQNAgent { ); params.insert( "replay_buffer_size".to_string(), - Value::from(self.config.replay_buffer_size), + Value::from(self.config.replay_buffer_capacity), ); params.insert( "batch_size".to_string(), @@ -183,14 +184,14 @@ impl Checkpointable for DQNAgent { "current_loss".to_string(), TryInto::::try_into(self.metrics.current_loss).unwrap_or(0.0), ); - metrics.insert("epsilon".to_string(), self.config.epsilon_start); // Current epsilon value + metrics.insert("epsilon".to_string(), self.config.epsilon_start as f64); // Current epsilon value metrics.insert( "replay_buffer_size".to_string(), self.get_replay_buffer_size() as f64, ); metrics.insert("average_reward".to_string(), 0.0); metrics.insert("success_rate".to_string(), 0.0); - metrics.insert("exploration_rate".to_string(), self.config.epsilon_start); + metrics.insert("exploration_rate".to_string(), self.config.epsilon_start as f64); metrics } @@ -235,7 +236,7 @@ impl DQNAgent { // In a real implementation, this would query the actual replay buffer // For now, return a reasonable default based on configuration let filled_ratio = 0.7; // Assume 70% filled - (self.config.replay_buffer_size as f64 * filled_ratio) as usize + (self.config.replay_buffer_capacity as f64 * filled_ratio) as usize } /// Get average reward from recent episodes diff --git a/ml/src/dqn/agent.rs b/ml/src/dqn/agent.rs index 7674f8ca4..1e3e2b2c4 100644 --- a/ml/src/dqn/agent.rs +++ b/ml/src/dqn/agent.rs @@ -18,6 +18,7 @@ use tracing::debug; use common::types::Price as IntegerPrice; use rust_decimal::Decimal; +use super::dqn::DQNConfig; use super::network::{QNetwork, QNetworkConfig}; use super::{Experience, ReplayBuffer, ReplayBufferConfig}; use crate::MLError; @@ -152,58 +153,7 @@ impl Default for TradingState { } } -/// `DQN` configuration for trading -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct DQNConfig { - /// State dimension (must match TradingState::dimension()) - pub state_dim: usize, - /// Number of actions (Buy, Sell, Hold) - pub num_actions: usize, - /// Hidden layer dimensions - pub hidden_dims: Vec, - /// Learning rate - pub learning_rate: f64, - /// Discount factor for future rewards - pub gamma: f64, - /// Experience replay buffer size - pub replay_buffer_size: usize, - /// Batch size for training - pub batch_size: usize, - /// Target network update frequency - pub target_update_freq: usize, - /// Exploration parameters - pub epsilon_start: f64, - pub epsilon_end: f64, - pub epsilon_decay: f64, - /// BUG #7 FIX: Minimum profit threshold (profit must exceed cost * factor, default 1.5 = 50% margin) - pub minimum_profit_factor: f32, - /// BUG #4 FIX: Soft target update coefficient (Polyak averaging rate, default 0.005) - pub tau: f64, - /// WAVE 30: L2 weight decay for AdamW optimizer (default: 1e-4, range: [1e-5, 1e-3]) - /// Prevents overfitting by penalizing large weights via L2 regularization. - pub weight_decay: f64, -} - -impl Default for DQNConfig { - fn default() -> Self { - Self { - state_dim: 51, // 45 market features + 6 portfolio features (Wave 23) - num_actions: 3, - hidden_dims: vec![128, 64, 32], - learning_rate: 0.001, - gamma: 0.99, - replay_buffer_size: 100_000, - batch_size: 32, - target_update_freq: 1000, - epsilon_start: 1.0, - epsilon_end: 0.01, - epsilon_decay: 0.995, - minimum_profit_factor: 1.5, // BUG #7 FIX: 50% margin above breakeven - tau: 0.005, // BUG #4 FIX: Polyak averaging coefficient (0.5% new weights per update) - weight_decay: 1e-4, // WAVE 30: Standard L2 regularization strength - } - } -} +// DQNConfig is imported from super::dqn (canonical definition with 49+ fields) /// `DQN` Agent metrics #[derive(Debug, Clone, Serialize, Deserialize)] @@ -275,9 +225,9 @@ impl DQNAgent { num_actions: config.num_actions, hidden_dims: config.hidden_dims.clone(), learning_rate: config.learning_rate, - epsilon_start: config.epsilon_start, - epsilon_end: config.epsilon_end, - epsilon_decay: config.epsilon_decay, + epsilon_start: config.epsilon_start as f64, + epsilon_end: config.epsilon_end as f64, + epsilon_decay: config.epsilon_decay as f64, target_update_freq: config.target_update_freq, dropout_prob: 0.2, dropout_schedule: None, @@ -293,7 +243,7 @@ impl DQNAgent { // Create replay buffer let buffer_config = ReplayBufferConfig { - capacity: config.replay_buffer_size, + capacity: config.replay_buffer_capacity, batch_size: config.batch_size, min_experiences: config.batch_size * 10, }; @@ -456,7 +406,7 @@ impl DQNAgent { // Target = reward + gamma * max(next_q) * (1 - done) let gamma_tensor = Tensor::from_vec( - vec![self.config.gamma as f32; batch_size], + vec![self.config.gamma; batch_size], batch_size, device, ) @@ -913,7 +863,7 @@ impl DQNAgent { self.estimate_parameter_count(), self.q_network.device_info(), self.replay_buffer.size(), - self.config.replay_buffer_size + self.config.replay_buffer_capacity ) } @@ -1123,9 +1073,20 @@ impl std::fmt::Debug for DQNAgent { mod tests { use super::*; + /// Create an agent-compatible DQNConfig with 3-action space (Buy/Sell/Hold) + fn agent_config() -> DQNConfig { + DQNConfig { + num_actions: 3, + hidden_dims: vec![128, 64, 32], + batch_size: 32, + replay_buffer_capacity: 100_000, + ..DQNConfig::default() + } + } + #[tokio::test] async fn test_dqn_agent_creation() -> Result<(), Box> { - let config = DQNConfig::default(); + let config = agent_config(); let agent = DQNAgent::new(config)?; assert_eq!(agent.get_epsilon(), 1.0); @@ -1148,7 +1109,7 @@ mod tests { #[tokio::test] async fn test_action_selection() -> Result<(), Box> { - let config = DQNConfig::default(); + let config = agent_config(); let mut agent = DQNAgent::new(config)?; let state = TradingState::default(); @@ -1165,7 +1126,7 @@ mod tests { #[tokio::test] async fn test_experience_storage() -> Result<(), Box> { - let config = DQNConfig::default(); + let config = agent_config(); let mut agent = DQNAgent::new(config)?; let experience = Experience::new( @@ -1184,7 +1145,7 @@ mod tests { #[tokio::test] async fn test_training_readiness() -> Result<(), Box> { - let config = DQNConfig::default(); + let config = agent_config(); let mut agent = DQNAgent::new(config)?; // Should not be ready initially @@ -1211,7 +1172,7 @@ mod tests { #[tokio::test] async fn test_training_statistics() -> Result<(), Box> { - let config = DQNConfig::default(); + let config = agent_config(); let mut agent = DQNAgent::new(config)?; // Update some statistics @@ -1229,7 +1190,7 @@ mod tests { #[tokio::test] async fn test_network_summary() -> Result<(), Box> { - let config = DQNConfig::default(); + let config = agent_config(); let agent = DQNAgent::new(config)?; let summary = agent.get_network_summary(); @@ -1314,7 +1275,7 @@ mod tests { hidden_dims: vec![64, 32], learning_rate: 0.01, gamma: 0.95, - replay_buffer_size: 50_000, + replay_buffer_capacity: 50_000, batch_size: 64, target_update_freq: 500, epsilon_start: 0.9, @@ -1323,6 +1284,7 @@ mod tests { minimum_profit_factor: 1.5, tau: 0.005, weight_decay: 1e-4, + ..DQNConfig::default() }; let agent = DQNAgent::new(config.clone())?; diff --git a/ml/src/dqn/demo_dqn.rs b/ml/src/dqn/demo_dqn.rs index 6a3deaaf6..b41dfadf4 100644 --- a/ml/src/dqn/demo_dqn.rs +++ b/ml/src/dqn/demo_dqn.rs @@ -3,7 +3,8 @@ //! This module provides a comprehensive demonstration of the 2025 production-ready //! DQN implementation for high-frequency trading. -use crate::dqn::agent::{DQNAgent, DQNConfig}; +use crate::dqn::agent::DQNAgent; +use crate::dqn::DQNConfig; use crate::safety::{MLSafetyConfig, MLSafetyManager}; use crate::MLError; use rust_decimal::Decimal; diff --git a/ml/src/dqn/dqn.rs b/ml/src/dqn/dqn.rs index d63dc36fc..e7b3aac1a 100644 --- a/ml/src/dqn/dqn.rs +++ b/ml/src/dqn/dqn.rs @@ -164,6 +164,13 @@ pub struct DQNConfig { // CVaR (Conditional Value at Risk) for risk-aware action selection pub use_cvar_action_selection: bool, pub cvar_alpha: f32, + + // Agent-level trading parameters (from legacy agent::DQNConfig) + /// BUG #7 FIX: Minimum profit threshold (profit must exceed cost * factor, default 1.5 = 50% margin) + pub minimum_profit_factor: f32, + /// WAVE 30: L2 weight decay for AdamW optimizer (default: 1e-4, range: [1e-5, 1e-3]) + /// Prevents overfitting by penalizing large weights via L2 regularization. + pub weight_decay: f64, } impl Default for DQNConfig { @@ -225,6 +232,10 @@ impl Default for DQNConfig { // CVaR: Disabled by default use_cvar_action_selection: false, cvar_alpha: 0.05, + + // Agent-level trading parameters + minimum_profit_factor: 1.5, // BUG #7 FIX: 50% margin above breakeven + weight_decay: 1e-4, // WAVE 30: Standard L2 regularization strength } } } @@ -304,6 +315,9 @@ impl DQNConfig { iqn_embedding_dim: 64, use_cvar_action_selection: false, cvar_alpha: 0.05, + + minimum_profit_factor: 1.5, + weight_decay: 1e-4, } } @@ -367,6 +381,9 @@ impl DQNConfig { iqn_embedding_dim: 64, use_cvar_action_selection: true, cvar_alpha: 0.05, + + minimum_profit_factor: 2.0, // Higher safety margin for conservative config + weight_decay: 1e-4, } } @@ -439,6 +456,9 @@ impl DQNConfig { iqn_embedding_dim: 64, use_cvar_action_selection: false, cvar_alpha: 0.05, + + minimum_profit_factor: 1.5, + weight_decay: 1e-4, } } } diff --git a/ml/src/examples.rs b/ml/src/examples.rs index 4d59748a7..dcc675e45 100644 --- a/ml/src/examples.rs +++ b/ml/src/examples.rs @@ -139,7 +139,8 @@ pub async fn run_example(config: ExampleConfig) -> Result Result { - use crate::dqn::agent::{DQNAgent, DQNConfig}; + use crate::dqn::agent::DQNAgent; + use crate::dqn::DQNConfig; // Configure DQN with real parameters let dqn_config = DQNConfig { @@ -149,7 +150,7 @@ async fn run_basic_dqn_example(config: &ExampleConfig) -> Result Result DQNConfig { iqn_embedding_dim: 64, use_cvar_action_selection: false, cvar_alpha: 0.05, + + minimum_profit_factor: 1.5, + weight_decay: 1e-4, } } diff --git a/ml/src/trainers/dqn/trainer.rs b/ml/src/trainers/dqn/trainer.rs index 11b27b673..56b32fcc4 100644 --- a/ml/src/trainers/dqn/trainer.rs +++ b/ml/src/trainers/dqn/trainer.rs @@ -356,6 +356,9 @@ impl DQNTrainer { iqn_embedding_dim: 64, // Fixed (not in search space) use_cvar_action_selection: false, cvar_alpha: 0.05, + + minimum_profit_factor: 1.5, + weight_decay: 1e-4, }; // Create DQN agent diff --git a/ml/tests/training_edge_cases.rs b/ml/tests/training_edge_cases.rs index 31d6554ae..b549cbce7 100644 --- a/ml/tests/training_edge_cases.rs +++ b/ml/tests/training_edge_cases.rs @@ -14,7 +14,8 @@ #![allow(unused_crate_dependencies)] use common::trading::MarketRegime; -use ml::dqn::agent::{DQNAgent, DQNConfig, TradingAction}; +use ml::dqn::agent::{DQNAgent, TradingAction}; +use ml::dqn::DQNConfig; use ml::dqn::experience::Experience; use ml::liquid::network::{LiquidNetwork, OutputLayerConfig}; use ml::liquid::training::{LiquidTrainer, LiquidTrainingConfig, TrainingBatch, TrainingSample}; @@ -66,7 +67,7 @@ async fn test_dqn_training_with_batch_size_one() -> Result<(), Box Result<(), Box ml::MLResult { - use ml::dqn::agent::{DQNAgent, DQNConfig}; + use ml::dqn::agent::DQNAgent; + use ml::dqn::DQNConfig; // DQN configuration matching paper trading config let config = DQNConfig { @@ -1274,12 +1275,13 @@ impl RealDQNModel { epsilon_start: 0.1, epsilon_end: 0.01, epsilon_decay: 0.995, - replay_buffer_size: 100000, + replay_buffer_capacity: 100000, batch_size: 128, target_update_freq: 1000, minimum_profit_factor: 1.5, tau: 0.005, weight_decay: 1e-4, + ..DQNConfig::default() }; let mut agent = DQNAgent::new(config)