gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: consolidate DQNConfig to single canonical definition

Browse Source 
Removed duplicate DQNConfig from agent.rs (13 fields, pre-Rainbow with
f64 gamma/epsilon) and adaptive-strategy stub (unit struct). Canonical
definition in dqn/dqn.rs now has 51 fields covering full Rainbow DQN
plus agent-level trading parameters (minimum_profit_factor, weight_decay).

Key changes:
- agent.rs imports DQNConfig from dqn.rs instead of defining its own
- Fixed f32/f64 type mismatches (epsilon_start/end/decay cast to f64
  where QNetworkConfig expects f64)
- Renamed replay_buffer_size -> replay_buffer_capacity across all callers
- Updated 13 files across ml, adaptive-strategy, and trading_service
- All 2009 ml tests pass, 0 clippy warnings in modified files

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-22 20:49:22 +01:00
parent 0fa7aa41c0
commit 42634014b6
13 changed files with 89 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
adaptive-strategy/src/models/deep_learning.rs
View File

@@ -24,8 +24,7 @@ use config::ml_config::Mamba2Config;
pub type AgentMetrics = u64;
/// Deep Q-Network agent for reinforcement learning
pub struct DQNAgent;
/// `Configuration` for `DQN` agent training
pub struct DQNConfig;
// DQNConfig: removed stub — canonical definition is ml::dqn::DQNConfig
/// Experience tuple for replay buffer
pub struct Experience;
/// Trading action space representation

3
ml/examples/evaluate_dqn_load_function.rs
View File

@@ -244,12 +244,13 @@ fn load_dqn_model(model_path: &Path, device_str: &str) -> Result<DQNAgent> {
hidden_dims: hidden_dims.clone(), // Clone to avoid move
learning_rate: 0.001, // Default (not used for inference)
gamma: 0.99, // Default (not used for inference)
replay_buffer_size: 100_000, // Default (not used for inference)
replay_buffer_capacity: 100_000, // Default (not used for inference)
batch_size: 32, // Default (not used for inference)
target_update_freq: 1000, // Default (not used for inference)
epsilon_start: 0.0, // Disable exploration for evaluation
epsilon_end: 0.0, // Disable exploration for evaluation
epsilon_decay: 1.0, // No decay needed for evaluation
..DQNConfig::default()
};
info!("Creating DQNAgent with configuration:");

3
ml/src/benchmark/dqn_benchmark.rs
View File

@@ -467,6 +467,9 @@ impl DqnBenchmarkRunner {
iqn_embedding_dim: 64,
use_cvar_action_selection: false,
cvar_alpha: 0.05,
minimum_profit_factor: 1.5,
weight_decay: 1e-4,
}
}

15
ml/src/checkpoint/model_implementations.rs
View File

@@ -15,7 +15,8 @@ use crate::MLError;
use rust_decimal::Decimal;
// Import all model types
use crate::dqn::agent::{DQNAgent, DQNConfig};
use crate::dqn::agent::DQNAgent;
use crate::dqn::DQNConfig;
use crate::liquid::LiquidNetworkConfig;
use crate::mamba::{Mamba2Config, Mamba2SSM};
use crate::tft::TFTConfig;
@@ -77,9 +78,9 @@ impl Checkpointable for DQNAgent {
.unwrap_or_default(),
target_network_weights: vec![],
replay_buffer_size: self.get_replay_buffer_size(),
replay_buffer_capacity: self.config.replay_buffer_size,
replay_buffer_capacity: self.config.replay_buffer_capacity,
average_reward: self.get_average_reward(),
epsilon: self.config.epsilon_start, // Use epsilon_start instead of epsilon
epsilon: self.config.epsilon_start as f64, // Use epsilon_start instead of epsilon
loss_history: vec![],
total_inferences: 0,
avg_inference_time_us: 0.0,
@@ -155,7 +156,7 @@ impl Checkpointable for DQNAgent {
);
params.insert(
"replay_buffer_size".to_string(),
Value::from(self.config.replay_buffer_size),
Value::from(self.config.replay_buffer_capacity),
);
params.insert(
"batch_size".to_string(),
@@ -183,14 +184,14 @@ impl Checkpointable for DQNAgent {
"current_loss".to_string(),
TryInto::<f64>::try_into(self.metrics.current_loss).unwrap_or(0.0),
);
metrics.insert("epsilon".to_string(), self.config.epsilon_start); // Current epsilon value
metrics.insert("epsilon".to_string(), self.config.epsilon_start as f64); // Current epsilon value
metrics.insert(
"replay_buffer_size".to_string(),
self.get_replay_buffer_size() as f64,
);
metrics.insert("average_reward".to_string(), 0.0);
metrics.insert("success_rate".to_string(), 0.0);
metrics.insert("exploration_rate".to_string(), self.config.epsilon_start);
metrics.insert("exploration_rate".to_string(), self.config.epsilon_start as f64);
metrics
}
@@ -235,7 +236,7 @@ impl DQNAgent {
// In a real implementation, this would query the actual replay buffer
// For now, return a reasonable default based on configuration
let filled_ratio = 0.7; // Assume 70% filled
(self.config.replay_buffer_size as f64 * filled_ratio) as usize
(self.config.replay_buffer_capacity as f64 * filled_ratio) as usize
}
/// Get average reward from recent episodes

92
ml/src/dqn/agent.rs
View File

@@ -18,6 +18,7 @@ use tracing::debug;
use common::types::Price as IntegerPrice;
use rust_decimal::Decimal;
use super::dqn::DQNConfig;
use super::network::{QNetwork, QNetworkConfig};
use super::{Experience, ReplayBuffer, ReplayBufferConfig};
use crate::MLError;
@@ -152,58 +153,7 @@ impl Default for TradingState {
}
}
/// `DQN` configuration for trading
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DQNConfig {
/// State dimension (must match TradingState::dimension())
pub state_dim: usize,
/// Number of actions (Buy, Sell, Hold)
pub num_actions: usize,
/// Hidden layer dimensions
pub hidden_dims: Vec<usize>,
/// Learning rate
pub learning_rate: f64,
/// Discount factor for future rewards
pub gamma: f64,
/// Experience replay buffer size
pub replay_buffer_size: usize,
/// Batch size for training
pub batch_size: usize,
/// Target network update frequency
pub target_update_freq: usize,
/// Exploration parameters
pub epsilon_start: f64,
pub epsilon_end: f64,
pub epsilon_decay: f64,
/// BUG #7 FIX: Minimum profit threshold (profit must exceed cost * factor, default 1.5 = 50% margin)
pub minimum_profit_factor: f32,
/// BUG #4 FIX: Soft target update coefficient (Polyak averaging rate, default 0.005)
pub tau: f64,
/// WAVE 30: L2 weight decay for AdamW optimizer (default: 1e-4, range: [1e-5, 1e-3])
/// Prevents overfitting by penalizing large weights via L2 regularization.
pub weight_decay: f64,
}
impl Default for DQNConfig {
fn default() -> Self {
Self {
state_dim: 51, // 45 market features + 6 portfolio features (Wave 23)
num_actions: 3,
hidden_dims: vec![128, 64, 32],
learning_rate: 0.001,
gamma: 0.99,
replay_buffer_size: 100_000,
batch_size: 32,
target_update_freq: 1000,
epsilon_start: 1.0,
epsilon_end: 0.01,
epsilon_decay: 0.995,
minimum_profit_factor: 1.5, // BUG #7 FIX: 50% margin above breakeven
tau: 0.005, // BUG #4 FIX: Polyak averaging coefficient (0.5% new weights per update)
weight_decay: 1e-4, // WAVE 30: Standard L2 regularization strength
}
}
}
// DQNConfig is imported from super::dqn (canonical definition with 49+ fields)
/// `DQN` Agent metrics
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -275,9 +225,9 @@ impl DQNAgent {
num_actions: config.num_actions,
hidden_dims: config.hidden_dims.clone(),
learning_rate: config.learning_rate,
epsilon_start: config.epsilon_start,
epsilon_end: config.epsilon_end,
epsilon_decay: config.epsilon_decay,
epsilon_start: config.epsilon_start as f64,
epsilon_end: config.epsilon_end as f64,
epsilon_decay: config.epsilon_decay as f64,
target_update_freq: config.target_update_freq,
dropout_prob: 0.2,
dropout_schedule: None,
@@ -293,7 +243,7 @@ impl DQNAgent {
// Create replay buffer
let buffer_config = ReplayBufferConfig {
capacity: config.replay_buffer_size,
capacity: config.replay_buffer_capacity,
batch_size: config.batch_size,
min_experiences: config.batch_size * 10,
};
@@ -456,7 +406,7 @@ impl DQNAgent {
// Target = reward + gamma * max(next_q) * (1 - done)
let gamma_tensor = Tensor::from_vec(
vec![self.config.gamma as f32; batch_size],
vec![self.config.gamma; batch_size],
batch_size,
device,
)
@@ -913,7 +863,7 @@ impl DQNAgent {
self.estimate_parameter_count(),
self.q_network.device_info(),
self.replay_buffer.size(),
self.config.replay_buffer_size
self.config.replay_buffer_capacity
)
}
@@ -1123,9 +1073,20 @@ impl std::fmt::Debug for DQNAgent {
mod tests {
use super::*;
/// Create an agent-compatible DQNConfig with 3-action space (Buy/Sell/Hold)
fn agent_config() -> DQNConfig {
DQNConfig {
num_actions: 3,
hidden_dims: vec![128, 64, 32],
batch_size: 32,
replay_buffer_capacity: 100_000,
..DQNConfig::default()
}
}
#[tokio::test]
async fn test_dqn_agent_creation() -> Result<(), Box<dyn std::error::Error>> {
let config = DQNConfig::default();
let config = agent_config();
let agent = DQNAgent::new(config)?;
assert_eq!(agent.get_epsilon(), 1.0);
@@ -1148,7 +1109,7 @@ mod tests {
#[tokio::test]
async fn test_action_selection() -> Result<(), Box<dyn std::error::Error>> {
let config = DQNConfig::default();
let config = agent_config();
let mut agent = DQNAgent::new(config)?;
let state = TradingState::default();
@@ -1165,7 +1126,7 @@ mod tests {
#[tokio::test]
async fn test_experience_storage() -> Result<(), Box<dyn std::error::Error>> {
let config = DQNConfig::default();
let config = agent_config();
let mut agent = DQNAgent::new(config)?;
let experience = Experience::new(
@@ -1184,7 +1145,7 @@ mod tests {
#[tokio::test]
async fn test_training_readiness() -> Result<(), Box<dyn std::error::Error>> {
let config = DQNConfig::default();
let config = agent_config();
let mut agent = DQNAgent::new(config)?;
// Should not be ready initially
@@ -1211,7 +1172,7 @@ mod tests {
#[tokio::test]
async fn test_training_statistics() -> Result<(), Box<dyn std::error::Error>> {
let config = DQNConfig::default();
let config = agent_config();
let mut agent = DQNAgent::new(config)?;
// Update some statistics
@@ -1229,7 +1190,7 @@ mod tests {
#[tokio::test]
async fn test_network_summary() -> Result<(), Box<dyn std::error::Error>> {
let config = DQNConfig::default();
let config = agent_config();
let agent = DQNAgent::new(config)?;
let summary = agent.get_network_summary();
@@ -1314,7 +1275,7 @@ mod tests {
hidden_dims: vec![64, 32],
learning_rate: 0.01,
gamma: 0.95,
replay_buffer_size: 50_000,
replay_buffer_capacity: 50_000,
batch_size: 64,
target_update_freq: 500,
epsilon_start: 0.9,
@@ -1323,6 +1284,7 @@ mod tests {
minimum_profit_factor: 1.5,
tau: 0.005,
weight_decay: 1e-4,
..DQNConfig::default()
};
let agent = DQNAgent::new(config.clone())?;

3
ml/src/dqn/demo_dqn.rs
View File

@@ -3,7 +3,8 @@
//! This module provides a comprehensive demonstration of the 2025 production-ready
//! DQN implementation for high-frequency trading.
use crate::dqn::agent::{DQNAgent, DQNConfig};
use crate::dqn::agent::DQNAgent;
use crate::dqn::DQNConfig;
use crate::safety::{MLSafetyConfig, MLSafetyManager};
use crate::MLError;
use rust_decimal::Decimal;

20
ml/src/dqn/dqn.rs
View File

@@ -164,6 +164,13 @@ pub struct DQNConfig {
// CVaR (Conditional Value at Risk) for risk-aware action selection
pub use_cvar_action_selection: bool,
pub cvar_alpha: f32,
// Agent-level trading parameters (from legacy agent::DQNConfig)
/// BUG #7 FIX: Minimum profit threshold (profit must exceed cost * factor, default 1.5 = 50% margin)
pub minimum_profit_factor: f32,
/// WAVE 30: L2 weight decay for AdamW optimizer (default: 1e-4, range: [1e-5, 1e-3])
/// Prevents overfitting by penalizing large weights via L2 regularization.
pub weight_decay: f64,
}
impl Default for DQNConfig {
@@ -225,6 +232,10 @@ impl Default for DQNConfig {
// CVaR: Disabled by default
use_cvar_action_selection: false,
cvar_alpha: 0.05,
// Agent-level trading parameters
minimum_profit_factor: 1.5, // BUG #7 FIX: 50% margin above breakeven
weight_decay: 1e-4, // WAVE 30: Standard L2 regularization strength
}
}
}
@@ -304,6 +315,9 @@ impl DQNConfig {
iqn_embedding_dim: 64,
use_cvar_action_selection: false,
cvar_alpha: 0.05,
minimum_profit_factor: 1.5,
weight_decay: 1e-4,
}
}
@@ -367,6 +381,9 @@ impl DQNConfig {
iqn_embedding_dim: 64,
use_cvar_action_selection: true,
cvar_alpha: 0.05,
minimum_profit_factor: 2.0, // Higher safety margin for conservative config
weight_decay: 1e-4,
}
}
@@ -439,6 +456,9 @@ impl DQNConfig {
iqn_embedding_dim: 64,
use_cvar_action_selection: false,
cvar_alpha: 0.05,
minimum_profit_factor: 1.5,
weight_decay: 1e-4,
}
}
}

18
ml/src/examples.rs
View File

@@ -139,7 +139,8 @@ pub async fn run_example(config: ExampleConfig) -> Result<ExampleResult, MLError
/// Run basic `DQN` example with actual Deep Q-Learning implementation
async fn run_basic_dqn_example(config: &ExampleConfig) -> Result<ExampleMetrics, MLError> {
use crate::dqn::agent::{DQNAgent, DQNConfig};
use crate::dqn::agent::DQNAgent;
use crate::dqn::DQNConfig;
// Configure DQN with real parameters
let dqn_config = DQNConfig {
@@ -149,7 +150,7 @@ async fn run_basic_dqn_example(config: &ExampleConfig) -> Result<ExampleMetrics,
learning_rate: 0.001,
gamma: 0.95,
batch_size: 32,
replay_buffer_size: 10000,
replay_buffer_capacity: 10000,
target_update_freq: 1000,
epsilon_start: 0.1,
epsilon_end: 0.01,
@@ -279,19 +280,20 @@ async fn run_rainbow_dqn_example(config: &ExampleConfig) -> Result<ExampleMetric
};
// TEMPORARILY USE BASIC DQN AGENT - Rainbow not yet implemented
use crate::dqn::agent::{DQNAgent, DQNConfig};
use crate::dqn::agent::DQNAgent;
use crate::dqn::DQNConfig;
let basic_config = DQNConfig {
state_dim: rainbow_config.state_dim,
num_actions: rainbow_config.num_actions,
hidden_dims: vec![128, 64],
learning_rate: rainbow_config.learning_rate,
gamma: rainbow_config.discount_factor,
gamma: rainbow_config.discount_factor as f32,
batch_size: rainbow_config.batch_size,
replay_buffer_size: rainbow_config.memory_size,
replay_buffer_capacity: rainbow_config.memory_size,
target_update_freq: rainbow_config.target_update_freq,
epsilon_start: rainbow_config.epsilon_start as f64,
epsilon_end: rainbow_config.epsilon_end as f64,
epsilon_decay: rainbow_config.epsilon_decay as f64,
epsilon_start: rainbow_config.epsilon_start as f32,
epsilon_end: rainbow_config.epsilon_end as f32,
epsilon_decay: rainbow_config.epsilon_decay as f32,
tau: 0.005, // BUG #4: Soft target update coefficient
..Default::default()
};

3
ml/src/integration/strategy_dqn_bridge.rs
View File

@@ -10,7 +10,8 @@ use std::sync::Arc;
use serde::{Deserialize, Serialize};
use tokio::sync::RwLock;
use crate::dqn::agent::{DQNAgent, DQNConfig};
use crate::dqn::agent::DQNAgent;
use crate::dqn::DQNConfig;
use crate::dqn::{Experience, TradingState};
use crate::MLError;

3
ml/src/trainers/dqn/config.rs
View File

@@ -880,6 +880,9 @@ pub(crate) fn dqn_config_2025() -> DQNConfig {
iqn_embedding_dim: 64,
use_cvar_action_selection: false,
cvar_alpha: 0.05,
minimum_profit_factor: 1.5,
weight_decay: 1e-4,
}
}

3
ml/src/trainers/dqn/trainer.rs
View File

@@ -356,6 +356,9 @@ impl DQNTrainer {
iqn_embedding_dim: 64, // Fixed (not in search space)
use_cvar_action_selection: false,
cvar_alpha: 0.05,
minimum_profit_factor: 1.5,
weight_decay: 1e-4,
};
// Create DQN agent

7
ml/tests/training_edge_cases.rs
View File

@@ -14,7 +14,8 @@
#![allow(unused_crate_dependencies)]
use common::trading::MarketRegime;
use ml::dqn::agent::{DQNAgent, DQNConfig, TradingAction};
use ml::dqn::agent::{DQNAgent, TradingAction};
use ml::dqn::DQNConfig;
use ml::dqn::experience::Experience;
use ml::liquid::network::{LiquidNetwork, OutputLayerConfig};
use ml::liquid::training::{LiquidTrainer, LiquidTrainingConfig, TrainingBatch, TrainingSample};
@@ -66,7 +67,7 @@ async fn test_dqn_training_with_batch_size_one() -> Result<(), Box<dyn std::erro
let config = DQNConfig {
state_dim: 52,
batch_size: 1,
replay_buffer_size: 1000,
replay_buffer_capacity: 1000,
..Default::default()
};
let mut agent = DQNAgent::new(config)?;
@@ -96,7 +97,7 @@ async fn test_dqn_training_with_large_batch_size() -> Result<(), Box<dyn std::er
let config = DQNConfig {
state_dim: 52,
batch_size: 512,
replay_buffer_size: 10_000,
replay_buffer_capacity: 10_000,
..Default::default()
};
let mut agent = DQNAgent::new(config)?;

6
services/trading_service/src/services/enhanced_ml.rs
View File

@@ -1262,7 +1262,8 @@ impl RealDQNModel {
model_id: String,
checkpoint_path: &std::path::Path,
) -> ml::MLResult<Self> {
use ml::dqn::agent::{DQNAgent, DQNConfig};
use ml::dqn::agent::DQNAgent;
use ml::dqn::DQNConfig;
// DQN configuration matching paper trading config
let config = DQNConfig {
@@ -1274,12 +1275,13 @@ impl RealDQNModel {
epsilon_start: 0.1,
epsilon_end: 0.01,
epsilon_decay: 0.995,
replay_buffer_size: 100000,
replay_buffer_capacity: 100000,
batch_size: 128,
target_update_freq: 1000,
minimum_profit_factor: 1.5,
tau: 0.005,
weight_decay: 1e-4,
..DQNConfig::default()
};
let mut agent = DQNAgent::new(config)