diff --git a/crates/ml/src/dqn/dqn.rs b/crates/ml/src/dqn/dqn.rs index 682ea550e..c6ad42c65 100644 --- a/crates/ml/src/dqn/dqn.rs +++ b/crates/ml/src/dqn/dqn.rs @@ -235,8 +235,8 @@ impl Default for DQNConfig { dueling_hidden_dim: 128, use_distributional: true, // BUG #36 FIXED: C51 re-enabled num_atoms: 51, - v_min: -10.0, // Widened: [-2,2] compressed Q-values, [-10,10] gives C51 room to separate 45 actions - v_max: 10.0, // Widened: [-2,2] compressed Q-values, [-10,10] gives C51 room to separate 45 actions + v_min: -10.0, // Widened: [-2,2] compressed Q-values, [-10,10] gives C51 room to separate actions + v_max: 10.0, // Widened: [-2,2] compressed Q-values, [-10,10] gives C51 room to separate actions use_noisy_nets: false, noisy_sigma_init: 0.5, enable_q_value_clipping: true, diff --git a/crates/ml/src/dqn/factored_q_network.rs b/crates/ml/src/dqn/factored_q_network.rs index 4f74c930c..140db8426 100644 --- a/crates/ml/src/dqn/factored_q_network.rs +++ b/crates/ml/src/dqn/factored_q_network.rs @@ -97,9 +97,9 @@ impl FactoredQNetwork { }) } - /// Forward pass: compute Q-values for all 45 actions directly + /// Forward pass: compute Q-values for all exposure actions /// - /// Returns q_values [batch, 45] + /// Returns q_values [batch, num_actions] pub fn forward(&self, state: &Tensor) -> Result { // DEBUG: Log input shape tracing::debug!("FactoredQNetwork input shape: {:?}", state.dims()); @@ -118,7 +118,7 @@ impl FactoredQNetwork { // DEBUG: Log hidden representation shape tracing::debug!("Hidden representation shape: {:?}", hidden.dims()); - // Joint head: hidden → 45 Q-values + // Joint head: hidden → Q-values (one per exposure level) let q_values = self .joint_head .forward(&hidden) @@ -137,14 +137,14 @@ impl FactoredQNetwork { } - /// Select greedy action (argmax on 45 Q-values) + /// Select greedy action (argmax on Q-values across exposure levels) pub fn select_greedy_action(&self, state: &Tensor) -> Result { let q_values = self.forward(state)?; // DEBUG: Log Q-value shape before argmax tracing::debug!("Pre-argmax Q-value shape: {:?}", q_values.dims()); - // Argmax across all 45 actions + // Argmax across exposure actions let action_idx = q_values .argmax(1) .map_err(|e| MLError::ModelError(format!("Argmax failed: {}", e)))? @@ -207,11 +207,12 @@ impl FactoredQNetwork { .to_vec2::() .map_err(|e| MLError::ModelError(format!("Failed to convert q_values to vec: {}", e)))?; - // Mask invalid actions (iterate all 45 action indices) + // Mask invalid exposure actions (0..5: Short100, Short50, Flat, Long50, Long100) for batch_idx in 0..batch_size { - for action_idx in 0..45 { - let action = FactoredAction::from_index(action_idx)?; - let target_position = action.exposure.target_exposure(); + for action_idx in 0..5 { + let exposure = crate::dqn::action_space::ExposureLevel::from_index(action_idx) + .map_err(|e| MLError::ModelError(format!("Invalid exposure index {}: {}", action_idx, e)))?; + let target_position = exposure.target_exposure(); // Check if this would exceed ±100% limit if (current_position + target_position).abs() > 1.0 { diff --git a/crates/ml/src/dqn/reward.rs b/crates/ml/src/dqn/reward.rs index 90056b564..fe6358248 100644 --- a/crates/ml/src/dqn/reward.rs +++ b/crates/ml/src/dqn/reward.rs @@ -522,7 +522,7 @@ impl RewardFunction { }; // Calculate diversity bonus (in-training regularization) - // Entropy is normalized [0,1] over 45 actions. Threshold 0.3 ≈ ~4 distinct actions. + // Entropy is normalized [0,1] over 5 exposure actions. Threshold 0.3 ≈ ~2 distinct actions. let entropy = calculate_entropy(recent_actions); let entropy_threshold = Decimal::try_from(0.3).unwrap_or(Decimal::ZERO); let diversity_bonus = if entropy < entropy_threshold { diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index 414036bab..d314e0627 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -424,8 +424,8 @@ impl Default for DQNParams { // ============================================================================= use_distributional: true, // ENABLED: BUG #36 fixed, full Rainbow DQN C51 num_atoms: 51, // Wave 2.3: Rainbow DQN standard - v_min: -10.0, // Widened for C51 action separation (45 actions need wider support) - v_max: 10.0, // Widened for C51 action separation (45 actions need wider support) + v_min: -10.0, // Widened for C51 action separation + v_max: 10.0, // Widened for C51 action separation use_noisy_nets: true, // Wave 8: Default ENABLED for full Rainbow DQN noisy_sigma_init: 0.5, // Wave 2.4: Rainbow DQN standard minimum_profit_factor: 1.5, // BUG #7: Default 50% margin above breakeven diff --git a/crates/ml/src/trainers/curriculum.rs b/crates/ml/src/trainers/curriculum.rs index 1ef9d6832..f01d5a91d 100644 --- a/crates/ml/src/trainers/curriculum.rs +++ b/crates/ml/src/trainers/curriculum.rs @@ -4,9 +4,9 @@ //! //! | Phase | Exposure Actions | Count | Regimes | Gate | //! |-------|-----------------|-------|---------|------| -//! | 1: Basic | Short50, Flat, Long50 | 27 | Ranging only | Sharpe > 0.5 for 3 folds | -//! | 2: FullPosition | All 5 exposures | 45 | Ranging + Trending | Sharpe > 0.3 for 2 folds | -//! | 3: AllRegimes | All 45 | 45 | All regimes | Terminal | +//! | 1: Basic | Short50, Flat, Long50 | 3 | Ranging only | Sharpe > 0.5 for 3 folds | +//! | 2: FullPosition | All 5 exposures | 5 | Ranging + Trending | Sharpe > 0.3 for 2 folds | +//! | 3: AllRegimes | All 5 | 5 | All regimes | Terminal | //! //! The scheduler tracks consecutive qualifying folds and advances the phase //! when the gate condition is met. Action masks combine curriculum restrictions