foxhunt/docs/agent7_proposed_implementation.rs

// PROPOSED IMPLEMENTATION: Overfitting Detection for DQN Hyperopt
// File: ml/src/hyperopt/adapters/dqn.rs
// Author: Agent 7 (Hive-Mind Swarm)
// Date: 2025-11-27

// ============================================================================
// CHANGE 1: Add ValidationMetrics Import (after line 56)
// ============================================================================

use crate::trainers::validation_metrics::{ValidationMetrics, EarlyStopCriteria};

// ============================================================================
// CHANGE 2: Add Helper Function (before impl HyperparameterOptimizable)
// ============================================================================

/// Calculate Shannon entropy of action distribution H = -Σ p_i log(p_i)
///
/// Used to measure exploration diversity in policy.
/// - 0.0: Deterministic (one action always chosen)
/// - 1.099: Uniform distribution over 3 actions (maximum entropy)
///
/// # Arguments
/// * `distribution` - Action probabilities [buy%, sell%, hold%]
///
/// # Returns
/// Entropy in bits (range: 0.0 to 1.099)
fn calculate_entropy(distribution: &[f32; 3]) -> f32 {
    distribution
        .iter()
        .filter(|&&p| p > 1e-8)  // Avoid log(0)
        .map(|&p| -p * p.log2())
        .sum()
}

// ============================================================================
// CHANGE 3: Add Overfitting Detection to train_with_params()
// ============================================================================

// This code should be inserted AFTER line ~1900 where trainer.train().await completes
// and BEFORE the final metrics are returned.

impl HyperparameterOptimizable for DQNTrainer {
    type Params = DQNParams;
    type Metrics = DQNMetrics;

    fn train_with_params(&mut self, params: Self::Params) -> Result<Self::Metrics, MLError> {
        // ... existing code up to trainer.train().await ...

        // ========================================================================
        // INSERTION POINT: After training completes, before returning metrics
        // ========================================================================

        // Get metrics from completed training
        let final_train_loss = trainer.get_avg_train_loss();
        let final_val_loss = trainer.get_best_val_loss();
        let final_q_value = trainer.get_avg_q_value();
        let final_epsilon = trainer.get_epsilon().await.unwrap_or(0.0);
        let epochs_completed = trainer.get_epochs_trained() as u32;

        // Extract gradient norm and Q-value std from training statistics
        let gradient_norm = trainer.get_final_gradient_norm().unwrap_or(0.0);
        let q_value_std = trainer.get_q_value_std().unwrap_or(1.0);

        // Get action distribution from trainer's final statistics
        let (buy_pct, sell_pct, hold_pct) = trainer.get_action_distribution()
            .unwrap_or((0.33, 0.33, 0.34));

        // ========================================================================
        // NEW CODE: Build ValidationMetrics history for overfitting detection
        // ========================================================================

        let mut val_metrics_history = Vec::new();

        // Get historical data from trainer
        let loss_history = trainer.get_loss_history();  // Vec<f64>
        let val_loss_history = trainer.get_val_loss_history();  // Vec<f64>
        let q_value_history = trainer.get_q_value_history();  // Vec<f64>

        let epochs_to_check = loss_history.len().min(val_loss_history.len());

        for epoch in 0..epochs_to_check {
            // Get train/val losses for this epoch
            let train_loss = loss_history[epoch] as f32;
            let val_loss = val_loss_history[epoch] as f32;
            let q_mean = q_value_history.get(epoch).copied().unwrap_or(0.0) as f32;

            // Estimate Q-value std (can be improved with per-epoch tracking)
            let q_std = (q_value_std * 0.1) as f32;  // Conservative estimate

            // Estimate action distribution (use final distribution as proxy)
            // TODO: Track per-epoch distributions in DQNTrainer for accuracy
            let action_dist = [buy_pct as f32, sell_pct as f32, hold_pct as f32];

            // Calculate policy entropy from action distribution
            let policy_entropy = calculate_entropy(&action_dist);

            // Get win rate and Sharpe from backtest (if available)
            let (win_rate, sharpe) = if let Some(ref backtest) = backtest_metrics {
                (backtest.win_rate as f32, backtest.sharpe_ratio as f32)
            } else {
                (0.5, 0.0)  // Default neutral values
            };

            // Get gradient norm (use final value as proxy for now)
            // TODO: Track per-epoch gradient norms in DQNTrainer
            let grad_norm = gradient_norm as f32;

            // Build ValidationMetrics for this epoch
            let vm = ValidationMetrics::new(
                epoch,
                train_loss,
                val_loss,
                q_mean,
                q_std,
                action_dist,
                policy_entropy,
                win_rate,
                sharpe,
                grad_norm,
            );

            val_metrics_history.push(vm);
        }

        // ========================================================================
        // NEW CODE: Check for overfitting using ValidationMetrics
        // ========================================================================

        if let Some(latest_vm) = val_metrics_history.last() {
            // Use EarlyStopCriteria::Overfitting to detect train/val divergence
            let criteria = EarlyStopCriteria::Overfitting;

            if let Some(reason) = criteria.should_stop(&latest_vm, &val_metrics_history) {
                tracing::warn!(
                    "⚠️  Trial {} PRUNED (overfitting detected): {}",
                    current_trial,
                    reason
                );

                // Log overfitting detection to training logs
                write_training_log_dqn(
                    &self.training_paths.logs_dir(),
                    &format!(
                        "Trial {} PRUNED (overfitting): {}\nTrain loss: {:.4}, Val loss: {:.4}, Ratio: {:.2}",
                        current_trial,
                        reason,
                        latest_vm.train_loss,
                        latest_vm.val_loss,
                        latest_vm.train_val_ratio()
                    ),
                )
                .ok();

                // Return heavily penalized metrics to prune this trial
                return Ok(DQNMetrics {
                    train_loss: latest_vm.train_loss as f64,
                    val_loss: latest_vm.val_loss as f64,
                    avg_q_value: latest_vm.q_value_mean as f64,
                    final_epsilon,
                    epochs_completed,
                    avg_episode_reward: -1000.0,  // Heavy penalty (will give objective = +1000)
                    buy_action_pct: latest_vm.action_distribution[0] as f64,
                    sell_action_pct: latest_vm.action_distribution[1] as f64,
                    hold_action_pct: latest_vm.action_distribution[2] as f64,
                    gradient_norm: latest_vm.gradient_norm as f64,
                    q_value_std: latest_vm.q_value_std as f64,
                    backtest_metrics: None,  // No backtest for pruned trials
                });
            } else {
                tracing::info!(
                    "✅ Trial {} passed overfitting check (train/val ratio: {:.2})",
                    current_trial,
                    latest_vm.train_val_ratio()
                );
            }
        }

        // ========================================================================
        // EXISTING CODE: Continue with normal metrics return
        // ========================================================================

        // Run backtest if enabled
        let backtest_metrics = if self.enable_backtest {
            // ... existing backtest code ...
        } else {
            None
        };

        // ... rest of existing code to build and return final metrics ...
    }
}

// ============================================================================
// EXPECTED BEHAVIOR
// ============================================================================

/*
SCENARIO 1: Trial with overfitting (train↓, val↑ for 5 epochs)

Epoch 10: train_loss=2.0, val_loss=2.0  ✓
Epoch 11: train_loss=1.8, val_loss=2.1  ✓
Epoch 12: train_loss=1.6, val_loss=2.2  ✓
Epoch 13: train_loss=1.4, val_loss=2.3  ✓
Epoch 14: train_loss=1.2, val_loss=2.4  ✓

→ is_overfitting() returns true
→ Trial PRUNED with avg_episode_reward = -1000.0
→ Log message: "⚠️  Trial 5 PRUNED (overfitting detected): train/val ratio: 2.00"
→ Hyperopt will not select this trial as best


SCENARIO 2: Trial with high train/val ratio

Epoch 20: train_loss=1.0, val_loss=3.5  (ratio = 3.5 > 2.0)

→ is_overfitting() returns true (Signal 2: ratio check)
→ Trial PRUNED immediately
→ Saves remaining epochs from being wasted


SCENARIO 3: Healthy trial (no overfitting)

Epoch 1-50: train_loss and val_loss both decreasing
train_loss=1.2, val_loss=1.5  (ratio = 0.8 < 2.0)

→ is_overfitting() returns false
→ Trial completes normally
→ Returns actual backtest metrics
→ May be selected as best trial by hyperopt
*/

// ============================================================================
// TESTING
// ============================================================================

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_calculate_entropy_uniform() {
        let uniform_dist = [0.33, 0.33, 0.34];
        let entropy = calculate_entropy(&uniform_dist);
        // log2(3) ≈ 1.585, so H(uniform) ≈ 1.585 * 0.33 ≈ 1.58
        assert!((entropy - 1.58).abs() < 0.1, "Entropy should be ~1.58 for uniform");
    }

    #[test]
    fn test_calculate_entropy_deterministic() {
        let deterministic = [0.0, 0.0, 1.0];
        let entropy = calculate_entropy(&deterministic);
        assert_eq!(entropy, 0.0, "Entropy should be 0 for deterministic");
    }

    #[test]
    fn test_calculate_entropy_mixed() {
        let mixed = [0.5, 0.3, 0.2];
        let entropy = calculate_entropy(&mixed);
        assert!(entropy > 0.0 && entropy < 1.6, "Entropy should be between 0 and log2(3)");
    }

    #[tokio::test]
    async fn test_hyperopt_detects_overfitting() {
        // This test would need a full integration test setup
        // Verifying that train_with_params() returns penalized metrics
        // when is_overfitting() triggers

        // Mock: Create DQNTrainer with test data
        // Mock: Create params that cause overfitting (high LR, low regularization)
        // Assert: metrics.avg_episode_reward == -1000.0
        // Assert: Log contains "PRUNED (overfitting)"
    }
}

// ============================================================================
// INTEGRATION CHECKLIST
// ============================================================================

/*
BEFORE MERGING:

1. ✓ Add import for ValidationMetrics, EarlyStopCriteria
2. ✓ Add calculate_entropy() helper function
3. ✓ Insert ValidationMetrics history construction in train_with_params()
4. ✓ Insert overfitting check before returning metrics
5. □ Verify DQNTrainer exposes required getters:
   - get_loss_history() -> &[f64]
   - get_val_loss_history() -> &[f64]
   - get_q_value_history() -> &[f64]
   - get_action_distribution() -> (f64, f64, f64)
   - get_final_gradient_norm() -> Option<f64>
   - get_q_value_std() -> Option<f64>
6. □ Run cargo test --package ml validation_metrics
7. □ Run hyperopt trial with known overfitting params
8. □ Verify log message appears
9. □ Verify penalized metrics returned
10. □ Update hyperopt documentation

NICE TO HAVE (Phase 2):
- Track per-epoch action distributions in DQNTrainer
- Track per-epoch gradient norms in DQNTrainer
- Expose ValidationMetrics natively from DQNTrainer
- Add comprehensive EarlyStopCriteria::All checking
*/