foxhunt/ml/tests/dqn_hyperopt_huber_loss_test.rs

//! Test suite for DQN Hyperopt Huber Loss Configuration
//!
//! Validates that Huber loss parameters are properly configured in hyperopt:
//! 1. DQNHyperparameters created from DQNParams have Huber loss enabled
//! 2. Default Huber loss values match production configuration (enabled, delta=1.0)
//! 3. Loss computation uses Huber loss during optimization
//!
//! This ensures hyperopt optimizes parameters for the CORRECT loss function
//! (matching production training which uses Huber loss, not MSE).

use ml::hyperopt::adapters::dqn::DQNParams;
use ml::trainers::dqn::DQNHyperparameters;

#[test]
fn test_hyperopt_dqn_hyperparameters_has_huber_loss_enabled() {
    // Test that DQNHyperparameters created from DQNParams has Huber loss enabled by default
    // This ensures hyperopt optimizes for Huber loss (matching production)

    let dqn_params = DQNParams {
        learning_rate: 0.0001,
        batch_size: 128,
        gamma: 0.99,
        epsilon_decay: 0.995,
        buffer_size: 100_000,
        movement_threshold: 0.02, // 2% default
    };

    // Create hyperparameters the same way the hyperopt adapter does
    // (mimicking the conversion in train_with_params)
    let hyperparams = DQNHyperparameters {
        learning_rate: dqn_params.learning_rate,
        batch_size: dqn_params.batch_size,
        gamma: dqn_params.gamma,
        epsilon_start: 1.0, // Fixed
        epsilon_end: 0.01,  // Fixed
        epsilon_decay: dqn_params.epsilon_decay,
        buffer_size: dqn_params.buffer_size,
        min_replay_size: dqn_params.batch_size * 2,
        epochs: 100,
        checkpoint_frequency: 10,
        early_stopping_enabled: true,
        q_value_floor: 0.5,
        min_loss_improvement_pct: 2.0,
        plateau_window: 5,
        min_epochs_before_stopping: 10,
        hold_penalty: -0.001,
        use_huber_loss: true, // CRITICAL: Must be enabled to match production
        huber_delta: 1.0,     // CRITICAL: Must match production delta
        use_double_dqn: true,
        gradient_clip_norm: Some(10.0),
        hold_penalty_weight: 0.01,
        movement_threshold: 0.02,
    };

    // Verify Huber loss is enabled (matching production)
    assert!(
        hyperparams.use_huber_loss,
        "Hyperopt must use Huber loss to match production training"
    );
    assert_eq!(
        hyperparams.huber_delta, 1.0,
        "Hyperopt delta must match production value (1.0)"
    );
}

#[test]
fn test_hyperopt_default_params_use_huber_loss() {
    // Test that default DQNParams lead to Huber loss configuration
    let default_params = DQNParams::default();

    // Create hyperparameters (same logic as hyperopt adapter)
    let hyperparams = DQNHyperparameters {
        learning_rate: default_params.learning_rate,
        batch_size: default_params.batch_size,
        gamma: default_params.gamma,
        epsilon_start: 1.0,
        epsilon_end: 0.01,
        epsilon_decay: default_params.epsilon_decay,
        buffer_size: default_params.buffer_size,
        min_replay_size: default_params.batch_size * 2,
        epochs: 100,
        checkpoint_frequency: 10,
        early_stopping_enabled: true,
        q_value_floor: 0.5,
        min_loss_improvement_pct: 2.0,
        plateau_window: 5,
        min_epochs_before_stopping: 10,
        hold_penalty: -0.001,
        use_huber_loss: true, // Default must be enabled
        huber_delta: 1.0,     // Default delta value
        use_double_dqn: true,
        gradient_clip_norm: Some(10.0),
        hold_penalty_weight: 0.01,
        movement_threshold: 0.02,
    };

    assert!(
        hyperparams.use_huber_loss,
        "Default hyperopt config must use Huber loss"
    );
    assert_eq!(hyperparams.huber_delta, 1.0, "Default delta must be 1.0");
}

#[test]
fn test_hyperopt_huber_loss_matches_production() {
    // Test that hyperopt Huber loss configuration exactly matches production script
    // Production script uses: --use-huber-loss --huber-delta 1.0

    // Simulate production hyperparameters from Trial #39 (best from Wave 1+2)
    let production_params = DQNParams {
        learning_rate: 0.000010,
        batch_size: 207,
        gamma: 0.950,
        epsilon_decay: 0.99900,
        buffer_size: 162_739,
        movement_threshold: 0.02, // Production: --movement-threshold 0.02
    };

    // Create hyperparameters with production Huber loss settings
    let hyperparams = DQNHyperparameters {
        learning_rate: production_params.learning_rate,
        batch_size: production_params.batch_size,
        gamma: production_params.gamma,
        epsilon_start: 1.0,
        epsilon_end: 0.01,
        epsilon_decay: production_params.epsilon_decay,
        buffer_size: production_params.buffer_size,
        min_replay_size: production_params.batch_size * 2,
        epochs: 500,
        checkpoint_frequency: 50,
        early_stopping_enabled: false, // Production uses --no-early-stopping
        q_value_floor: 0.5,
        min_loss_improvement_pct: 2.0,
        plateau_window: 5,
        min_epochs_before_stopping: 10,
        hold_penalty: -0.001,
        use_huber_loss: true,          // Production flag: --use-huber-loss
        huber_delta: 1.0,              // Production flag: --huber-delta 1.0
        use_double_dqn: true,          // Production flag: --use-double-dqn
        gradient_clip_norm: Some(1.0), // Production flag: --gradient-clip-norm 1.0
        hold_penalty_weight: 0.01,     // Production flag: --hold-penalty-weight 0.01
        movement_threshold: 0.02,      // Production flag: --movement-threshold 0.02
    };

    // Verify exact match with production configuration
    assert_eq!(
        hyperparams.use_huber_loss, true,
        "Must match production: --use-huber-loss"
    );
    assert_eq!(
        hyperparams.huber_delta, 1.0,
        "Must match production: --huber-delta 1.0"
    );
    assert_eq!(
        hyperparams.use_double_dqn, true,
        "Must match production: --use-double-dqn"
    );
    assert_eq!(
        hyperparams.gradient_clip_norm,
        Some(1.0),
        "Must match production: --gradient-clip-norm 1.0"
    );
}

#[test]
fn test_huber_loss_backwards_compatibility() {
    // Test that if use_huber_loss=false, system falls back to MSE
    // This ensures backwards compatibility if anyone needs MSE mode

    let params = DQNParams::default();

    // Create hyperparameters with Huber loss disabled
    let hyperparams = DQNHyperparameters {
        learning_rate: params.learning_rate,
        batch_size: params.batch_size,
        gamma: params.gamma,
        epsilon_start: 1.0,
        epsilon_end: 0.01,
        epsilon_decay: params.epsilon_decay,
        buffer_size: params.buffer_size,
        min_replay_size: params.batch_size * 2,
        epochs: 100,
        checkpoint_frequency: 10,
        early_stopping_enabled: true,
        q_value_floor: 0.5,
        min_loss_improvement_pct: 2.0,
        plateau_window: 5,
        min_epochs_before_stopping: 10,
        hold_penalty: -0.001,
        use_huber_loss: false, // Explicitly disabled for backwards compat
        huber_delta: 1.0,
        use_double_dqn: true,
        gradient_clip_norm: Some(10.0),
        hold_penalty_weight: 0.01,
        movement_threshold: 0.02,
    };

    assert!(
        !hyperparams.use_huber_loss,
        "Should support MSE mode when explicitly disabled"
    );
}

#[test]
fn test_huber_delta_value_range() {
    // Test that various Huber delta values can be configured
    // Common range for trading: 0.5 to 2.0

    let test_deltas = vec![0.5, 1.0, 1.5, 2.0];

    for delta in test_deltas {
        let params = DQNParams::default();

        let hyperparams = DQNHyperparameters {
            learning_rate: params.learning_rate,
            batch_size: params.batch_size,
            gamma: params.gamma,
            epsilon_start: 1.0,
            epsilon_end: 0.01,
            epsilon_decay: params.epsilon_decay,
            buffer_size: params.buffer_size,
            min_replay_size: params.batch_size * 2,
            epochs: 100,
            checkpoint_frequency: 10,
            early_stopping_enabled: true,
            q_value_floor: 0.5,
            min_loss_improvement_pct: 2.0,
            plateau_window: 5,
            min_epochs_before_stopping: 10,
            hold_penalty: -0.001,
            use_huber_loss: true,
            huber_delta: delta,
            use_double_dqn: true,
            gradient_clip_norm: Some(10.0),
            hold_penalty_weight: 0.01,
            movement_threshold: 0.02,
        };

        assert_eq!(
            hyperparams.huber_delta, delta,
            "Delta value should match: {}",
            delta
        );
    }
}