foxhunt/ml/tests/ppo_hyperopt_integration_test.rs

//! Integration tests for PPO hyperparameter optimization workflow
//!
//! These tests validate the complete hyperopt workflow, including:
//! - Multi-step parameter transformations (PPOParams ↔ continuous)
//! - Multi-parameter interactions (value LR + minibatch, policy LR + entropy)
//! - Statistical parameter space sampling (1000 random samples)
//! - Edge case combinations (min/max boundaries)
//! - Clamping behavior (out-of-bounds handling)
//! - Log-scale precision (extreme value preservation)
//!
//! Unlike unit tests (which test individual functions), these tests validate
//! the integration between the hyperopt adapter and the optimization engine.

use ml::hyperopt::adapters::ppo::{PPOParams, PPOTrainer};
use ml::hyperopt::traits::{HyperparameterOptimizable, ParameterSpace};
use rand::Rng;

/// Helper: Validate that PPOParams are within expected bounds (with floating point tolerance)
fn assert_params_valid(params: &PPOParams) {
    // Log-scale parameters (with 1e-9 tolerance for floating point errors)
    let tolerance = 1e-9;
    assert!(
        params.policy_learning_rate >= 1e-6 - tolerance
            && params.policy_learning_rate <= 5e-5 + tolerance,
        "Policy LR out of bounds: {}",
        params.policy_learning_rate
    );
    assert!(
        params.value_learning_rate >= 1e-5 - tolerance
            && params.value_learning_rate <= 5e-3 + tolerance,
        "Value LR out of bounds: {}",
        params.value_learning_rate
    );
    assert!(
        params.entropy_coeff >= 0.001 - tolerance && params.entropy_coeff <= 0.1 + tolerance,
        "Entropy coeff out of bounds: {}",
        params.entropy_coeff
    );

    // Linear-scale parameters
    assert!(
        params.clip_epsilon >= 0.1 && params.clip_epsilon <= 0.3,
        "Clip epsilon out of bounds: {}",
        params.clip_epsilon
    );
    assert!(
        params.value_loss_coeff >= 0.5 && params.value_loss_coeff <= 2.0,
        "Value loss coeff out of bounds: {}",
        params.value_loss_coeff
    );
    assert!(
        params.minibatch_size >= 64 && params.minibatch_size <= 230,
        "Minibatch size out of bounds: {}",
        params.minibatch_size
    );

    // No NaN/Inf
    assert!(
        params.policy_learning_rate.is_finite(),
        "Policy LR is not finite"
    );
    assert!(
        params.value_learning_rate.is_finite(),
        "Value LR is not finite"
    );
    assert!(
        params.clip_epsilon.is_finite(),
        "Clip epsilon is not finite"
    );
    assert!(
        params.value_loss_coeff.is_finite(),
        "Value loss coeff is not finite"
    );
    assert!(
        params.entropy_coeff.is_finite(),
        "Entropy coeff is not finite"
    );
}

#[test]
fn test_full_hyperopt_workflow_simulation() {
    // Simulate real hyperopt optimizer behavior (multi-step roundtrip)
    // This tests that repeated conversions (as would happen during optimization)
    // don't introduce drift or numerical instability

    let mut rng = rand::thread_rng();

    // Start with default parameters
    let mut current_params = PPOParams::default();

    // Simulate 10 optimizer iterations
    for iteration in 0..10 {
        // Convert to continuous space
        let mut continuous = current_params.to_continuous();
        assert_eq!(continuous.len(), 6, "Expected 6 continuous parameters");

        // Simulate optimizer mutation (small random perturbations)
        // This mimics what an optimizer like PSO or GP would do
        for param_idx in 0..continuous.len() {
            let bounds = PPOParams::continuous_bounds();
            let (lower, upper) = bounds[param_idx];
            let range = upper - lower;

            // Add small noise (±5% of range)
            let noise = rng.gen_range(-0.05 * range..0.05 * range);
            continuous[param_idx] += noise;

            // Clamp to bounds (optimizer would do this)
            continuous[param_idx] = continuous[param_idx].clamp(lower, upper);
        }

        // Convert back to PPOParams
        current_params = PPOParams::from_continuous(&continuous)
            .expect("Failed to convert continuous to PPOParams");

        // Validate parameters are still valid
        assert_params_valid(&current_params);

        println!(
            "Iteration {}: policy_lr={:.6}, value_lr={:.6}, minibatch={}",
            iteration,
            current_params.policy_learning_rate,
            current_params.value_learning_rate,
            current_params.minibatch_size
        );
    }
}

#[test]
fn test_multi_parameter_interaction() {
    // Test that changing one parameter doesn't corrupt others
    // This validates parameter independence in the continuous encoding

    // Test matrix 1: Value LR × Minibatch Size (Wave 1 expanded both)
    let value_lrs = vec![1e-5, 1e-3, 5e-3]; // Min, mid, max
    let minibatch_sizes = vec![64, 147, 230]; // Min, mid, max

    for &value_lr in &value_lrs {
        for &minibatch_size in &minibatch_sizes {
            let params = PPOParams {
                policy_learning_rate: 1e-5, // Fixed
                value_learning_rate: value_lr,
                clip_epsilon: 0.2,
                value_loss_coeff: 1.0,
                entropy_coeff: 0.01,
                minibatch_size,
            };

            let continuous = params.to_continuous();
            let recovered = PPOParams::from_continuous(&continuous).unwrap();

            // Verify value LR and minibatch size are preserved
            assert!(
                (recovered.value_learning_rate - value_lr).abs() / value_lr < 1e-6,
                "Value LR mismatch: expected {}, got {}",
                value_lr,
                recovered.value_learning_rate
            );
            assert_eq!(
                recovered.minibatch_size, minibatch_size,
                "Minibatch size mismatch: expected {}, got {}",
                minibatch_size, recovered.minibatch_size
            );

            // Verify other params are unchanged
            assert!(
                (recovered.policy_learning_rate - 1e-5).abs() / 1e-5 < 1e-6,
                "Policy LR changed unexpectedly"
            );
        }
    }

    // Test matrix 2: Policy LR × Entropy Coeff (both log-scale)
    let policy_lrs = vec![1e-6, 2e-5, 5e-5]; // Min, mid, max
    let entropy_coeffs = vec![0.001, 0.05, 0.1]; // Min, mid, max

    for &policy_lr in &policy_lrs {
        for &entropy_coeff in &entropy_coeffs {
            let params = PPOParams {
                policy_learning_rate: policy_lr,
                value_learning_rate: 1e-4, // Fixed
                clip_epsilon: 0.2,
                value_loss_coeff: 1.0,
                entropy_coeff,
                minibatch_size: 128,
            };

            let continuous = params.to_continuous();
            let recovered = PPOParams::from_continuous(&continuous).unwrap();

            // Verify policy LR and entropy coeff are preserved
            assert!(
                (recovered.policy_learning_rate - policy_lr).abs() / policy_lr < 1e-6,
                "Policy LR mismatch: expected {}, got {}",
                policy_lr,
                recovered.policy_learning_rate
            );
            assert!(
                (recovered.entropy_coeff - entropy_coeff).abs() / entropy_coeff < 1e-6,
                "Entropy coeff mismatch: expected {}, got {}",
                entropy_coeff,
                recovered.entropy_coeff
            );
        }
    }
}

#[test]
fn test_parameter_space_sampling() {
    // Statistical validation: sample 1000 random points and verify all are valid
    // This catches issues that single-point tests miss

    let mut rng = rand::thread_rng();
    let bounds = PPOParams::continuous_bounds();
    let num_samples = 1000;

    let mut policy_lr_samples = Vec::new();
    let mut value_lr_samples = Vec::new();

    for _ in 0..num_samples {
        // Sample random point from continuous bounds
        let continuous: Vec<f64> = bounds
            .iter()
            .map(|(lower, upper)| rng.gen_range(*lower..*upper))
            .collect();

        // Convert to PPOParams
        let params = PPOParams::from_continuous(&continuous)
            .expect("Failed to convert sampled continuous params");

        // Validate parameters
        assert_params_valid(&params);

        // Collect samples for distribution analysis
        policy_lr_samples.push(params.policy_learning_rate);
        value_lr_samples.push(params.value_learning_rate);
    }

    // Verify log-scale distribution (more samples in lower linear half)
    // For policy LR (1e-6 to 5e-5), median should be ~sqrt(1e-6 * 5e-5) = 7.07e-6
    let mut sorted_policy_lr = policy_lr_samples.clone();
    sorted_policy_lr.sort_by(|a, b| a.partial_cmp(b).unwrap());
    let median_policy_lr = sorted_policy_lr[num_samples / 2];

    assert!(
        median_policy_lr < 2e-5,
        "Policy LR median too high (not log-distributed): {}",
        median_policy_lr
    );

    // For value LR (1e-5 to 5e-3), median should be ~sqrt(1e-5 * 5e-3) = 2.24e-4
    let mut sorted_value_lr = value_lr_samples.clone();
    sorted_value_lr.sort_by(|a, b| a.partial_cmp(b).unwrap());
    let median_value_lr = sorted_value_lr[num_samples / 2];

    assert!(
        median_value_lr < 2e-3,
        "Value LR median too high (not log-distributed): {}",
        median_value_lr
    );

    println!("Policy LR median: {:.6}", median_policy_lr);
    println!("Value LR median: {:.6}", median_value_lr);
    println!("All {} samples valid", num_samples);
}

#[test]
fn test_edge_case_combinations() {
    // Boundary condition testing: min/max combinations
    // Note: Log-scale params are NOT clamped by from_continuous(), so we must
    // use the continuous bounds (ln values), not the parameter bounds
    let bounds = PPOParams::continuous_bounds();
    let continuous1 = vec![
        bounds[0].0, // Min policy LR (log)
        bounds[1].0, // Min value LR (log)
        0.1,         // Min clip epsilon
        0.5,         // Min value loss coeff
        bounds[4].0, // Min entropy coeff (log)
        230.0,       // Max minibatch
    ];
    let params1 = PPOParams::from_continuous(&continuous1).unwrap();
    assert_params_valid(&params1);
    assert_eq!(params1.minibatch_size, 230);
    assert!(
        (params1.value_learning_rate - 1e-5).abs() / 1e-5 < 1e-6,
        "Min value LR not preserved"
    );

    // Combo 2: Max value LR + min minibatch
    let continuous2 = vec![
        bounds[0].1, // Max policy LR (log)
        bounds[1].1, // Max value LR (log)
        0.3,         // Max clip epsilon
        2.0,         // Max value loss coeff
        bounds[4].1, // Max entropy coeff (log)
        64.0,        // Min minibatch
    ];
    let params2 = PPOParams::from_continuous(&continuous2).unwrap();
    assert_params_valid(&params2);
    assert_eq!(params2.minibatch_size, 64);
    assert!(
        (params2.value_learning_rate - 5e-3).abs() / 5e-3 < 1e-6,
        "Max value LR not preserved"
    );

    // Combo 3: Min policy LR + max entropy
    let continuous3 = vec![
        bounds[0].0,   // Min policy LR (log)
        1e-4_f64.ln(), // Mid value LR
        0.2,           // Mid clip epsilon
        1.0,           // Mid value loss coeff
        bounds[4].1,   // Max entropy coeff (log)
        128.0,         // Mid minibatch
    ];
    let params3 = PPOParams::from_continuous(&continuous3).unwrap();
    assert_params_valid(&params3);
    assert!(
        (params3.policy_learning_rate - 1e-6).abs() / 1e-6 < 1e-6,
        "Min policy LR not preserved"
    );
    assert!(
        (params3.entropy_coeff - 0.1).abs() / 0.1 < 1e-6,
        "Max entropy coeff not preserved"
    );

    // Combo 4: All mins
    let continuous_all_mins = vec![
        bounds[0].0, // Min policy LR (log)
        bounds[1].0, // Min value LR (log)
        bounds[2].0, // Min clip epsilon
        bounds[3].0, // Min value loss coeff
        bounds[4].0, // Min entropy coeff (log)
        bounds[5].0, // Min minibatch
    ];
    let params_all_mins = PPOParams::from_continuous(&continuous_all_mins).unwrap();
    assert_params_valid(&params_all_mins);

    // Combo 5: All maxes
    let continuous_all_maxes = vec![
        bounds[0].1, // Max policy LR (log)
        bounds[1].1, // Max value LR (log)
        bounds[2].1, // Max clip epsilon
        bounds[3].1, // Max value loss coeff
        bounds[4].1, // Max entropy coeff (log)
        bounds[5].1, // Max minibatch
    ];
    let params_all_maxes = PPOParams::from_continuous(&continuous_all_maxes).unwrap();
    assert_params_valid(&params_all_maxes);
}

#[test]
fn test_clamping_behavior() {
    // Validate out-of-bounds handling (clamping)

    // Test clip_epsilon clamping (linear scale: 0.1 to 0.3)
    let test_cases_clip = vec![
        (-1.0, 0.1), // Far below → clamp to min
        (0.05, 0.1), // Below → clamp to min
        (0.2, 0.2),  // Within bounds → unchanged
        (0.5, 0.3),  // Above → clamp to max
        (2.0, 0.3),  // Far above → clamp to max
    ];

    for (input, expected) in test_cases_clip {
        let continuous = vec![
            1e-5_f64.ln(), // Valid policy LR
            1e-4_f64.ln(), // Valid value LR
            input,         // Test clip_epsilon
            1.0,           // Valid value loss coeff
            0.01_f64.ln(), // Valid entropy coeff
            128.0,         // Valid minibatch
        ];
        let params = PPOParams::from_continuous(&continuous).unwrap();
        assert!(
            (params.clip_epsilon - expected).abs() < 1e-9,
            "Clip epsilon clamping failed: input={}, expected={}, got={}",
            input,
            expected,
            params.clip_epsilon
        );
    }

    // Test value_loss_coeff clamping (linear scale: 0.5 to 2.0)
    let test_cases_value_loss = vec![
        (0.0, 0.5),  // Far below → clamp to min
        (0.3, 0.5),  // Below → clamp to min
        (1.0, 1.0),  // Within bounds → unchanged
        (3.0, 2.0),  // Above → clamp to max
        (10.0, 2.0), // Far above → clamp to max
    ];

    for (input, expected) in test_cases_value_loss {
        let continuous = vec![
            1e-5_f64.ln(), // Valid policy LR
            1e-4_f64.ln(), // Valid value LR
            0.2,           // Valid clip epsilon
            input,         // Test value_loss_coeff
            0.01_f64.ln(), // Valid entropy coeff
            128.0,         // Valid minibatch
        ];
        let params = PPOParams::from_continuous(&continuous).unwrap();
        assert!(
            (params.value_loss_coeff - expected).abs() < 1e-9,
            "Value loss coeff clamping failed: input={}, expected={}, got={}",
            input,
            expected,
            params.value_loss_coeff
        );
    }

    // Test minibatch_size clamping (linear scale: 64 to 230)
    let test_cases_minibatch = vec![
        (0.0, 64),    // Far below → clamp to min
        (50.0, 64),   // Below → clamp to min
        (128.0, 128), // Within bounds → unchanged
        (300.0, 230), // Above → clamp to max
        (500.0, 230), // Far above → clamp to max
    ];

    for (input, expected) in test_cases_minibatch {
        let continuous = vec![
            1e-5_f64.ln(), // Valid policy LR
            1e-4_f64.ln(), // Valid value LR
            0.2,           // Valid clip epsilon
            1.0,           // Valid value loss coeff
            0.01_f64.ln(), // Valid entropy coeff
            input,         // Test minibatch_size
        ];
        let params = PPOParams::from_continuous(&continuous).unwrap();
        assert_eq!(
            params.minibatch_size, expected,
            "Minibatch size clamping failed: input={}, expected={}, got={}",
            input, expected, params.minibatch_size
        );
    }

    // Test log-scale params: extreme values should not panic (but will be out of bounds)
    // NOTE: from_continuous() does NOT clamp log-scale params, so we expect them to be
    // outside the valid range. The optimizer is responsible for keeping values in bounds.
    let extreme_low = vec![
        -100.0, // Extremely low log value (1e-43)
        -100.0, // Extremely low log value
        0.2,    // Valid clip epsilon
        1.0,    // Valid value loss coeff
        -100.0, // Extremely low log value
        128.0,  // Valid minibatch
    ];
    let params_extreme_low = PPOParams::from_continuous(&extreme_low).unwrap();
    // Log-scale params will be out of bounds, but should not panic
    assert!(params_extreme_low.policy_learning_rate.is_finite());
    assert!(params_extreme_low.value_learning_rate.is_finite());
    assert!(params_extreme_low.entropy_coeff.is_finite());

    let extreme_high = vec![
        100.0, // Extremely high log value (2.7e43)
        100.0, // Extremely high log value
        0.2,   // Valid clip epsilon
        1.0,   // Valid value loss coeff
        100.0, // Extremely high log value
        128.0, // Valid minibatch
    ];
    let params_extreme_high = PPOParams::from_continuous(&extreme_high).unwrap();
    // Log-scale params will be out of bounds, but should not panic
    assert!(params_extreme_high.policy_learning_rate.is_finite());
    assert!(params_extreme_high.value_learning_rate.is_finite());
    assert!(params_extreme_high.entropy_coeff.is_finite());
}

#[test]
fn test_log_scale_precision() {
    // Verify log-scale preserves extreme values (no loss of precision)

    // Test 1: Min policy LR (1e-6)
    let params_min_policy = PPOParams {
        policy_learning_rate: 1e-6,
        value_learning_rate: 1e-4,
        clip_epsilon: 0.2,
        value_loss_coeff: 1.0,
        entropy_coeff: 0.01,
        minibatch_size: 128,
    };
    let continuous = params_min_policy.to_continuous();
    let recovered = PPOParams::from_continuous(&continuous).unwrap();
    let relative_error = (recovered.policy_learning_rate - 1e-6).abs() / 1e-6;
    assert!(
        relative_error < 1e-6,
        "Min policy LR precision lost: expected 1e-6, got {}, relative error={}",
        recovered.policy_learning_rate,
        relative_error
    );

    // Test 2: Min value LR (1e-5)
    let params_min_value = PPOParams {
        policy_learning_rate: 1e-5,
        value_learning_rate: 1e-5,
        clip_epsilon: 0.2,
        value_loss_coeff: 1.0,
        entropy_coeff: 0.01,
        minibatch_size: 128,
    };
    let continuous = params_min_value.to_continuous();
    let recovered = PPOParams::from_continuous(&continuous).unwrap();
    let relative_error = (recovered.value_learning_rate - 1e-5).abs() / 1e-5;
    assert!(
        relative_error < 1e-6,
        "Min value LR precision lost: expected 1e-5, got {}, relative error={}",
        recovered.value_learning_rate,
        relative_error
    );

    // Test 3: Min entropy coeff (0.001)
    let params_min_entropy = PPOParams {
        policy_learning_rate: 1e-5,
        value_learning_rate: 1e-4,
        clip_epsilon: 0.2,
        value_loss_coeff: 1.0,
        entropy_coeff: 0.001,
        minibatch_size: 128,
    };
    let continuous = params_min_entropy.to_continuous();
    let recovered = PPOParams::from_continuous(&continuous).unwrap();
    let relative_error = (recovered.entropy_coeff - 0.001).abs() / 0.001;
    assert!(
        relative_error < 1e-6,
        "Min entropy coeff precision lost: expected 0.001, got {}, relative error={}",
        recovered.entropy_coeff,
        relative_error
    );

    // Test 4: Verify no values become 0.0 after roundtrip
    assert!(
        recovered.policy_learning_rate > 0.0,
        "Policy LR became zero after roundtrip"
    );
    assert!(
        recovered.value_learning_rate > 0.0,
        "Value LR became zero after roundtrip"
    );
    assert!(
        recovered.entropy_coeff > 0.0,
        "Entropy coeff became zero after roundtrip"
    );
}

#[test]
fn test_objective_function_with_realistic_params() {
    // Test objective function with realistic PPOParams from hyperopt trials
    // This validates that the objective function works correctly with
    // parameters that would be produced by the optimizer

    use ml::hyperopt::adapters::ppo::PPOMetrics;

    // Simulate metrics from a good trial (high reward)
    let good_metrics = PPOMetrics {
        policy_loss: 0.5,
        value_loss: 0.3,
        val_policy_loss: 0.4,
        val_value_loss: 0.2,
        combined_loss: 0.8,
        avg_episode_reward: 150.0, // High reward
        episodes_completed: 1000,
    };

    let good_objective = PPOTrainer::extract_objective(&good_metrics);

    // Simulate metrics from a poor trial (low reward)
    let poor_metrics = PPOMetrics {
        policy_loss: 0.3,
        value_loss: 0.2,
        val_policy_loss: 0.25,
        val_value_loss: 0.15,
        combined_loss: 0.5,
        avg_episode_reward: 20.0, // Low reward
        episodes_completed: 1000,
    };

    let poor_objective = PPOTrainer::extract_objective(&poor_metrics);

    // Verify: higher reward → lower objective (better for minimization)
    assert!(
        good_objective < poor_objective,
        "Good trial should have lower objective: good={}, poor={}",
        good_objective,
        poor_objective
    );

    // Verify: objective is negative of reward
    assert_eq!(good_objective, -150.0);
    assert_eq!(poor_objective, -20.0);
}