foxhunt/ml/tests/dqn_hyperopt_json_loading_test.rs

//! Integration tests for DQN hyperopt JSON loading functionality
//!
//! Tests the ability to load hyperparameters from JSON files for production
//! deployment and hyperopt result persistence.

use anyhow::Result;
use serde_json;
use std::fs;
use std::io::Write;
use tempfile::NamedTempFile;

use ml::hyperopt::adapters::dqn::{BestTrialExport, DQNParams};

#[cfg(test)]
mod hyperopt_json_loading_tests {
    use super::*;

    /// Test 1: Load valid example_trial26.json and verify all parameters override defaults
    #[test]
    fn test_load_valid_json_overrides_defaults() {
        // Load the example JSON file (path relative to ml/ directory where test runs)
        let json_path = "hyperopt_results/example_trial26.json";
        let json_content = fs::read_to_string(json_path)
            .expect("Failed to read example_trial26.json");

        // Parse the BestTrialExport
        let trial: BestTrialExport = serde_json::from_str(&json_content)
            .expect("Failed to parse example_trial26.json");

        // Verify metadata fields
        assert_eq!(trial.trial_number, 26, "Trial number mismatch");
        assert_eq!(trial.sharpe, 0.7743, "Sharpe ratio mismatch");
        assert_eq!(trial.win_rate, 51.22, "Win rate mismatch");
        assert_eq!(trial.max_drawdown, 0.63, "Max drawdown mismatch");
        assert_eq!(trial.total_return, 2.31, "Total return mismatch");
        assert_eq!(trial.gradient_clip_norm, 10.0, "Gradient clip norm mismatch");

        // Verify all 21 hyperparameters were loaded correctly (not defaults)
        let params = trial.hyperparameters;
        let defaults = DQNParams::default();

        // Core parameters
        assert_eq!(params.learning_rate, 0.00001, "Learning rate should be 0.00001");
        assert_ne!(params.learning_rate, defaults.learning_rate, "Learning rate should override default");

        assert_eq!(params.batch_size, 59, "Batch size should be 59");
        assert_ne!(params.batch_size, defaults.batch_size, "Batch size should override default");

        assert_eq!(params.gamma, 0.961042, "Gamma should be 0.961042");
        assert_ne!(params.gamma, defaults.gamma, "Gamma should override default");

        assert_eq!(params.buffer_size, 92399, "Buffer size should be 92399");
        assert_ne!(params.buffer_size, defaults.buffer_size, "Buffer size should override default");

        assert_eq!(params.hold_penalty_weight, 0.5, "Hold penalty weight should be 0.5");
        assert_ne!(params.hold_penalty_weight, defaults.hold_penalty_weight, "Hold penalty should override default");

        assert_eq!(params.max_position_absolute, 10.0, "Max position should be 10.0");
        assert_ne!(params.max_position_absolute, defaults.max_position_absolute, "Max position should override default");

        // Loss/regularization parameters
        assert_eq!(params.huber_delta, 10.0, "Huber delta should be 10.0");
        assert_eq!(params.entropy_coefficient, 0.01, "Entropy coefficient should be 0.01");
        assert_eq!(params.transaction_cost_multiplier, 1.0, "Transaction cost multiplier should be 1.0");

        // PER parameters
        assert_eq!(params.use_per, true, "PER should be enabled");
        assert_eq!(params.per_alpha, 0.6, "PER alpha should be 0.6");
        assert_eq!(params.per_beta_start, 0.4, "PER beta start should be 0.4");

        // Dueling DQN parameters
        assert_eq!(params.use_dueling, true, "Dueling should be enabled");
        assert_eq!(params.dueling_hidden_dim, 128, "Dueling hidden dim should be 128");

        // Multi-step and soft updates
        assert_eq!(params.n_steps, 3, "N-steps should be 3");
        assert_eq!(params.tau, 0.001, "Tau should be 0.001");

        // Distributional RL parameters
        assert_eq!(params.use_distributional, true, "Distributional should be enabled");
        assert_eq!(params.num_atoms, 51, "Num atoms should be 51");
        assert_eq!(params.v_min, -2.0, "V_min should be -2.0");
        assert_eq!(params.v_max, 2.0, "V_max should be 2.0");

        // Noisy Networks parameters
        assert_eq!(params.use_noisy_nets, true, "Noisy nets should be enabled");
        assert_eq!(params.noisy_sigma_init, 0.5, "Noisy sigma init should be 0.5");

        // Bug #7 fix
        assert_eq!(params.minimum_profit_factor, 1.5, "Minimum profit factor should be 1.5");

        println!("✅ All 21 hyperparameters successfully loaded from JSON");
    }

    /// Test 2: Attempt to load nonexistent JSON file returns error
    #[test]
    fn test_load_nonexistent_json_returns_error() {
        let result = fs::read_to_string("hyperopt_results/nonexistent_trial.json");

        assert!(result.is_err(), "Loading nonexistent file should return error");

        let error = result.unwrap_err();
        let error_msg = format!("{}", error);
        assert!(
            error_msg.contains("No such file") || error_msg.contains("not found"),
            "Error message should indicate file not found, got: {}",
            error_msg
        );

        println!("✅ Nonexistent file properly returns error");
    }

    /// Test 3: Attempt to load invalid JSON returns error
    #[test]
    fn test_load_invalid_json_returns_error() {
        // Create temporary file with invalid JSON
        let mut temp_file = NamedTempFile::new().expect("Failed to create temp file");
        writeln!(temp_file, r#"{{"broken": }}"#).expect("Failed to write invalid JSON");

        let temp_path = temp_file.path();
        let json_content = fs::read_to_string(temp_path).expect("Failed to read temp file");

        let result: Result<BestTrialExport, _> = serde_json::from_str(&json_content);

        assert!(result.is_err(), "Parsing invalid JSON should return error");

        let error = result.unwrap_err();
        let error_msg = format!("{}", error);
        assert!(
            error_msg.contains("expected value") || error_msg.contains("EOF"),
            "Error message should indicate JSON parse error, got: {}",
            error_msg
        );

        println!("✅ Invalid JSON properly returns parse error");
    }

    /// Test 4: Load JSON with missing required fields
    #[test]
    fn test_load_json_missing_required_field() {
        // Create temporary file with JSON missing 'learning_rate'
        let mut temp_file = NamedTempFile::new().expect("Failed to create temp file");
        let incomplete_json = r#"{
            "trial_number": 1,
            "sharpe": 0.5,
            "win_rate": 50.0,
            "max_drawdown": 1.0,
            "total_return": 1.0,
            "hyperparameters": {
                "batch_size": 64,
                "gamma": 0.99
            },
            "timestamp": "2025-11-22T00:00:00Z",
            "gradient_clip_norm": 10.0
        }"#;
        writeln!(temp_file, "{}", incomplete_json).expect("Failed to write incomplete JSON");

        let temp_path = temp_file.path();
        let json_content = fs::read_to_string(temp_path).expect("Failed to read temp file");

        let result: Result<BestTrialExport, _> = serde_json::from_str(&json_content);

        assert!(
            result.is_err(),
            "Parsing JSON with missing required fields should return error"
        );

        let error = result.unwrap_err();
        let error_msg = format!("{}", error);
        assert!(
            error_msg.contains("missing field") || error_msg.contains("learning_rate"),
            "Error should indicate missing field, got: {}",
            error_msg
        );

        println!("✅ Missing required field properly returns error");
    }

    /// Test 5: Verify JSON roundtrip consistency
    #[test]
    fn test_json_roundtrip_consistency() {
        // Create a BestTrialExport with known values
        let original_params = DQNParams {
            learning_rate: 0.00005,
            batch_size: 100,
            gamma: 0.98,
            buffer_size: 80000,
            hold_penalty_weight: 1.5,
            max_position_absolute: 5.0,
            huber_delta: 15.0,
            entropy_coefficient: 0.05,
            transaction_cost_multiplier: 1.2,
            use_per: true,
            per_alpha: 0.7,
            per_beta_start: 0.5,
            use_dueling: false,
            dueling_hidden_dim: 256,
            n_steps: 5,
            tau: 0.005,
            use_distributional: false,
            num_atoms: 101,
            v_min: -5.0,
            v_max: 5.0,
            use_noisy_nets: false,
            noisy_sigma_init: 0.3,
            minimum_profit_factor: 1.8,
        };

        let original_trial = BestTrialExport {
            trial_number: 99,
            sharpe: 1.234,
            win_rate: 55.67,
            max_drawdown: 2.34,
            total_return: 5.67,
            hyperparameters: original_params.clone(),
            timestamp: "2025-11-22T12:34:56Z".to_string(),
            gradient_clip_norm: 50.0,
        };

        // Serialize to JSON
        let json_str = serde_json::to_string_pretty(&original_trial)
            .expect("Failed to serialize to JSON");

        // Deserialize back
        let roundtrip_trial: BestTrialExport = serde_json::from_str(&json_str)
            .expect("Failed to deserialize from JSON");

        // Verify all fields match exactly
        assert_eq!(roundtrip_trial.trial_number, original_trial.trial_number);
        assert_eq!(roundtrip_trial.sharpe, original_trial.sharpe);
        assert_eq!(roundtrip_trial.win_rate, original_trial.win_rate);
        assert_eq!(roundtrip_trial.max_drawdown, original_trial.max_drawdown);
        assert_eq!(roundtrip_trial.total_return, original_trial.total_return);
        assert_eq!(roundtrip_trial.gradient_clip_norm, original_trial.gradient_clip_norm);
        assert_eq!(roundtrip_trial.timestamp, original_trial.timestamp);

        // Verify all hyperparameters
        let params = roundtrip_trial.hyperparameters;
        assert_eq!(params.learning_rate, 0.00005);
        assert_eq!(params.batch_size, 100);
        assert_eq!(params.gamma, 0.98);
        assert_eq!(params.buffer_size, 80000);
        assert_eq!(params.hold_penalty_weight, 1.5);
        assert_eq!(params.max_position_absolute, 5.0);
        assert_eq!(params.huber_delta, 15.0);
        assert_eq!(params.entropy_coefficient, 0.05);
        assert_eq!(params.transaction_cost_multiplier, 1.2);
        assert_eq!(params.use_per, true);
        assert_eq!(params.per_alpha, 0.7);
        assert_eq!(params.per_beta_start, 0.5);
        assert_eq!(params.use_dueling, false);
        assert_eq!(params.dueling_hidden_dim, 256);
        assert_eq!(params.n_steps, 5);
        assert_eq!(params.tau, 0.005);
        assert_eq!(params.use_distributional, false);
        assert_eq!(params.num_atoms, 101);
        assert_eq!(params.v_min, -5.0);
        assert_eq!(params.v_max, 5.0);
        assert_eq!(params.use_noisy_nets, false);
        assert_eq!(params.noisy_sigma_init, 0.3);
        assert_eq!(params.minimum_profit_factor, 1.8);

        println!("✅ JSON roundtrip maintains all 21 hyperparameters with perfect fidelity");
    }

    /// Test 6: Verify timestamp parsing
    #[test]
    fn test_timestamp_format_validation() {
        let json_path = "hyperopt_results/example_trial26.json";
        let json_content = fs::read_to_string(json_path)
            .expect("Failed to read example_trial26.json");

        let trial: BestTrialExport = serde_json::from_str(&json_content)
            .expect("Failed to parse JSON");

        // Verify timestamp is in ISO 8601 format
        assert!(!trial.timestamp.is_empty(), "Timestamp should not be empty");
        assert!(
            trial.timestamp.contains('T') && trial.timestamp.contains('Z'),
            "Timestamp should be in ISO 8601 format (YYYY-MM-DDTHH:MM:SSZ), got: {}",
            trial.timestamp
        );

        println!("✅ Timestamp properly formatted as ISO 8601: {}", trial.timestamp);
    }

    /// Test 7: Verify all boolean flags are properly serialized
    #[test]
    fn test_boolean_flags_serialization() {
        let params_all_true = DQNParams {
            use_per: true,
            use_dueling: true,
            use_distributional: true,
            use_noisy_nets: true,
            ..Default::default()
        };

        let trial_true = BestTrialExport {
            trial_number: 1,
            sharpe: 1.0,
            win_rate: 50.0,
            max_drawdown: 1.0,
            total_return: 1.0,
            hyperparameters: params_all_true,
            timestamp: "2025-11-22T00:00:00Z".to_string(),
            gradient_clip_norm: 10.0,
        };

        let json_true = serde_json::to_string(&trial_true).expect("Failed to serialize");
        let roundtrip_true: BestTrialExport = serde_json::from_str(&json_true)
            .expect("Failed to deserialize");

        assert_eq!(roundtrip_true.hyperparameters.use_per, true);
        assert_eq!(roundtrip_true.hyperparameters.use_dueling, true);
        assert_eq!(roundtrip_true.hyperparameters.use_distributional, true);
        assert_eq!(roundtrip_true.hyperparameters.use_noisy_nets, true);

        // Test with all false
        let params_all_false = DQNParams {
            use_per: false,
            use_dueling: false,
            use_distributional: false,
            use_noisy_nets: false,
            ..Default::default()
        };

        let trial_false = BestTrialExport {
            trial_number: 2,
            sharpe: 0.5,
            win_rate: 48.0,
            max_drawdown: 2.0,
            total_return: 0.5,
            hyperparameters: params_all_false,
            timestamp: "2025-11-22T00:00:00Z".to_string(),
            gradient_clip_norm: 10.0,
        };

        let json_false = serde_json::to_string(&trial_false).expect("Failed to serialize");
        let roundtrip_false: BestTrialExport = serde_json::from_str(&json_false)
            .expect("Failed to deserialize");

        assert_eq!(roundtrip_false.hyperparameters.use_per, false);
        assert_eq!(roundtrip_false.hyperparameters.use_dueling, false);
        assert_eq!(roundtrip_false.hyperparameters.use_distributional, false);
        assert_eq!(roundtrip_false.hyperparameters.use_noisy_nets, false);

        println!("✅ Boolean flags (use_per, use_dueling, use_distributional, use_noisy_nets) serialize correctly");
    }

    /// Test 8: Verify numeric bounds are preserved
    #[test]
    fn test_numeric_bounds_preservation() {
        // Test edge case values
        let params_edge = DQNParams {
            learning_rate: 0.0001,     // Upper bound from hyperopt
            batch_size: 230,            // Max for RTX 3050 Ti
            gamma: 0.99,                // Upper bound
            buffer_size: 100_000,       // Upper bound
            hold_penalty_weight: 5.0,   // Upper bound
            max_position_absolute: 10.0, // Upper bound
            huber_delta: 2.0,           // Upper bound
            entropy_coefficient: 0.1,   // Upper bound
            transaction_cost_multiplier: 2.0, // Upper bound
            per_alpha: 0.8,             // Upper bound
            per_beta_start: 0.6,        // Upper bound
            dueling_hidden_dim: 512,    // Upper bound
            n_steps: 10,                // Upper bound
            tau: 0.01,                  // Upper bound
            num_atoms: 201,             // Upper bound
            v_min: -2000.0,             // Lower bound
            v_max: 2000.0,              // Upper bound
            noisy_sigma_init: 1.0,      // Upper bound
            minimum_profit_factor: 2.0, // Upper bound
            ..Default::default()
        };

        let trial = BestTrialExport {
            trial_number: 999,
            sharpe: 5.0,                // Extremely high (but possible)
            win_rate: 99.99,            // Near perfect
            max_drawdown: 0.01,         // Minimal
            total_return: 100.0,        // Exceptional
            hyperparameters: params_edge,
            timestamp: "2025-11-22T23:59:59Z".to_string(),
            gradient_clip_norm: 1000.0, // High gradient clipping
        };

        let json = serde_json::to_string_pretty(&trial).expect("Failed to serialize");
        let roundtrip: BestTrialExport = serde_json::from_str(&json)
            .expect("Failed to deserialize");

        // Verify edge values preserved exactly
        assert_eq!(roundtrip.hyperparameters.learning_rate, 0.0001);
        assert_eq!(roundtrip.hyperparameters.batch_size, 230);
        assert_eq!(roundtrip.hyperparameters.v_min, -2000.0);
        assert_eq!(roundtrip.hyperparameters.v_max, 2000.0);
        assert_eq!(roundtrip.sharpe, 5.0);
        assert_eq!(roundtrip.gradient_clip_norm, 1000.0);

        println!("✅ Numeric bounds and edge values preserved with full precision");
    }
}