foxhunt/ml/tests/performance_regression_tests.rs

//! Performance Regression Detection Tests (TDD)
//!
//! Test-Driven Development approach for automated performance regression detection.
//! These tests SHOULD FAIL initially, then pass after implementation.
//!
//! Coverage:
//! - Baseline saving/loading
//! - Regression detection (>10% threshold)
//! - Metric tracking (DBN load, feature extraction, training, inference)
//! - CI integration readiness

use ml::benchmark::{PerformanceTracker, PerformanceBaseline, PerformanceMetrics, RegressionResult};
use std::path::PathBuf;
use tempfile::TempDir;

#[tokio::test]
async fn test_save_baseline() {
    let temp_dir = TempDir::new().expect("Failed to create temp dir");
    let baseline_path = temp_dir.path().join("baseline.json");

    let mut tracker = PerformanceTracker::new(baseline_path.clone());

    // Create sample metrics
    let metrics = PerformanceMetrics {
        dbn_load_time_ms: 0.70,
        feature_extraction_time_ms: 5.2,
        training_step_time_ms: 120.0,
        inference_latency_us: 45.0,
        throughput_samples_per_sec: 1000.0,
        memory_usage_mb: 250.0,
        timestamp: chrono::Utc::now(),
        git_commit: "abc123".to_string(),
        model_type: "DQN".to_string(),
    };

    tracker.record_metrics(metrics.clone()).await.expect("Failed to record metrics");
    tracker.save_baseline().await.expect("Failed to save baseline");

    // Verify file exists
    assert!(baseline_path.exists(), "Baseline file should exist");

    // Verify can load back
    let loaded = PerformanceTracker::load_baseline(&baseline_path).await;
    assert!(loaded.is_ok(), "Should load baseline successfully");
}

#[tokio::test]
async fn test_load_baseline() {
    let temp_dir = TempDir::new().expect("Failed to create temp dir");
    let baseline_path = temp_dir.path().join("baseline.json");

    // Create and save baseline
    let mut tracker = PerformanceTracker::new(baseline_path.clone());
    let metrics = PerformanceMetrics {
        dbn_load_time_ms: 0.70,
        feature_extraction_time_ms: 5.2,
        training_step_time_ms: 120.0,
        inference_latency_us: 45.0,
        throughput_samples_per_sec: 1000.0,
        memory_usage_mb: 250.0,
        timestamp: chrono::Utc::now(),
        git_commit: "abc123".to_string(),
        model_type: "DQN".to_string(),
    };

    tracker.record_metrics(metrics.clone()).await.expect("Failed to record");
    tracker.save_baseline().await.expect("Failed to save");

    // Load baseline
    let baseline = PerformanceTracker::load_baseline(&baseline_path).await.expect("Failed to load");

    assert_eq!(baseline.model_type, "DQN");
    assert_eq!(baseline.dbn_load_time_ms, 0.70);
    assert_eq!(baseline.feature_extraction_time_ms, 5.2);
    assert_eq!(baseline.training_step_time_ms, 120.0);
    assert_eq!(baseline.inference_latency_us, 45.0);
}

#[tokio::test]
async fn test_no_regression_when_within_threshold() {
    let temp_dir = TempDir::new().expect("Failed to create temp dir");
    let baseline_path = temp_dir.path().join("baseline.json");

    let mut tracker = PerformanceTracker::new(baseline_path.clone());

    // Save baseline
    let baseline_metrics = PerformanceMetrics {
        dbn_load_time_ms: 0.70,
        feature_extraction_time_ms: 5.0,
        training_step_time_ms: 100.0,
        inference_latency_us: 50.0,
        throughput_samples_per_sec: 1000.0,
        memory_usage_mb: 250.0,
        timestamp: chrono::Utc::now(),
        git_commit: "baseline".to_string(),
        model_type: "DQN".to_string(),
    };

    tracker.record_metrics(baseline_metrics).await.expect("Failed to record baseline");
    tracker.save_baseline().await.expect("Failed to save baseline");

    // New metrics within 10% threshold (5% slower is OK)
    let new_metrics = PerformanceMetrics {
        dbn_load_time_ms: 0.73, // 4.3% slower - OK
        feature_extraction_time_ms: 5.2, // 4% slower - OK
        training_step_time_ms: 105.0, // 5% slower - OK
        inference_latency_us: 52.0, // 4% slower - OK
        throughput_samples_per_sec: 980.0, // 2% slower - OK
        memory_usage_mb: 260.0, // 4% increase - OK
        timestamp: chrono::Utc::now(),
        git_commit: "new".to_string(),
        model_type: "DQN".to_string(),
    };

    tracker.record_metrics(new_metrics).await.expect("Failed to record new");

    let result = tracker.check_regression().await.expect("Failed to check regression");

    assert!(!result.has_regression, "Should not detect regression within threshold");
    assert!(result.regressions.is_empty(), "Should have no regression items");
}

#[tokio::test]
async fn test_detect_regression_above_threshold() {
    let temp_dir = TempDir::new().expect("Failed to create temp dir");
    let baseline_path = temp_dir.path().join("baseline.json");

    let mut tracker = PerformanceTracker::new(baseline_path.clone());

    // Save baseline
    let baseline_metrics = PerformanceMetrics {
        dbn_load_time_ms: 0.70,
        feature_extraction_time_ms: 5.0,
        training_step_time_ms: 100.0,
        inference_latency_us: 50.0,
        throughput_samples_per_sec: 1000.0,
        memory_usage_mb: 250.0,
        timestamp: chrono::Utc::now(),
        git_commit: "baseline".to_string(),
        model_type: "DQN".to_string(),
    };

    tracker.record_metrics(baseline_metrics).await.expect("Failed to record baseline");
    tracker.save_baseline().await.expect("Failed to save baseline");

    // New metrics with >10% regression (15% slower)
    let new_metrics = PerformanceMetrics {
        dbn_load_time_ms: 0.81, // 15.7% slower - REGRESSION
        feature_extraction_time_ms: 5.8, // 16% slower - REGRESSION
        training_step_time_ms: 120.0, // 20% slower - REGRESSION
        inference_latency_us: 60.0, // 20% slower - REGRESSION
        throughput_samples_per_sec: 850.0, // 15% slower - REGRESSION
        memory_usage_mb: 290.0, // 16% increase - REGRESSION
        timestamp: chrono::Utc::now(),
        git_commit: "regression".to_string(),
        model_type: "DQN".to_string(),
    };

    tracker.record_metrics(new_metrics).await.expect("Failed to record new");

    let result = tracker.check_regression().await.expect("Failed to check regression");

    assert!(result.has_regression, "Should detect regression above threshold");
    assert!(!result.regressions.is_empty(), "Should have regression items");

    // Check specific regressions detected
    assert!(result.regressions.iter().any(|r| r.metric == "dbn_load_time_ms"), "Should detect DBN load regression");
    assert!(result.regressions.iter().any(|r| r.metric == "training_step_time_ms"), "Should detect training regression");
    assert!(result.regressions.iter().any(|r| r.metric == "inference_latency_us"), "Should detect inference regression");
}

#[tokio::test]
async fn test_track_dbn_load_time() {
    let temp_dir = TempDir::new().expect("Failed to create temp dir");
    let baseline_path = temp_dir.path().join("baseline.json");

    let mut tracker = PerformanceTracker::new(baseline_path);

    let metrics = PerformanceMetrics {
        dbn_load_time_ms: 0.70, // From CLAUDE.md: 0.70ms for 1,674 bars
        feature_extraction_time_ms: 5.0,
        training_step_time_ms: 100.0,
        inference_latency_us: 50.0,
        throughput_samples_per_sec: 1000.0,
        memory_usage_mb: 250.0,
        timestamp: chrono::Utc::now(),
        git_commit: "test".to_string(),
        model_type: "DQN".to_string(),
    };

    tracker.record_metrics(metrics.clone()).await.expect("Failed to record");

    let recorded = tracker.get_latest_metrics().expect("Should have metrics");
    assert_eq!(recorded.dbn_load_time_ms, 0.70);
}

#[tokio::test]
async fn test_track_feature_extraction_time() {
    let temp_dir = TempDir::new().expect("Failed to create temp dir");
    let baseline_path = temp_dir.path().join("baseline.json");

    let mut tracker = PerformanceTracker::new(baseline_path);

    let metrics = PerformanceMetrics {
        dbn_load_time_ms: 0.70,
        feature_extraction_time_ms: 5.2, // Custom feature extraction time
        training_step_time_ms: 100.0,
        inference_latency_us: 50.0,
        throughput_samples_per_sec: 1000.0,
        memory_usage_mb: 250.0,
        timestamp: chrono::Utc::now(),
        git_commit: "test".to_string(),
        model_type: "DQN".to_string(),
    };

    tracker.record_metrics(metrics).await.expect("Failed to record");

    let recorded = tracker.get_latest_metrics().expect("Should have metrics");
    assert_eq!(recorded.feature_extraction_time_ms, 5.2);
}

#[tokio::test]
async fn test_track_training_step_time() {
    let temp_dir = TempDir::new().expect("Failed to create temp dir");
    let baseline_path = temp_dir.path().join("baseline.json");

    let mut tracker = PerformanceTracker::new(baseline_path);

    let metrics = PerformanceMetrics {
        dbn_load_time_ms: 0.70,
        feature_extraction_time_ms: 5.0,
        training_step_time_ms: 120.0, // Training step time
        inference_latency_us: 50.0,
        throughput_samples_per_sec: 1000.0,
        memory_usage_mb: 250.0,
        timestamp: chrono::Utc::now(),
        git_commit: "test".to_string(),
        model_type: "DQN".to_string(),
    };

    tracker.record_metrics(metrics).await.expect("Failed to record");

    let recorded = tracker.get_latest_metrics().expect("Should have metrics");
    assert_eq!(recorded.training_step_time_ms, 120.0);
}

#[tokio::test]
async fn test_track_inference_latency() {
    let temp_dir = TempDir::new().expect("Failed to create temp dir");
    let baseline_path = temp_dir.path().join("baseline.json");

    let mut tracker = PerformanceTracker::new(baseline_path);

    let metrics = PerformanceMetrics {
        dbn_load_time_ms: 0.70,
        feature_extraction_time_ms: 5.0,
        training_step_time_ms: 100.0,
        inference_latency_us: 45.0, // From CLAUDE.md: <50μs target
        throughput_samples_per_sec: 1000.0,
        memory_usage_mb: 250.0,
        timestamp: chrono::Utc::now(),
        git_commit: "test".to_string(),
        model_type: "DQN".to_string(),
    };

    tracker.record_metrics(metrics).await.expect("Failed to record");

    let recorded = tracker.get_latest_metrics().expect("Should have metrics");
    assert_eq!(recorded.inference_latency_us, 45.0);
}

#[tokio::test]
async fn test_multiple_models_independent_baselines() {
    let temp_dir = TempDir::new().expect("Failed to create temp dir");
    let dqn_baseline = temp_dir.path().join("dqn_baseline.json");
    let ppo_baseline = temp_dir.path().join("ppo_baseline.json");

    // DQN tracker
    let mut dqn_tracker = PerformanceTracker::new(dqn_baseline);
    let dqn_metrics = PerformanceMetrics {
        dbn_load_time_ms: 0.70,
        feature_extraction_time_ms: 5.0,
        training_step_time_ms: 100.0,
        inference_latency_us: 50.0,
        throughput_samples_per_sec: 1000.0,
        memory_usage_mb: 150.0,
        timestamp: chrono::Utc::now(),
        git_commit: "test".to_string(),
        model_type: "DQN".to_string(),
    };
    dqn_tracker.record_metrics(dqn_metrics).await.expect("Failed to record DQN");
    dqn_tracker.save_baseline().await.expect("Failed to save DQN baseline");

    // PPO tracker
    let mut ppo_tracker = PerformanceTracker::new(ppo_baseline);
    let ppo_metrics = PerformanceMetrics {
        dbn_load_time_ms: 0.70,
        feature_extraction_time_ms: 5.0,
        training_step_time_ms: 150.0, // PPO slower than DQN
        inference_latency_us: 60.0,
        throughput_samples_per_sec: 800.0,
        memory_usage_mb: 200.0, // PPO more memory
        timestamp: chrono::Utc::now(),
        git_commit: "test".to_string(),
        model_type: "PPO".to_string(),
    };
    ppo_tracker.record_metrics(ppo_metrics).await.expect("Failed to record PPO");
    ppo_tracker.save_baseline().await.expect("Failed to save PPO baseline");

    // Verify independent baselines
    let dqn_baseline_loaded = dqn_tracker.get_latest_metrics().expect("Should have DQN metrics");
    let ppo_baseline_loaded = ppo_tracker.get_latest_metrics().expect("Should have PPO metrics");

    assert_eq!(dqn_baseline_loaded.model_type, "DQN");
    assert_eq!(ppo_baseline_loaded.model_type, "PPO");
    assert_ne!(dqn_baseline_loaded.memory_usage_mb, ppo_baseline_loaded.memory_usage_mb);
}

#[tokio::test]
async fn test_regression_result_format_for_ci() {
    let temp_dir = TempDir::new().expect("Failed to create temp dir");
    let baseline_path = temp_dir.path().join("baseline.json");

    let mut tracker = PerformanceTracker::new(baseline_path);

    // Baseline
    let baseline_metrics = PerformanceMetrics {
        dbn_load_time_ms: 0.70,
        feature_extraction_time_ms: 5.0,
        training_step_time_ms: 100.0,
        inference_latency_us: 50.0,
        throughput_samples_per_sec: 1000.0,
        memory_usage_mb: 250.0,
        timestamp: chrono::Utc::now(),
        git_commit: "baseline".to_string(),
        model_type: "DQN".to_string(),
    };
    tracker.record_metrics(baseline_metrics).await.expect("Failed to record baseline");
    tracker.save_baseline().await.expect("Failed to save baseline");

    // Regression
    let new_metrics = PerformanceMetrics {
        dbn_load_time_ms: 1.0, // 42.9% slower - REGRESSION
        feature_extraction_time_ms: 5.0,
        training_step_time_ms: 100.0,
        inference_latency_us: 50.0,
        throughput_samples_per_sec: 1000.0,
        memory_usage_mb: 250.0,
        timestamp: chrono::Utc::now(),
        git_commit: "new".to_string(),
        model_type: "DQN".to_string(),
    };
    tracker.record_metrics(new_metrics).await.expect("Failed to record new");

    let result = tracker.check_regression().await.expect("Failed to check regression");

    // Verify CI-friendly format
    assert!(result.has_regression);
    assert!(!result.summary.is_empty());
    assert!(!result.regressions.is_empty());

    for regression in &result.regressions {
        assert!(!regression.metric.is_empty());
        assert!(regression.baseline_value > 0.0);
        assert!(regression.current_value > 0.0);
        assert!(regression.percent_change > 10.0);
        assert!(!regression.description.is_empty());
    }
}

#[tokio::test]
async fn test_ci_exit_code_on_regression() {
    let temp_dir = TempDir::new().expect("Failed to create temp dir");
    let baseline_path = temp_dir.path().join("baseline.json");

    let mut tracker = PerformanceTracker::new(baseline_path);

    // Baseline
    let baseline_metrics = PerformanceMetrics {
        dbn_load_time_ms: 0.70,
        feature_extraction_time_ms: 5.0,
        training_step_time_ms: 100.0,
        inference_latency_us: 50.0,
        throughput_samples_per_sec: 1000.0,
        memory_usage_mb: 250.0,
        timestamp: chrono::Utc::now(),
        git_commit: "baseline".to_string(),
        model_type: "DQN".to_string(),
    };
    tracker.record_metrics(baseline_metrics).await.expect("Failed");
    tracker.save_baseline().await.expect("Failed");

    // Regression
    let new_metrics = PerformanceMetrics {
        dbn_load_time_ms: 1.0,
        feature_extraction_time_ms: 5.0,
        training_step_time_ms: 100.0,
        inference_latency_us: 50.0,
        throughput_samples_per_sec: 1000.0,
        memory_usage_mb: 250.0,
        timestamp: chrono::Utc::now(),
        git_commit: "new".to_string(),
        model_type: "DQN".to_string(),
    };
    tracker.record_metrics(new_metrics).await.expect("Failed");

    let result = tracker.check_regression().await.expect("Failed");

    // CI should fail with exit code 1 when has_regression is true
    assert!(result.has_regression);
    assert_eq!(result.exit_code(), 1);
}

#[tokio::test]
async fn test_ci_exit_code_on_success() {
    let temp_dir = TempDir::new().expect("Failed to create temp dir");
    let baseline_path = temp_dir.path().join("baseline.json");

    let mut tracker = PerformanceTracker::new(baseline_path);

    // Baseline
    let baseline_metrics = PerformanceMetrics {
        dbn_load_time_ms: 0.70,
        feature_extraction_time_ms: 5.0,
        training_step_time_ms: 100.0,
        inference_latency_us: 50.0,
        throughput_samples_per_sec: 1000.0,
        memory_usage_mb: 250.0,
        timestamp: chrono::Utc::now(),
        git_commit: "baseline".to_string(),
        model_type: "DQN".to_string(),
    };
    tracker.record_metrics(baseline_metrics).await.expect("Failed");
    tracker.save_baseline().await.expect("Failed");

    // No regression
    let new_metrics = PerformanceMetrics {
        dbn_load_time_ms: 0.72, // 2.9% - OK
        feature_extraction_time_ms: 5.0,
        training_step_time_ms: 100.0,
        inference_latency_us: 50.0,
        throughput_samples_per_sec: 1000.0,
        memory_usage_mb: 250.0,
        timestamp: chrono::Utc::now(),
        git_commit: "new".to_string(),
        model_type: "DQN".to_string(),
    };
    tracker.record_metrics(new_metrics).await.expect("Failed");

    let result = tracker.check_regression().await.expect("Failed");

    // CI should succeed with exit code 0
    assert!(!result.has_regression);
    assert_eq!(result.exit_code(), 0);
}