foxhunt/crates/ml/src/benchmark/performance_tracker.rs

//! Performance Regression Detection for ML Training Pipeline
//!
//! Automated performance regression detection system that tracks key metrics
//! across the training pipeline and fails CI builds when performance degrades
//! beyond acceptable thresholds.
//!
//! # Tracked Metrics
//!
//! - **DBN Load Time**: Real market data loading from DBN files (target: <10ms)
//! - **Feature Extraction**: Technical indicator calculation (16 features)
//! - **Training Step**: Single training iteration time
//! - **Inference Latency**: Model prediction time (target: <50μs)
//! - **Throughput**: Samples processed per second
//! - **Memory Usage**: Peak memory consumption in MB
//!
//! # Regression Threshold
//!
//! Fail CI if any metric regresses by >10% compared to baseline
//!
//! # Usage
//!
//! ```rust,no_run
//! use ml::benchmark::{PerformanceTracker, PerformanceMetrics};
//! use std::path::PathBuf;
//!
//! #[tokio::main]
//! async fn main() -> Result<(), Box<dyn std::error::Error>> {
//!     let baseline_path = PathBuf::from("baseline.json");
//!     let mut tracker = PerformanceTracker::new(baseline_path);
//!
//!     // Record metrics from training run
//!     let metrics = PerformanceMetrics {
//!         dbn_load_time_ms: 0.70,
//!         feature_extraction_time_ms: 5.2,
//!         training_step_time_ms: 120.0,
//!         inference_latency_us: 45.0,
//!         throughput_samples_per_sec: 1000.0,
//!         memory_usage_mb: 250.0,
//!         timestamp: chrono::Utc::now(),
//!         git_commit: "abc123".to_owned(),
//!         model_type: "DQN".to_owned(),
//!     };
//!
//!     tracker.record_metrics(metrics).await?;
//!     tracker.save_baseline().await?;
//!
//!     // Check for regressions in PR
//!     let result = tracker.check_regression().await?;
//!     if result.has_regression {
//!         eprintln!("Performance regression detected!");
//!         std::process::exit(1);
//!     }
//!
//!     Ok(())
//! }
//! ```

use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use std::fmt::Write as _;
use std::path::PathBuf;
use tokio::fs;
use tracing::{info, warn};

/// Performance metrics for a single training run
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PerformanceMetrics {
    /// DBN data loading time in milliseconds
    pub dbn_load_time_ms: f64,
    /// Feature extraction time in milliseconds
    pub feature_extraction_time_ms: f64,
    /// Training step time in milliseconds
    pub training_step_time_ms: f64,
    /// Inference latency in microseconds
    pub inference_latency_us: f64,
    /// Throughput in samples per second
    pub throughput_samples_per_sec: f64,
    /// Memory usage in megabytes
    pub memory_usage_mb: f64,
    /// Timestamp of measurement
    pub timestamp: DateTime<Utc>,
    /// Git commit hash
    pub git_commit: String,
    /// Model type (DQN, PPO, MAMBA-2, TFT)
    pub model_type: String,
}

/// Performance baseline saved to disk
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PerformanceBaseline {
    /// Model type
    pub model_type: String,
    /// DBN load time baseline
    pub dbn_load_time_ms: f64,
    /// Feature extraction baseline
    pub feature_extraction_time_ms: f64,
    /// Training step baseline
    pub training_step_time_ms: f64,
    /// Inference latency baseline
    pub inference_latency_us: f64,
    /// Throughput baseline
    pub throughput_samples_per_sec: f64,
    /// Memory usage baseline
    pub memory_usage_mb: f64,
    /// Baseline timestamp
    pub timestamp: DateTime<Utc>,
    /// Git commit of baseline
    pub git_commit: String,
}

impl From<PerformanceMetrics> for PerformanceBaseline {
    fn from(metrics: PerformanceMetrics) -> Self {
        Self {
            model_type: metrics.model_type,
            dbn_load_time_ms: metrics.dbn_load_time_ms,
            feature_extraction_time_ms: metrics.feature_extraction_time_ms,
            training_step_time_ms: metrics.training_step_time_ms,
            inference_latency_us: metrics.inference_latency_us,
            throughput_samples_per_sec: metrics.throughput_samples_per_sec,
            memory_usage_mb: metrics.memory_usage_mb,
            timestamp: metrics.timestamp,
            git_commit: metrics.git_commit,
        }
    }
}

/// Single regression item
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RegressionItem {
    /// Metric name
    pub metric: String,
    /// Baseline value
    pub baseline_value: f64,
    /// Current value
    pub current_value: f64,
    /// Percent change (positive = regression/slower)
    pub percent_change: f64,
    /// Human-readable description
    pub description: String,
}

/// Result of regression check
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RegressionResult {
    /// True if any regression detected
    pub has_regression: bool,
    /// List of regressions found
    pub regressions: Vec<RegressionItem>,
    /// Summary message
    pub summary: String,
    /// Current metrics
    pub current: PerformanceMetrics,
    /// Baseline metrics
    pub baseline: PerformanceBaseline,
}

impl RegressionResult {
    /// Get exit code for CI (0 = success, 1 = regression)
    pub fn exit_code(&self) -> i32 {
        if self.has_regression {
            1
        } else {
            0
        }
    }
}

/// Performance tracker for regression detection
#[derive(Debug)]
pub struct PerformanceTracker {
    /// Path to baseline file
    baseline_path: PathBuf,
    /// Current metrics (in-memory)
    current_metrics: Option<PerformanceMetrics>,
    /// Regression threshold (10% default)
    threshold_percent: f64,
}

impl PerformanceTracker {
    /// Create new performance tracker
    pub fn new(baseline_path: PathBuf) -> Self {
        Self {
            baseline_path,
            current_metrics: None,
            threshold_percent: 10.0, // 10% regression threshold
        }
    }

    /// Create tracker with custom threshold
    pub fn with_threshold(baseline_path: PathBuf, threshold_percent: f64) -> Self {
        Self {
            baseline_path,
            current_metrics: None,
            threshold_percent,
        }
    }

    /// Record performance metrics
    pub async fn record_metrics(
        &mut self,
        metrics: PerformanceMetrics,
    ) -> Result<(), std::io::Error> {
        info!(
            "Recording performance metrics for {}: DBN={:.2}ms, Features={:.2}ms, Training={:.2}ms, Inference={:.2}μs",
            metrics.model_type,
            metrics.dbn_load_time_ms,
            metrics.feature_extraction_time_ms,
            metrics.training_step_time_ms,
            metrics.inference_latency_us
        );

        self.current_metrics = Some(metrics);
        Ok(())
    }

    /// Save current metrics as baseline
    pub async fn save_baseline(&self) -> Result<(), std::io::Error> {
        let metrics = self.current_metrics.as_ref().ok_or_else(|| {
            std::io::Error::new(std::io::ErrorKind::NotFound, "No metrics recorded yet")
        })?;

        let baseline: PerformanceBaseline = metrics.clone().into();
        let json = serde_json::to_string_pretty(&baseline)
            .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;

        fs::write(&self.baseline_path, json).await?;

        info!(
            "Saved performance baseline for {} to {}",
            baseline.model_type,
            self.baseline_path.display()
        );

        Ok(())
    }

    /// Load baseline from disk
    pub async fn load_baseline(path: &PathBuf) -> Result<PerformanceBaseline, std::io::Error> {
        let json = fs::read_to_string(path).await?;
        let baseline: PerformanceBaseline = serde_json::from_str(&json)
            .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;

        info!(
            "Loaded performance baseline for {} from {}",
            baseline.model_type,
            path.display()
        );

        Ok(baseline)
    }

    /// Get latest recorded metrics
    pub fn get_latest_metrics(&self) -> Option<&PerformanceMetrics> {
        self.current_metrics.as_ref()
    }

    /// Check for performance regressions
    pub async fn check_regression(&self) -> Result<RegressionResult, std::io::Error> {
        let current = self.current_metrics.as_ref().ok_or_else(|| {
            std::io::Error::new(std::io::ErrorKind::NotFound, "No current metrics recorded")
        })?;

        let baseline = Self::load_baseline(&self.baseline_path).await?;

        let mut regressions = Vec::new();

        // Check DBN load time
        self.check_metric(
            "dbn_load_time_ms",
            baseline.dbn_load_time_ms,
            current.dbn_load_time_ms,
            "DBN data loading time",
            &mut regressions,
        );

        // Check feature extraction time
        self.check_metric(
            "feature_extraction_time_ms",
            baseline.feature_extraction_time_ms,
            current.feature_extraction_time_ms,
            "Feature extraction time",
            &mut regressions,
        );

        // Check training step time
        self.check_metric(
            "training_step_time_ms",
            baseline.training_step_time_ms,
            current.training_step_time_ms,
            "Training step time",
            &mut regressions,
        );

        // Check inference latency
        self.check_metric(
            "inference_latency_us",
            baseline.inference_latency_us,
            current.inference_latency_us,
            "Inference latency",
            &mut regressions,
        );

        // Check throughput (inverse: lower is worse)
        self.check_metric_inverse(
            "throughput_samples_per_sec",
            baseline.throughput_samples_per_sec,
            current.throughput_samples_per_sec,
            "Throughput",
            &mut regressions,
        );

        // Check memory usage
        self.check_metric(
            "memory_usage_mb",
            baseline.memory_usage_mb,
            current.memory_usage_mb,
            "Memory usage",
            &mut regressions,
        );

        let has_regression = !regressions.is_empty();

        let summary = if has_regression {
            let count = regressions.len();
            let metrics_list: Vec<String> = regressions.iter().map(|r| r.metric.clone()).collect();
            format!(
                "Performance regression detected: {} metric(s) degraded by >{}%: {}",
                count,
                self.threshold_percent,
                metrics_list.join(", ")
            )
        } else {
            format!(
                "No performance regression detected (threshold: {}%)",
                self.threshold_percent
            )
        };

        if has_regression {
            warn!("{}", summary);
            for regression in &regressions {
                warn!(
                    "  - {}: {:.2} → {:.2} ({:+.1}%)",
                    regression.metric,
                    regression.baseline_value,
                    regression.current_value,
                    regression.percent_change
                );
            }
        } else {
            info!("{}", summary);
        }

        Ok(RegressionResult {
            has_regression,
            regressions,
            summary,
            current: current.clone(),
            baseline,
        })
    }

    /// Check single metric for regression (higher = worse)
    fn check_metric(
        &self,
        metric_name: &str,
        baseline_value: f64,
        current_value: f64,
        description: &str,
        regressions: &mut Vec<RegressionItem>,
    ) {
        if baseline_value <= 0.0 {
            return; // Skip invalid baseline
        }

        let percent_change = ((current_value - baseline_value) / baseline_value) * 100.0;

        if percent_change > self.threshold_percent {
            regressions.push(RegressionItem {
                metric: metric_name.to_string(),
                baseline_value,
                current_value,
                percent_change,
                description: format!(
                    "{} increased by {:.1}% ({:.2} → {:.2})",
                    description, percent_change, baseline_value, current_value
                ),
            });
        }
    }

    /// Check metric where lower is worse (e.g., throughput)
    fn check_metric_inverse(
        &self,
        metric_name: &str,
        baseline_value: f64,
        current_value: f64,
        description: &str,
        regressions: &mut Vec<RegressionItem>,
    ) {
        if baseline_value <= 0.0 {
            return; // Skip invalid baseline
        }

        let percent_change = ((baseline_value - current_value) / baseline_value) * 100.0;

        if percent_change > self.threshold_percent {
            regressions.push(RegressionItem {
                metric: metric_name.to_string(),
                baseline_value,
                current_value,
                percent_change,
                description: format!(
                    "{} decreased by {:.1}% ({:.2} → {:.2})",
                    description, percent_change, baseline_value, current_value
                ),
            });
        }
    }

    /// Generate CI-friendly report
    pub fn generate_ci_report(result: &RegressionResult) -> String {
        let mut report = String::new();

        report.push_str("# Performance Regression Check\n\n");

        if result.has_regression {
            report.push_str("## ❌ Regression Detected\n\n");
            _ = writeln!(report, "{}\n", result.summary);

            report.push_str("### Regressions\n\n");
            report.push_str("| Metric | Baseline | Current | Change |\n");
            report.push_str("|--------|----------|---------|--------|\n");

            for regression in &result.regressions {
                _ = writeln!(
                    report,
                    "| {} | {:.2} | {:.2} | {:+.1}% |",
                    regression.metric,
                    regression.baseline_value,
                    regression.current_value,
                    regression.percent_change
                );
            }

            report.push_str("\n### Details\n\n");
            for regression in &result.regressions {
                _ = writeln!(report, "- {}", regression.description);
            }
        } else {
            report.push_str("## ✅ No Regression\n\n");
            _ = writeln!(report, "{}\n", result.summary);

            report.push_str("### Metrics\n\n");
            report.push_str("| Metric | Baseline | Current | Change |\n");
            report.push_str("|--------|----------|---------|--------|\n");

            let metrics = vec![
                (
                    "dbn_load_time_ms",
                    result.baseline.dbn_load_time_ms,
                    result.current.dbn_load_time_ms,
                ),
                (
                    "feature_extraction_time_ms",
                    result.baseline.feature_extraction_time_ms,
                    result.current.feature_extraction_time_ms,
                ),
                (
                    "training_step_time_ms",
                    result.baseline.training_step_time_ms,
                    result.current.training_step_time_ms,
                ),
                (
                    "inference_latency_us",
                    result.baseline.inference_latency_us,
                    result.current.inference_latency_us,
                ),
                (
                    "throughput_samples_per_sec",
                    result.baseline.throughput_samples_per_sec,
                    result.current.throughput_samples_per_sec,
                ),
                (
                    "memory_usage_mb",
                    result.baseline.memory_usage_mb,
                    result.current.memory_usage_mb,
                ),
            ];

            for (name, baseline, current) in metrics {
                let percent_change = if baseline > 0.0 {
                    ((current - baseline) / baseline) * 100.0
                } else {
                    0.0
                };

                _ = writeln!(
                    report,
                    "| {} | {:.2} | {:.2} | {:+.1}% |",
                    name, baseline, current, percent_change
                );
            }
        }

        _ = writeln!(
            report,
            "\n**Baseline**: {} (commit: {})",
            result.baseline.timestamp.format("%Y-%m-%d %H:%M:%S"),
            result.baseline.git_commit
        );
        _ = writeln!(
            report,
            "**Current**: {} (commit: {})",
            result.current.timestamp.format("%Y-%m-%d %H:%M:%S"),
            result.current.git_commit
        );

        report
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use tempfile::TempDir;

    #[tokio::test]
    async fn test_create_tracker() {
        let temp_dir = TempDir::new().unwrap();
        let baseline_path = temp_dir.path().join("baseline.json");
        let tracker = PerformanceTracker::new(baseline_path);

        assert_eq!(tracker.threshold_percent, 10.0);
        assert!(tracker.current_metrics.is_none());
    }

    #[tokio::test]
    async fn test_custom_threshold() {
        let temp_dir = TempDir::new().unwrap();
        let baseline_path = temp_dir.path().join("baseline.json");
        let tracker = PerformanceTracker::with_threshold(baseline_path, 15.0);

        assert_eq!(tracker.threshold_percent, 15.0);
    }

    #[tokio::test]
    async fn test_record_and_get_metrics() {
        let temp_dir = TempDir::new().unwrap();
        let baseline_path = temp_dir.path().join("baseline.json");
        let mut tracker = PerformanceTracker::new(baseline_path);

        let metrics = PerformanceMetrics {
            dbn_load_time_ms: 0.70,
            feature_extraction_time_ms: 5.0,
            training_step_time_ms: 100.0,
            inference_latency_us: 50.0,
            throughput_samples_per_sec: 1000.0,
            memory_usage_mb: 250.0,
            timestamp: Utc::now(),
            git_commit: "test".to_owned(),
            model_type: "DQN".to_owned(),
        };

        tracker.record_metrics(metrics.clone()).await.unwrap();

        let recorded = tracker.get_latest_metrics().unwrap();
        assert_eq!(recorded.dbn_load_time_ms, 0.70);
        assert_eq!(recorded.model_type, "DQN");
    }
}