foxhunt/ml/examples/check_performance_regression.rs

//! Performance Regression Checker for CI
//!
//! Compares current performance metrics against baseline and detects regressions.
//! Exits with code 1 if regression detected (>10% degradation).
//!
//! Usage:
//! ```bash
//! cargo run --release -p ml --example check_performance_regression -- \
//!   --baseline baseline.json \
//!   --current current.json \
//!   --output report.md
//! ```

use anyhow::{Context, Result};
use ml::benchmark::{PerformanceBaseline, PerformanceMetrics, PerformanceTracker, RegressionResult};
use std::path::PathBuf;
use std::process;
use structopt::StructOpt;
use tracing::{error, info, Level};
use tracing_subscriber::FmtSubscriber;

/// CLI options
#[derive(Debug, StructOpt)]
#[structopt(
    name = "check_performance_regression",
    about = "Check for performance regressions against baseline"
)]
struct Opts {
    /// Baseline metrics file
    #[structopt(long)]
    baseline: String,

    /// Current metrics file
    #[structopt(long)]
    current: String,

    /// Output report file (Markdown)
    #[structopt(long)]
    output: String,

    /// Regression threshold percentage (default: 10%)
    #[structopt(long, default_value = "10.0")]
    threshold: f64,

    /// Verbose logging
    #[structopt(short, long)]
    verbose: bool,
}

#[tokio::main]
async fn main() -> Result<()> {
    let opts = Opts::from_args();

    // Initialize logging
    let level = if opts.verbose {
        Level::DEBUG
    } else {
        Level::INFO
    };

    let subscriber = FmtSubscriber::builder().with_max_level(level).finish();
    tracing::subscriber::set_global_default(subscriber)
        .context("Failed to set tracing subscriber")?;

    info!("Checking performance regression");
    info!("Baseline: {}", opts.baseline);
    info!("Current: {}", opts.current);
    info!("Threshold: {}%", opts.threshold);

    // Load baseline
    let baseline_path = PathBuf::from(&opts.baseline);
    let baseline = PerformanceTracker::load_baseline(&baseline_path)
        .await
        .context("Failed to load baseline")?;

    info!("Loaded baseline: {} ({})", baseline.model_type, baseline.git_commit);

    // Load current metrics
    let current_path = PathBuf::from(&opts.current);
    let current_baseline = PerformanceTracker::load_baseline(&current_path)
        .await
        .context("Failed to load current metrics")?;

    // Convert baseline to metrics for tracker
    let current_metrics = PerformanceMetrics {
        dbn_load_time_ms: current_baseline.dbn_load_time_ms,
        feature_extraction_time_ms: current_baseline.feature_extraction_time_ms,
        training_step_time_ms: current_baseline.training_step_time_ms,
        inference_latency_us: current_baseline.inference_latency_us,
        throughput_samples_per_sec: current_baseline.throughput_samples_per_sec,
        memory_usage_mb: current_baseline.memory_usage_mb,
        timestamp: current_baseline.timestamp,
        git_commit: current_baseline.git_commit.clone(),
        model_type: current_baseline.model_type.clone(),
    };

    info!("Loaded current: {} ({})", current_metrics.model_type, current_metrics.git_commit);

    // Create tracker with custom threshold
    let mut tracker = PerformanceTracker::with_threshold(baseline_path.clone(), opts.threshold);
    tracker.record_metrics(current_metrics).await?;

    // Check for regressions
    let result = tracker.check_regression().await
        .context("Failed to check regression")?;

    // Generate report
    let report = PerformanceTracker::generate_ci_report(&result);

    // Save report
    let output_path = PathBuf::from(&opts.output);
    if let Some(parent) = output_path.parent() {
        tokio::fs::create_dir_all(parent).await?;
    }
    tokio::fs::write(&output_path, &report).await?;

    info!("Report saved to {}", opts.output);

    // Print summary
    println!("\n{}", result.summary);

    if result.has_regression {
        error!("❌ Performance regression detected!");
        println!("\n### Regressions Found:");
        for regression in &result.regressions {
            println!(
                "  - {}: {:.2} → {:.2} ({:+.1}%)",
                regression.metric,
                regression.baseline_value,
                regression.current_value,
                regression.percent_change
            );
        }
        println!("\nSee {} for full report", opts.output);

        // Exit with error code for CI
        process::exit(result.exit_code());
    } else {
        info!("✅ No performance regression detected");
        println!("\nAll metrics within {}% threshold", opts.threshold);

        // Exit successfully
        process::exit(0);
    }
}