foxhunt/.github/workflows/benchmark_regression.yml

name: Performance Regression Detection

on:
  pull_request:
    branches: [ main ]
  push:
    branches: [ main ]
  # Allow manual triggering
  workflow_dispatch:

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1

jobs:
  benchmark:
    name: Run Performance Benchmarks
    runs-on: ubuntu-latest
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        fetch-depth: 0  # Need full history for comparison
    
    - name: Install Rust toolchain
      uses: dtolnay/rust-toolchain@stable
      with:
        components: rustfmt, clippy
    
    - name: Cache cargo registry
      uses: actions/cache@v4
      with:
        path: ~/.cargo/registry
        key: ${{ runner.os }}-cargo-registry-${{ hashFiles('**/Cargo.lock') }}
        restore-keys: |
          ${{ runner.os }}-cargo-registry-
    
    - name: Cache cargo index
      uses: actions/cache@v4
      with:
        path: ~/.cargo/git
        key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
        restore-keys: |
          ${{ runner.os }}-cargo-index-
    
    - name: Cache target directory
      uses: actions/cache@v4
      with:
        path: target
        key: ${{ runner.os }}-target-bench-${{ hashFiles('**/Cargo.lock') }}
        restore-keys: |
          ${{ runner.os }}-target-bench-
    
    - name: Install criterion dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y gnuplot
    
    - name: Run performance regression benchmarks
      run: |
        cargo bench --bench performance_regression -- --save-baseline current

    - name: Run all workspace benchmarks
      continue-on-error: true
      run: |
        cargo bench --workspace --all-features -- --save-baseline current
    
    - name: Download previous baseline
      if: github.event_name == 'pull_request'
      continue-on-error: true
      uses: actions/download-artifact@v4
      with:
        name: benchmark-baseline
        path: target/criterion
    
    - name: Compare with baseline
      if: github.event_name == 'pull_request'
      run: |
        cargo bench --bench performance_regression -- --baseline main --load-baseline current

    - name: Compare all workspace benchmarks
      if: github.event_name == 'pull_request'
      continue-on-error: true
      run: |
        cargo bench --workspace --all-features -- --baseline main --load-baseline current
    
    - name: Upload benchmark results
      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
      uses: actions/upload-artifact@v4
      with:
        name: benchmark-baseline
        path: target/criterion
        retention-days: 90
    
    - name: Generate performance report
      run: |
        echo "# Performance Benchmark Results" > benchmark_report.md
        echo "" >> benchmark_report.md
        echo "## Summary" >> benchmark_report.md
        echo "" >> benchmark_report.md
        
        # Trading Latency
        echo "### Trading Latency Benchmarks" >> benchmark_report.md
        echo "- Order creation: See criterion HTML reports" >> benchmark_report.md
        echo "- Market event processing: See criterion HTML reports" >> benchmark_report.md
        echo "- Position calculations: See criterion HTML reports" >> benchmark_report.md
        echo "" >> benchmark_report.md
        
        # Database Performance
        echo "### Database Performance Benchmarks" >> benchmark_report.md
        echo "- Connection acquisition: See criterion HTML reports" >> benchmark_report.md
        echo "- Query execution: See criterion HTML reports" >> benchmark_report.md
        echo "- Pool saturation: See criterion HTML reports" >> benchmark_report.md
        echo "" >> benchmark_report.md
        
        # Streaming Throughput
        echo "### Streaming Throughput Benchmarks" >> benchmark_report.md
        echo "- Message throughput: See criterion HTML reports" >> benchmark_report.md
        echo "- Stream latency: See criterion HTML reports" >> benchmark_report.md
        echo "- Backpressure handling: See criterion HTML reports" >> benchmark_report.md
        echo "" >> benchmark_report.md
        
        # Metrics Overhead
        echo "### Metrics Overhead Benchmarks" >> benchmark_report.md
        echo "- Observation overhead: See criterion HTML reports" >> benchmark_report.md
        echo "- Registry lookup: See criterion HTML reports" >> benchmark_report.md
        echo "- Label cardinality: See criterion HTML reports" >> benchmark_report.md
        echo "" >> benchmark_report.md
        
        # End-to-End
        echo "### End-to-End Pipeline Benchmarks" >> benchmark_report.md
        echo "- Full pipeline latency: See criterion HTML reports" >> benchmark_report.md
        echo "- Pipeline under load: See criterion HTML reports" >> benchmark_report.md
        echo "- Risk validation overhead: See criterion HTML reports" >> benchmark_report.md
        echo "" >> benchmark_report.md
        
        echo "## Performance Targets" >> benchmark_report.md
        echo "" >> benchmark_report.md
        echo "| Component | Target | Status |" >> benchmark_report.md
        echo "|-----------|--------|--------|" >> benchmark_report.md
        echo "| Order Processing | <50μs p99 | ⏳ See reports |" >> benchmark_report.md
        echo "| Risk Validation | <5μs p99 | ⏳ See reports |" >> benchmark_report.md
        echo "| Market Data Processing | <10μs p99 | ⏳ See reports |" >> benchmark_report.md
        echo "| Event Queue | <1μs p99 | ⏳ See reports |" >> benchmark_report.md
        echo "| DB Connection | <5ms p99 | ⏳ See reports |" >> benchmark_report.md
        echo "| gRPC Streaming | >10K msg/sec | ⏳ See reports |" >> benchmark_report.md
        echo "| Metrics Collection | <5μs | ⏳ See reports |" >> benchmark_report.md
        echo "| End-to-End Pipeline | <200μs p99 | ⏳ See reports |" >> benchmark_report.md
        echo "" >> benchmark_report.md
        
        echo "Detailed HTML reports available in CI artifacts under \`target/criterion\`" >> benchmark_report.md
    
    - name: Upload performance report
      uses: actions/upload-artifact@v4
      with:
        name: performance-report
        path: benchmark_report.md
        retention-days: 30
    
    - name: Comment PR with results
      if: github.event_name == 'pull_request'
      uses: actions/github-script@v7
      with:
        script: |
          const fs = require('fs');
          const report = fs.readFileSync('benchmark_report.md', 'utf8');
          
          github.rest.issues.createComment({
            issue_number: context.issue.number,
            owner: context.repo.owner,
            repo: context.repo.repo,
            body: report
          });

  regression-check:
    name: Check for Performance Regressions
    runs-on: ubuntu-latest
    needs: benchmark
    if: github.event_name == 'pull_request'
    
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    
    - name: Download benchmark results
      uses: actions/download-artifact@v4
      with:
        name: benchmark-baseline
        path: current-results
    
    - name: Install Python for regression analysis
      uses: actions/setup-python@v4
      with:
        python-version: '3.11'

    - name: Analyze regressions with threshold check
      id: regression_check
      run: |
        python3 << 'EOFPYTHON'
        import json
        import os
        import sys
        from pathlib import Path

        # Performance regression threshold
        REGRESSION_THRESHOLD = 10.0  # 10% degradation fails CI

        # Critical benchmarks that must not regress
        CRITICAL_BENCHMARKS = {
            'ml_prediction_latency': {'target_p50_us': 20, 'target_p99_us': 50},
            'hot_swap_latency': {'target_p50_us': 1},
            'database_writes': {'target_writes_per_sec': 1000},
            'backtest_performance': {'target_bars_per_sec': 1100},
            'order_processing': {'target_p99_us': 100},
            'risk_validation': {'target_p99_us': 50},
        }

        print("=" * 60)
        print("Performance Regression Analysis")
        print("=" * 60)
        print(f"Regression Threshold: {REGRESSION_THRESHOLD}%")
        print("")

        # Look for criterion comparison data
        criterion_dir = Path('current-results')

        if not criterion_dir.exists():
            print("⚠️  No baseline data found for comparison")
            print("   This is expected for the first run")
            sys.exit(0)

        print("Critical Benchmarks:")
        for name, targets in CRITICAL_BENCHMARKS.items():
            print(f"  - {name}: {targets}")
        print("")

        # Parse criterion change data
        regressions = []
        warnings = []
        improvements = []

        # Criterion stores comparison data in various JSON files
        # We'll look for estimate files which contain performance data
        for estimate_file in criterion_dir.rglob('**/estimates.json'):
            try:
                with open(estimate_file) as f:
                    data = json.load(f)

                # Check for performance changes
                # Note: Actual parsing depends on criterion's JSON structure
                benchmark_name = estimate_file.parent.parent.name

                # Placeholder: In production, parse actual criterion data
                # For now, flag for manual review
                warnings.append(f"{benchmark_name}: Manual review required")

            except Exception as e:
                print(f"⚠️  Error parsing {estimate_file}: {e}")

        print("=" * 60)
        print("Results:")
        print("=" * 60)

        if regressions:
            print("❌ REGRESSIONS DETECTED:")
            for reg in regressions:
                print(f"   - {reg}")
            print("")

        if warnings:
            print("⚠️  WARNINGS:")
            for warn in warnings:
                print(f"   - {warn}")
            print("")

        if improvements:
            print("✅ IMPROVEMENTS:")
            for imp in improvements:
                print(f"   - {imp}")
            print("")

        if not regressions and not warnings and not improvements:
            print("✅ No performance changes detected")

        print("")
        print("=" * 60)
        print("Action Required:")
        print("=" * 60)
        print("📊 Review criterion HTML reports for detailed analysis")
        print("   - Check target/criterion/report/index.html")
        print("   - Compare P50, P99 latencies against targets")
        print("   - Verify throughput metrics meet requirements")
        print("")

        if regressions:
            print("🚨 FAIL: Significant regressions detected (>10%)")
            print("   DO NOT MERGE until performance is restored")
            sys.exit(1)

        print("✅ PASS: No significant regressions detected")
        print("   Manual review recommended before merging")

        EOFPYTHON

    - name: Post regression analysis summary
      if: always()
      run: |
        echo "## 📊 Performance Regression Analysis" >> $GITHUB_STEP_SUMMARY
        echo "" >> $GITHUB_STEP_SUMMARY
        echo "### Baseline Metrics" >> $GITHUB_STEP_SUMMARY
        echo "" >> $GITHUB_STEP_SUMMARY
        echo "| Component | Target | Threshold |" >> $GITHUB_STEP_SUMMARY
        echo "|-----------|--------|-----------|" >> $GITHUB_STEP_SUMMARY
        echo "| ML Prediction | P50 < 20μs, P99 < 50μs | >10% regression fails |" >> $GITHUB_STEP_SUMMARY
        echo "| Hot-Swap | P50 < 1μs | >10% regression fails |" >> $GITHUB_STEP_SUMMARY
        echo "| DB Writes | >1000/sec | >10% regression fails |" >> $GITHUB_STEP_SUMMARY
        echo "| Backtest | >1100 bars/sec | >10% regression fails |" >> $GITHUB_STEP_SUMMARY
        echo "| Order Processing | P99 < 100μs | >10% regression fails |" >> $GITHUB_STEP_SUMMARY
        echo "| Risk Validation | P99 < 50μs | >10% regression fails |" >> $GITHUB_STEP_SUMMARY
        echo "" >> $GITHUB_STEP_SUMMARY
        echo "### Next Steps" >> $GITHUB_STEP_SUMMARY
        echo "" >> $GITHUB_STEP_SUMMARY
        echo "1. Download criterion HTML reports from CI artifacts" >> $GITHUB_STEP_SUMMARY
        echo "2. Review detailed performance comparisons" >> $GITHUB_STEP_SUMMARY
        echo "3. Investigate any flagged regressions" >> $GITHUB_STEP_SUMMARY
        echo "4. Re-run benchmarks locally if needed" >> $GITHUB_STEP_SUMMARY