foxhunt/scripts/record_baseline_metrics.sh

#!/bin/bash
#
# Performance Baseline Metrics Recording Script
#
# This script records baseline performance metrics for the Foxhunt HFT system.
# Baselines are saved and used for regression detection in CI.
#
# Usage:
#   ./scripts/record_baseline_metrics.sh [baseline-name]
#
# Examples:
#   ./scripts/record_baseline_metrics.sh main       # Record main branch baseline
#   ./scripts/record_baseline_metrics.sh v1.0.0     # Record version baseline
#   ./scripts/record_baseline_metrics.sh            # Record 'current' baseline

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Configuration
BASELINE_NAME="${1:-current}"
BASELINE_DIR="target/criterion/baselines"
METRICS_DIR="performance_metrics"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)

echo -e "${BLUE}=================================================${NC}"
echo -e "${BLUE}Performance Baseline Metrics Recording${NC}"
echo -e "${BLUE}=================================================${NC}"
echo ""
echo -e "${GREEN}Baseline name:${NC} $BASELINE_NAME"
echo -e "${GREEN}Timestamp:${NC} $TIMESTAMP"
echo ""

# Create metrics directory
mkdir -p "$METRICS_DIR"

# Function to record system info
record_system_info() {
    echo -e "${YELLOW}Recording system information...${NC}"

    cat > "$METRICS_DIR/system_info_${BASELINE_NAME}_${TIMESTAMP}.txt" <<EOF
Performance Baseline - System Information
==========================================
Baseline: $BASELINE_NAME
Timestamp: $TIMESTAMP
Date: $(date)

Hardware:
---------
CPU: $(lscpu | grep "Model name" | sed 's/Model name:\s*//')
Cores: $(nproc)
Memory: $(free -h | grep Mem | awk '{print $2}')
GPU: $(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null || echo "No GPU detected")

Software:
---------
OS: $(uname -a)
Rust: $(rustc --version)
Cargo: $(cargo --version)

Git:
----
Commit: $(git rev-parse HEAD)
Branch: $(git rev-parse --abbrev-ref HEAD)
Status: $(git status --short | wc -l) files changed
EOF

    echo -e "${GREEN}✓ System info recorded${NC}"
}

# Function to run benchmarks and save baseline
run_benchmarks() {
    echo ""
    echo -e "${YELLOW}Running performance regression benchmarks...${NC}"
    echo -e "${BLUE}This will take approximately 5-10 minutes${NC}"
    echo ""

    # Run with baseline save
    if cargo bench --bench performance_regression -- --save-baseline "$BASELINE_NAME"; then
        echo ""
        echo -e "${GREEN}✓ Benchmarks completed successfully${NC}"
        return 0
    else
        echo ""
        echo -e "${RED}✗ Benchmarks failed${NC}"
        return 1
    fi
}

# Function to extract key metrics
extract_metrics() {
    echo ""
    echo -e "${YELLOW}Extracting key performance metrics...${NC}"

    local metrics_file="$METRICS_DIR/metrics_${BASELINE_NAME}_${TIMESTAMP}.json"

    cat > "$metrics_file" <<EOF
{
  "baseline": "$BASELINE_NAME",
  "timestamp": "$TIMESTAMP",
  "date": "$(date -Iseconds)",
  "git_commit": "$(git rev-parse HEAD)",
  "git_branch": "$(git rev-parse --abbrev-ref HEAD)",
  "metrics": {
    "ml_prediction_latency": {
      "description": "ML model prediction latency",
      "target_p50_us": 20,
      "target_p99_us": 50,
      "note": "See criterion HTML report for actual values"
    },
    "hot_swap_latency": {
      "description": "Model hot-swap latency",
      "target_p50_us": 1,
      "note": "See criterion HTML report for actual values"
    },
    "database_writes": {
      "description": "Database write throughput",
      "target_writes_per_sec": 1000,
      "note": "See criterion HTML report for actual values"
    },
    "backtest_performance": {
      "description": "Backtest bar processing rate",
      "target_bars_per_sec": 1100,
      "note": "See criterion HTML report for actual values"
    },
    "order_processing": {
      "description": "Order processing latency",
      "target_p99_us": 100,
      "note": "See criterion HTML report for actual values"
    },
    "risk_validation": {
      "description": "Risk validation latency",
      "target_p99_us": 50,
      "note": "See criterion HTML report for actual values"
    }
  },
  "baseline_location": "$BASELINE_DIR/$BASELINE_NAME"
}
EOF

    echo -e "${GREEN}✓ Metrics extracted to: $metrics_file${NC}"
}

# Function to generate baseline summary
generate_summary() {
    echo ""
    echo -e "${YELLOW}Generating baseline summary...${NC}"

    local summary_file="$METRICS_DIR/baseline_summary_${BASELINE_NAME}.md"

    cat > "$summary_file" <<EOF
# Performance Baseline: $BASELINE_NAME

**Timestamp**: $TIMESTAMP
**Date**: $(date)
**Git Commit**: $(git rev-parse --short HEAD)
**Git Branch**: $(git rev-parse --abbrev-ref HEAD)

## System Configuration

- **CPU**: $(lscpu | grep "Model name" | sed 's/Model name:\s*//' | xargs)
- **Cores**: $(nproc)
- **Memory**: $(free -h | grep Mem | awk '{print $2}')
- **GPU**: $(nvidia-smi --query-gpu=name --format=csv,noheader 2>/dev/null || echo "No GPU detected")
- **OS**: $(uname -s) $(uname -r)
- **Rust**: $(rustc --version | awk '{print $2}')

## Baseline Metrics

| Component | Target | Status |
|-----------|--------|--------|
| ML Prediction Latency | P50 < 20μs, P99 < 50μs | ⏳ See HTML Report |
| Hot-Swap Latency | P50 < 1μs | ⏳ See HTML Report |
| Database Writes | >1000/sec | ⏳ See HTML Report |
| Backtest Performance | >1100 bars/sec | ⏳ See HTML Report |
| Order Processing | P99 < 100μs | ⏳ See HTML Report |
| Risk Validation | P99 < 50μs | ⏳ See HTML Report |

## Usage

### Compare Against This Baseline

\`\`\`bash
# Run benchmarks and compare
cargo bench --bench performance_regression -- --baseline $BASELINE_NAME

# View detailed HTML report
open target/criterion/report/index.html
\`\`\`

### Detect Regressions

Regressions are flagged when performance degrades >10% from baseline:

- **Red** (❌): Significant regression detected
- **Yellow** (⚠️): Borderline regression
- **Green** (✅): Performance maintained or improved

## Files

- **Baseline data**: \`$BASELINE_DIR/$BASELINE_NAME/\`
- **Metrics JSON**: \`$METRICS_DIR/metrics_${BASELINE_NAME}_${TIMESTAMP}.json\`
- **System info**: \`$METRICS_DIR/system_info_${BASELINE_NAME}_${TIMESTAMP}.txt\`
- **HTML reports**: \`target/criterion/\`

## Notes

- Criterion automatically saves detailed statistics
- HTML reports include latency distributions and percentiles
- Use \`--baseline $BASELINE_NAME\` to compare future runs
- Baselines are stored in \`target/criterion/baselines/\`
EOF

    echo -e "${GREEN}✓ Summary generated: $summary_file${NC}"
}

# Function to create comparison script
create_comparison_script() {
    echo ""
    echo -e "${YELLOW}Creating comparison helper script...${NC}"

    local script_file="$METRICS_DIR/compare_with_${BASELINE_NAME}.sh"

    cat > "$script_file" <<'EOFSCRIPT'
#!/bin/bash
# Auto-generated comparison script

BASELINE_NAME="BASELINE_PLACEHOLDER"

echo "Comparing current performance against baseline: $BASELINE_NAME"
echo ""

cargo bench --bench performance_regression -- --baseline "$BASELINE_NAME"

echo ""
echo "View detailed HTML report:"
echo "  open target/criterion/report/index.html"
EOFSCRIPT

    sed -i "s/BASELINE_PLACEHOLDER/$BASELINE_NAME/" "$script_file"
    chmod +x "$script_file"

    echo -e "${GREEN}✓ Comparison script created: $script_file${NC}"
}

# Main execution
main() {
    echo -e "${BLUE}Step 1/5: Recording system information${NC}"
    record_system_info

    echo ""
    echo -e "${BLUE}Step 2/5: Running benchmarks${NC}"
    if ! run_benchmarks; then
        echo -e "${RED}Failed to complete benchmarks. Exiting.${NC}"
        exit 1
    fi

    echo ""
    echo -e "${BLUE}Step 3/5: Extracting metrics${NC}"
    extract_metrics

    echo ""
    echo -e "${BLUE}Step 4/5: Generating summary${NC}"
    generate_summary

    echo ""
    echo -e "${BLUE}Step 5/5: Creating comparison helpers${NC}"
    create_comparison_script

    echo ""
    echo -e "${GREEN}=================================================${NC}"
    echo -e "${GREEN}✓ Baseline '$BASELINE_NAME' recorded successfully${NC}"
    echo -e "${GREEN}=================================================${NC}"
    echo ""
    echo -e "${BLUE}Next steps:${NC}"
    echo -e "  1. View HTML report: ${YELLOW}open target/criterion/report/index.html${NC}"
    echo -e "  2. Compare later: ${YELLOW}cargo bench --bench performance_regression -- --baseline $BASELINE_NAME${NC}"
    echo -e "  3. Review summary: ${YELLOW}cat $METRICS_DIR/baseline_summary_${BASELINE_NAME}.md${NC}"
    echo ""
    echo -e "${BLUE}Files created:${NC}"
    echo -e "  - Baseline: ${YELLOW}$BASELINE_DIR/$BASELINE_NAME/${NC}"
    echo -e "  - Metrics: ${YELLOW}$METRICS_DIR/metrics_${BASELINE_NAME}_${TIMESTAMP}.json${NC}"
    echo -e "  - Summary: ${YELLOW}$METRICS_DIR/baseline_summary_${BASELINE_NAME}.md${NC}"
    echo -e "  - System: ${YELLOW}$METRICS_DIR/system_info_${BASELINE_NAME}_${TIMESTAMP}.txt${NC}"
    echo ""
}

# Run main function
main "$@"