#!/bin/bash # # Performance Baseline Metrics Recording Script # # This script records baseline performance metrics for the Foxhunt HFT system. # Baselines are saved and used for regression detection in CI. # # Usage: # ./scripts/record_baseline_metrics.sh [baseline-name] # # Examples: # ./scripts/record_baseline_metrics.sh main # Record main branch baseline # ./scripts/record_baseline_metrics.sh v1.0.0 # Record version baseline # ./scripts/record_baseline_metrics.sh # Record 'current' baseline set -euo pipefail # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Configuration BASELINE_NAME="${1:-current}" BASELINE_DIR="target/criterion/baselines" METRICS_DIR="performance_metrics" TIMESTAMP=$(date +%Y%m%d_%H%M%S) echo -e "${BLUE}=================================================${NC}" echo -e "${BLUE}Performance Baseline Metrics Recording${NC}" echo -e "${BLUE}=================================================${NC}" echo "" echo -e "${GREEN}Baseline name:${NC} $BASELINE_NAME" echo -e "${GREEN}Timestamp:${NC} $TIMESTAMP" echo "" # Create metrics directory mkdir -p "$METRICS_DIR" # Function to record system info record_system_info() { echo -e "${YELLOW}Recording system information...${NC}" cat > "$METRICS_DIR/system_info_${BASELINE_NAME}_${TIMESTAMP}.txt" </dev/null || echo "No GPU detected") Software: --------- OS: $(uname -a) Rust: $(rustc --version) Cargo: $(cargo --version) Git: ---- Commit: $(git rev-parse HEAD) Branch: $(git rev-parse --abbrev-ref HEAD) Status: $(git status --short | wc -l) files changed EOF echo -e "${GREEN}✓ System info recorded${NC}" } # Function to run benchmarks and save baseline run_benchmarks() { echo "" echo -e "${YELLOW}Running performance regression benchmarks...${NC}" echo -e "${BLUE}This will take approximately 5-10 minutes${NC}" echo "" # Run with baseline save if cargo bench --bench performance_regression -- --save-baseline "$BASELINE_NAME"; then echo "" echo -e "${GREEN}✓ Benchmarks completed successfully${NC}" return 0 else echo "" echo -e "${RED}✗ Benchmarks failed${NC}" return 1 fi } # Function to extract key metrics extract_metrics() { echo "" echo -e "${YELLOW}Extracting key performance metrics...${NC}" local metrics_file="$METRICS_DIR/metrics_${BASELINE_NAME}_${TIMESTAMP}.json" cat > "$metrics_file" < "$summary_file" </dev/null || echo "No GPU detected") - **OS**: $(uname -s) $(uname -r) - **Rust**: $(rustc --version | awk '{print $2}') ## Baseline Metrics | Component | Target | Status | |-----------|--------|--------| | ML Prediction Latency | P50 < 20μs, P99 < 50μs | ⏳ See HTML Report | | Hot-Swap Latency | P50 < 1μs | ⏳ See HTML Report | | Database Writes | >1000/sec | ⏳ See HTML Report | | Backtest Performance | >1100 bars/sec | ⏳ See HTML Report | | Order Processing | P99 < 100μs | ⏳ See HTML Report | | Risk Validation | P99 < 50μs | ⏳ See HTML Report | ## Usage ### Compare Against This Baseline \`\`\`bash # Run benchmarks and compare cargo bench --bench performance_regression -- --baseline $BASELINE_NAME # View detailed HTML report open target/criterion/report/index.html \`\`\` ### Detect Regressions Regressions are flagged when performance degrades >10% from baseline: - **Red** (❌): Significant regression detected - **Yellow** (⚠️): Borderline regression - **Green** (✅): Performance maintained or improved ## Files - **Baseline data**: \`$BASELINE_DIR/$BASELINE_NAME/\` - **Metrics JSON**: \`$METRICS_DIR/metrics_${BASELINE_NAME}_${TIMESTAMP}.json\` - **System info**: \`$METRICS_DIR/system_info_${BASELINE_NAME}_${TIMESTAMP}.txt\` - **HTML reports**: \`target/criterion/\` ## Notes - Criterion automatically saves detailed statistics - HTML reports include latency distributions and percentiles - Use \`--baseline $BASELINE_NAME\` to compare future runs - Baselines are stored in \`target/criterion/baselines/\` EOF echo -e "${GREEN}✓ Summary generated: $summary_file${NC}" } # Function to create comparison script create_comparison_script() { echo "" echo -e "${YELLOW}Creating comparison helper script...${NC}" local script_file="$METRICS_DIR/compare_with_${BASELINE_NAME}.sh" cat > "$script_file" <<'EOFSCRIPT' #!/bin/bash # Auto-generated comparison script BASELINE_NAME="BASELINE_PLACEHOLDER" echo "Comparing current performance against baseline: $BASELINE_NAME" echo "" cargo bench --bench performance_regression -- --baseline "$BASELINE_NAME" echo "" echo "View detailed HTML report:" echo " open target/criterion/report/index.html" EOFSCRIPT sed -i "s/BASELINE_PLACEHOLDER/$BASELINE_NAME/" "$script_file" chmod +x "$script_file" echo -e "${GREEN}✓ Comparison script created: $script_file${NC}" } # Main execution main() { echo -e "${BLUE}Step 1/5: Recording system information${NC}" record_system_info echo "" echo -e "${BLUE}Step 2/5: Running benchmarks${NC}" if ! run_benchmarks; then echo -e "${RED}Failed to complete benchmarks. Exiting.${NC}" exit 1 fi echo "" echo -e "${BLUE}Step 3/5: Extracting metrics${NC}" extract_metrics echo "" echo -e "${BLUE}Step 4/5: Generating summary${NC}" generate_summary echo "" echo -e "${BLUE}Step 5/5: Creating comparison helpers${NC}" create_comparison_script echo "" echo -e "${GREEN}=================================================${NC}" echo -e "${GREEN}✓ Baseline '$BASELINE_NAME' recorded successfully${NC}" echo -e "${GREEN}=================================================${NC}" echo "" echo -e "${BLUE}Next steps:${NC}" echo -e " 1. View HTML report: ${YELLOW}open target/criterion/report/index.html${NC}" echo -e " 2. Compare later: ${YELLOW}cargo bench --bench performance_regression -- --baseline $BASELINE_NAME${NC}" echo -e " 3. Review summary: ${YELLOW}cat $METRICS_DIR/baseline_summary_${BASELINE_NAME}.md${NC}" echo "" echo -e "${BLUE}Files created:${NC}" echo -e " - Baseline: ${YELLOW}$BASELINE_DIR/$BASELINE_NAME/${NC}" echo -e " - Metrics: ${YELLOW}$METRICS_DIR/metrics_${BASELINE_NAME}_${TIMESTAMP}.json${NC}" echo -e " - Summary: ${YELLOW}$METRICS_DIR/baseline_summary_${BASELINE_NAME}.md${NC}" echo -e " - System: ${YELLOW}$METRICS_DIR/system_info_${BASELINE_NAME}_${TIMESTAMP}.txt${NC}" echo "" } # Run main function main "$@"