#!/bin/bash # Performance Benchmarking Script for Foxhunt HFT Trading System # Comprehensive performance validation for sub-microsecond latency requirements # # This script validates system performance against HFT benchmarks and provides # detailed metrics for latency, throughput, and resource utilization. set -euo pipefail # Configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" BENCHMARK_LOG="/home/jgrusewski/Work/foxhunt/logs/benchmark-$(date +%s).log" RESULTS_DIR="/opt/foxhunt/benchmark-results" FOXHUNT_CORE_ENDPOINT="http://localhost:8080" FOXHUNT_TLI_ENDPOINT="http://localhost:8081" GRPC_ENDPOINT="localhost:50051" # Performance thresholds for HFT MAX_LATENCY_US=30 MIN_THROUGHPUT_OPS=1000 MAX_CPU_PERCENT=80 MAX_MEMORY_PERCENT=75 MAX_JITTER_US=10 # Test parameters WARMUP_DURATION=30 TEST_DURATION=60 CONCURRENT_CONNECTIONS=100 ORDERS_PER_SECOND=1000 # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # Create directories mkdir -p "$(dirname "$BENCHMARK_LOG")" mkdir -p "$RESULTS_DIR" log() { echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1" | tee -a "$BENCHMARK_LOG" } error() { echo -e "${RED}[ERROR]${NC} $1" | tee -a "$BENCHMARK_LOG" } success() { echo -e "${GREEN}[SUCCESS]${NC} $1" | tee -a "$BENCHMARK_LOG" } warning() { echo -e "${YELLOW}[WARNING]${NC} $1" | tee -a "$BENCHMARK_LOG" } usage() { echo "Usage: $0 [options]" echo "Options:" echo " --duration Test duration (default: 60)" echo " --warmup Warmup duration (default: 30)" echo " --connections Concurrent connections (default: 100)" echo " --rate Orders per second (default: 1000)" echo " --baseline Establish performance baseline" echo " --compare Compare against baseline" echo " --report-only Generate report from existing results" echo " --help Show this help" exit 1 } # ============================================================================= # SYSTEM RESOURCE MONITORING # ============================================================================= start_resource_monitoring() { log "Starting resource monitoring..." # CPU monitoring { while true; do echo "$(date +%s),$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | sed 's/%us,//')" sleep 1 done } > "$RESULTS_DIR/cpu_usage.csv" & CPU_MONITOR_PID=$! # Memory monitoring { while true; do echo "$(date +%s),$(free | grep Mem | awk '{printf "%.1f", $3/$2 * 100.0}')" sleep 1 done } > "$RESULTS_DIR/memory_usage.csv" & MEMORY_MONITOR_PID=$! # Network monitoring if command -v iftop &> /dev/null; then { iftop -t -s 1 -L 1000 2>/dev/null | grep -E "^\s*[0-9]" | while read line; do echo "$(date +%s),$line" done } > "$RESULTS_DIR/network_usage.csv" & NETWORK_MONITOR_PID=$! fi log "Resource monitoring started" } stop_resource_monitoring() { log "Stopping resource monitoring..." # Stop all monitoring processes [ "${CPU_MONITOR_PID:-}" ] && kill $CPU_MONITOR_PID 2>/dev/null || true [ "${MEMORY_MONITOR_PID:-}" ] && kill $MEMORY_MONITOR_PID 2>/dev/null || true [ "${NETWORK_MONITOR_PID:-}" ] && kill $NETWORK_MONITOR_PID 2>/dev/null || true sleep 2 log "Resource monitoring stopped" } # ============================================================================= # LATENCY BENCHMARKING # ============================================================================= benchmark_latency() { log "Running latency benchmark..." local results_file="$RESULTS_DIR/latency_results.json" # Test REST API latency if command -v curl &> /dev/null; then log "Testing REST API latency..." { echo "[" for i in $(seq 1 1000); do start_time=$(date +%s%N) if curl -s --max-time 1 "$FOXHUNT_CORE_ENDPOINT/health" >/dev/null 2>&1; then end_time=$(date +%s%N) latency_ns=$((end_time - start_time)) latency_us=$((latency_ns / 1000)) echo " {\"request\": $i, \"latency_us\": $latency_us, \"timestamp\": $(date +%s)}," fi [ $((i % 100)) -eq 0 ] && log "Completed $i/1000 latency tests" done echo " {\"end\": true}" echo "]" } > "$results_file" # Calculate statistics local avg_latency min_latency max_latency p95_latency p99_latency avg_latency=$(jq '[.[] | select(.latency_us != null) | .latency_us] | add / length' "$results_file" 2>/dev/null || echo "0") min_latency=$(jq '[.[] | select(.latency_us != null) | .latency_us] | min' "$results_file" 2>/dev/null || echo "0") max_latency=$(jq '[.[] | select(.latency_us != null) | .latency_us] | max' "$results_file" 2>/dev/null || echo "0") p95_latency=$(jq '[.[] | select(.latency_us != null) | .latency_us] | sort | .[((length * 0.95) | floor)]' "$results_file" 2>/dev/null || echo "0") p99_latency=$(jq '[.[] | select(.latency_us != null) | .latency_us] | sort | .[((length * 0.99) | floor)]' "$results_file" 2>/dev/null || echo "0") log "REST API Latency Results:" log " Average: ${avg_latency}μs" log " Minimum: ${min_latency}μs" log " Maximum: ${max_latency}μs" log " 95th percentile: ${p95_latency}μs" log " 99th percentile: ${p99_latency}μs" # Check against thresholds if (( $(echo "$avg_latency <= $MAX_LATENCY_US" | bc -l) )); then success "Average latency within threshold: ${avg_latency}μs ≤ ${MAX_LATENCY_US}μs" else error "Average latency exceeds threshold: ${avg_latency}μs > ${MAX_LATENCY_US}μs" fi fi } # ============================================================================= # THROUGHPUT BENCHMARKING # ============================================================================= benchmark_throughput() { log "Running throughput benchmark..." local results_file="$RESULTS_DIR/throughput_results.json" if command -v wrk &> /dev/null; then log "Using wrk for HTTP throughput testing..." # Run wrk benchmark wrk -t 4 -c $CONCURRENT_CONNECTIONS -d ${TEST_DURATION}s --latency \ -s <(cat <<'EOF' wrk.method = "POST" wrk.body = '{"symbol":"AAPL","side":"buy","quantity":100,"price":150.00}' wrk.headers["Content-Type"] = "application/json" EOF ) "$FOXHUNT_CORE_ENDPOINT/orders" > "$RESULTS_DIR/wrk_output.txt" # Parse wrk results local requests_per_sec requests_per_sec=$(grep "Requests/sec:" "$RESULTS_DIR/wrk_output.txt" | awk '{print $2}' | cut -d'.' -f1) log "HTTP Throughput: $requests_per_sec requests/sec" if [ "$requests_per_sec" -ge "$MIN_THROUGHPUT_OPS" ]; then success "Throughput within threshold: $requests_per_sec ≥ $MIN_THROUGHPUT_OPS ops/sec" else error "Throughput below threshold: $requests_per_sec < $MIN_THROUGHPUT_OPS ops/sec" fi elif command -v ab &> /dev/null; then log "Using Apache Bench for HTTP throughput testing..." # Create test data file echo '{"symbol":"AAPL","side":"buy","quantity":100,"price":150.00}' > "$RESULTS_DIR/order_data.json" # Run Apache Bench ab -n 10000 -c $CONCURRENT_CONNECTIONS -T 'application/json' \ -p "$RESULTS_DIR/order_data.json" \ "$FOXHUNT_CORE_ENDPOINT/orders" > "$RESULTS_DIR/ab_output.txt" # Parse AB results local requests_per_sec requests_per_sec=$(grep "Requests per second:" "$RESULTS_DIR/ab_output.txt" | awk '{print $4}' | cut -d'.' -f1) log "HTTP Throughput: $requests_per_sec requests/sec" if [ "$requests_per_sec" -ge "$MIN_THROUGHPUT_OPS" ]; then success "Throughput within threshold: $requests_per_sec ≥ $MIN_THROUGHPUT_OPS ops/sec" else error "Throughput below threshold: $requests_per_sec < $MIN_THROUGHPUT_OPS ops/sec" fi else warning "No HTTP benchmarking tool available (wrk or ab required)" fi } # ============================================================================= # GRPC BENCHMARKING # ============================================================================= benchmark_grpc() { log "Running gRPC benchmark..." if command -v ghz &> /dev/null; then log "Using ghz for gRPC benchmarking..." # Run gRPC benchmark ghz --insecure \ --proto="/opt/foxhunt/proto/trading.proto" \ --call="trading.TradingService/PlaceOrder" \ -d '{"symbol":"AAPL","side":"BUY","quantity":100,"price":150.00}' \ -c $CONCURRENT_CONNECTIONS \ -n 10000 \ --timeout=10s \ "$GRPC_ENDPOINT" > "$RESULTS_DIR/grpc_results.json" # Parse results if [ -f "$RESULTS_DIR/grpc_results.json" ]; then local avg_latency total_requests rps avg_latency=$(jq -r '.average // "0"' "$RESULTS_DIR/grpc_results.json" | sed 's/ms//' 2>/dev/null || echo "0") total_requests=$(jq -r '.count // "0"' "$RESULTS_DIR/grpc_results.json" 2>/dev/null || echo "0") rps=$(jq -r '.rps // "0"' "$RESULTS_DIR/grpc_results.json" 2>/dev/null || echo "0") log "gRPC Results:" log " Average latency: ${avg_latency}ms" log " Total requests: $total_requests" log " Requests per second: $rps" fi elif command -v grpcurl &> /dev/null; then log "Using grpcurl for basic gRPC connectivity test..." # Basic connectivity test if grpcurl -plaintext "$GRPC_ENDPOINT" list >/dev/null 2>&1; then success "gRPC service is accessible" # Simple latency test local start_time end_time latency_ms start_time=$(date +%s%N) grpcurl -plaintext -d '{"symbol":"AAPL","side":"BUY","quantity":100,"price":150.00}' \ "$GRPC_ENDPOINT" trading.TradingService/PlaceOrder >/dev/null 2>&1 || true end_time=$(date +%s%N) latency_ms=$(((end_time - start_time) / 1000000)) log "gRPC single request latency: ${latency_ms}ms" else error "gRPC service not accessible" fi else warning "No gRPC benchmarking tool available (ghz or grpcurl required)" fi } # ============================================================================= # MEMORY AND CACHE BENCHMARKING # ============================================================================= benchmark_memory() { log "Running memory and cache benchmark..." # Memory bandwidth test if command -v sysbench &> /dev/null; then log "Testing memory bandwidth with sysbench..." sysbench memory \ --memory-block-size=1M \ --memory-total-size=10G \ --memory-oper=write \ run > "$RESULTS_DIR/memory_bandwidth.txt" local bandwidth bandwidth=$(grep "MiB/sec" "$RESULTS_DIR/memory_bandwidth.txt" | awk '{print $2}' | head -1) log "Memory write bandwidth: ${bandwidth} MiB/sec" elif command -v dd &> /dev/null; then log "Testing memory with dd..." # Simple memory test local bandwidth bandwidth=$(dd if=/dev/zero of=/dev/null bs=1M count=1000 2>&1 | grep -o '[0-9.]* GB/s' | head -1 || echo "unknown") log "Memory bandwidth (dd): $bandwidth" fi # Cache latency test (if available) if command -v lmbench &> /dev/null; then log "Testing cache latency with lmbench..." lat_mem_rd 1000 2 > "$RESULTS_DIR/cache_latency.txt" || true fi } # ============================================================================= # NETWORK BENCHMARKING # ============================================================================= benchmark_network() { log "Running network benchmark..." # Network latency test (localhost) if command -v ping &> /dev/null; then local avg_latency avg_latency=$(ping -c 100 -i 0.01 localhost 2>/dev/null | tail -1 | awk -F '/' '{print $5}' || echo "0") log "Localhost network latency: ${avg_latency}ms" # Convert to microseconds for comparison local latency_us latency_us=$(echo "$avg_latency * 1000" | bc -l 2>/dev/null | cut -d'.' -f1 || echo "0") if [ "$latency_us" -lt 100 ]; then success "Network latency acceptable: ${latency_us}μs" else warning "High network latency: ${latency_us}μs" fi fi # Network bandwidth test (if iperf3 available) if command -v iperf3 &> /dev/null; then log "Testing network bandwidth with iperf3..." # This would require an iperf3 server, skip for localhost testing log "iperf3 available but skipping (requires server setup)" fi } # ============================================================================= # JITTER AND STABILITY TESTING # ============================================================================= benchmark_jitter() { log "Running jitter and stability benchmark..." local jitter_file="$RESULTS_DIR/jitter_results.csv" echo "timestamp,latency_us" > "$jitter_file" # Collect latency measurements for jitter analysis for i in $(seq 1 1000); do local start_time end_time latency_us start_time=$(date +%s%N) curl -s --max-time 1 "$FOXHUNT_CORE_ENDPOINT/health" >/dev/null 2>&1 || true end_time=$(date +%s%N) latency_us=$(((end_time - start_time) / 1000)) echo "$(date +%s%N),$latency_us" >> "$jitter_file" # Brief pause to avoid overwhelming the system sleep 0.001 done # Calculate jitter (standard deviation) if command -v python3 &> /dev/null; then local jitter_us jitter_us=$(python3 -c " import csv import statistics latencies = [] with open('$jitter_file', 'r') as f: reader = csv.DictReader(f) for row in reader: latencies.append(float(row['latency_us'])) if latencies: print(f'{statistics.stdev(latencies):.2f}') else: print('0') " 2>/dev/null || echo "0") log "Latency jitter (std dev): ${jitter_us}μs" if (( $(echo "$jitter_us <= $MAX_JITTER_US" | bc -l) )); then success "Jitter within threshold: ${jitter_us}μs ≤ ${MAX_JITTER_US}μs" else warning "High jitter detected: ${jitter_us}μs > ${MAX_JITTER_US}μs" fi fi } # ============================================================================= # BASELINE AND COMPARISON # ============================================================================= save_baseline() { local baseline_file="$RESULTS_DIR/baseline_$(date +%Y%m%d_%H%M%S).json" log "Saving performance baseline to: $baseline_file" # Create baseline JSON cat > "$baseline_file" </dev/null || echo "0"), "p95_latency_us": $(jq '[.[] | select(.latency_us != null) | .latency_us] | sort | .[((length * 0.95) | floor)]' "$RESULTS_DIR/latency_results.json" 2>/dev/null || echo "0"), "throughput_ops": $(grep -o '[0-9]*' "$RESULTS_DIR/wrk_output.txt" 2>/dev/null | head -1 || echo "0"), "jitter_us": $(tail -1 "$RESULTS_DIR/jitter_results.csv" 2>/dev/null | cut -d',' -f2 || echo "0") } } EOF success "Baseline saved: $baseline_file" } compare_with_baseline() { local baseline_file="$1" if [ ! -f "$baseline_file" ]; then error "Baseline file not found: $baseline_file" return 1 fi log "Comparing current results with baseline: $baseline_file" # Load baseline metrics local baseline_latency baseline_throughput baseline_jitter baseline_latency=$(jq -r '.performance_metrics.avg_latency_us' "$baseline_file" 2>/dev/null || echo "0") baseline_throughput=$(jq -r '.performance_metrics.throughput_ops' "$baseline_file" 2>/dev/null || echo "0") baseline_jitter=$(jq -r '.performance_metrics.jitter_us' "$baseline_file" 2>/dev/null || echo "0") # Get current metrics local current_latency current_throughput current_jitter current_latency=$(jq '[.[] | select(.latency_us != null) | .latency_us] | add / length' "$RESULTS_DIR/latency_results.json" 2>/dev/null || echo "0") current_throughput=$(grep -o '[0-9]*' "$RESULTS_DIR/wrk_output.txt" 2>/dev/null | head -1 || echo "0") current_jitter=$(tail -1 "$RESULTS_DIR/jitter_results.csv" 2>/dev/null | cut -d',' -f2 || echo "0") log "Performance Comparison:" log " Latency: ${current_latency}μs (baseline: ${baseline_latency}μs)" log " Throughput: ${current_throughput} ops/sec (baseline: ${baseline_throughput} ops/sec)" log " Jitter: ${current_jitter}μs (baseline: ${baseline_jitter}μs)" # Calculate percentage changes if command -v python3 &> /dev/null; then python3 -c " baseline_lat = float('$baseline_latency') current_lat = float('$current_latency') baseline_thr = float('$baseline_throughput') current_thr = float('$current_throughput') if baseline_lat > 0: lat_change = ((current_lat - baseline_lat) / baseline_lat) * 100 print(f'Latency change: {lat_change:+.2f}%') if baseline_thr > 0: thr_change = ((current_thr - baseline_thr) / baseline_thr) * 100 print(f'Throughput change: {thr_change:+.2f}%') " fi } # ============================================================================= # REPORT GENERATION # ============================================================================= generate_report() { local report_file="$RESULTS_DIR/benchmark_report_$(date +%Y%m%d_%H%M%S).html" log "Generating performance report: $report_file" cat > "$report_file" < Foxhunt HFT Performance Benchmark Report

Foxhunt HFT Performance Benchmark Report

Generated: $(date)

System: $(hostname) - $(uname -r)

Performance Summary

MetricValueThresholdStatus
Average Latency$(jq '[.[] | select(.latency_us != null) | .latency_us] | add / length' "$RESULTS_DIR/latency_results.json" 2>/dev/null || echo "N/A")μs≤ ${MAX_LATENCY_US}μs$(if (( $(jq '[.[] | select(.latency_us != null) | .latency_us] | add / length' "$RESULTS_DIR/latency_results.json" 2>/dev/null || echo "999") <= MAX_LATENCY_US )); then echo "PASS"; else echo "FAIL"; fi)

Test Configuration

  • Test Duration: ${TEST_DURATION} seconds
  • Warmup Duration: ${WARMUP_DURATION} seconds
  • Concurrent Connections: ${CONCURRENT_CONNECTIONS}
  • Target Rate: ${ORDERS_PER_SECOND} ops/sec

Raw Results

Detailed results are available in the following files:

EOF success "Report generated: $report_file" } # ============================================================================= # MAIN EXECUTION # ============================================================================= main() { local baseline_mode=false local compare_baseline="" local report_only=false # Parse arguments while [[ $# -gt 0 ]]; do case $1 in --duration) TEST_DURATION="$2" shift 2 ;; --warmup) WARMUP_DURATION="$2" shift 2 ;; --connections) CONCURRENT_CONNECTIONS="$2" shift 2 ;; --rate) ORDERS_PER_SECOND="$2" shift 2 ;; --baseline) baseline_mode=true shift ;; --compare) compare_baseline="$2" shift 2 ;; --report-only) report_only=true shift ;; -h|--help) usage ;; *) error "Unknown option: $1" usage ;; esac done # Print banner echo "============================================================" echo " Foxhunt HFT Performance Benchmark" echo "============================================================" echo log "Starting performance benchmark..." log "Test duration: ${TEST_DURATION}s" log "Warmup duration: ${WARMUP_DURATION}s" log "Concurrent connections: $CONCURRENT_CONNECTIONS" log "Target rate: $ORDERS_PER_SECOND ops/sec" # Report only mode if [ "$report_only" = true ]; then generate_report exit 0 fi # Check if services are running if ! curl -f -s "$FOXHUNT_CORE_ENDPOINT/health" >/dev/null 2>&1; then error "Foxhunt core service is not responding at $FOXHUNT_CORE_ENDPOINT" exit 1 fi # Warmup phase if [ "$WARMUP_DURATION" -gt 0 ]; then log "Starting warmup phase (${WARMUP_DURATION}s)..." sleep "$WARMUP_DURATION" log "Warmup completed" fi # Start resource monitoring start_resource_monitoring # Run benchmarks benchmark_latency benchmark_throughput benchmark_grpc benchmark_memory benchmark_network benchmark_jitter # Stop resource monitoring stop_resource_monitoring # Save baseline if requested if [ "$baseline_mode" = true ]; then save_baseline fi # Compare with baseline if provided if [ -n "$compare_baseline" ]; then compare_with_baseline "$compare_baseline" fi # Generate report generate_report log "Performance benchmark completed" log "Results directory: $RESULTS_DIR" log "Benchmark log: $BENCHMARK_LOG" success "Benchmark completed successfully" } # Execute main function main "$@"