#!/bin/bash # # Comprehensive Load Test for Foxhunt Trading Service # # Tests: # 1. Performance benchmarks (baseline) # 2. Stress tests (concurrent load) # 3. Database performance # 4. Resource monitoring # set -e echo "=======================================================================" echo " FOXHUNT TRADING SERVICE - COMPREHENSIVE LOAD TEST" echo "=======================================================================" echo "" echo "Target: Trading Service" echo " gRPC Port: 50052" echo " Health Port: 8081" echo " Metrics Port: 9092" echo "" # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' # No Color # Check if services are running echo "🔍 Checking service availability..." if docker ps | grep -q foxhunt-trading-service; then echo -e "${GREEN}✅ Trading Service is running${NC}" else echo -e "${RED}❌ Trading Service is NOT running${NC}" echo "Start with: docker-compose up -d trading_service" exit 1 fi if docker ps | grep -q foxhunt-postgres; then echo -e "${GREEN}✅ PostgreSQL is running${NC}" else echo -e "${RED}❌ PostgreSQL is NOT running${NC}" exit 1 fi # Check Prometheus metrics echo "" echo "📊 Checking Prometheus metrics..." if curl -s http://localhost:9092/metrics > /dev/null 2>&1; then echo -e "${GREEN}✅ Metrics endpoint accessible${NC}" # Extract key metrics TOTAL_ORDERS=$(curl -s http://localhost:9092/metrics | grep "trading_orders_total" | grep -v "#" | head -1 | awk '{print $2}' || echo "0") TOTAL_LATENCY=$(curl -s http://localhost:9092/metrics | grep "trading_total_latency_seconds" | grep -v "#" | head -1 | awk '{print $2}' || echo "0") echo " Current orders total: $TOTAL_ORDERS" echo " Current latency total: $TOTAL_LATENCY seconds" else echo -e "${YELLOW}⚠️ Metrics endpoint not accessible${NC}" fi echo "" echo "=======================================================================" echo " TEST 1: PERFORMANCE BENCHMARKS (Baseline)" echo "=======================================================================" echo "" echo "Running performance_and_stress_tests..." cargo test --package tests --test performance_and_stress_tests --release -- --nocapture --test-threads=1 2>&1 | tee /tmp/perf_tests.log echo "" echo "=======================================================================" echo " TEST 2: HFT BENCHMARKS" echo "=======================================================================" echo "" echo "Running HFT performance benchmarks..." cargo test --package tests --test "*/hft_benchmarks" --release -- --nocapture --test-threads=1 2>&1 | tee /tmp/hft_benchmarks.log echo "" echo "=======================================================================" echo " TEST 3: CRITICAL PATH TESTS" echo "=======================================================================" echo "" echo "Running critical path performance tests..." cargo test --package tests --test "*/critical_path_tests" --release -- --nocapture --test-threads=1 2>&1 | tee /tmp/critical_path.log echo "" echo "=======================================================================" echo " TEST 4: DATABASE PERFORMANCE" echo "=======================================================================" echo "" echo "Running database performance tests..." cargo test --package tests --test database_pool_performance --release -- --nocapture --test-threads=1 2>&1 | tee /tmp/db_perf.log echo "" echo "=======================================================================" echo " TEST 5: GRPC STREAMING LOAD TEST" echo "=======================================================================" echo "" echo "Running gRPC streaming load test..." cargo test --package tests --test grpc_streaming_load_test --release -- --nocapture --test-threads=1 2>&1 | tee /tmp/grpc_load.log echo "" echo "=======================================================================" echo " TEST 6: E2E LATENCY MEASUREMENT" echo "=======================================================================" echo "" echo "Running end-to-end latency tests..." cargo test --package tests --test e2e_latency_measurement --release -- --nocapture --test-threads=1 2>&1 | tee /tmp/e2e_latency.log echo "" echo "=======================================================================" echo " TEST 7: RESOURCE MONITORING" echo "=======================================================================" echo "" # Check final metrics after tests echo "📊 Final Prometheus metrics:" FINAL_ORDERS=$(curl -s http://localhost:9092/metrics | grep "trading_orders_total" | grep -v "#" | head -1 | awk '{print $2}' || echo "0") FINAL_LATENCY=$(curl -s http://localhost:9092/metrics | grep "trading_total_latency_seconds" | grep -v "#" | head -1 | awk '{print $2}' || echo "0") echo " Orders processed: $FINAL_ORDERS" echo " Total latency: $FINAL_LATENCY seconds" if [ "$FINAL_ORDERS" != "0" ]; then AVG_LATENCY=$(echo "scale=6; $FINAL_LATENCY / $FINAL_ORDERS" | bc) AVG_LATENCY_MS=$(echo "scale=2; $AVG_LATENCY * 1000" | bc) echo " Average latency: ${AVG_LATENCY_MS}ms" fi # Check Docker stats echo "" echo "🖥️ Docker resource usage:" docker stats --no-stream foxhunt-trading-service foxhunt-postgres | tail -2 echo "" echo "=======================================================================" echo " PERFORMANCE SUMMARY" echo "=======================================================================" echo "" # Parse test results PERF_PASSED=$(grep -c "test result: ok" /tmp/perf_tests.log 2>/dev/null || echo "0") HFT_PASSED=$(grep -c "test result: ok" /tmp/hft_benchmarks.log 2>/dev/null || echo "0") CRITICAL_PASSED=$(grep -c "test result: ok" /tmp/critical_path.log 2>/dev/null || echo "0") DB_PASSED=$(grep -c "test result: ok" /tmp/db_perf.log 2>/dev/null || echo "0") GRPC_PASSED=$(grep -c "test result: ok" /tmp/grpc_load.log 2>/dev/null || echo "0") E2E_PASSED=$(grep -c "test result: ok" /tmp/e2e_latency.log 2>/dev/null || echo "0") TOTAL_TESTS=6 PASSED_TESTS=$((PERF_PASSED + HFT_PASSED + CRITICAL_PASSED + DB_PASSED + GRPC_PASSED + E2E_PASSED)) echo "Test Results:" echo " Performance Tests: ${PERF_PASSED}/1" echo " HFT Benchmarks: ${HFT_PASSED}/1" echo " Critical Path Tests: ${CRITICAL_PASSED}/1" echo " Database Performance: ${DB_PASSED}/1" echo " gRPC Load Tests: ${GRPC_PASSED}/1" echo " E2E Latency Tests: ${E2E_PASSED}/1" echo "" echo " TOTAL: ${PASSED_TESTS}/${TOTAL_TESTS} test suites passed" echo "" # Extract performance metrics from test logs echo "Performance Metrics (from test logs):" echo "" # Order processing latency if grep -q "Order Processing Latency Results" /tmp/perf_tests.log 2>/dev/null; then echo " Order Processing:" grep -A 3 "Order Processing Latency Results" /tmp/perf_tests.log | tail -3 fi # Lock-free performance if grep -q "Lock-Free Ring Buffer Performance" /tmp/perf_tests.log 2>/dev/null; then echo " Lock-Free Throughput:" grep -A 4 "Lock-Free Ring Buffer Performance" /tmp/perf_tests.log | grep "Throughput:" | head -1 fi # Concurrent processing if grep -q "Concurrent Order Processing Stress Test" /tmp/perf_tests.log 2>/dev/null; then echo " Concurrent Processing:" grep -A 4 "Concurrent Order Processing Stress Test" /tmp/perf_tests.log | grep "Throughput:" | head -1 fi echo "" echo "=======================================================================" echo " PRODUCTION READINESS ASSESSMENT" echo "=======================================================================" echo "" # Assess production readiness READY=0 # Check 1: Test pass rate if [ "$PASSED_TESTS" -ge 5 ]; then echo -e "${GREEN}✅ Test Coverage: ${PASSED_TESTS}/${TOTAL_TESTS} tests passed (>= 83%)${NC}" READY=$((READY + 1)) elif [ "$PASSED_TESTS" -ge 4 ]; then echo -e "${YELLOW}⚠️ Test Coverage: ${PASSED_TESTS}/${TOTAL_TESTS} tests passed (>= 67%)${NC}" READY=$((READY + 1)) else echo -e "${RED}❌ Test Coverage: ${PASSED_TESTS}/${TOTAL_TESTS} tests passed (< 67%)${NC}" fi # Check 2: Service health if docker ps | grep -q "foxhunt-trading-service.*healthy"; then echo -e "${GREEN}✅ Service Health: Trading Service healthy${NC}" READY=$((READY + 1)) else echo -e "${RED}❌ Service Health: Trading Service unhealthy${NC}" fi # Check 3: Metrics availability if curl -s http://localhost:9092/metrics > /dev/null 2>&1; then echo -e "${GREEN}✅ Monitoring: Prometheus metrics accessible${NC}" READY=$((READY + 1)) else echo -e "${RED}❌ Monitoring: Prometheus metrics not accessible${NC}" fi echo "" echo "Production Readiness Score: ${READY}/3" echo "" if [ "$READY" -eq 3 ]; then echo -e "${GREEN}🎉 PRODUCTION READY!${NC}" exit 0 elif [ "$READY" -ge 2 ]; then echo -e "${YELLOW}⚠️ Acceptable for production with monitoring${NC}" exit 0 else echo -e "${RED}❌ Not ready for production deployment${NC}" exit 1 fi