8d89fe80ff
- Archive: 85 agent .txt files → docs/archive/agents/legacy_txt/ - Scripts: Move 110 shell scripts → scripts/ (keep deploy.sh in root) - Models: Move 18 .safetensors → ml/models/checkpoints/training_artifacts/ - Delete: 34 directories (~33GB freed) - target/, coverage_*, test artifacts - Build: Clean 14 build artifacts (.rlib, .o, .pid, binaries) - Tests: Move 14 .rs files → tests/standalone/ - SQL: Move 5 files → sql/ (keep init-db*.sql for Docker) - Wave 153: Archive to docs/archive/historical/wave153/ - Docs: Archive 9 markdown files to wave_d/reports/ and historical/ Total impact: ~34GB freed (both waves), root directory cleaned from 583 to ~40 essential files Directory count reduced from 65 to 31 (52% reduction) All historical data preserved in organized archive structure
236 lines
8.8 KiB
Bash
Executable File
236 lines
8.8 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# Comprehensive Load Test for Foxhunt Trading Service
|
|
#
|
|
# Tests:
|
|
# 1. Performance benchmarks (baseline)
|
|
# 2. Stress tests (concurrent load)
|
|
# 3. Database performance
|
|
# 4. Resource monitoring
|
|
#
|
|
|
|
set -e
|
|
|
|
echo "======================================================================="
|
|
echo " FOXHUNT TRADING SERVICE - COMPREHENSIVE LOAD TEST"
|
|
echo "======================================================================="
|
|
echo ""
|
|
echo "Target: Trading Service"
|
|
echo " gRPC Port: 50052"
|
|
echo " Health Port: 8081"
|
|
echo " Metrics Port: 9092"
|
|
echo ""
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Check if services are running
|
|
echo "🔍 Checking service availability..."
|
|
if docker ps | grep -q foxhunt-trading-service; then
|
|
echo -e "${GREEN}✅ Trading Service is running${NC}"
|
|
else
|
|
echo -e "${RED}❌ Trading Service is NOT running${NC}"
|
|
echo "Start with: docker-compose up -d trading_service"
|
|
exit 1
|
|
fi
|
|
|
|
if docker ps | grep -q foxhunt-postgres; then
|
|
echo -e "${GREEN}✅ PostgreSQL is running${NC}"
|
|
else
|
|
echo -e "${RED}❌ PostgreSQL is NOT running${NC}"
|
|
exit 1
|
|
fi
|
|
|
|
# Check Prometheus metrics
|
|
echo ""
|
|
echo "📊 Checking Prometheus metrics..."
|
|
if curl -s http://localhost:9092/metrics > /dev/null 2>&1; then
|
|
echo -e "${GREEN}✅ Metrics endpoint accessible${NC}"
|
|
|
|
# Extract key metrics
|
|
TOTAL_ORDERS=$(curl -s http://localhost:9092/metrics | grep "trading_orders_total" | grep -v "#" | head -1 | awk '{print $2}' || echo "0")
|
|
TOTAL_LATENCY=$(curl -s http://localhost:9092/metrics | grep "trading_total_latency_seconds" | grep -v "#" | head -1 | awk '{print $2}' || echo "0")
|
|
|
|
echo " Current orders total: $TOTAL_ORDERS"
|
|
echo " Current latency total: $TOTAL_LATENCY seconds"
|
|
else
|
|
echo -e "${YELLOW}⚠️ Metrics endpoint not accessible${NC}"
|
|
fi
|
|
|
|
echo ""
|
|
echo "======================================================================="
|
|
echo " TEST 1: PERFORMANCE BENCHMARKS (Baseline)"
|
|
echo "======================================================================="
|
|
echo ""
|
|
echo "Running performance_and_stress_tests..."
|
|
cargo test --package tests --test performance_and_stress_tests --release -- --nocapture --test-threads=1 2>&1 | tee /tmp/perf_tests.log
|
|
|
|
echo ""
|
|
echo "======================================================================="
|
|
echo " TEST 2: HFT BENCHMARKS"
|
|
echo "======================================================================="
|
|
echo ""
|
|
echo "Running HFT performance benchmarks..."
|
|
cargo test --package tests --test "*/hft_benchmarks" --release -- --nocapture --test-threads=1 2>&1 | tee /tmp/hft_benchmarks.log
|
|
|
|
echo ""
|
|
echo "======================================================================="
|
|
echo " TEST 3: CRITICAL PATH TESTS"
|
|
echo "======================================================================="
|
|
echo ""
|
|
echo "Running critical path performance tests..."
|
|
cargo test --package tests --test "*/critical_path_tests" --release -- --nocapture --test-threads=1 2>&1 | tee /tmp/critical_path.log
|
|
|
|
echo ""
|
|
echo "======================================================================="
|
|
echo " TEST 4: DATABASE PERFORMANCE"
|
|
echo "======================================================================="
|
|
echo ""
|
|
echo "Running database performance tests..."
|
|
cargo test --package tests --test database_pool_performance --release -- --nocapture --test-threads=1 2>&1 | tee /tmp/db_perf.log
|
|
|
|
echo ""
|
|
echo "======================================================================="
|
|
echo " TEST 5: GRPC STREAMING LOAD TEST"
|
|
echo "======================================================================="
|
|
echo ""
|
|
echo "Running gRPC streaming load test..."
|
|
cargo test --package tests --test grpc_streaming_load_test --release -- --nocapture --test-threads=1 2>&1 | tee /tmp/grpc_load.log
|
|
|
|
echo ""
|
|
echo "======================================================================="
|
|
echo " TEST 6: E2E LATENCY MEASUREMENT"
|
|
echo "======================================================================="
|
|
echo ""
|
|
echo "Running end-to-end latency tests..."
|
|
cargo test --package tests --test e2e_latency_measurement --release -- --nocapture --test-threads=1 2>&1 | tee /tmp/e2e_latency.log
|
|
|
|
echo ""
|
|
echo "======================================================================="
|
|
echo " TEST 7: RESOURCE MONITORING"
|
|
echo "======================================================================="
|
|
echo ""
|
|
|
|
# Check final metrics after tests
|
|
echo "📊 Final Prometheus metrics:"
|
|
FINAL_ORDERS=$(curl -s http://localhost:9092/metrics | grep "trading_orders_total" | grep -v "#" | head -1 | awk '{print $2}' || echo "0")
|
|
FINAL_LATENCY=$(curl -s http://localhost:9092/metrics | grep "trading_total_latency_seconds" | grep -v "#" | head -1 | awk '{print $2}' || echo "0")
|
|
|
|
echo " Orders processed: $FINAL_ORDERS"
|
|
echo " Total latency: $FINAL_LATENCY seconds"
|
|
|
|
if [ "$FINAL_ORDERS" != "0" ]; then
|
|
AVG_LATENCY=$(echo "scale=6; $FINAL_LATENCY / $FINAL_ORDERS" | bc)
|
|
AVG_LATENCY_MS=$(echo "scale=2; $AVG_LATENCY * 1000" | bc)
|
|
echo " Average latency: ${AVG_LATENCY_MS}ms"
|
|
fi
|
|
|
|
# Check Docker stats
|
|
echo ""
|
|
echo "🖥️ Docker resource usage:"
|
|
docker stats --no-stream foxhunt-trading-service foxhunt-postgres | tail -2
|
|
|
|
echo ""
|
|
echo "======================================================================="
|
|
echo " PERFORMANCE SUMMARY"
|
|
echo "======================================================================="
|
|
echo ""
|
|
|
|
# Parse test results
|
|
PERF_PASSED=$(grep -c "test result: ok" /tmp/perf_tests.log 2>/dev/null || echo "0")
|
|
HFT_PASSED=$(grep -c "test result: ok" /tmp/hft_benchmarks.log 2>/dev/null || echo "0")
|
|
CRITICAL_PASSED=$(grep -c "test result: ok" /tmp/critical_path.log 2>/dev/null || echo "0")
|
|
DB_PASSED=$(grep -c "test result: ok" /tmp/db_perf.log 2>/dev/null || echo "0")
|
|
GRPC_PASSED=$(grep -c "test result: ok" /tmp/grpc_load.log 2>/dev/null || echo "0")
|
|
E2E_PASSED=$(grep -c "test result: ok" /tmp/e2e_latency.log 2>/dev/null || echo "0")
|
|
|
|
TOTAL_TESTS=6
|
|
PASSED_TESTS=$((PERF_PASSED + HFT_PASSED + CRITICAL_PASSED + DB_PASSED + GRPC_PASSED + E2E_PASSED))
|
|
|
|
echo "Test Results:"
|
|
echo " Performance Tests: ${PERF_PASSED}/1"
|
|
echo " HFT Benchmarks: ${HFT_PASSED}/1"
|
|
echo " Critical Path Tests: ${CRITICAL_PASSED}/1"
|
|
echo " Database Performance: ${DB_PASSED}/1"
|
|
echo " gRPC Load Tests: ${GRPC_PASSED}/1"
|
|
echo " E2E Latency Tests: ${E2E_PASSED}/1"
|
|
echo ""
|
|
echo " TOTAL: ${PASSED_TESTS}/${TOTAL_TESTS} test suites passed"
|
|
echo ""
|
|
|
|
# Extract performance metrics from test logs
|
|
echo "Performance Metrics (from test logs):"
|
|
echo ""
|
|
|
|
# Order processing latency
|
|
if grep -q "Order Processing Latency Results" /tmp/perf_tests.log 2>/dev/null; then
|
|
echo " Order Processing:"
|
|
grep -A 3 "Order Processing Latency Results" /tmp/perf_tests.log | tail -3
|
|
fi
|
|
|
|
# Lock-free performance
|
|
if grep -q "Lock-Free Ring Buffer Performance" /tmp/perf_tests.log 2>/dev/null; then
|
|
echo " Lock-Free Throughput:"
|
|
grep -A 4 "Lock-Free Ring Buffer Performance" /tmp/perf_tests.log | grep "Throughput:" | head -1
|
|
fi
|
|
|
|
# Concurrent processing
|
|
if grep -q "Concurrent Order Processing Stress Test" /tmp/perf_tests.log 2>/dev/null; then
|
|
echo " Concurrent Processing:"
|
|
grep -A 4 "Concurrent Order Processing Stress Test" /tmp/perf_tests.log | grep "Throughput:" | head -1
|
|
fi
|
|
|
|
echo ""
|
|
echo "======================================================================="
|
|
echo " PRODUCTION READINESS ASSESSMENT"
|
|
echo "======================================================================="
|
|
echo ""
|
|
|
|
# Assess production readiness
|
|
READY=0
|
|
|
|
# Check 1: Test pass rate
|
|
if [ "$PASSED_TESTS" -ge 5 ]; then
|
|
echo -e "${GREEN}✅ Test Coverage: ${PASSED_TESTS}/${TOTAL_TESTS} tests passed (>= 83%)${NC}"
|
|
READY=$((READY + 1))
|
|
elif [ "$PASSED_TESTS" -ge 4 ]; then
|
|
echo -e "${YELLOW}⚠️ Test Coverage: ${PASSED_TESTS}/${TOTAL_TESTS} tests passed (>= 67%)${NC}"
|
|
READY=$((READY + 1))
|
|
else
|
|
echo -e "${RED}❌ Test Coverage: ${PASSED_TESTS}/${TOTAL_TESTS} tests passed (< 67%)${NC}"
|
|
fi
|
|
|
|
# Check 2: Service health
|
|
if docker ps | grep -q "foxhunt-trading-service.*healthy"; then
|
|
echo -e "${GREEN}✅ Service Health: Trading Service healthy${NC}"
|
|
READY=$((READY + 1))
|
|
else
|
|
echo -e "${RED}❌ Service Health: Trading Service unhealthy${NC}"
|
|
fi
|
|
|
|
# Check 3: Metrics availability
|
|
if curl -s http://localhost:9092/metrics > /dev/null 2>&1; then
|
|
echo -e "${GREEN}✅ Monitoring: Prometheus metrics accessible${NC}"
|
|
READY=$((READY + 1))
|
|
else
|
|
echo -e "${RED}❌ Monitoring: Prometheus metrics not accessible${NC}"
|
|
fi
|
|
|
|
echo ""
|
|
echo "Production Readiness Score: ${READY}/3"
|
|
echo ""
|
|
|
|
if [ "$READY" -eq 3 ]; then
|
|
echo -e "${GREEN}🎉 PRODUCTION READY!${NC}"
|
|
exit 0
|
|
elif [ "$READY" -ge 2 ]; then
|
|
echo -e "${YELLOW}⚠️ Acceptable for production with monitoring${NC}"
|
|
exit 0
|
|
else
|
|
echo -e "${RED}❌ Not ready for production deployment${NC}"
|
|
exit 1
|
|
fi
|