#!/bin/bash # Cross-Service Integration Test Script # Tests actual cross-service communication flows set -e echo "================================" echo "Cross-Service Integration Tests" echo "================================" echo "" # Colors RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' # No Color # Test results TOTAL_TESTS=0 PASSED_TESTS=0 FAILED_TESTS=0 function test_service_health() { echo "Test 1: Service Health Checks" echo "==============================" TOTAL_TESTS=$((TOTAL_TESTS + 4)) # API Gateway if curl -s -f http://localhost:9091/health > /dev/null 2>&1; then echo -e "${GREEN}✓${NC} API Gateway (9091) - Healthy" PASSED_TESTS=$((PASSED_TESTS + 1)) else echo -e "${RED}✗${NC} API Gateway (9091) - Unhealthy" FAILED_TESTS=$((FAILED_TESTS + 1)) fi # Trading Service if curl -s -f http://localhost:9092/health > /dev/null 2>&1; then echo -e "${GREEN}✓${NC} Trading Service (9092) - Healthy" PASSED_TESTS=$((PASSED_TESTS + 1)) else echo -e "${RED}✗${NC} Trading Service (9092) - Unhealthy" FAILED_TESTS=$((FAILED_TESTS + 1)) fi # Backtesting Service if curl -s -f http://localhost:8083/health > /dev/null 2>&1; then echo -e "${GREEN}✓${NC} Backtesting Service (8083) - Healthy" PASSED_TESTS=$((PASSED_TESTS + 1)) else echo -e "${RED}✗${NC} Backtesting Service (8083) - Unhealthy" FAILED_TESTS=$((FAILED_TESTS + 1)) fi # ML Training Service if curl -s -f http://localhost:8095/health > /dev/null 2>&1; then echo -e "${GREEN}✓${NC} ML Training Service (8095) - Healthy" PASSED_TESTS=$((PASSED_TESTS + 1)) else echo -e "${RED}✗${NC} ML Training Service (8095) - Unhealthy" FAILED_TESTS=$((FAILED_TESTS + 1)) fi echo "" } function test_database_connection() { echo "Test 2: PostgreSQL Connectivity" echo "===============================" TOTAL_TESTS=$((TOTAL_TESTS + 1)) if psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -c "SELECT 1" > /dev/null 2>&1; then echo -e "${GREEN}✓${NC} PostgreSQL connection successful" PASSED_TESTS=$((PASSED_TESTS + 1)) # Check key tables TABLES=$(psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='public'") echo " Found $TABLES tables in public schema" else echo -e "${RED}✗${NC} PostgreSQL connection failed" FAILED_TESTS=$((FAILED_TESTS + 1)) fi echo "" } function test_redis_connection() { echo "Test 3: Redis Connectivity" echo "==========================" TOTAL_TESTS=$((TOTAL_TESTS + 1)) if redis-cli -h localhost -p 6379 PING 2>/dev/null | grep -q PONG; then echo -e "${GREEN}✓${NC} Redis connection successful" PASSED_TESTS=$((PASSED_TESTS + 1)) # Test set/get redis-cli -h localhost -p 6379 SET test_key "integration_test" > /dev/null 2>&1 VALUE=$(redis-cli -h localhost -p 6379 GET test_key 2>/dev/null) if [ "$VALUE" = "integration_test" ]; then echo " Redis SET/GET working" fi redis-cli -h localhost -p 6379 DEL test_key > /dev/null 2>&1 else echo -e "${RED}✗${NC} Redis connection failed" FAILED_TESTS=$((FAILED_TESTS + 1)) fi echo "" } function test_grpc_ports() { echo "Test 4: gRPC Port Availability" echo "==============================" TOTAL_TESTS=$((TOTAL_TESTS + 4)) # API Gateway gRPC if lsof -i :50051 2>/dev/null | grep -q LISTEN; then echo -e "${GREEN}✓${NC} API Gateway gRPC (50051) - Listening" PASSED_TESTS=$((PASSED_TESTS + 1)) else echo -e "${RED}✗${NC} API Gateway gRPC (50051) - Not listening" FAILED_TESTS=$((FAILED_TESTS + 1)) fi # Trading Service gRPC if lsof -i :50052 2>/dev/null | grep -q LISTEN; then echo -e "${GREEN}✓${NC} Trading Service gRPC (50052) - Listening" PASSED_TESTS=$((PASSED_TESTS + 1)) else echo -e "${RED}✗${NC} Trading Service gRPC (50052) - Not listening" FAILED_TESTS=$((FAILED_TESTS + 1)) fi # Backtesting Service gRPC if lsof -i :50053 2>/dev/null | grep -q LISTEN; then echo -e "${GREEN}✓${NC} Backtesting Service gRPC (50053) - Listening" PASSED_TESTS=$((PASSED_TESTS + 1)) else echo -e "${RED}✗${NC} Backtesting Service gRPC (50053) - Not listening" FAILED_TESTS=$((FAILED_TESTS + 1)) fi # ML Training Service gRPC if lsof -i :50054 2>/dev/null | grep -q LISTEN; then echo -e "${GREEN}✓${NC} ML Training Service gRPC (50054) - Listening" PASSED_TESTS=$((PASSED_TESTS + 1)) else echo -e "${RED}✗${NC} ML Training Service gRPC (50054) - Not listening" FAILED_TESTS=$((FAILED_TESTS + 1)) fi echo "" } function test_metrics_endpoints() { echo "Test 5: Prometheus Metrics Endpoints" echo "====================================" TOTAL_TESTS=$((TOTAL_TESTS + 4)) # API Gateway metrics if curl -s http://localhost:9091/metrics | grep -q "# TYPE"; then echo -e "${GREEN}✓${NC} API Gateway metrics (9091) - Available" PASSED_TESTS=$((PASSED_TESTS + 1)) else echo -e "${RED}✗${NC} API Gateway metrics (9091) - Unavailable" FAILED_TESTS=$((FAILED_TESTS + 1)) fi # Trading Service metrics if curl -s http://localhost:9092/metrics | grep -q "# TYPE"; then echo -e "${GREEN}✓${NC} Trading Service metrics (9092) - Available" PASSED_TESTS=$((PASSED_TESTS + 1)) else echo -e "${RED}✗${NC} Trading Service metrics (9092) - Unavailable" FAILED_TESTS=$((FAILED_TESTS + 1)) fi # Backtesting Service metrics if curl -s http://localhost:9093/metrics | grep -q "# TYPE"; then echo -e "${GREEN}✓${NC} Backtesting Service metrics (9093) - Available" PASSED_TESTS=$((PASSED_TESTS + 1)) else echo -e "${RED}✗${NC} Backtesting Service metrics (9093) - Unavailable" FAILED_TESTS=$((FAILED_TESTS + 1)) fi # ML Training Service metrics if curl -s http://localhost:9094/metrics | grep -q "# TYPE"; then echo -e "${GREEN}✓${NC} ML Training Service metrics (9094) - Available" PASSED_TESTS=$((PASSED_TESTS + 1)) else echo -e "${RED}✗${NC} ML Training Service metrics (9094) - Unavailable" FAILED_TESTS=$((FAILED_TESTS + 1)) fi echo "" } function test_prometheus_targets() { echo "Test 6: Prometheus Service Discovery" echo "====================================" TOTAL_TESTS=$((TOTAL_TESTS + 1)) if curl -s http://localhost:9090/api/v1/targets 2>/dev/null | grep -q '"health":"up"'; then echo -e "${GREEN}✓${NC} Prometheus has healthy targets" PASSED_TESTS=$((PASSED_TESTS + 1)) # Count healthy targets HEALTHY=$(curl -s http://localhost:9090/api/v1/targets 2>/dev/null | grep -o '"health":"up"' | wc -l) echo " $HEALTHY services reporting to Prometheus" else echo -e "${RED}✗${NC} No healthy Prometheus targets found" FAILED_TESTS=$((FAILED_TESTS + 1)) fi echo "" } function test_parquet_files() { echo "Test 7: Parquet Test Data Availability" echo "======================================" TOTAL_TESTS=$((TOTAL_TESTS + 1)) PARQUET_COUNT=$(find /home/jgrusewski/Work/foxhunt/test_data -name "*.parquet" 2>/dev/null | wc -l) if [ "$PARQUET_COUNT" -gt 0 ]; then echo -e "${GREEN}✓${NC} Found $PARQUET_COUNT Parquet test files" PASSED_TESTS=$((PASSED_TESTS + 1)) else echo -e "${YELLOW}⚠${NC} No Parquet test files found" FAILED_TESTS=$((FAILED_TESTS + 1)) fi echo "" } function test_database_orders() { echo "Test 8: Database Order Persistence" echo "==================================" TOTAL_TESTS=$((TOTAL_TESTS + 1)) ORDER_COUNT=$(psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -t -c "SELECT COUNT(*) FROM orders" 2>/dev/null | tr -d ' ') if [ $? -eq 0 ]; then echo -e "${GREEN}✓${NC} Orders table accessible: $ORDER_COUNT orders" PASSED_TESTS=$((PASSED_TESTS + 1)) else echo -e "${RED}✗${NC} Failed to query orders table" FAILED_TESTS=$((FAILED_TESTS + 1)) fi echo "" } function measure_latency() { echo "Test 9: Inter-Service Latency Measurement" echo "=========================================" TOTAL_TESTS=$((TOTAL_TESTS + 4)) # API Gateway START=$(date +%s%N) curl -s http://localhost:9091/health > /dev/null 2>&1 END=$(date +%s%N) LATENCY=$(( (END - START) / 1000000 )) if [ $LATENCY -lt 100 ]; then echo -e "${GREEN}✓${NC} API Gateway health: ${LATENCY}ms" PASSED_TESTS=$((PASSED_TESTS + 1)) else echo -e "${YELLOW}⚠${NC} API Gateway health: ${LATENCY}ms (>100ms)" PASSED_TESTS=$((PASSED_TESTS + 1)) fi # Trading Service START=$(date +%s%N) curl -s http://localhost:9092/health > /dev/null 2>&1 END=$(date +%s%N) LATENCY=$(( (END - START) / 1000000 )) if [ $LATENCY -lt 100 ]; then echo -e "${GREEN}✓${NC} Trading Service health: ${LATENCY}ms" PASSED_TESTS=$((PASSED_TESTS + 1)) else echo -e "${YELLOW}⚠${NC} Trading Service health: ${LATENCY}ms (>100ms)" PASSED_TESTS=$((PASSED_TESTS + 1)) fi # Backtesting Service START=$(date +%s%N) curl -s http://localhost:8083/health > /dev/null 2>&1 END=$(date +%s%N) LATENCY=$(( (END - START) / 1000000 )) if [ $LATENCY -lt 100 ]; then echo -e "${GREEN}✓${NC} Backtesting Service health: ${LATENCY}ms" PASSED_TESTS=$((PASSED_TESTS + 1)) else echo -e "${YELLOW}⚠${NC} Backtesting Service health: ${LATENCY}ms (>100ms)" PASSED_TESTS=$((PASSED_TESTS + 1)) fi # ML Training Service START=$(date +%s%N) curl -s http://localhost:8095/health > /dev/null 2>&1 END=$(date +%s%N) LATENCY=$(( (END - START) / 1000000 )) if [ $LATENCY -lt 100 ]; then echo -e "${GREEN}✓${NC} ML Training Service health: ${LATENCY}ms" PASSED_TESTS=$((PASSED_TESTS + 1)) else echo -e "${YELLOW}⚠${NC} ML Training Service health: ${LATENCY}ms (>100ms)" PASSED_TESTS=$((PASSED_TESTS + 1)) fi echo "" } # Run all tests test_service_health test_database_connection test_redis_connection test_grpc_ports test_metrics_endpoints test_prometheus_targets test_parquet_files test_database_orders measure_latency # Summary echo "================================" echo "Test Summary" echo "================================" echo "Total Tests: $TOTAL_TESTS" echo -e "${GREEN}Passed: $PASSED_TESTS${NC}" echo -e "${RED}Failed: $FAILED_TESTS${NC}" echo "" PASS_RATE=$(awk "BEGIN {printf \"%.1f\", ($PASSED_TESTS / $TOTAL_TESTS) * 100}") echo "Pass Rate: $PASS_RATE%" echo "" if [ $FAILED_TESTS -eq 0 ]; then echo -e "${GREEN}✓ ALL TESTS PASSED${NC}" exit 0 else echo -e "${RED}✗ SOME TESTS FAILED${NC}" exit 1 fi