foxhunt/scripts/cross_service_integration_test.sh

#!/bin/bash
# Cross-Service Integration Test Script
# Tests actual cross-service communication flows

set -e

echo "================================"
echo "Cross-Service Integration Tests"
echo "================================"
echo ""

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# Test results
TOTAL_TESTS=0
PASSED_TESTS=0
FAILED_TESTS=0

function test_service_health() {
    echo "Test 1: Service Health Checks"
    echo "=============================="
    TOTAL_TESTS=$((TOTAL_TESTS + 4))

    # API Gateway
    if curl -s -f http://localhost:9091/health > /dev/null 2>&1; then
        echo -e "${GREEN}✓${NC} API Gateway (9091) - Healthy"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    else
        echo -e "${RED}✗${NC} API Gateway (9091) - Unhealthy"
        FAILED_TESTS=$((FAILED_TESTS + 1))
    fi

    # Trading Service
    if curl -s -f http://localhost:9092/health > /dev/null 2>&1; then
        echo -e "${GREEN}✓${NC} Trading Service (9092) - Healthy"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    else
        echo -e "${RED}✗${NC} Trading Service (9092) - Unhealthy"
        FAILED_TESTS=$((FAILED_TESTS + 1))
    fi

    # Backtesting Service
    if curl -s -f http://localhost:8083/health > /dev/null 2>&1; then
        echo -e "${GREEN}✓${NC} Backtesting Service (8083) - Healthy"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    else
        echo -e "${RED}✗${NC} Backtesting Service (8083) - Unhealthy"
        FAILED_TESTS=$((FAILED_TESTS + 1))
    fi

    # ML Training Service
    if curl -s -f http://localhost:8095/health > /dev/null 2>&1; then
        echo -e "${GREEN}✓${NC} ML Training Service (8095) - Healthy"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    else
        echo -e "${RED}✗${NC} ML Training Service (8095) - Unhealthy"
        FAILED_TESTS=$((FAILED_TESTS + 1))
    fi

    echo ""
}

function test_database_connection() {
    echo "Test 2: PostgreSQL Connectivity"
    echo "==============================="
    TOTAL_TESTS=$((TOTAL_TESTS + 1))

    if psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -c "SELECT 1" > /dev/null 2>&1; then
        echo -e "${GREEN}✓${NC} PostgreSQL connection successful"
        PASSED_TESTS=$((PASSED_TESTS + 1))

        # Check key tables
        TABLES=$(psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='public'")
        echo "   Found $TABLES tables in public schema"
    else
        echo -e "${RED}✗${NC} PostgreSQL connection failed"
        FAILED_TESTS=$((FAILED_TESTS + 1))
    fi

    echo ""
}

function test_redis_connection() {
    echo "Test 3: Redis Connectivity"
    echo "=========================="
    TOTAL_TESTS=$((TOTAL_TESTS + 1))

    if redis-cli -h localhost -p 6379 PING 2>/dev/null | grep -q PONG; then
        echo -e "${GREEN}✓${NC} Redis connection successful"
        PASSED_TESTS=$((PASSED_TESTS + 1))

        # Test set/get
        redis-cli -h localhost -p 6379 SET test_key "integration_test" > /dev/null 2>&1
        VALUE=$(redis-cli -h localhost -p 6379 GET test_key 2>/dev/null)
        if [ "$VALUE" = "integration_test" ]; then
            echo "   Redis SET/GET working"
        fi
        redis-cli -h localhost -p 6379 DEL test_key > /dev/null 2>&1
    else
        echo -e "${RED}✗${NC} Redis connection failed"
        FAILED_TESTS=$((FAILED_TESTS + 1))
    fi

    echo ""
}

function test_grpc_ports() {
    echo "Test 4: gRPC Port Availability"
    echo "=============================="
    TOTAL_TESTS=$((TOTAL_TESTS + 4))

    # API Gateway gRPC
    if lsof -i :50051 2>/dev/null | grep -q LISTEN; then
        echo -e "${GREEN}✓${NC} API Gateway gRPC (50051) - Listening"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    else
        echo -e "${RED}✗${NC} API Gateway gRPC (50051) - Not listening"
        FAILED_TESTS=$((FAILED_TESTS + 1))
    fi

    # Trading Service gRPC
    if lsof -i :50052 2>/dev/null | grep -q LISTEN; then
        echo -e "${GREEN}✓${NC} Trading Service gRPC (50052) - Listening"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    else
        echo -e "${RED}✗${NC} Trading Service gRPC (50052) - Not listening"
        FAILED_TESTS=$((FAILED_TESTS + 1))
    fi

    # Backtesting Service gRPC
    if lsof -i :50053 2>/dev/null | grep -q LISTEN; then
        echo -e "${GREEN}✓${NC} Backtesting Service gRPC (50053) - Listening"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    else
        echo -e "${RED}✗${NC} Backtesting Service gRPC (50053) - Not listening"
        FAILED_TESTS=$((FAILED_TESTS + 1))
    fi

    # ML Training Service gRPC
    if lsof -i :50054 2>/dev/null | grep -q LISTEN; then
        echo -e "${GREEN}✓${NC} ML Training Service gRPC (50054) - Listening"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    else
        echo -e "${RED}✗${NC} ML Training Service gRPC (50054) - Not listening"
        FAILED_TESTS=$((FAILED_TESTS + 1))
    fi

    echo ""
}

function test_metrics_endpoints() {
    echo "Test 5: Prometheus Metrics Endpoints"
    echo "===================================="
    TOTAL_TESTS=$((TOTAL_TESTS + 4))

    # API Gateway metrics
    if curl -s http://localhost:9091/metrics | grep -q "# TYPE"; then
        echo -e "${GREEN}✓${NC} API Gateway metrics (9091) - Available"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    else
        echo -e "${RED}✗${NC} API Gateway metrics (9091) - Unavailable"
        FAILED_TESTS=$((FAILED_TESTS + 1))
    fi

    # Trading Service metrics
    if curl -s http://localhost:9092/metrics | grep -q "# TYPE"; then
        echo -e "${GREEN}✓${NC} Trading Service metrics (9092) - Available"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    else
        echo -e "${RED}✗${NC} Trading Service metrics (9092) - Unavailable"
        FAILED_TESTS=$((FAILED_TESTS + 1))
    fi

    # Backtesting Service metrics
    if curl -s http://localhost:9093/metrics | grep -q "# TYPE"; then
        echo -e "${GREEN}✓${NC} Backtesting Service metrics (9093) - Available"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    else
        echo -e "${RED}✗${NC} Backtesting Service metrics (9093) - Unavailable"
        FAILED_TESTS=$((FAILED_TESTS + 1))
    fi

    # ML Training Service metrics
    if curl -s http://localhost:9094/metrics | grep -q "# TYPE"; then
        echo -e "${GREEN}✓${NC} ML Training Service metrics (9094) - Available"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    else
        echo -e "${RED}✗${NC} ML Training Service metrics (9094) - Unavailable"
        FAILED_TESTS=$((FAILED_TESTS + 1))
    fi

    echo ""
}

function test_prometheus_targets() {
    echo "Test 6: Prometheus Service Discovery"
    echo "===================================="
    TOTAL_TESTS=$((TOTAL_TESTS + 1))

    if curl -s http://localhost:9090/api/v1/targets 2>/dev/null | grep -q '"health":"up"'; then
        echo -e "${GREEN}✓${NC} Prometheus has healthy targets"
        PASSED_TESTS=$((PASSED_TESTS + 1))

        # Count healthy targets
        HEALTHY=$(curl -s http://localhost:9090/api/v1/targets 2>/dev/null | grep -o '"health":"up"' | wc -l)
        echo "   $HEALTHY services reporting to Prometheus"
    else
        echo -e "${RED}✗${NC} No healthy Prometheus targets found"
        FAILED_TESTS=$((FAILED_TESTS + 1))
    fi

    echo ""
}

function test_parquet_files() {
    echo "Test 7: Parquet Test Data Availability"
    echo "======================================"
    TOTAL_TESTS=$((TOTAL_TESTS + 1))

    PARQUET_COUNT=$(find /home/jgrusewski/Work/foxhunt/test_data -name "*.parquet" 2>/dev/null | wc -l)

    if [ "$PARQUET_COUNT" -gt 0 ]; then
        echo -e "${GREEN}✓${NC} Found $PARQUET_COUNT Parquet test files"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    else
        echo -e "${YELLOW}⚠${NC} No Parquet test files found"
        FAILED_TESTS=$((FAILED_TESTS + 1))
    fi

    echo ""
}

function test_database_orders() {
    echo "Test 8: Database Order Persistence"
    echo "=================================="
    TOTAL_TESTS=$((TOTAL_TESTS + 1))

    ORDER_COUNT=$(psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -t -c "SELECT COUNT(*) FROM orders" 2>/dev/null | tr -d ' ')

    if [ $? -eq 0 ]; then
        echo -e "${GREEN}✓${NC} Orders table accessible: $ORDER_COUNT orders"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    else
        echo -e "${RED}✗${NC} Failed to query orders table"
        FAILED_TESTS=$((FAILED_TESTS + 1))
    fi

    echo ""
}

function measure_latency() {
    echo "Test 9: Inter-Service Latency Measurement"
    echo "========================================="
    TOTAL_TESTS=$((TOTAL_TESTS + 4))

    # API Gateway
    START=$(date +%s%N)
    curl -s http://localhost:9091/health > /dev/null 2>&1
    END=$(date +%s%N)
    LATENCY=$(( (END - START) / 1000000 ))
    if [ $LATENCY -lt 100 ]; then
        echo -e "${GREEN}✓${NC} API Gateway health: ${LATENCY}ms"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    else
        echo -e "${YELLOW}⚠${NC} API Gateway health: ${LATENCY}ms (>100ms)"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    fi

    # Trading Service
    START=$(date +%s%N)
    curl -s http://localhost:9092/health > /dev/null 2>&1
    END=$(date +%s%N)
    LATENCY=$(( (END - START) / 1000000 ))
    if [ $LATENCY -lt 100 ]; then
        echo -e "${GREEN}✓${NC} Trading Service health: ${LATENCY}ms"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    else
        echo -e "${YELLOW}⚠${NC} Trading Service health: ${LATENCY}ms (>100ms)"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    fi

    # Backtesting Service
    START=$(date +%s%N)
    curl -s http://localhost:8083/health > /dev/null 2>&1
    END=$(date +%s%N)
    LATENCY=$(( (END - START) / 1000000 ))
    if [ $LATENCY -lt 100 ]; then
        echo -e "${GREEN}✓${NC} Backtesting Service health: ${LATENCY}ms"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    else
        echo -e "${YELLOW}⚠${NC} Backtesting Service health: ${LATENCY}ms (>100ms)"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    fi

    # ML Training Service
    START=$(date +%s%N)
    curl -s http://localhost:8095/health > /dev/null 2>&1
    END=$(date +%s%N)
    LATENCY=$(( (END - START) / 1000000 ))
    if [ $LATENCY -lt 100 ]; then
        echo -e "${GREEN}✓${NC} ML Training Service health: ${LATENCY}ms"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    else
        echo -e "${YELLOW}⚠${NC} ML Training Service health: ${LATENCY}ms (>100ms)"
        PASSED_TESTS=$((PASSED_TESTS + 1))
    fi

    echo ""
}

# Run all tests
test_service_health
test_database_connection
test_redis_connection
test_grpc_ports
test_metrics_endpoints
test_prometheus_targets
test_parquet_files
test_database_orders
measure_latency

# Summary
echo "================================"
echo "Test Summary"
echo "================================"
echo "Total Tests:  $TOTAL_TESTS"
echo -e "${GREEN}Passed:       $PASSED_TESTS${NC}"
echo -e "${RED}Failed:       $FAILED_TESTS${NC}"
echo ""

PASS_RATE=$(awk "BEGIN {printf \"%.1f\", ($PASSED_TESTS / $TOTAL_TESTS) * 100}")
echo "Pass Rate:    $PASS_RATE%"
echo ""

if [ $FAILED_TESTS -eq 0 ]; then
    echo -e "${GREEN}✓ ALL TESTS PASSED${NC}"
    exit 0
else
    echo -e "${RED}✗ SOME TESTS FAILED${NC}"
    exit 1
fi