foxhunt/scripts/test_service_integration.sh

#!/bin/bash
# Wave 105 Agent 4: Multi-Service Integration Testing Script
# Tests all 4 gRPC services together and validates inter-service communication

set -e

# Color codes for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Service configuration
API_GATEWAY_PORT=50051
TRADING_SERVICE_PORT=50052
BACKTESTING_SERVICE_PORT=50053
ML_TRAINING_SERVICE_PORT=50054

# Track test results
TESTS_PASSED=0
TESTS_FAILED=0
TESTS_TOTAL=0

# Test result tracking
test_result() {
    local test_name=$1
    local result=$2

    TESTS_TOTAL=$((TESTS_TOTAL + 1))

    if [ $result -eq 0 ]; then
        echo -e "${GREEN}✓ PASS${NC}: $test_name"
        TESTS_PASSED=$((TESTS_PASSED + 1))
    else
        echo -e "${RED}✗ FAIL${NC}: $test_name"
        TESTS_FAILED=$((TESTS_FAILED + 1))
    fi
}

print_header() {
    echo -e "\n${BLUE}========================================${NC}"
    echo -e "${BLUE}$1${NC}"
    echo -e "${BLUE}========================================${NC}\n"
}

print_summary() {
    echo -e "\n${BLUE}========================================${NC}"
    echo -e "${BLUE}TEST SUMMARY${NC}"
    echo -e "${BLUE}========================================${NC}"
    echo -e "Total Tests: ${TESTS_TOTAL}"
    echo -e "${GREEN}Passed: ${TESTS_PASSED}${NC}"
    echo -e "${RED}Failed: ${TESTS_FAILED}${NC}"

    if [ $TESTS_FAILED -eq 0 ]; then
        echo -e "\n${GREEN}All tests passed!${NC}\n"
        return 0
    else
        echo -e "\n${RED}Some tests failed!${NC}\n"
        return 1
    fi
}

# Step 1: Check Docker and docker-compose
print_header "Step 1: Checking Prerequisites"

if ! command -v docker &> /dev/null; then
    echo -e "${RED}ERROR: Docker not found${NC}"
    exit 1
fi
test_result "Docker installed" 0

if ! command -v docker-compose &> /dev/null && ! docker compose version &> /dev/null; then
    echo -e "${RED}ERROR: docker-compose not found${NC}"
    exit 1
fi
test_result "docker-compose installed" 0

if ! command -v grpcurl &> /dev/null; then
    echo -e "${YELLOW}WARNING: grpcurl not found (install for gRPC testing)${NC}"
    test_result "grpcurl installed" 1
else
    test_result "grpcurl installed" 0
fi

# Step 2: Start infrastructure services
print_header "Step 2: Starting Infrastructure Services"

echo "Starting postgres, redis, vault, influxdb..."
docker-compose up -d postgres redis vault influxdb

# Wait for infrastructure to be healthy
echo "Waiting for infrastructure health checks..."
sleep 10

# Check infrastructure health
docker-compose ps | grep -E "(postgres|redis|vault|influxdb)" | grep -i "up" > /dev/null
test_result "Infrastructure services running" $?

# Step 3: Build application services
print_header "Step 3: Building Application Services"

echo -e "${YELLOW}NOTE: Building Rust services will take 10-20 minutes...${NC}"
echo "Building trading_service..."
docker-compose build trading_service 2>&1 | tail -5
test_result "trading_service build" $?

echo "Building backtesting_service..."
docker-compose build backtesting_service 2>&1 | tail -5
test_result "backtesting_service build" $?

echo "Building ml_training_service..."
docker-compose build ml_training_service 2>&1 | tail -5
test_result "ml_training_service build" $?

echo "Building api_gateway..."
docker-compose build api_gateway 2>&1 | tail -5
test_result "api_gateway build" $?

# Step 4: Start application services
print_header "Step 4: Starting Application Services"

echo "Starting backend services (trading, backtesting, ml_training)..."
docker-compose up -d trading_service backtesting_service ml_training_service

echo "Waiting for backend services to initialize (30s)..."
sleep 30

# Check backend service health
docker-compose ps trading_service | grep -i "up" > /dev/null
test_result "trading_service running" $?

docker-compose ps backtesting_service | grep -i "up" > /dev/null
test_result "backtesting_service running" $?

docker-compose ps ml_training_service | grep -i "up" > /dev/null
test_result "ml_training_service running" $?

echo "Starting api_gateway..."
docker-compose up -d api_gateway

echo "Waiting for api_gateway to initialize (20s)..."
sleep 20

docker-compose ps api_gateway | grep -i "up" > /dev/null
test_result "api_gateway running" $?

# Step 5: Health check all services
print_header "Step 5: gRPC Health Checks"

if command -v grpcurl &> /dev/null; then
    # API Gateway (port 50051)
    grpcurl -plaintext -max-time 5 localhost:$API_GATEWAY_PORT grpc.health.v1.Health/Check > /dev/null 2>&1
    test_result "api_gateway health check (port $API_GATEWAY_PORT)" $?

    # Trading Service (port 50052)
    grpcurl -plaintext -max-time 5 localhost:$TRADING_SERVICE_PORT grpc.health.v1.Health/Check > /dev/null 2>&1
    test_result "trading_service health check (port $TRADING_SERVICE_PORT)" $?

    # Backtesting Service (port 50053)
    grpcurl -plaintext -max-time 5 localhost:$BACKTESTING_SERVICE_PORT grpc.health.v1.Health/Check > /dev/null 2>&1
    test_result "backtesting_service health check (port $BACKTESTING_SERVICE_PORT)" $?

    # ML Training Service (port 50054)
    grpcurl -plaintext -max-time 5 localhost:$ML_TRAINING_SERVICE_PORT grpc.health.v1.Health/Check > /dev/null 2>&1
    test_result "ml_training_service health check (port $ML_TRAINING_SERVICE_PORT)" $?
else
    echo -e "${YELLOW}Skipping gRPC health checks (grpcurl not installed)${NC}"
fi

# Step 6: Check service logs for errors
print_header "Step 6: Checking Service Logs"

echo "Checking api_gateway logs for errors..."
if docker-compose logs --tail=50 api_gateway | grep -i "error\|panic\|fatal" | grep -v "test" > /dev/null; then
    echo -e "${YELLOW}WARNING: Found errors in api_gateway logs${NC}"
    docker-compose logs --tail=20 api_gateway | grep -i "error\|panic\|fatal" | grep -v "test"
    test_result "api_gateway clean logs" 1
else
    test_result "api_gateway clean logs" 0
fi

echo "Checking trading_service logs for errors..."
if docker-compose logs --tail=50 trading_service | grep -i "error\|panic\|fatal" | grep -v "test" > /dev/null; then
    echo -e "${YELLOW}WARNING: Found errors in trading_service logs${NC}"
    docker-compose logs --tail=20 trading_service | grep -i "error\|panic\|fatal" | grep -v "test"
    test_result "trading_service clean logs" 1
else
    test_result "trading_service clean logs" 0
fi

echo "Checking backtesting_service logs for errors..."
if docker-compose logs --tail=50 backtesting_service | grep -i "error\|panic\|fatal" | grep -v "test" > /dev/null; then
    echo -e "${YELLOW}WARNING: Found errors in backtesting_service logs${NC}"
    docker-compose logs --tail=20 backtesting_service | grep -i "error\|panic\|fatal" | grep -v "test"
    test_result "backtesting_service clean logs" 1
else
    test_result "backtesting_service clean logs" 0
fi

echo "Checking ml_training_service logs for errors..."
if docker-compose logs --tail=50 ml_training_service | grep -i "error\|panic\|fatal" | grep -v "test" > /dev/null; then
    echo -e "${YELLOW}WARNING: Found errors in ml_training_service logs${NC}"
    docker-compose logs --tail=20 ml_training_service | grep -i "error\|panic\|fatal" | grep -v "test"
    test_result "ml_training_service clean logs" 1
else
    test_result "ml_training_service clean logs" 0
fi

# Step 7: Test service discovery/connectivity
print_header "Step 7: Service Network Connectivity"

echo "Testing api_gateway → trading_service connectivity..."
docker-compose exec -T api_gateway sh -c "nc -zv trading_service 50051" &> /dev/null
test_result "api_gateway can reach trading_service" $?

echo "Testing api_gateway → backtesting_service connectivity..."
docker-compose exec -T api_gateway sh -c "nc -zv backtesting_service 50052" &> /dev/null
test_result "api_gateway can reach backtesting_service" $?

echo "Testing api_gateway → ml_training_service connectivity..."
docker-compose exec -T api_gateway sh -c "nc -zv ml_training_service 50053" &> /dev/null
test_result "api_gateway can reach ml_training_service" $?

# Step 8: Metrics endpoints
print_header "Step 8: Prometheus Metrics Endpoints"

curl -s http://localhost:9091/metrics > /dev/null 2>&1
test_result "api_gateway metrics (port 9091)" $?

curl -s http://localhost:9092/metrics > /dev/null 2>&1
test_result "trading_service metrics (port 9092)" $?

curl -s http://localhost:9093/metrics > /dev/null 2>&1
test_result "backtesting_service metrics (port 9093)" $?

curl -s http://localhost:9094/metrics > /dev/null 2>&1
test_result "ml_training_service metrics (port 9094)" $?

# Step 9: Test failover (optional - commented out for safety)
print_header "Step 9: Failover Testing (Manual)"

echo -e "${YELLOW}Failover tests require manual execution:${NC}"
echo "  1. Stop trading_service: docker-compose stop trading_service"
echo "  2. Check api_gateway logs: docker-compose logs --tail=50 api_gateway"
echo "  3. Verify graceful degradation (should see connection errors but no crashes)"
echo "  4. Restart trading_service: docker-compose start trading_service"
echo "  5. Verify recovery: docker-compose logs --tail=20 trading_service"

# Print final summary
print_summary

# Cleanup instructions
echo -e "\n${BLUE}========================================${NC}"
echo -e "${BLUE}CLEANUP${NC}"
echo -e "${BLUE}========================================${NC}"
echo "To stop all services:"
echo "  docker-compose down"
echo ""
echo "To stop and remove volumes:"
echo "  docker-compose down -v"
echo ""
echo "To view logs:"
echo "  docker-compose logs -f [service_name]"
echo ""