foxhunt/scripts/test_service_startup.sh

#!/bin/bash
# WAVE 105 AGENT 10: Service Startup Validation Test Script
#
# This script tests if each service can start and reach healthy state within 60 seconds.
# Usage: ./scripts/test_service_startup.sh [service_name]
#   service_name: trading_service, backtesting_service, ml_training_service, api_gateway, or "all"

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Configuration
TIMEOUT_SECONDS=60
LOG_DIR="logs/startup_tests"
mkdir -p "$LOG_DIR"

# Prerequisites check
check_prerequisites() {
    echo -e "${BLUE}Checking prerequisites...${NC}"

    # Check PostgreSQL
    if ! pg_isready -q 2>/dev/null; then
        echo -e "${RED}❌ PostgreSQL is not running${NC}"
        echo "Start with: sudo systemctl start postgresql"
        return 1
    fi
    echo -e "${GREEN}✓ PostgreSQL is running${NC}"

    # Check Redis
    if ! redis-cli ping >/dev/null 2>&1; then
        echo -e "${YELLOW}⚠️  Redis is not running (required for api_gateway and trading_service)${NC}"
        echo "Start with: sudo systemctl start redis"
    else
        echo -e "${GREEN}✓ Redis is running${NC}"
    fi

    # Check database exists
    if ! psql -lqt | cut -d \| -f 1 | grep -qw foxhunt 2>/dev/null; then
        echo -e "${YELLOW}⚠️  Database 'foxhunt' not found${NC}"
        echo "Create with: createdb foxhunt"
    else
        echo -e "${GREEN}✓ Database 'foxhunt' exists${NC}"
    fi

    # Check JWT secret
    if [ ! -f "${JWT_SECRET_FILE:-}" ]; then
        JWT_SECRET_FILE="/tmp/jwt_secret.key"
        if [ ! -f "$JWT_SECRET_FILE" ]; then
            echo -e "${YELLOW}⚠️  Generating JWT secret...${NC}"
            openssl rand -base64 64 > "$JWT_SECRET_FILE"
            chmod 600 "$JWT_SECRET_FILE"
        fi
        echo -e "${GREEN}✓ JWT secret at $JWT_SECRET_FILE${NC}"
    fi

    # Check model cache directory
    MODEL_CACHE_DIR="${MODEL_CACHE_DIR:-/tmp/foxhunt/model_cache}"
    if [ ! -d "$MODEL_CACHE_DIR" ]; then
        echo -e "${YELLOW}⚠️  Creating model cache directory...${NC}"
        mkdir -p "$MODEL_CACHE_DIR"
    fi
    echo -e "${GREEN}✓ Model cache directory at $MODEL_CACHE_DIR${NC}"

    echo ""
}

# Setup environment variables
setup_env() {
    export DATABASE_URL="${DATABASE_URL:-postgresql://localhost/foxhunt}"
    export REDIS_URL="${REDIS_URL:-redis://localhost:6379}"
    export JWT_SECRET_FILE="${JWT_SECRET_FILE:-/tmp/jwt_secret.key}"
    export ENVIRONMENT="development"
    export ENABLE_HTTP2_OPTIMIZATIONS="false"  # Simplify testing
    export REQUIRE_MTLS="false"  # Disable mTLS for testing
    export MODEL_CACHE_DIR="${MODEL_CACHE_DIR:-/tmp/foxhunt/model_cache}"
    export RUST_LOG="${RUST_LOG:-info}"
}

# Test single service startup
test_service() {
    local service_name=$1
    local binary_path="target/debug/$service_name"
    local port=${2:-50051}
    local log_file="$LOG_DIR/${service_name}_$(date +%Y%m%d_%H%M%S).log"

    echo -e "${BLUE}Testing $service_name startup...${NC}"

    # Check if binary exists
    if [ ! -f "$binary_path" ]; then
        echo -e "${RED}❌ Binary not found: $binary_path${NC}"
        echo -e "${YELLOW}   Build with: cargo build -p $service_name${NC}"
        return 1
    fi

    local binary_size=$(du -h "$binary_path" | cut -f1)
    echo -e "${GREEN}✓ Binary exists: $binary_path ($binary_size)${NC}"

    # Set service-specific port
    export GRPC_PORT=$port

    # Start service in background
    echo "Starting $service_name on port $port..."
    echo "Logs: $log_file"

    local start_time=$(date +%s)
    "$binary_path" > "$log_file" 2>&1 &
    local pid=$!

    echo "PID: $pid"

    # Wait for startup (check logs for ready message)
    local elapsed=0
    local ready=false

    while [ $elapsed -lt $TIMEOUT_SECONDS ]; do
        sleep 1
        elapsed=$((elapsed + 1))

        # Check if process is still running
        if ! kill -0 $pid 2>/dev/null; then
            echo -e "${RED}❌ Process died during startup${NC}"
            echo "Last 20 lines of log:"
            tail -20 "$log_file"
            return 1
        fi

        # Check for ready/listening messages in logs
        if grep -qi "listening\|ready\|started" "$log_file"; then
            ready=true
            break
        fi

        # Show progress
        if [ $((elapsed % 5)) -eq 0 ]; then
            echo "  Waiting... ${elapsed}s elapsed"
        fi
    done

    local end_time=$(date +%s)
    local startup_time=$((end_time - start_time))

    # Kill the service
    echo "Stopping service (PID: $pid)..."
    kill $pid 2>/dev/null || true
    sleep 2
    kill -9 $pid 2>/dev/null || true

    if [ "$ready" = true ]; then
        echo -e "${GREEN}✅ $service_name started successfully in ${startup_time}s${NC}"

        # Show startup log excerpt
        echo ""
        echo "Startup log excerpt:"
        grep -i "initialized\|ready\|listening\|started" "$log_file" | head -10
        echo ""

        return 0
    else
        echo -e "${RED}❌ $service_name failed to start within ${TIMEOUT_SECONDS}s${NC}"
        echo "Last 30 lines of log:"
        tail -30 "$log_file"
        return 1
    fi
}

# Main execution
main() {
    local service=${1:-all}

    echo "========================================="
    echo "Service Startup Validation Test"
    echo "Wave 105 Agent 10"
    echo "========================================="
    echo ""

    check_prerequisites || exit 1
    setup_env

    echo "Environment:"
    echo "  DATABASE_URL: $DATABASE_URL"
    echo "  REDIS_URL: $REDIS_URL"
    echo "  JWT_SECRET_FILE: $JWT_SECRET_FILE"
    echo "  MODEL_CACHE_DIR: $MODEL_CACHE_DIR"
    echo ""

    local success_count=0
    local fail_count=0

    if [ "$service" = "all" ]; then
        # Test all services
        services=("backtesting_service:50052" "ml_training_service:50053" "trading_service:50051" "api_gateway:50051")

        for svc_port in "${services[@]}"; do
            IFS=':' read -r svc port <<< "$svc_port"
            echo ""
            echo "========================================="
            if test_service "$svc" "$port"; then
                ((success_count++))
            else
                ((fail_count++))
            fi
        done
    else
        # Test single service
        local port=${2:-50051}
        if test_service "$service" "$port"; then
            ((success_count++))
        else
            ((fail_count++))
        fi
    fi

    # Summary
    echo ""
    echo "========================================="
    echo "SUMMARY"
    echo "========================================="
    echo -e "${GREEN}Success: $success_count${NC}"
    echo -e "${RED}Failed: $fail_count${NC}"
    echo ""

    if [ $fail_count -eq 0 ]; then
        echo -e "${GREEN}✅ All services started successfully!${NC}"
        exit 0
    else
        echo -e "${RED}❌ Some services failed to start${NC}"
        exit 1
    fi
}

# Run main with all arguments
main "$@"