foxhunt/deployment/deploy_staging.sh

#!/bin/bash
# =============================================================================
# FOXHUNT STAGING DEPLOYMENT SCRIPT
# =============================================================================
# Deploys the Foxhunt HFT system to staging environment and validates
# operational readiness.
#
# Usage:
#   ./deployment/deploy_staging.sh [command]
#
# Commands:
#   deploy    - Deploy all services (default)
#   start     - Start existing deployment
#   stop      - Stop all services
#   restart   - Restart all services
#   status    - Check deployment status
#   logs      - Follow logs from all services
#   health    - Run health checks
#   cleanup   - Remove all staging resources
#
# =============================================================================

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Script directory
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"

# Configuration
COMPOSE_FILE="${PROJECT_ROOT}/docker-compose.staging.yml"
ENV_FILE="${PROJECT_ROOT}/.env.staging"
LOG_DIR="${PROJECT_ROOT}/logs/staging"

# Logging functions
log_info() {
    echo -e "${BLUE}[INFO]${NC} $1"
}

log_success() {
    echo -e "${GREEN}[SUCCESS]${NC} $1"
}

log_warning() {
    echo -e "${YELLOW}[WARNING]${NC} $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

# Check prerequisites
check_prerequisites() {
    log_info "Checking prerequisites..."

    # Check Docker
    if ! command -v docker &> /dev/null; then
        log_error "Docker is not installed"
        exit 1
    fi

    # Check Docker Compose
    if ! command -v docker-compose &> /dev/null; then
        log_error "Docker Compose is not installed"
        exit 1
    fi

    # Check if Docker daemon is running
    if ! docker info &> /dev/null; then
        log_error "Docker daemon is not running"
        exit 1
    fi

    # Check compose file exists
    if [ ! -f "$COMPOSE_FILE" ]; then
        log_error "Docker Compose file not found: $COMPOSE_FILE"
        exit 1
    fi

    # Check environment file
    if [ ! -f "$ENV_FILE" ]; then
        log_warning "Environment file not found: $ENV_FILE"
        log_info "Using default values from docker-compose.staging.yml"
    fi

    log_success "Prerequisites check passed"
}

# Create necessary directories
setup_directories() {
    log_info "Setting up directories..."

    mkdir -p "$LOG_DIR"
    mkdir -p "${PROJECT_ROOT}/data/staging"
    mkdir -p "${PROJECT_ROOT}/config/monitoring"

    log_success "Directories created"
}

# Deploy services
deploy_services() {
    log_info "Deploying staging environment..."

    # Pull latest images
    log_info "Pulling latest images..."
    docker-compose -f "$COMPOSE_FILE" pull --quiet

    # Build custom images
    log_info "Building service images..."
    if [ -f "$ENV_FILE" ]; then
        docker-compose -f "$COMPOSE_FILE" --env-file "$ENV_FILE" build --parallel
    else
        docker-compose -f "$COMPOSE_FILE" build --parallel
    fi

    # Start services
    log_info "Starting services..."
    if [ -f "$ENV_FILE" ]; then
        docker-compose -f "$COMPOSE_FILE" --env-file "$ENV_FILE" up -d
    else
        docker-compose -f "$COMPOSE_FILE" up -d
    fi

    log_success "Services deployed"
}

# Start services
start_services() {
    log_info "Starting staging services..."

    if [ -f "$ENV_FILE" ]; then
        docker-compose -f "$COMPOSE_FILE" --env-file "$ENV_FILE" start
    else
        docker-compose -f "$COMPOSE_FILE" start
    fi

    log_success "Services started"
}

# Stop services
stop_services() {
    log_info "Stopping staging services..."

    docker-compose -f "$COMPOSE_FILE" stop

    log_success "Services stopped"
}

# Restart services
restart_services() {
    log_info "Restarting staging services..."

    stop_services
    sleep 5
    start_services

    log_success "Services restarted"
}

# Check service status
check_status() {
    log_info "Checking service status..."
    echo ""

    docker-compose -f "$COMPOSE_FILE" ps

    echo ""
    log_info "Service health:"

    # Check each service health
    services=("postgres" "redis" "trading-service" "backtesting-service" "ml-training-service" "prometheus" "grafana")

    for service in "${services[@]}"; do
        container="foxhunt-${service}-staging"
        if docker ps --filter "name=$container" --filter "status=running" | grep -q "$container"; then
            health=$(docker inspect --format='{{.State.Health.Status}}' "$container" 2>/dev/null || echo "no healthcheck")
            if [ "$health" = "healthy" ]; then
                echo -e "  ${GREEN}✓${NC} $service: healthy"
            elif [ "$health" = "unhealthy" ]; then
                echo -e "  ${RED}✗${NC} $service: unhealthy"
            elif [ "$health" = "starting" ]; then
                echo -e "  ${YELLOW}⋯${NC} $service: starting"
            else
                echo -e "  ${BLUE}?${NC} $service: running (no healthcheck)"
            fi
        else
            echo -e "  ${RED}✗${NC} $service: not running"
        fi
    done

    echo ""
}

# Follow logs
follow_logs() {
    log_info "Following logs (Ctrl+C to exit)..."

    docker-compose -f "$COMPOSE_FILE" logs -f --tail=100
}

# Run health checks
run_health_checks() {
    log_info "Running comprehensive health checks..."
    echo ""

    # Wait for services to be ready
    log_info "Waiting for services to be ready (60s timeout)..."
    sleep 10

    local timeout=60
    local elapsed=0

    while [ $elapsed -lt $timeout ]; do
        all_healthy=true

        for service in "trading-service" "backtesting-service" "ml-training-service"; do
            container="foxhunt-${service}-staging"
            health=$(docker inspect --format='{{.State.Health.Status}}' "$container" 2>/dev/null || echo "unknown")
            if [ "$health" != "healthy" ]; then
                all_healthy=false
                break
            fi
        done

        if [ "$all_healthy" = true ]; then
            log_success "All services are healthy"
            break
        fi

        sleep 5
        elapsed=$((elapsed + 5))
        echo -n "."
    done

    echo ""

    if [ "$all_healthy" != true ]; then
        log_warning "Some services are not healthy after ${timeout}s"
    fi

    # Test PostgreSQL connectivity
    log_info "Testing PostgreSQL connectivity..."
    if docker exec foxhunt-postgres-staging pg_isready -U foxhunt_staging -d foxhunt_staging &>/dev/null; then
        log_success "PostgreSQL is ready"
    else
        log_error "PostgreSQL is not ready"
    fi

    # Test Redis connectivity
    log_info "Testing Redis connectivity..."
    if docker exec foxhunt-redis-staging redis-cli ping | grep -q "PONG"; then
        log_success "Redis is ready"
    else
        log_error "Redis is not ready"
    fi

    # Check gRPC endpoints
    log_info "Checking gRPC health endpoints..."

    # Trading Service
    if command -v grpc_health_probe &> /dev/null; then
        if grpc_health_probe -addr=localhost:50051 &>/dev/null; then
            log_success "Trading Service gRPC is healthy"
        else
            log_warning "Trading Service gRPC health check failed (may need grpc_health_probe)"
        fi
    else
        log_info "  Trading Service: localhost:50051 (install grpc_health_probe to verify)"
    fi

    # Backtesting Service
    if command -v grpc_health_probe &> /dev/null; then
        if grpc_health_probe -addr=localhost:50052 &>/dev/null; then
            log_success "Backtesting Service gRPC is healthy"
        else
            log_warning "Backtesting Service gRPC health check failed"
        fi
    else
        log_info "  Backtesting Service: localhost:50052 (install grpc_health_probe to verify)"
    fi

    # ML Training Service
    if command -v grpc_health_probe &> /dev/null; then
        if grpc_health_probe -addr=localhost:50053 &>/dev/null; then
            log_success "ML Training Service gRPC is healthy"
        else
            log_warning "ML Training Service gRPC health check failed"
        fi
    else
        log_info "  ML Training Service: localhost:50053 (install grpc_health_probe to verify)"
    fi

    # Check Prometheus
    log_info "Checking Prometheus..."
    if curl -sf http://localhost:9090/-/healthy &>/dev/null; then
        log_success "Prometheus is healthy"

        # Check Prometheus targets
        targets=$(curl -s http://localhost:9090/api/v1/targets | grep -o '"health":"up"' | wc -l)
        log_info "  Active targets: $targets"
    else
        log_error "Prometheus is not healthy"
    fi

    # Check Grafana
    log_info "Checking Grafana..."
    if curl -sf http://localhost:3001/api/health &>/dev/null; then
        log_success "Grafana is healthy"
        log_info "  URL: http://localhost:3001 (admin / check .env.staging for password)"
    else
        log_error "Grafana is not healthy"
    fi

    echo ""
    log_info "Health check summary:"
    echo "  PostgreSQL:          localhost:5433"
    echo "  Redis:               localhost:6380"
    echo "  Trading Service:     localhost:50051 (gRPC), localhost:8081 (HTTP), localhost:9001 (metrics)"
    echo "  Backtesting Service: localhost:50052 (gRPC), localhost:8082 (HTTP), localhost:9002 (metrics)"
    echo "  ML Training Service: localhost:50053 (gRPC), localhost:8083 (HTTP), localhost:9003 (metrics)"
    echo "  Prometheus:          http://localhost:9090"
    echo "  Grafana:             http://localhost:3001"
    echo ""
}

# Cleanup deployment
cleanup_deployment() {
    log_warning "This will remove all staging containers, networks, and volumes"
    read -p "Are you sure? (yes/no): " -r
    echo

    if [[ ! $REPLY =~ ^[Yy][Ee][Ss]$ ]]; then
        log_info "Cleanup cancelled"
        return
    fi

    log_info "Stopping and removing services..."
    docker-compose -f "$COMPOSE_FILE" down -v --remove-orphans

    log_info "Removing staging logs..."
    rm -rf "$LOG_DIR"

    log_success "Cleanup complete"
}

# Main command handler
main() {
    local command="${1:-deploy}"

    case "$command" in
        deploy)
            check_prerequisites
            setup_directories
            deploy_services
            echo ""
            log_info "Waiting for services to initialize..."
            sleep 15
            run_health_checks
            ;;
        start)
            check_prerequisites
            start_services
            ;;
        stop)
            stop_services
            ;;
        restart)
            restart_services
            ;;
        status)
            check_status
            ;;
        logs)
            follow_logs
            ;;
        health)
            run_health_checks
            ;;
        cleanup)
            cleanup_deployment
            ;;
        *)
            echo "Usage: $0 {deploy|start|stop|restart|status|logs|health|cleanup}"
            echo ""
            echo "Commands:"
            echo "  deploy    - Deploy all services (default)"
            echo "  start     - Start existing deployment"
            echo "  stop      - Stop all services"
            echo "  restart   - Restart all services"
            echo "  status    - Check deployment status"
            echo "  logs      - Follow logs from all services"
            echo "  health    - Run health checks"
            echo "  cleanup   - Remove all staging resources"
            exit 1
            ;;
    esac
}

# Run main function
main "$@"