foxhunt/scripts/grpc_load_test_wave78.sh

#!/bin/bash

# Wave 78: gRPC Load Testing Script for Foxhunt HFT Services
# Tests Trading Service, Backtesting Service, ML Training Service, and API Gateway
# Uses ghz (https://ghz.sh) for gRPC load testing

set -e

# Color output for better readability
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
FOXHUNT_ROOT="$(dirname "$SCRIPT_DIR")"
RESULTS_DIR="$FOXHUNT_ROOT/test_results/grpc_load_tests"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)

# Ensure PATH includes Go binaries
export PATH="$HOME/go/bin:$HOME/go-install/go/bin:$PATH"

# Service configurations
TRADING_SERVICE_HOST="localhost:50051"
BACKTESTING_SERVICE_HOST="localhost:50052"
ML_TRAINING_SERVICE_HOST="localhost:50053"
API_GATEWAY_HOST="localhost:50050"

# Proto file locations
TRADING_PROTO="$FOXHUNT_ROOT/services/trading_service/proto/trading.proto"
ML_TRAINING_PROTO="$FOXHUNT_ROOT/services/ml_training_service/proto/ml_training.proto"
HEALTH_PROTO="$FOXHUNT_ROOT/tli/proto/health.proto"

# Test parameters (configurable)
DURATION="${DURATION:-10s}"          # Test duration
RPS="${RPS:-100}"                    # Requests per second
CONNECTIONS="${CONNECTIONS:-10}"     # Number of connections
CONCURRENCY="${CONCURRENCY:-50}"     # Number of concurrent workers

echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Foxhunt gRPC Load Testing - Wave 78${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""

# Create results directory
mkdir -p "$RESULTS_DIR"

# Function to check if service is running
check_service() {
    local host=$1
    local service_name=$2

    echo -e "${YELLOW}Checking $service_name at $host...${NC}"

    if nc -z -w 5 $(echo $host | cut -d: -f1) $(echo $host | cut -d: -f2) 2>/dev/null; then
        echo -e "${GREEN}✓ $service_name is running${NC}"
        return 0
    else
        echo -e "${RED}✗ $service_name is not running${NC}"
        return 1
    fi
}

# Function to run ghz test
run_ghz_test() {
    local service_name=$1
    local host=$2
    local proto_file=$3
    local method=$4
    local data=$5
    local output_file="$RESULTS_DIR/${service_name}_${TIMESTAMP}.json"

    echo ""
    echo -e "${BLUE}Testing $service_name - $method${NC}"
    echo -e "${YELLOW}Target: $host${NC}"
    echo -e "${YELLOW}Duration: $DURATION | RPS: $RPS | Connections: $CONNECTIONS | Concurrency: $CONCURRENCY${NC}"

    # Run ghz with comprehensive metrics
    $HOME/go/bin/ghz \
        --insecure \
        --proto "$proto_file" \
        --call "$method" \
        -d "$data" \
        --duration "$DURATION" \
        --rps "$RPS" \
        --connections "$CONNECTIONS" \
        --concurrency "$CONCURRENCY" \
        --format json \
        --output "$output_file" \
        "$host" \
        2>&1 | tee >(cat >&2)

    # Also save human-readable output
    $HOME/go/bin/ghz \
        --insecure \
        --proto "$proto_file" \
        --call "$method" \
        -d "$data" \
        --duration "$DURATION" \
        --rps "$RPS" \
        --connections "$CONNECTIONS" \
        --concurrency "$CONCURRENCY" \
        --format pretty \
        "$host" \
        > "$RESULTS_DIR/${service_name}_${TIMESTAMP}.txt"

    echo -e "${GREEN}✓ Results saved to $output_file${NC}"

    # Display summary
    if [ -f "$output_file" ]; then
        echo ""
        echo -e "${BLUE}Summary:${NC}"
        cat "$output_file" | jq -r '
            "Total requests: " + (.count | tostring) +
            "\nAverage latency: " + (.average | tostring) + "ms" +
            "\nP50 latency: " + (.latencies.P50 | tostring) + "ms" +
            "\nP95 latency: " + (.latencies.P95 | tostring) + "ms" +
            "\nP99 latency: " + (.latencies.P99 | tostring) + "ms" +
            "\nMax latency: " + (.latencies.max | tostring) + "ms" +
            "\nRPS: " + (.rps | tostring) +
            "\nErrors: " + (if .errorDistribution then (.errorDistribution | length | tostring) else "0" end)
        ' 2>/dev/null || echo "Summary parsing failed (check JSON file directly)"
    fi
}

# Function to test Trading Service
test_trading_service() {
    echo ""
    echo -e "${BLUE}========================================${NC}"
    echo -e "${BLUE}Testing Trading Service (Port 50051)${NC}"
    echo -e "${BLUE}========================================${NC}"

    if ! check_service "$TRADING_SERVICE_HOST" "Trading Service"; then
        echo -e "${RED}Skipping Trading Service tests${NC}"
        return 1
    fi

    # Test 1: GetOrderStatus (simple unary RPC)
    run_ghz_test \
        "trading_service_order_status" \
        "$TRADING_SERVICE_HOST" \
        "$TRADING_PROTO" \
        "trading.TradingService.GetOrderStatus" \
        '{"order_id": "test-order-123"}'

    # Test 2: GetPositions (unary RPC with optional fields)
    run_ghz_test \
        "trading_service_positions" \
        "$TRADING_SERVICE_HOST" \
        "$TRADING_PROTO" \
        "trading.TradingService.GetPositions" \
        '{"account_id": "test-account", "symbol": "AAPL"}'

    # Test 3: GetPortfolioSummary
    run_ghz_test \
        "trading_service_portfolio" \
        "$TRADING_SERVICE_HOST" \
        "$TRADING_PROTO" \
        "trading.TradingService.GetPortfolioSummary" \
        '{"account_id": "test-account"}'

    # Test 4: GetOrderBook
    run_ghz_test \
        "trading_service_orderbook" \
        "$TRADING_SERVICE_HOST" \
        "$TRADING_PROTO" \
        "trading.TradingService.GetOrderBook" \
        '{"symbol": "AAPL", "depth": 10}'
}

# Function to test ML Training Service
test_ml_training_service() {
    echo ""
    echo -e "${BLUE}========================================${NC}"
    echo -e "${BLUE}Testing ML Training Service (Port 50053)${NC}"
    echo -e "${BLUE}========================================${NC}"

    if ! check_service "$ML_TRAINING_SERVICE_HOST" "ML Training Service"; then
        echo -e "${RED}Skipping ML Training Service tests${NC}"
        return 1
    fi

    # Test 1: HealthCheck
    run_ghz_test \
        "ml_training_service_health" \
        "$ML_TRAINING_SERVICE_HOST" \
        "$ML_TRAINING_PROTO" \
        "ml_training.MLTrainingService.HealthCheck" \
        '{}'

    # Test 2: ListAvailableModels
    run_ghz_test \
        "ml_training_service_models" \
        "$ML_TRAINING_SERVICE_HOST" \
        "$ML_TRAINING_PROTO" \
        "ml_training.MLTrainingService.ListAvailableModels" \
        '{}'

    # Test 3: ListTrainingJobs (with pagination)
    run_ghz_test \
        "ml_training_service_jobs" \
        "$ML_TRAINING_SERVICE_HOST" \
        "$ML_TRAINING_PROTO" \
        "ml_training.MLTrainingService.ListTrainingJobs" \
        '{"page": 1, "page_size": 10}'
}

# Function to test Health endpoints
test_health_endpoints() {
    echo ""
    echo -e "${BLUE}========================================${NC}"
    echo -e "${BLUE}Testing Health Endpoints (All Services)${NC}"
    echo -e "${BLUE}========================================${NC}"

    # Trading Service Health
    if check_service "$TRADING_SERVICE_HOST" "Trading Service"; then
        run_ghz_test \
            "health_trading" \
            "$TRADING_SERVICE_HOST" \
            "$HEALTH_PROTO" \
            "grpc.health.v1.Health.Check" \
            '{"service": ""}'
    fi

    # ML Training Service Health (already has HealthCheck in its proto)
    # Covered above

    # API Gateway Health
    if check_service "$API_GATEWAY_HOST" "API Gateway"; then
        run_ghz_test \
            "health_gateway" \
            "$API_GATEWAY_HOST" \
            "$HEALTH_PROTO" \
            "grpc.health.v1.Health.Check" \
            '{"service": ""}'
    fi
}

# High-load stress test
stress_test_trading_service() {
    echo ""
    echo -e "${BLUE}========================================${NC}"
    echo -e "${BLUE}STRESS TEST: Trading Service${NC}"
    echo -e "${BLUE}========================================${NC}"

    if ! check_service "$TRADING_SERVICE_HOST" "Trading Service"; then
        echo -e "${RED}Skipping stress test${NC}"
        return 1
    fi

    echo -e "${YELLOW}Running high-load test: 1000 RPS, 100 connections, 30s duration${NC}"

    # Override parameters for stress test
    $HOME/go/bin/ghz \
        --insecure \
        --proto "$TRADING_PROTO" \
        --call "trading.TradingService.GetOrderStatus" \
        -d '{"order_id": "stress-test-order"}' \
        --duration "30s" \
        --rps 1000 \
        --connections 100 \
        --concurrency 200 \
        --format json \
        --output "$RESULTS_DIR/stress_test_trading_${TIMESTAMP}.json" \
        "$TRADING_SERVICE_HOST" \
        2>&1 | tee >(cat >&2)

    $HOME/go/bin/ghz \
        --insecure \
        --proto "$TRADING_PROTO" \
        --call "trading.TradingService.GetOrderStatus" \
        -d '{"order_id": "stress-test-order"}' \
        --duration "30s" \
        --rps 1000 \
        --connections 100 \
        --concurrency 200 \
        --format pretty \
        "$TRADING_SERVICE_HOST" \
        > "$RESULTS_DIR/stress_test_trading_${TIMESTAMP}.txt"

    echo -e "${GREEN}✓ Stress test results saved${NC}"
}

# Main execution
main() {
    local test_mode="${1:-all}"

    echo -e "${BLUE}Test mode: $test_mode${NC}"
    echo ""

    case "$test_mode" in
        trading)
            test_trading_service
            ;;
        ml)
            test_ml_training_service
            ;;
        health)
            test_health_endpoints
            ;;
        stress)
            stress_test_trading_service
            ;;
        all)
            test_trading_service
            test_ml_training_service
            test_health_endpoints
            ;;
        *)
            echo -e "${RED}Unknown test mode: $test_mode${NC}"
            echo "Usage: $0 [trading|ml|health|stress|all]"
            exit 1
            ;;
    esac

    echo ""
    echo -e "${GREEN}========================================${NC}"
    echo -e "${GREEN}All tests completed!${NC}"
    echo -e "${GREEN}Results saved to: $RESULTS_DIR${NC}"
    echo -e "${GREEN}========================================${NC}"
    echo ""
    echo -e "${YELLOW}To view detailed results:${NC}"
    echo "  - JSON: cat $RESULTS_DIR/*_${TIMESTAMP}.json | jq ."
    echo "  - Text: cat $RESULTS_DIR/*_${TIMESTAMP}.txt"
    echo ""
    echo -e "${YELLOW}To run stress test:${NC}"
    echo "  $0 stress"
}

# Run main function with arguments
main "$@"