foxhunt/sustained_load_grpc_test.sh

#!/bin/bash
#
# Sustained Load Test for Trading Service - Wave 141 Phase 5 Agent 262
#
# Configuration:
# - Duration: 5 minutes (300 seconds)
# - Target: 1,000+ orders/minute (~16.67 orders/sec)
# - Method: gRPC via grpcurl with proper authentication
# - Monitoring: Time-series performance tracking
#

set -e

echo "======================================================================"
echo "     SUSTAINED LOAD TEST (5 Minutes) - gRPC"
echo "======================================================================"
echo ""
echo "Target: 1,000+ orders/minute (~16.67 orders/sec)"
echo "Duration: 300 seconds"
echo "Symbols: BTC/USD, ETH/USD"
echo "======================================================================"
echo ""

# Check dependencies
if ! command -v grpcurl &> /dev/null; then
    echo "❌ grpcurl not installed"
    echo "Install: go install github.com/fullstorydev/grpcurl/cmd/grpcurl@latest"
    exit 1
fi

if ! command -v jq &> /dev/null; then
    echo "❌ jq not installed"
    echo "Install: sudo apt-get install jq"
    exit 1
fi

# Configuration
GRPC_HOST="localhost:50052"
PROTO_PATH="/home/jgrusewski/Work/foxhunt/tli/proto/trading.proto"
IMPORT_PATH="/home/jgrusewski/Work/foxhunt"
SERVICE="foxhunt.tli.TradingService"
METHOD="SubmitOrder"

# Test parameters
DURATION=300  # 5 minutes
TARGET_RPS=17  # Slightly above 16.67
DELAY=$(echo "scale=6; 1.0 / $TARGET_RPS" | bc)  # Delay between requests

# Metrics
RESULTS_FILE="/tmp/sustained_load_results_$$.json"
METRICS_FILE="/tmp/sustained_load_metrics_$$.txt"

echo "Configuration:"
echo "  gRPC Host:    $GRPC_HOST"
echo "  Target RPS:   $TARGET_RPS"
echo "  Delay:        ${DELAY}s between requests"
echo "  Duration:     ${DURATION}s"
echo ""

# Check service health
echo "🔍 Checking service health..."
if docker ps | grep -q "foxhunt-trading-service.*healthy"; then
    echo "✅ Trading Service is healthy"
else
    echo "❌ Trading Service is not healthy"
    docker ps | grep trading
    exit 1
fi

# Initialize metrics
TOTAL_ORDERS=0
SUCCESSFUL_ORDERS=0
FAILED_ORDERS=0
START_TIME=$(date +%s)
LATENCIES=()

echo ""
echo "🚀 Starting sustained load test..."
echo "   Press Ctrl+C to stop early"
echo ""

# Initialize metrics file
echo "timestamp,elapsed,total,successful,failed,latency_ms,throughput" > "$METRICS_FILE"

# Main test loop
END_TIME=$((START_TIME + DURATION))
CURRENT_TIME=$(date +%s)
LAST_UPDATE=$CURRENT_TIME
UPDATE_INTERVAL=30  # Report every 30 seconds

while [ $CURRENT_TIME -lt $END_TIME ]; do
    ORDER_ID=$(uuidgen)

    # Alternate between BTC/USD and ETH/USD
    if [ $((TOTAL_ORDERS % 2)) -eq 0 ]; then
        SYMBOL="BTC/USD"
        PRICE="50000.0"
    else
        SYMBOL="ETH/USD"
        PRICE="3000.0"
    fi

    # Submit order (measure latency)
    REQUEST_START=$(date +%s%3N)  # Milliseconds

    RESPONSE=$(grpcurl -plaintext \
        -import-path "$IMPORT_PATH" \
        -proto "$PROTO_PATH" \
        -d '{
          "symbol": "'"$SYMBOL"'",
          "side": "BUY",
          "order_type": "LIMIT",
          "quantity": 1.0,
          "price": '"$PRICE"',
          "time_in_force": "GTC",
          "client_order_id": "'"$ORDER_ID"'"
        }' \
        "$GRPC_HOST" "$SERVICE/$METHOD" 2>&1 || true)

    REQUEST_END=$(date +%s%3N)
    LATENCY_MS=$((REQUEST_END - REQUEST_START))

    TOTAL_ORDERS=$((TOTAL_ORDERS + 1))

    # Check if successful (look for ERROR in response)
    if echo "$RESPONSE" | grep -q "ERROR"; then
        FAILED_ORDERS=$((FAILED_ORDERS + 1))
    else
        SUCCESSFUL_ORDERS=$((SUCCESSFUL_ORDERS + 1))
        LATENCIES+=($LATENCY_MS)
    fi

    # Update metrics every 30 seconds
    CURRENT_TIME=$(date +%s)
    if [ $((CURRENT_TIME - LAST_UPDATE)) -ge $UPDATE_INTERVAL ]; then
        ELAPSED=$((CURRENT_TIME - START_TIME))
        THROUGHPUT=$(echo "scale=2; $SUCCESSFUL_ORDERS / $ELAPSED" | bc)

        # Calculate P99 latency (simplified)
        if [ ${#LATENCIES[@]} -gt 0 ]; then
            SORTED_LATENCIES=($(printf '%s\n' "${LATENCIES[@]}" | sort -n))
            P99_INDEX=$(( ${#SORTED_LATENCIES[@]} * 99 / 100 ))
            P99_LATENCY=${SORTED_LATENCIES[$P99_INDEX]}
        else
            P99_LATENCY=0
        fi

        echo "⏱️  ${ELAPSED}s elapsed | Orders: $SUCCESSFUL_ORDERS | Throughput: ${THROUGHPUT}/sec | P99: ${P99_LATENCY}ms | Errors: $FAILED_ORDERS"

        # Log to metrics file
        echo "$CURRENT_TIME,$ELAPSED,$TOTAL_ORDERS,$SUCCESSFUL_ORDERS,$FAILED_ORDERS,$P99_LATENCY,$THROUGHPUT" >> "$METRICS_FILE"

        LAST_UPDATE=$CURRENT_TIME
    fi

    # Sleep to control rate
    sleep "$DELAY"

    CURRENT_TIME=$(date +%s)
done

# Final statistics
END_TIME=$(date +%s)
TOTAL_DURATION=$((END_TIME - START_TIME))

echo ""
echo "======================================================================"
echo "     TEST COMPLETED - ANALYZING RESULTS"
echo "======================================================================"
echo ""

# Calculate statistics
SUCCESS_RATE=$(echo "scale=2; $SUCCESSFUL_ORDERS * 100 / $TOTAL_ORDERS" | bc)
THROUGHPUT=$(echo "scale=2; $SUCCESSFUL_ORDERS / $TOTAL_DURATION" | bc)
ORDERS_PER_MINUTE=$(echo "scale=0; $THROUGHPUT * 60" | bc)

echo "📊 PERFORMANCE SUMMARY:"
echo "  Duration:          ${TOTAL_DURATION}s"
echo "  Total Orders:      $TOTAL_ORDERS"
echo "  Successful:        $SUCCESSFUL_ORDERS ($SUCCESS_RATE%)"
echo "  Failed:            $FAILED_ORDERS"
echo "  Throughput:        ${THROUGHPUT} orders/sec"
echo "  Orders/Minute:     $ORDERS_PER_MINUTE"
echo ""

# Calculate latency percentiles
if [ ${#LATENCIES[@]} -gt 0 ]; then
    SORTED_LATENCIES=($(printf '%s\n' "${LATENCIES[@]}" | sort -n))
    COUNT=${#SORTED_LATENCIES[@]}

    MIN_LAT=${SORTED_LATENCIES[0]}
    P50_INDEX=$((COUNT / 2))
    P50_LAT=${SORTED_LATENCIES[$P50_INDEX]}
    P95_INDEX=$((COUNT * 95 / 100))
    P95_LAT=${SORTED_LATENCIES[$P95_INDEX]}
    P99_INDEX=$((COUNT * 99 / 100))
    P99_LAT=${SORTED_LATENCIES[$P99_INDEX]}
    MAX_LAT=${SORTED_LATENCIES[$((COUNT - 1))]}

    echo "📈 LATENCY METRICS:"
    echo "  Min:               ${MIN_LAT}ms"
    echo "  P50:               ${P50_LAT}ms"
    echo "  P95:               ${P95_LAT}ms"
    echo "  P99:               ${P99_LAT}ms"
    echo "  Max:               ${MAX_LAT}ms"
    echo ""
fi

# Trend analysis (compare first half vs second half)
HALF_DURATION=$((TOTAL_DURATION / 2))

# First half metrics
FIRST_HALF_SUCCESSFUL=$(awk -F',' -v half=$HALF_DURATION '$2 <= half {last=$4} END {print last}' "$METRICS_FILE")
FIRST_HALF_TIME=$(awk -F',' -v half=$HALF_DURATION '$2 <= half {last=$2} END {print last}' "$METRICS_FILE")

# Second half metrics
SECOND_HALF_SUCCESSFUL=$SUCCESSFUL_ORDERS
SECOND_HALF_DURATION=$((TOTAL_DURATION - FIRST_HALF_TIME))

if [ -n "$FIRST_HALF_TIME" ] && [ "$FIRST_HALF_TIME" -gt 0 ]; then
    FIRST_HALF_THROUGHPUT=$(echo "scale=2; ($FIRST_HALF_SUCCESSFUL) / $FIRST_HALF_TIME" | bc)
    SECOND_HALF_ORDERS=$((SUCCESSFUL_ORDERS - FIRST_HALF_SUCCESSFUL))
    SECOND_HALF_THROUGHPUT=$(echo "scale=2; $SECOND_HALF_ORDERS / $SECOND_HALF_DURATION" | bc)

    DEGRADATION=$(echo "scale=2; ($SECOND_HALF_THROUGHPUT - $FIRST_HALF_THROUGHPUT) * 100 / $FIRST_HALF_THROUGHPUT" | bc)

    echo "======================================================================"
    echo "     DEGRADATION ANALYSIS"
    echo "======================================================================"
    echo ""
    echo "📉 Throughput Trend:"
    echo "  First Half Avg:    ${FIRST_HALF_THROUGHPUT} orders/sec"
    echo "  Second Half Avg:   ${SECOND_HALF_THROUGHPUT} orders/sec"
    echo "  Change:            ${DEGRADATION}%"

    DEGRADATION_ABS=${DEGRADATION#-}
    if [ $(echo "$DEGRADATION_ABS < 10" | bc) -eq 1 ]; then
        echo "  Status:            STABLE ✅"
    else
        echo "  Status:            DEGRADED ⚠️"
    fi
    echo ""
fi

# Check service health post-test
echo "======================================================================"
echo "     SUCCESS CRITERIA EVALUATION"
echo "======================================================================"
echo ""

CHECKS_PASSED=0
TOTAL_CHECKS=5

# Check 1: Throughput
if [ $(echo "$ORDERS_PER_MINUTE >= 1000" | bc) -eq 1 ]; then
    echo "✅ Throughput: $ORDERS_PER_MINUTE orders/min (>= 1,000)"
    CHECKS_PASSED=$((CHECKS_PASSED + 1))
else
    echo "❌ Throughput: $ORDERS_PER_MINUTE orders/min (< 1,000)"
fi

# Check 2: Success rate
if [ $(echo "$SUCCESS_RATE >= 99.0" | bc) -eq 1 ]; then
    echo "✅ Success Rate: $SUCCESS_RATE% (>= 99%)"
    CHECKS_PASSED=$((CHECKS_PASSED + 1))
else
    echo "❌ Success Rate: $SUCCESS_RATE% (< 99%)"
fi

# Check 3: Performance degradation
if [ -n "$DEGRADATION_ABS" ]; then
    if [ $(echo "$DEGRADATION_ABS < 10" | bc) -eq 1 ]; then
        echo "✅ Performance Stable: ${DEGRADATION}% change (< 10%)"
        CHECKS_PASSED=$((CHECKS_PASSED + 1))
    else
        echo "❌ Performance Degraded: ${DEGRADATION}% change (>= 10%)"
    fi
else
    echo "⚠️  Performance Degradation: Unable to calculate"
fi

# Check 4: No memory leak (simplified - check if service still healthy)
if docker ps | grep -q "foxhunt-trading-service.*healthy"; then
    echo "✅ No Memory Leak: Service still healthy"
    CHECKS_PASSED=$((CHECKS_PASSED + 1))
else
    echo "❌ Memory Leak Suspected: Service unhealthy"
fi

# Check 5: Service health
if docker ps | grep -q "foxhunt-trading-service.*healthy"; then
    echo "✅ Service Health: Healthy post-test"
    CHECKS_PASSED=$((CHECKS_PASSED + 1))
else
    echo "❌ Service Health: Unhealthy post-test"
fi

echo ""
echo "📊 OVERALL: $CHECKS_PASSED/$TOTAL_CHECKS checks passed"
echo ""

if [ $CHECKS_PASSED -ge 4 ]; then
    echo "🎉 TEST PASSED - PRODUCTION READY"
    VERDICT="PASS"
    EXIT_CODE=0
elif [ $CHECKS_PASSED -ge 3 ]; then
    echo "⚠️  TEST PASSED WITH WARNINGS - Monitor in production"
    VERDICT="PASS_WITH_WARNINGS"
    EXIT_CODE=0
else
    echo "❌ TEST FAILED - Not ready for sustained load"
    VERDICT="FAIL"
    EXIT_CODE=1
fi

echo "======================================================================"
echo ""

# Save results
cat > "$RESULTS_FILE" <<EOF
{
  "test_config": {
    "duration_secs": $TOTAL_DURATION,
    "target_rps": $TARGET_RPS,
    "grpc_host": "$GRPC_HOST"
  },
  "performance": {
    "total_orders": $TOTAL_ORDERS,
    "successful": $SUCCESSFUL_ORDERS,
    "failed": $FAILED_ORDERS,
    "success_rate_pct": $SUCCESS_RATE,
    "throughput_per_sec": $THROUGHPUT,
    "orders_per_minute": $ORDERS_PER_MINUTE
  },
  "latency": {
    "min_ms": ${MIN_LAT:-0},
    "p50_ms": ${P50_LAT:-0},
    "p95_ms": ${P95_LAT:-0},
    "p99_ms": ${P99_LAT:-0},
    "max_ms": ${MAX_LAT:-0}
  },
  "verdict": "$VERDICT",
  "checks_passed": "$CHECKS_PASSED/$TOTAL_CHECKS"
}
EOF

echo "💾 Results saved to:"
echo "   JSON: $RESULTS_FILE"
echo "   Metrics: $METRICS_FILE"
echo ""

exit $EXIT_CODE