foxhunt/scripts/python/testing/concurrent_connection_test.py

#!/usr/bin/env python3
"""
Concurrent Connection Load Test for Foxhunt HFT Trading System
Tests system behavior under 10, 50, 100, and 200+ concurrent gRPC connections
"""

import asyncio
import time
import statistics
from dataclasses import dataclass
from typing import List, Optional
import grpc
import sys
from concurrent.futures import ThreadPoolExecutor

# JWT token for authentication (same as used in previous tests)
JWT_TOKEN = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ0ZXN0X3VzZXIiLCJleHAiOjk5OTk5OTk5OTksImp0aSI6InRlc3RfdXNlciIsInJvbGVzIjpbInRyYWRlciJdLCJwZXJtaXNzaW9ucyI6WyJ0cmFkZSJdfQ.kHXiGFA0JjGmW66SiFRTUvzd2S6mOGN8pDfr9VJ8mAM"

@dataclass
class ConnectionMetrics:
    """Metrics for a single connection attempt"""
    connection_time_ms: float
    request_latency_ms: float
    success: bool
    error_message: Optional[str] = None

@dataclass
class LoadTestResult:
    """Aggregated results for a load test run"""
    concurrent_connections: int
    total_requests: int
    successful_requests: int
    failed_requests: int
    connection_time_p50: float
    connection_time_p95: float
    connection_time_p99: float
    latency_p50: float
    latency_p95: float
    latency_p99: float
    error_rate: float
    total_duration_secs: float
    throughput_rps: float

async def execute_single_connection(endpoint: str, connection_id: int) -> ConnectionMetrics:
    """Execute a single connection and request to test connection behavior"""
    connection_start = time.time()
    
    try:
        # Create gRPC channel with connection settings
        channel = grpc.aio.insecure_channel(
            endpoint,
            options=[
                ('grpc.max_receive_message_length', 100 * 1024 * 1024),
                ('grpc.max_send_message_length', 100 * 1024 * 1024),
                ('grpc.keepalive_time_ms', 30000),
                ('grpc.keepalive_timeout_ms', 20000),
                ('grpc.http2.max_pings_without_data', 0),
                ('grpc.keepalive_permit_without_calls', 1),
            ]
        )
        
        connection_time = (time.time() - connection_start) * 1000  # ms
        
        # Execute a simple unary RPC call to test the connection
        request_start = time.time()
        
        # Simple channel state check
        await channel.channel_ready()
        
        request_latency = (time.time() - request_start) * 1000  # ms
        
        await channel.close()
        
        return ConnectionMetrics(
            connection_time_ms=connection_time,
            request_latency_ms=request_latency,
            success=True,
            error_message=None
        )
        
    except Exception as e:
        connection_time = (time.time() - connection_start) * 1000
        return ConnectionMetrics(
            connection_time_ms=connection_time,
            request_latency_ms=0,
            success=False,
            error_message=str(e)
        )

async def run_concurrent_load_test(endpoint: str, concurrent_connections: int) -> LoadTestResult:
    """Run load test with specified number of concurrent connections"""
    print(f"\n{'='*60}")
    print(f"Testing with {concurrent_connections} concurrent connections")
    print(f"{'='*60}")
    
    test_start = time.time()
    
    # Create concurrent tasks
    tasks = [
        execute_single_connection(endpoint, i)
        for i in range(concurrent_connections)
    ]
    
    # Execute all tasks concurrently
    metrics = await asyncio.gather(*tasks)
    
    total_duration = time.time() - test_start
    
    # Calculate statistics
    connection_times = [m.connection_time_ms for m in metrics]
    latencies = [m.request_latency_ms for m in metrics if m.success]
    
    successful = sum(1 for m in metrics if m.success)
    failed = len(metrics) - successful
    error_rate = (failed / len(metrics)) * 100.0
    
    def percentile(values: List[float], p: float) -> float:
        if not values:
            return 0.0
        sorted_values = sorted(values)
        idx = int((len(sorted_values) - 1) * p)
        return sorted_values[idx]
    
    conn_p50 = percentile(connection_times, 0.50)
    conn_p95 = percentile(connection_times, 0.95)
    conn_p99 = percentile(connection_times, 0.99)
    
    lat_p50 = percentile(latencies, 0.50) if latencies else 0
    lat_p95 = percentile(latencies, 0.95) if latencies else 0
    lat_p99 = percentile(latencies, 0.99) if latencies else 0
    
    throughput = successful / total_duration if total_duration > 0 else 0
    
    return LoadTestResult(
        concurrent_connections=concurrent_connections,
        total_requests=len(metrics),
        successful_requests=successful,
        failed_requests=failed,
        connection_time_p50=conn_p50,
        connection_time_p95=conn_p95,
        connection_time_p99=conn_p99,
        latency_p50=lat_p50,
        latency_p95=lat_p95,
        latency_p99=lat_p99,
        error_rate=error_rate,
        total_duration_secs=total_duration,
        throughput_rps=throughput
    )

def print_result(result: LoadTestResult):
    """Print detailed results for a test run"""
    success_pct = (result.successful_requests / result.total_requests) * 100.0
    
    print(f"\nResults for {result.concurrent_connections} concurrent connections:")
    print(f"  Total Requests:      {result.total_requests}")
    print(f"  Successful:          {result.successful_requests} ({success_pct:.1f}%)")
    print(f"  Failed:              {result.failed_requests} ({result.error_rate:.1f}%)")
    print(f"  Duration:            {result.total_duration_secs:.2f}s")
    print(f"  Throughput:          {result.throughput_rps:.2f} conn/s")
    print(f"\n  Connection Times:")
    print(f"    P50:               {result.connection_time_p50:.2f}ms")
    print(f"    P95:               {result.connection_time_p95:.2f}ms")
    print(f"    P99:               {result.connection_time_p99:.2f}ms")
    print(f"\n  Request Latencies:")
    print(f"    P50:               {result.latency_p50:.2f}ms")
    print(f"    P95:               {result.latency_p95:.2f}ms")
    print(f"    P99:               {result.latency_p99:.2f}ms")

async def main():
    """Main test execution function"""
    print("\n┌" + "─"*70 + "┐")
    print("│  Foxhunt HFT Trading System - Concurrent Connection Load Test     │")
    print("└" + "─"*70 + "┘")
    
    # Test against Trading Service directly (port 50052)
    endpoint = "localhost:50052"
    test_levels = [10, 50, 100, 200]
    
    all_results = []
    
    for connections in test_levels:
        result = await run_concurrent_load_test(endpoint, connections)
        print_result(result)
        all_results.append(result)
        
        # Add delay between tests
        if connections < 200:
            print("\nWaiting 10 seconds before next test...")
            await asyncio.sleep(10)
    
    # Print summary table
    print("\n\n┌" + "─"*100 + "┐")
    print("│  SUMMARY TABLE - ALL TEST LEVELS" + " "*50 + "│")
    print("├" + "─"*100 + "┤")
    print("│ Connections │ Success │ Error % │ Conn P99 │ Lat P50 │ Lat P95 │ Lat P99 │ Throughput │")
    print("├" + "─"*100 + "┤")
    
    for result in all_results:
        success_pct = (result.successful_requests / result.total_requests) * 100.0
        print(f"│ {result.concurrent_connections:11d} │ {success_pct:6.1f}% │ {result.error_rate:6.1f}% │ "
              f"{result.connection_time_p99:7.1f}ms │ {result.latency_p50:6.1f}ms │ "
              f"{result.latency_p95:6.1f}ms │ {result.latency_p99:6.1f}ms │ "
              f"{result.throughput_rps:8.1f} c/s │")
    
    print("└" + "─"*100 + "┘")
    
    # Determine PASS/FAIL based on success criteria
    result_100 = next((r for r in all_results if r.concurrent_connections == 100), None)
    
    if result_100:
        pass_criteria = (
            result_100.error_rate < 1.0 and
            result_100.latency_p99 < 100
        )
        
        if pass_criteria:
            print("\n✅ TEST PASSED - System successfully handled 100+ concurrent connections")
            print("   - Error rate < 1%")
            print("   - P99 latency < 100ms")
        else:
            print("\n❌ TEST FAILED - System did not meet success criteria")
            print(f"   - Error rate: {result_100.error_rate:.2f}% (required: <1%)")
            print(f"   - P99 latency: {result_100.latency_p99:.2f}ms (required: <100ms)")
    
    return all_results

if __name__ == "__main__":
    asyncio.run(main())