foxhunt/sustained_load_test.py

#!/usr/bin/env python3
"""
Sustained Load Test for Trading Service
Wave 141 Phase 5 - Agent 262

Test Configuration:
- Duration: 5 minutes (300 seconds)
- Target: 1,000+ orders/minute (~16.67 orders/sec)
- Monitoring: Time-series performance, resource usage, memory leaks
- Symbols: BTC/USD, ETH/USD rotation

Metrics Tracked:
- Throughput over time
- Latency degradation
- Memory usage trend
- CPU usage trend
- Error rate
- Service health
"""

import asyncio
import time
import statistics
import sys
import uuid
import json
from datetime import datetime
from concurrent.futures import ThreadPoolExecutor
from typing import List, Dict, Tuple
import requests
import threading

# Time-series metrics storage
class TimeSeriesMetrics:
    def __init__(self):
        self.data_points = []  # List of (timestamp, metrics_dict)
        self.lock = threading.Lock()

    def record_data_point(self, metrics: Dict):
        with self.lock:
            self.data_points.append({
                'timestamp': time.time(),
                'datetime': datetime.now().isoformat(),
                **metrics
            })

    def get_time_series(self, metric_name: str) -> List[Tuple[float, float]]:
        """Get time series for a specific metric"""
        with self.lock:
            return [(dp['timestamp'], dp.get(metric_name, 0))
                    for dp in self.data_points if metric_name in dp]

    def analyze_trend(self, metric_name: str) -> Dict:
        """Analyze trend for degradation detection"""
        series = self.get_time_series(metric_name)
        if len(series) < 2:
            return {'trend': 'insufficient_data'}

        # Split into first half and second half
        mid = len(series) // 2
        first_half = [val for _, val in series[:mid]]
        second_half = [val for _, val in series[mid:]]

        first_avg = statistics.mean(first_half) if first_half else 0
        second_avg = statistics.mean(second_half) if second_half else 0

        if first_avg == 0:
            degradation_pct = 0
        else:
            degradation_pct = ((second_avg - first_avg) / first_avg) * 100

        return {
            'first_half_avg': first_avg,
            'second_half_avg': second_avg,
            'degradation_pct': degradation_pct,
            'trend': 'stable' if abs(degradation_pct) < 10 else 'degraded'
        }

class PerformanceMetrics:
    def __init__(self):
        self.latencies_ms = []
        self.successful = 0
        self.failed = 0
        self.start_time = None
        self.end_time = None
        self.lock = threading.Lock()

    def record_success(self, latency_ms: float):
        with self.lock:
            self.latencies_ms.append(latency_ms)
            self.successful += 1

    def record_failure(self):
        with self.lock:
            self.failed += 1

    def get_current_stats(self) -> Dict:
        """Get current statistics snapshot"""
        with self.lock:
            if not self.latencies_ms:
                return {
                    'total': self.successful + self.failed,
                    'successful': self.successful,
                    'failed': self.failed,
                    'min_lat': 0, 'p50': 0, 'p95': 0, 'p99': 0, 'max_lat': 0
                }

            sorted_lat = sorted(self.latencies_ms)
            n = len(sorted_lat)

            return {
                'total': self.successful + self.failed,
                'successful': self.successful,
                'failed': self.failed,
                'min_lat': sorted_lat[0],
                'p50': sorted_lat[n // 2],
                'p95': sorted_lat[int(n * 0.95)] if n > 20 else sorted_lat[-1],
                'p99': sorted_lat[int(n * 0.99)] if n > 100 else sorted_lat[-1],
                'max_lat': sorted_lat[-1]
            }

def submit_order_http(order_id: str, symbol: str) -> Tuple[bool, float]:
    """Submit order via HTTP to Trading Service"""
    url = "http://localhost:8081/api/v1/orders"

    payload = {
        "order_id": order_id,
        "symbol": symbol,
        "side": "buy",
        "order_type": "limit",
        "quantity": "1.0",
        "price": "50000.0" if "BTC" in symbol else "3000.0",
        "time_in_force": "gtc"
    }

    start = time.time()
    try:
        response = requests.post(url, json=payload, timeout=10)
        latency_ms = (time.time() - start) * 1000
        return response.status_code == 200, latency_ms
    except Exception as e:
        latency_ms = (time.time() - start) * 1000
        return False, latency_ms

def get_service_metrics() -> Dict:
    """Fetch current service metrics from Prometheus endpoint"""
    try:
        response = requests.get("http://localhost:9092/metrics", timeout=2)
        metrics = {}

        for line in response.text.split('\n'):
            if line.startswith('#') or not line.strip():
                continue

            if 'trading_orders_total' in line:
                metrics['orders_total'] = float(line.split()[-1])
            elif 'process_resident_memory_bytes' in line:
                metrics['memory_bytes'] = float(line.split()[-1])
            elif 'process_cpu_seconds_total' in line:
                metrics['cpu_seconds'] = float(line.split()[-1])

        return metrics
    except:
        return {}

def get_docker_stats() -> Dict:
    """Get Docker container resource usage"""
    try:
        import subprocess
        result = subprocess.run(
            ['docker', 'stats', '--no-stream', '--format',
             'json', 'foxhunt-trading-service'],
            capture_output=True, text=True, timeout=3
        )

        if result.returncode == 0 and result.stdout:
            stats = json.loads(result.stdout)
            return {
                'cpu_pct': stats.get('CPUPerc', '0%').rstrip('%'),
                'memory_usage': stats.get('MemUsage', '0B'),
                'memory_pct': stats.get('MemPerc', '0%').rstrip('%')
            }
    except:
        pass

    return {}

def monitor_resources(time_series: TimeSeriesMetrics, metrics: PerformanceMetrics,
                      shutdown_flag: List[bool]):
    """Background thread to monitor system resources"""
    interval = 5  # Sample every 5 seconds

    while not shutdown_flag[0]:
        # Get current performance stats
        stats = metrics.get_current_stats()

        # Get service metrics
        service_metrics = get_service_metrics()

        # Get Docker stats
        docker_stats = get_docker_stats()

        # Calculate current throughput
        elapsed = time.time() - metrics.start_time if metrics.start_time else 1
        throughput = stats['successful'] / elapsed if elapsed > 0 else 0

        # Record data point
        time_series.record_data_point({
            'throughput': throughput,
            'latency_p50': stats['p50'],
            'latency_p99': stats['p99'],
            'success_count': stats['successful'],
            'failure_count': stats['failed'],
            'memory_bytes': service_metrics.get('memory_bytes', 0),
            'cpu_pct': float(docker_stats.get('cpu_pct', 0)),
            'memory_pct': float(docker_stats.get('memory_pct', 0))
        })

        time.sleep(interval)

async def run_sustained_load_test():
    """Run 5-minute sustained load test"""
    print("\n" + "="*70)
    print("     SUSTAINED LOAD TEST (5 Minutes)")
    print("="*70)
    print(f"Target: 1,000+ orders/minute (~16.67 orders/sec)")
    print(f"Duration: 300 seconds")
    print(f"Symbols: BTC/USD, ETH/USD")
    print("="*70 + "\n")

    test_duration_secs = 300
    num_clients = 8
    target_orders_per_sec = 17  # Slightly above 16.67 target
    orders_per_client_per_sec = target_orders_per_sec / num_clients

    metrics = PerformanceMetrics()
    time_series = TimeSeriesMetrics()
    shutdown_flag = [False]

    symbols = ["BTC/USD", "ETH/USD"]

    print(f"🚀 Starting {num_clients} clients for {test_duration_secs} seconds...")
    print(f"🎯 Target: {target_orders_per_sec:.1f} orders/sec total\n")

    # Start resource monitoring thread
    monitor_thread = threading.Thread(
        target=monitor_resources,
        args=(time_series, metrics, shutdown_flag),
        daemon=True
    )

    metrics.start_time = time.time()
    monitor_thread.start()

    # Print progress updates
    def print_progress():
        last_update = time.time()
        while not shutdown_flag[0]:
            current = time.time()
            if current - last_update >= 30:  # Update every 30 seconds
                elapsed = current - metrics.start_time
                stats = metrics.get_current_stats()
                throughput = stats['successful'] / elapsed if elapsed > 0 else 0

                print(f"⏱️  {elapsed:.0f}s elapsed | "
                      f"Orders: {stats['successful']} | "
                      f"Throughput: {throughput:.1f}/sec | "
                      f"P99: {stats['p99']:.1f}ms | "
                      f"Errors: {stats['failed']}")

                last_update = current
            time.sleep(1)

    progress_thread = threading.Thread(target=print_progress, daemon=True)
    progress_thread.start()

    # Client workers
    def client_worker(client_id: int):
        order_count = 0
        delay_secs = 1.0 / orders_per_client_per_sec

        while not shutdown_flag[0]:
            order_id = str(uuid.uuid4())
            symbol = symbols[order_count % len(symbols)]

            success, latency_ms = submit_order_http(order_id, symbol)

            if success:
                metrics.record_success(latency_ms)
            else:
                metrics.record_failure()

            order_count += 1
            time.sleep(delay_secs)

    # Start client threads
    with ThreadPoolExecutor(max_workers=num_clients) as executor:
        futures = [executor.submit(client_worker, i) for i in range(num_clients)]

        # Run for specified duration
        time.sleep(test_duration_secs)
        shutdown_flag[0] = True

        # Wait for all clients to finish
        for future in futures:
            try:
                future.result(timeout=5)
            except:
                pass

    metrics.end_time = time.time()

    # Final statistics
    print("\n" + "="*70)
    print("     TEST COMPLETED - ANALYZING RESULTS")
    print("="*70 + "\n")

    # Performance summary
    stats = metrics.get_current_stats()
    duration = metrics.end_time - metrics.start_time
    throughput = stats['successful'] / duration if duration > 0 else 0
    success_rate = (stats['successful'] / stats['total'] * 100) if stats['total'] > 0 else 0

    print("📊 PERFORMANCE SUMMARY:")
    print(f"  Duration:          {duration:.2f}s")
    print(f"  Total Orders:      {stats['total']}")
    print(f"  Successful:        {stats['successful']} ({success_rate:.2f}%)")
    print(f"  Failed:            {stats['failed']}")
    print(f"  Throughput:        {throughput:.1f} orders/sec")
    print(f"  Orders/Minute:     {throughput * 60:.0f}")
    print()
    print("📈 LATENCY METRICS:")
    print(f"  Min:               {stats['min_lat']:.2f}ms")
    print(f"  P50:               {stats['p50']:.2f}ms")
    print(f"  P95:               {stats['p95']:.2f}ms")
    print(f"  P99:               {stats['p99']:.2f}ms")
    print(f"  Max:               {stats['max_lat']:.2f}ms")

    # Trend analysis
    print("\n" + "="*70)
    print("     DEGRADATION ANALYSIS")
    print("="*70 + "\n")

    throughput_trend = time_series.analyze_trend('throughput')
    latency_trend = time_series.analyze_trend('latency_p99')
    memory_trend = time_series.analyze_trend('memory_bytes')

    print(f"📉 Throughput Trend:")
    print(f"  First Half Avg:    {throughput_trend['first_half_avg']:.1f} orders/sec")
    print(f"  Second Half Avg:   {throughput_trend['second_half_avg']:.1f} orders/sec")
    print(f"  Change:            {throughput_trend['degradation_pct']:+.1f}%")
    print(f"  Status:            {throughput_trend['trend'].upper()}")

    print(f"\n⏱️  Latency Trend (P99):")
    print(f"  First Half Avg:    {latency_trend['first_half_avg']:.1f}ms")
    print(f"  Second Half Avg:   {latency_trend['second_half_avg']:.1f}ms")
    print(f"  Change:            {latency_trend['degradation_pct']:+.1f}%")
    print(f"  Status:            {latency_trend['trend'].upper()}")

    print(f"\n💾 Memory Trend:")
    if memory_trend['first_half_avg'] > 0:
        print(f"  First Half Avg:    {memory_trend['first_half_avg']/1024/1024:.1f}MB")
        print(f"  Second Half Avg:   {memory_trend['second_half_avg']/1024/1024:.1f}MB")
        print(f"  Change:            {memory_trend['degradation_pct']:+.1f}%")
        print(f"  Status:            {memory_trend['trend'].upper()}")
    else:
        print(f"  Status:            No memory data available")

    # Pass/Fail Assessment
    print("\n" + "="*70)
    print("     SUCCESS CRITERIA EVALUATION")
    print("="*70 + "\n")

    passed_checks = 0
    total_checks = 5

    # Check 1: Sustained throughput
    orders_per_minute = throughput * 60
    if orders_per_minute >= 1000:
        print(f"✅ Throughput: {orders_per_minute:.0f} orders/min (>= 1,000)")
        passed_checks += 1
    else:
        print(f"❌ Throughput: {orders_per_minute:.0f} orders/min (< 1,000)")

    # Check 2: Success rate
    if success_rate >= 99.0:
        print(f"✅ Success Rate: {success_rate:.2f}% (>= 99%)")
        passed_checks += 1
    else:
        print(f"❌ Success Rate: {success_rate:.2f}% (< 99%)")

    # Check 3: No performance degradation
    if abs(throughput_trend['degradation_pct']) < 10:
        print(f"✅ Performance Stable: {throughput_trend['degradation_pct']:+.1f}% change (< 10%)")
        passed_checks += 1
    else:
        print(f"❌ Performance Degraded: {throughput_trend['degradation_pct']:+.1f}% change (>= 10%)")

    # Check 4: No memory leak
    if memory_trend['first_half_avg'] == 0 or abs(memory_trend['degradation_pct']) < 20:
        print(f"✅ No Memory Leak: {memory_trend['degradation_pct']:+.1f}% change (< 20%)")
        passed_checks += 1
    else:
        print(f"❌ Memory Leak Detected: {memory_trend['degradation_pct']:+.1f}% change (>= 20%)")

    # Check 5: Service health
    try:
        health_response = requests.get("http://localhost:8081/health", timeout=5)
        if health_response.status_code == 200:
            print(f"✅ Service Health: Healthy post-test")
            passed_checks += 1
        else:
            print(f"❌ Service Health: Unhealthy post-test")
    except:
        print(f"❌ Service Health: Unreachable post-test")

    print(f"\n📊 OVERALL: {passed_checks}/{total_checks} checks passed\n")

    if passed_checks >= 4:
        print("🎉 TEST PASSED - PRODUCTION READY")
        verdict = "PASS"
    elif passed_checks >= 3:
        print("⚠️  TEST PASSED WITH WARNINGS - Monitor in production")
        verdict = "PASS_WITH_WARNINGS"
    else:
        print("❌ TEST FAILED - Not ready for sustained load")
        verdict = "FAIL"

    print("="*70 + "\n")

    # Save detailed report
    report = {
        'test_config': {
            'duration_secs': test_duration_secs,
            'num_clients': num_clients,
            'target_orders_per_sec': target_orders_per_sec,
            'symbols': symbols
        },
        'performance': {
            'duration': duration,
            'total_orders': stats['total'],
            'successful': stats['successful'],
            'failed': stats['failed'],
            'success_rate_pct': success_rate,
            'throughput_per_sec': throughput,
            'orders_per_minute': orders_per_minute
        },
        'latency': {
            'min_ms': stats['min_lat'],
            'p50_ms': stats['p50'],
            'p95_ms': stats['p95'],
            'p99_ms': stats['p99'],
            'max_ms': stats['max_lat']
        },
        'trends': {
            'throughput': throughput_trend,
            'latency_p99': latency_trend,
            'memory': memory_trend
        },
        'time_series': time_series.data_points,
        'verdict': verdict,
        'checks_passed': f"{passed_checks}/{total_checks}"
    }

    with open('/home/jgrusewski/Work/foxhunt/sustained_load_test_results.json', 'w') as f:
        json.dump(report, f, indent=2)

    print("💾 Detailed results saved to: sustained_load_test_results.json\n")

    return verdict, report

if __name__ == "__main__":
    try:
        verdict, report = asyncio.run(run_sustained_load_test())
        sys.exit(0 if verdict == "PASS" else 1)
    except KeyboardInterrupt:
        print("\n⚠️  Test interrupted by user")
        sys.exit(2)
    except Exception as e:
        print(f"\n❌ Test failed with error: {e}")
        import traceback
        traceback.print_exc()
        sys.exit(1)