433af5c25d
- Docker: Delete 23 deprecated Dockerfiles, fix CI/CD to use Dockerfile.foxhunt-build - Config: Remove 36 .env files, keep 4 essential, delete config/environments/ - Docs: Archive 614 Wave D files to docs/archive/wave_d/, 95% reduction in root - Scripts: Delete 56 deprecated scripts, keep 58 production-critical (49% reduction) - Python: Organize 37 scripts into scripts/python/ subdirectories, delete ml/python/ - Build: Remove 1GB artifacts, delete old venvs, clean Python cache from git - Migrations: Delete deprecated directory (4,432 lines), remove duplicate database/migrations/ - Infrastructure: Delete deployment/ (61 files), docs/scripts/ (8 files) Total impact: ~2,500 files cleaned, 750MB+ space freed, zero production impact All deleted scripts backed up to archives. runpod/ and tests/runpod/ preserved. data_acquisition_service retained per user request.
484 lines
16 KiB
Python
Executable File
484 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Sustained Load Test for Trading Service
|
|
Wave 141 Phase 5 - Agent 262
|
|
|
|
Test Configuration:
|
|
- Duration: 5 minutes (300 seconds)
|
|
- Target: 1,000+ orders/minute (~16.67 orders/sec)
|
|
- Monitoring: Time-series performance, resource usage, memory leaks
|
|
- Symbols: BTC/USD, ETH/USD rotation
|
|
|
|
Metrics Tracked:
|
|
- Throughput over time
|
|
- Latency degradation
|
|
- Memory usage trend
|
|
- CPU usage trend
|
|
- Error rate
|
|
- Service health
|
|
"""
|
|
|
|
import asyncio
|
|
import time
|
|
import statistics
|
|
import sys
|
|
import uuid
|
|
import json
|
|
from datetime import datetime
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from typing import List, Dict, Tuple
|
|
import requests
|
|
import threading
|
|
|
|
# Time-series metrics storage
|
|
class TimeSeriesMetrics:
|
|
def __init__(self):
|
|
self.data_points = [] # List of (timestamp, metrics_dict)
|
|
self.lock = threading.Lock()
|
|
|
|
def record_data_point(self, metrics: Dict):
|
|
with self.lock:
|
|
self.data_points.append({
|
|
'timestamp': time.time(),
|
|
'datetime': datetime.now().isoformat(),
|
|
**metrics
|
|
})
|
|
|
|
def get_time_series(self, metric_name: str) -> List[Tuple[float, float]]:
|
|
"""Get time series for a specific metric"""
|
|
with self.lock:
|
|
return [(dp['timestamp'], dp.get(metric_name, 0))
|
|
for dp in self.data_points if metric_name in dp]
|
|
|
|
def analyze_trend(self, metric_name: str) -> Dict:
|
|
"""Analyze trend for degradation detection"""
|
|
series = self.get_time_series(metric_name)
|
|
if len(series) < 2:
|
|
return {'trend': 'insufficient_data'}
|
|
|
|
# Split into first half and second half
|
|
mid = len(series) // 2
|
|
first_half = [val for _, val in series[:mid]]
|
|
second_half = [val for _, val in series[mid:]]
|
|
|
|
first_avg = statistics.mean(first_half) if first_half else 0
|
|
second_avg = statistics.mean(second_half) if second_half else 0
|
|
|
|
if first_avg == 0:
|
|
degradation_pct = 0
|
|
else:
|
|
degradation_pct = ((second_avg - first_avg) / first_avg) * 100
|
|
|
|
return {
|
|
'first_half_avg': first_avg,
|
|
'second_half_avg': second_avg,
|
|
'degradation_pct': degradation_pct,
|
|
'trend': 'stable' if abs(degradation_pct) < 10 else 'degraded'
|
|
}
|
|
|
|
class PerformanceMetrics:
|
|
def __init__(self):
|
|
self.latencies_ms = []
|
|
self.successful = 0
|
|
self.failed = 0
|
|
self.start_time = None
|
|
self.end_time = None
|
|
self.lock = threading.Lock()
|
|
|
|
def record_success(self, latency_ms: float):
|
|
with self.lock:
|
|
self.latencies_ms.append(latency_ms)
|
|
self.successful += 1
|
|
|
|
def record_failure(self):
|
|
with self.lock:
|
|
self.failed += 1
|
|
|
|
def get_current_stats(self) -> Dict:
|
|
"""Get current statistics snapshot"""
|
|
with self.lock:
|
|
if not self.latencies_ms:
|
|
return {
|
|
'total': self.successful + self.failed,
|
|
'successful': self.successful,
|
|
'failed': self.failed,
|
|
'min_lat': 0, 'p50': 0, 'p95': 0, 'p99': 0, 'max_lat': 0
|
|
}
|
|
|
|
sorted_lat = sorted(self.latencies_ms)
|
|
n = len(sorted_lat)
|
|
|
|
return {
|
|
'total': self.successful + self.failed,
|
|
'successful': self.successful,
|
|
'failed': self.failed,
|
|
'min_lat': sorted_lat[0],
|
|
'p50': sorted_lat[n // 2],
|
|
'p95': sorted_lat[int(n * 0.95)] if n > 20 else sorted_lat[-1],
|
|
'p99': sorted_lat[int(n * 0.99)] if n > 100 else sorted_lat[-1],
|
|
'max_lat': sorted_lat[-1]
|
|
}
|
|
|
|
def submit_order_http(order_id: str, symbol: str) -> Tuple[bool, float]:
|
|
"""Submit order via HTTP to Trading Service"""
|
|
url = "http://localhost:8081/api/v1/orders"
|
|
|
|
payload = {
|
|
"order_id": order_id,
|
|
"symbol": symbol,
|
|
"side": "buy",
|
|
"order_type": "limit",
|
|
"quantity": "1.0",
|
|
"price": "50000.0" if "BTC" in symbol else "3000.0",
|
|
"time_in_force": "gtc"
|
|
}
|
|
|
|
start = time.time()
|
|
try:
|
|
response = requests.post(url, json=payload, timeout=10)
|
|
latency_ms = (time.time() - start) * 1000
|
|
return response.status_code == 200, latency_ms
|
|
except Exception as e:
|
|
latency_ms = (time.time() - start) * 1000
|
|
return False, latency_ms
|
|
|
|
def get_service_metrics() -> Dict:
|
|
"""Fetch current service metrics from Prometheus endpoint"""
|
|
try:
|
|
response = requests.get("http://localhost:9092/metrics", timeout=2)
|
|
metrics = {}
|
|
|
|
for line in response.text.split('\n'):
|
|
if line.startswith('#') or not line.strip():
|
|
continue
|
|
|
|
if 'trading_orders_total' in line:
|
|
metrics['orders_total'] = float(line.split()[-1])
|
|
elif 'process_resident_memory_bytes' in line:
|
|
metrics['memory_bytes'] = float(line.split()[-1])
|
|
elif 'process_cpu_seconds_total' in line:
|
|
metrics['cpu_seconds'] = float(line.split()[-1])
|
|
|
|
return metrics
|
|
except:
|
|
return {}
|
|
|
|
def get_docker_stats() -> Dict:
|
|
"""Get Docker container resource usage"""
|
|
try:
|
|
import subprocess
|
|
result = subprocess.run(
|
|
['docker', 'stats', '--no-stream', '--format',
|
|
'json', 'foxhunt-trading-service'],
|
|
capture_output=True, text=True, timeout=3
|
|
)
|
|
|
|
if result.returncode == 0 and result.stdout:
|
|
stats = json.loads(result.stdout)
|
|
return {
|
|
'cpu_pct': stats.get('CPUPerc', '0%').rstrip('%'),
|
|
'memory_usage': stats.get('MemUsage', '0B'),
|
|
'memory_pct': stats.get('MemPerc', '0%').rstrip('%')
|
|
}
|
|
except:
|
|
pass
|
|
|
|
return {}
|
|
|
|
def monitor_resources(time_series: TimeSeriesMetrics, metrics: PerformanceMetrics,
|
|
shutdown_flag: List[bool]):
|
|
"""Background thread to monitor system resources"""
|
|
interval = 5 # Sample every 5 seconds
|
|
|
|
while not shutdown_flag[0]:
|
|
# Get current performance stats
|
|
stats = metrics.get_current_stats()
|
|
|
|
# Get service metrics
|
|
service_metrics = get_service_metrics()
|
|
|
|
# Get Docker stats
|
|
docker_stats = get_docker_stats()
|
|
|
|
# Calculate current throughput
|
|
elapsed = time.time() - metrics.start_time if metrics.start_time else 1
|
|
throughput = stats['successful'] / elapsed if elapsed > 0 else 0
|
|
|
|
# Record data point
|
|
time_series.record_data_point({
|
|
'throughput': throughput,
|
|
'latency_p50': stats['p50'],
|
|
'latency_p99': stats['p99'],
|
|
'success_count': stats['successful'],
|
|
'failure_count': stats['failed'],
|
|
'memory_bytes': service_metrics.get('memory_bytes', 0),
|
|
'cpu_pct': float(docker_stats.get('cpu_pct', 0)),
|
|
'memory_pct': float(docker_stats.get('memory_pct', 0))
|
|
})
|
|
|
|
time.sleep(interval)
|
|
|
|
async def run_sustained_load_test():
|
|
"""Run 5-minute sustained load test"""
|
|
print("\n" + "="*70)
|
|
print(" SUSTAINED LOAD TEST (5 Minutes)")
|
|
print("="*70)
|
|
print(f"Target: 1,000+ orders/minute (~16.67 orders/sec)")
|
|
print(f"Duration: 300 seconds")
|
|
print(f"Symbols: BTC/USD, ETH/USD")
|
|
print("="*70 + "\n")
|
|
|
|
test_duration_secs = 300
|
|
num_clients = 8
|
|
target_orders_per_sec = 17 # Slightly above 16.67 target
|
|
orders_per_client_per_sec = target_orders_per_sec / num_clients
|
|
|
|
metrics = PerformanceMetrics()
|
|
time_series = TimeSeriesMetrics()
|
|
shutdown_flag = [False]
|
|
|
|
symbols = ["BTC/USD", "ETH/USD"]
|
|
|
|
print(f"🚀 Starting {num_clients} clients for {test_duration_secs} seconds...")
|
|
print(f"🎯 Target: {target_orders_per_sec:.1f} orders/sec total\n")
|
|
|
|
# Start resource monitoring thread
|
|
monitor_thread = threading.Thread(
|
|
target=monitor_resources,
|
|
args=(time_series, metrics, shutdown_flag),
|
|
daemon=True
|
|
)
|
|
|
|
metrics.start_time = time.time()
|
|
monitor_thread.start()
|
|
|
|
# Print progress updates
|
|
def print_progress():
|
|
last_update = time.time()
|
|
while not shutdown_flag[0]:
|
|
current = time.time()
|
|
if current - last_update >= 30: # Update every 30 seconds
|
|
elapsed = current - metrics.start_time
|
|
stats = metrics.get_current_stats()
|
|
throughput = stats['successful'] / elapsed if elapsed > 0 else 0
|
|
|
|
print(f"⏱️ {elapsed:.0f}s elapsed | "
|
|
f"Orders: {stats['successful']} | "
|
|
f"Throughput: {throughput:.1f}/sec | "
|
|
f"P99: {stats['p99']:.1f}ms | "
|
|
f"Errors: {stats['failed']}")
|
|
|
|
last_update = current
|
|
time.sleep(1)
|
|
|
|
progress_thread = threading.Thread(target=print_progress, daemon=True)
|
|
progress_thread.start()
|
|
|
|
# Client workers
|
|
def client_worker(client_id: int):
|
|
order_count = 0
|
|
delay_secs = 1.0 / orders_per_client_per_sec
|
|
|
|
while not shutdown_flag[0]:
|
|
order_id = str(uuid.uuid4())
|
|
symbol = symbols[order_count % len(symbols)]
|
|
|
|
success, latency_ms = submit_order_http(order_id, symbol)
|
|
|
|
if success:
|
|
metrics.record_success(latency_ms)
|
|
else:
|
|
metrics.record_failure()
|
|
|
|
order_count += 1
|
|
time.sleep(delay_secs)
|
|
|
|
# Start client threads
|
|
with ThreadPoolExecutor(max_workers=num_clients) as executor:
|
|
futures = [executor.submit(client_worker, i) for i in range(num_clients)]
|
|
|
|
# Run for specified duration
|
|
time.sleep(test_duration_secs)
|
|
shutdown_flag[0] = True
|
|
|
|
# Wait for all clients to finish
|
|
for future in futures:
|
|
try:
|
|
future.result(timeout=5)
|
|
except:
|
|
pass
|
|
|
|
metrics.end_time = time.time()
|
|
|
|
# Final statistics
|
|
print("\n" + "="*70)
|
|
print(" TEST COMPLETED - ANALYZING RESULTS")
|
|
print("="*70 + "\n")
|
|
|
|
# Performance summary
|
|
stats = metrics.get_current_stats()
|
|
duration = metrics.end_time - metrics.start_time
|
|
throughput = stats['successful'] / duration if duration > 0 else 0
|
|
success_rate = (stats['successful'] / stats['total'] * 100) if stats['total'] > 0 else 0
|
|
|
|
print("📊 PERFORMANCE SUMMARY:")
|
|
print(f" Duration: {duration:.2f}s")
|
|
print(f" Total Orders: {stats['total']}")
|
|
print(f" Successful: {stats['successful']} ({success_rate:.2f}%)")
|
|
print(f" Failed: {stats['failed']}")
|
|
print(f" Throughput: {throughput:.1f} orders/sec")
|
|
print(f" Orders/Minute: {throughput * 60:.0f}")
|
|
print()
|
|
print("📈 LATENCY METRICS:")
|
|
print(f" Min: {stats['min_lat']:.2f}ms")
|
|
print(f" P50: {stats['p50']:.2f}ms")
|
|
print(f" P95: {stats['p95']:.2f}ms")
|
|
print(f" P99: {stats['p99']:.2f}ms")
|
|
print(f" Max: {stats['max_lat']:.2f}ms")
|
|
|
|
# Trend analysis
|
|
print("\n" + "="*70)
|
|
print(" DEGRADATION ANALYSIS")
|
|
print("="*70 + "\n")
|
|
|
|
throughput_trend = time_series.analyze_trend('throughput')
|
|
latency_trend = time_series.analyze_trend('latency_p99')
|
|
memory_trend = time_series.analyze_trend('memory_bytes')
|
|
|
|
print(f"📉 Throughput Trend:")
|
|
print(f" First Half Avg: {throughput_trend['first_half_avg']:.1f} orders/sec")
|
|
print(f" Second Half Avg: {throughput_trend['second_half_avg']:.1f} orders/sec")
|
|
print(f" Change: {throughput_trend['degradation_pct']:+.1f}%")
|
|
print(f" Status: {throughput_trend['trend'].upper()}")
|
|
|
|
print(f"\n⏱️ Latency Trend (P99):")
|
|
print(f" First Half Avg: {latency_trend['first_half_avg']:.1f}ms")
|
|
print(f" Second Half Avg: {latency_trend['second_half_avg']:.1f}ms")
|
|
print(f" Change: {latency_trend['degradation_pct']:+.1f}%")
|
|
print(f" Status: {latency_trend['trend'].upper()}")
|
|
|
|
print(f"\n💾 Memory Trend:")
|
|
if memory_trend['first_half_avg'] > 0:
|
|
print(f" First Half Avg: {memory_trend['first_half_avg']/1024/1024:.1f}MB")
|
|
print(f" Second Half Avg: {memory_trend['second_half_avg']/1024/1024:.1f}MB")
|
|
print(f" Change: {memory_trend['degradation_pct']:+.1f}%")
|
|
print(f" Status: {memory_trend['trend'].upper()}")
|
|
else:
|
|
print(f" Status: No memory data available")
|
|
|
|
# Pass/Fail Assessment
|
|
print("\n" + "="*70)
|
|
print(" SUCCESS CRITERIA EVALUATION")
|
|
print("="*70 + "\n")
|
|
|
|
passed_checks = 0
|
|
total_checks = 5
|
|
|
|
# Check 1: Sustained throughput
|
|
orders_per_minute = throughput * 60
|
|
if orders_per_minute >= 1000:
|
|
print(f"✅ Throughput: {orders_per_minute:.0f} orders/min (>= 1,000)")
|
|
passed_checks += 1
|
|
else:
|
|
print(f"❌ Throughput: {orders_per_minute:.0f} orders/min (< 1,000)")
|
|
|
|
# Check 2: Success rate
|
|
if success_rate >= 99.0:
|
|
print(f"✅ Success Rate: {success_rate:.2f}% (>= 99%)")
|
|
passed_checks += 1
|
|
else:
|
|
print(f"❌ Success Rate: {success_rate:.2f}% (< 99%)")
|
|
|
|
# Check 3: No performance degradation
|
|
if abs(throughput_trend['degradation_pct']) < 10:
|
|
print(f"✅ Performance Stable: {throughput_trend['degradation_pct']:+.1f}% change (< 10%)")
|
|
passed_checks += 1
|
|
else:
|
|
print(f"❌ Performance Degraded: {throughput_trend['degradation_pct']:+.1f}% change (>= 10%)")
|
|
|
|
# Check 4: No memory leak
|
|
if memory_trend['first_half_avg'] == 0 or abs(memory_trend['degradation_pct']) < 20:
|
|
print(f"✅ No Memory Leak: {memory_trend['degradation_pct']:+.1f}% change (< 20%)")
|
|
passed_checks += 1
|
|
else:
|
|
print(f"❌ Memory Leak Detected: {memory_trend['degradation_pct']:+.1f}% change (>= 20%)")
|
|
|
|
# Check 5: Service health
|
|
try:
|
|
health_response = requests.get("http://localhost:8081/health", timeout=5)
|
|
if health_response.status_code == 200:
|
|
print(f"✅ Service Health: Healthy post-test")
|
|
passed_checks += 1
|
|
else:
|
|
print(f"❌ Service Health: Unhealthy post-test")
|
|
except:
|
|
print(f"❌ Service Health: Unreachable post-test")
|
|
|
|
print(f"\n📊 OVERALL: {passed_checks}/{total_checks} checks passed\n")
|
|
|
|
if passed_checks >= 4:
|
|
print("🎉 TEST PASSED - PRODUCTION READY")
|
|
verdict = "PASS"
|
|
elif passed_checks >= 3:
|
|
print("⚠️ TEST PASSED WITH WARNINGS - Monitor in production")
|
|
verdict = "PASS_WITH_WARNINGS"
|
|
else:
|
|
print("❌ TEST FAILED - Not ready for sustained load")
|
|
verdict = "FAIL"
|
|
|
|
print("="*70 + "\n")
|
|
|
|
# Save detailed report
|
|
report = {
|
|
'test_config': {
|
|
'duration_secs': test_duration_secs,
|
|
'num_clients': num_clients,
|
|
'target_orders_per_sec': target_orders_per_sec,
|
|
'symbols': symbols
|
|
},
|
|
'performance': {
|
|
'duration': duration,
|
|
'total_orders': stats['total'],
|
|
'successful': stats['successful'],
|
|
'failed': stats['failed'],
|
|
'success_rate_pct': success_rate,
|
|
'throughput_per_sec': throughput,
|
|
'orders_per_minute': orders_per_minute
|
|
},
|
|
'latency': {
|
|
'min_ms': stats['min_lat'],
|
|
'p50_ms': stats['p50'],
|
|
'p95_ms': stats['p95'],
|
|
'p99_ms': stats['p99'],
|
|
'max_ms': stats['max_lat']
|
|
},
|
|
'trends': {
|
|
'throughput': throughput_trend,
|
|
'latency_p99': latency_trend,
|
|
'memory': memory_trend
|
|
},
|
|
'time_series': time_series.data_points,
|
|
'verdict': verdict,
|
|
'checks_passed': f"{passed_checks}/{total_checks}"
|
|
}
|
|
|
|
with open('/home/jgrusewski/Work/foxhunt/sustained_load_test_results.json', 'w') as f:
|
|
json.dump(report, f, indent=2)
|
|
|
|
print("💾 Detailed results saved to: sustained_load_test_results.json\n")
|
|
|
|
return verdict, report
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
verdict, report = asyncio.run(run_sustained_load_test())
|
|
sys.exit(0 if verdict == "PASS" else 1)
|
|
except KeyboardInterrupt:
|
|
print("\n⚠️ Test interrupted by user")
|
|
sys.exit(2)
|
|
except Exception as e:
|
|
print(f"\n❌ Test failed with error: {e}")
|
|
import traceback
|
|
traceback.print_exc()
|
|
sys.exit(1)
|