chore: Major codebase cleanup - remove deprecated files and organize structure
- Docker: Delete 23 deprecated Dockerfiles, fix CI/CD to use Dockerfile.foxhunt-build - Config: Remove 36 .env files, keep 4 essential, delete config/environments/ - Docs: Archive 614 Wave D files to docs/archive/wave_d/, 95% reduction in root - Scripts: Delete 56 deprecated scripts, keep 58 production-critical (49% reduction) - Python: Organize 37 scripts into scripts/python/ subdirectories, delete ml/python/ - Build: Remove 1GB artifacts, delete old venvs, clean Python cache from git - Migrations: Delete deprecated directory (4,432 lines), remove duplicate database/migrations/ - Infrastructure: Delete deployment/ (61 files), docs/scripts/ (8 files) Total impact: ~2,500 files cleaned, 750MB+ space freed, zero production impact All deleted scripts backed up to archives. runpod/ and tests/runpod/ preserved. data_acquisition_service retained per user request.
This commit is contained in:
jgrusewski
483
scripts/python/testing/sustained_load_test.py
Executable file
483
scripts/python/testing/sustained_load_test.py
Executable file
@@ -0,0 +1,483 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Sustained Load Test for Trading Service
|
||||
Wave 141 Phase 5 - Agent 262
|
||||
|
||||
Test Configuration:
|
||||
- Duration: 5 minutes (300 seconds)
|
||||
- Target: 1,000+ orders/minute (~16.67 orders/sec)
|
||||
- Monitoring: Time-series performance, resource usage, memory leaks
|
||||
- Symbols: BTC/USD, ETH/USD rotation
|
||||
|
||||
Metrics Tracked:
|
||||
- Throughput over time
|
||||
- Latency degradation
|
||||
- Memory usage trend
|
||||
- CPU usage trend
|
||||
- Error rate
|
||||
- Service health
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
import statistics
|
||||
import sys
|
||||
import uuid
|
||||
import json
|
||||
from datetime import datetime
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from typing import List, Dict, Tuple
|
||||
import requests
|
||||
import threading
|
||||
|
||||
# Time-series metrics storage
|
||||
class TimeSeriesMetrics:
|
||||
def __init__(self):
|
||||
self.data_points = [] # List of (timestamp, metrics_dict)
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def record_data_point(self, metrics: Dict):
|
||||
with self.lock:
|
||||
self.data_points.append({
|
||||
'timestamp': time.time(),
|
||||
'datetime': datetime.now().isoformat(),
|
||||
**metrics
|
||||
})
|
||||
|
||||
def get_time_series(self, metric_name: str) -> List[Tuple[float, float]]:
|
||||
"""Get time series for a specific metric"""
|
||||
with self.lock:
|
||||
return [(dp['timestamp'], dp.get(metric_name, 0))
|
||||
for dp in self.data_points if metric_name in dp]
|
||||
|
||||
def analyze_trend(self, metric_name: str) -> Dict:
|
||||
"""Analyze trend for degradation detection"""
|
||||
series = self.get_time_series(metric_name)
|
||||
if len(series) < 2:
|
||||
return {'trend': 'insufficient_data'}
|
||||
|
||||
# Split into first half and second half
|
||||
mid = len(series) // 2
|
||||
first_half = [val for _, val in series[:mid]]
|
||||
second_half = [val for _, val in series[mid:]]
|
||||
|
||||
first_avg = statistics.mean(first_half) if first_half else 0
|
||||
second_avg = statistics.mean(second_half) if second_half else 0
|
||||
|
||||
if first_avg == 0:
|
||||
degradation_pct = 0
|
||||
else:
|
||||
degradation_pct = ((second_avg - first_avg) / first_avg) * 100
|
||||
|
||||
return {
|
||||
'first_half_avg': first_avg,
|
||||
'second_half_avg': second_avg,
|
||||
'degradation_pct': degradation_pct,
|
||||
'trend': 'stable' if abs(degradation_pct) < 10 else 'degraded'
|
||||
}
|
||||
|
||||
class PerformanceMetrics:
|
||||
def __init__(self):
|
||||
self.latencies_ms = []
|
||||
self.successful = 0
|
||||
self.failed = 0
|
||||
self.start_time = None
|
||||
self.end_time = None
|
||||
self.lock = threading.Lock()
|
||||
|
||||
def record_success(self, latency_ms: float):
|
||||
with self.lock:
|
||||
self.latencies_ms.append(latency_ms)
|
||||
self.successful += 1
|
||||
|
||||
def record_failure(self):
|
||||
with self.lock:
|
||||
self.failed += 1
|
||||
|
||||
def get_current_stats(self) -> Dict:
|
||||
"""Get current statistics snapshot"""
|
||||
with self.lock:
|
||||
if not self.latencies_ms:
|
||||
return {
|
||||
'total': self.successful + self.failed,
|
||||
'successful': self.successful,
|
||||
'failed': self.failed,
|
||||
'min_lat': 0, 'p50': 0, 'p95': 0, 'p99': 0, 'max_lat': 0
|
||||
}
|
||||
|
||||
sorted_lat = sorted(self.latencies_ms)
|
||||
n = len(sorted_lat)
|
||||
|
||||
return {
|
||||
'total': self.successful + self.failed,
|
||||
'successful': self.successful,
|
||||
'failed': self.failed,
|
||||
'min_lat': sorted_lat[0],
|
||||
'p50': sorted_lat[n // 2],
|
||||
'p95': sorted_lat[int(n * 0.95)] if n > 20 else sorted_lat[-1],
|
||||
'p99': sorted_lat[int(n * 0.99)] if n > 100 else sorted_lat[-1],
|
||||
'max_lat': sorted_lat[-1]
|
||||
}
|
||||
|
||||
def submit_order_http(order_id: str, symbol: str) -> Tuple[bool, float]:
|
||||
"""Submit order via HTTP to Trading Service"""
|
||||
url = "http://localhost:8081/api/v1/orders"
|
||||
|
||||
payload = {
|
||||
"order_id": order_id,
|
||||
"symbol": symbol,
|
||||
"side": "buy",
|
||||
"order_type": "limit",
|
||||
"quantity": "1.0",
|
||||
"price": "50000.0" if "BTC" in symbol else "3000.0",
|
||||
"time_in_force": "gtc"
|
||||
}
|
||||
|
||||
start = time.time()
|
||||
try:
|
||||
response = requests.post(url, json=payload, timeout=10)
|
||||
latency_ms = (time.time() - start) * 1000
|
||||
return response.status_code == 200, latency_ms
|
||||
except Exception as e:
|
||||
latency_ms = (time.time() - start) * 1000
|
||||
return False, latency_ms
|
||||
|
||||
def get_service_metrics() -> Dict:
|
||||
"""Fetch current service metrics from Prometheus endpoint"""
|
||||
try:
|
||||
response = requests.get("http://localhost:9092/metrics", timeout=2)
|
||||
metrics = {}
|
||||
|
||||
for line in response.text.split('\n'):
|
||||
if line.startswith('#') or not line.strip():
|
||||
continue
|
||||
|
||||
if 'trading_orders_total' in line:
|
||||
metrics['orders_total'] = float(line.split()[-1])
|
||||
elif 'process_resident_memory_bytes' in line:
|
||||
metrics['memory_bytes'] = float(line.split()[-1])
|
||||
elif 'process_cpu_seconds_total' in line:
|
||||
metrics['cpu_seconds'] = float(line.split()[-1])
|
||||
|
||||
return metrics
|
||||
except:
|
||||
return {}
|
||||
|
||||
def get_docker_stats() -> Dict:
|
||||
"""Get Docker container resource usage"""
|
||||
try:
|
||||
import subprocess
|
||||
result = subprocess.run(
|
||||
['docker', 'stats', '--no-stream', '--format',
|
||||
'json', 'foxhunt-trading-service'],
|
||||
capture_output=True, text=True, timeout=3
|
||||
)
|
||||
|
||||
if result.returncode == 0 and result.stdout:
|
||||
stats = json.loads(result.stdout)
|
||||
return {
|
||||
'cpu_pct': stats.get('CPUPerc', '0%').rstrip('%'),
|
||||
'memory_usage': stats.get('MemUsage', '0B'),
|
||||
'memory_pct': stats.get('MemPerc', '0%').rstrip('%')
|
||||
}
|
||||
except:
|
||||
pass
|
||||
|
||||
return {}
|
||||
|
||||
def monitor_resources(time_series: TimeSeriesMetrics, metrics: PerformanceMetrics,
|
||||
shutdown_flag: List[bool]):
|
||||
"""Background thread to monitor system resources"""
|
||||
interval = 5 # Sample every 5 seconds
|
||||
|
||||
while not shutdown_flag[0]:
|
||||
# Get current performance stats
|
||||
stats = metrics.get_current_stats()
|
||||
|
||||
# Get service metrics
|
||||
service_metrics = get_service_metrics()
|
||||
|
||||
# Get Docker stats
|
||||
docker_stats = get_docker_stats()
|
||||
|
||||
# Calculate current throughput
|
||||
elapsed = time.time() - metrics.start_time if metrics.start_time else 1
|
||||
throughput = stats['successful'] / elapsed if elapsed > 0 else 0
|
||||
|
||||
# Record data point
|
||||
time_series.record_data_point({
|
||||
'throughput': throughput,
|
||||
'latency_p50': stats['p50'],
|
||||
'latency_p99': stats['p99'],
|
||||
'success_count': stats['successful'],
|
||||
'failure_count': stats['failed'],
|
||||
'memory_bytes': service_metrics.get('memory_bytes', 0),
|
||||
'cpu_pct': float(docker_stats.get('cpu_pct', 0)),
|
||||
'memory_pct': float(docker_stats.get('memory_pct', 0))
|
||||
})
|
||||
|
||||
time.sleep(interval)
|
||||
|
||||
async def run_sustained_load_test():
|
||||
"""Run 5-minute sustained load test"""
|
||||
print("\n" + "="*70)
|
||||
print(" SUSTAINED LOAD TEST (5 Minutes)")
|
||||
print("="*70)
|
||||
print(f"Target: 1,000+ orders/minute (~16.67 orders/sec)")
|
||||
print(f"Duration: 300 seconds")
|
||||
print(f"Symbols: BTC/USD, ETH/USD")
|
||||
print("="*70 + "\n")
|
||||
|
||||
test_duration_secs = 300
|
||||
num_clients = 8
|
||||
target_orders_per_sec = 17 # Slightly above 16.67 target
|
||||
orders_per_client_per_sec = target_orders_per_sec / num_clients
|
||||
|
||||
metrics = PerformanceMetrics()
|
||||
time_series = TimeSeriesMetrics()
|
||||
shutdown_flag = [False]
|
||||
|
||||
symbols = ["BTC/USD", "ETH/USD"]
|
||||
|
||||
print(f"🚀 Starting {num_clients} clients for {test_duration_secs} seconds...")
|
||||
print(f"🎯 Target: {target_orders_per_sec:.1f} orders/sec total\n")
|
||||
|
||||
# Start resource monitoring thread
|
||||
monitor_thread = threading.Thread(
|
||||
target=monitor_resources,
|
||||
args=(time_series, metrics, shutdown_flag),
|
||||
daemon=True
|
||||
)
|
||||
|
||||
metrics.start_time = time.time()
|
||||
monitor_thread.start()
|
||||
|
||||
# Print progress updates
|
||||
def print_progress():
|
||||
last_update = time.time()
|
||||
while not shutdown_flag[0]:
|
||||
current = time.time()
|
||||
if current - last_update >= 30: # Update every 30 seconds
|
||||
elapsed = current - metrics.start_time
|
||||
stats = metrics.get_current_stats()
|
||||
throughput = stats['successful'] / elapsed if elapsed > 0 else 0
|
||||
|
||||
print(f"⏱️ {elapsed:.0f}s elapsed | "
|
||||
f"Orders: {stats['successful']} | "
|
||||
f"Throughput: {throughput:.1f}/sec | "
|
||||
f"P99: {stats['p99']:.1f}ms | "
|
||||
f"Errors: {stats['failed']}")
|
||||
|
||||
last_update = current
|
||||
time.sleep(1)
|
||||
|
||||
progress_thread = threading.Thread(target=print_progress, daemon=True)
|
||||
progress_thread.start()
|
||||
|
||||
# Client workers
|
||||
def client_worker(client_id: int):
|
||||
order_count = 0
|
||||
delay_secs = 1.0 / orders_per_client_per_sec
|
||||
|
||||
while not shutdown_flag[0]:
|
||||
order_id = str(uuid.uuid4())
|
||||
symbol = symbols[order_count % len(symbols)]
|
||||
|
||||
success, latency_ms = submit_order_http(order_id, symbol)
|
||||
|
||||
if success:
|
||||
metrics.record_success(latency_ms)
|
||||
else:
|
||||
metrics.record_failure()
|
||||
|
||||
order_count += 1
|
||||
time.sleep(delay_secs)
|
||||
|
||||
# Start client threads
|
||||
with ThreadPoolExecutor(max_workers=num_clients) as executor:
|
||||
futures = [executor.submit(client_worker, i) for i in range(num_clients)]
|
||||
|
||||
# Run for specified duration
|
||||
time.sleep(test_duration_secs)
|
||||
shutdown_flag[0] = True
|
||||
|
||||
# Wait for all clients to finish
|
||||
for future in futures:
|
||||
try:
|
||||
future.result(timeout=5)
|
||||
except:
|
||||
pass
|
||||
|
||||
metrics.end_time = time.time()
|
||||
|
||||
# Final statistics
|
||||
print("\n" + "="*70)
|
||||
print(" TEST COMPLETED - ANALYZING RESULTS")
|
||||
print("="*70 + "\n")
|
||||
|
||||
# Performance summary
|
||||
stats = metrics.get_current_stats()
|
||||
duration = metrics.end_time - metrics.start_time
|
||||
throughput = stats['successful'] / duration if duration > 0 else 0
|
||||
success_rate = (stats['successful'] / stats['total'] * 100) if stats['total'] > 0 else 0
|
||||
|
||||
print("📊 PERFORMANCE SUMMARY:")
|
||||
print(f" Duration: {duration:.2f}s")
|
||||
print(f" Total Orders: {stats['total']}")
|
||||
print(f" Successful: {stats['successful']} ({success_rate:.2f}%)")
|
||||
print(f" Failed: {stats['failed']}")
|
||||
print(f" Throughput: {throughput:.1f} orders/sec")
|
||||
print(f" Orders/Minute: {throughput * 60:.0f}")
|
||||
print()
|
||||
print("📈 LATENCY METRICS:")
|
||||
print(f" Min: {stats['min_lat']:.2f}ms")
|
||||
print(f" P50: {stats['p50']:.2f}ms")
|
||||
print(f" P95: {stats['p95']:.2f}ms")
|
||||
print(f" P99: {stats['p99']:.2f}ms")
|
||||
print(f" Max: {stats['max_lat']:.2f}ms")
|
||||
|
||||
# Trend analysis
|
||||
print("\n" + "="*70)
|
||||
print(" DEGRADATION ANALYSIS")
|
||||
print("="*70 + "\n")
|
||||
|
||||
throughput_trend = time_series.analyze_trend('throughput')
|
||||
latency_trend = time_series.analyze_trend('latency_p99')
|
||||
memory_trend = time_series.analyze_trend('memory_bytes')
|
||||
|
||||
print(f"📉 Throughput Trend:")
|
||||
print(f" First Half Avg: {throughput_trend['first_half_avg']:.1f} orders/sec")
|
||||
print(f" Second Half Avg: {throughput_trend['second_half_avg']:.1f} orders/sec")
|
||||
print(f" Change: {throughput_trend['degradation_pct']:+.1f}%")
|
||||
print(f" Status: {throughput_trend['trend'].upper()}")
|
||||
|
||||
print(f"\n⏱️ Latency Trend (P99):")
|
||||
print(f" First Half Avg: {latency_trend['first_half_avg']:.1f}ms")
|
||||
print(f" Second Half Avg: {latency_trend['second_half_avg']:.1f}ms")
|
||||
print(f" Change: {latency_trend['degradation_pct']:+.1f}%")
|
||||
print(f" Status: {latency_trend['trend'].upper()}")
|
||||
|
||||
print(f"\n💾 Memory Trend:")
|
||||
if memory_trend['first_half_avg'] > 0:
|
||||
print(f" First Half Avg: {memory_trend['first_half_avg']/1024/1024:.1f}MB")
|
||||
print(f" Second Half Avg: {memory_trend['second_half_avg']/1024/1024:.1f}MB")
|
||||
print(f" Change: {memory_trend['degradation_pct']:+.1f}%")
|
||||
print(f" Status: {memory_trend['trend'].upper()}")
|
||||
else:
|
||||
print(f" Status: No memory data available")
|
||||
|
||||
# Pass/Fail Assessment
|
||||
print("\n" + "="*70)
|
||||
print(" SUCCESS CRITERIA EVALUATION")
|
||||
print("="*70 + "\n")
|
||||
|
||||
passed_checks = 0
|
||||
total_checks = 5
|
||||
|
||||
# Check 1: Sustained throughput
|
||||
orders_per_minute = throughput * 60
|
||||
if orders_per_minute >= 1000:
|
||||
print(f"✅ Throughput: {orders_per_minute:.0f} orders/min (>= 1,000)")
|
||||
passed_checks += 1
|
||||
else:
|
||||
print(f"❌ Throughput: {orders_per_minute:.0f} orders/min (< 1,000)")
|
||||
|
||||
# Check 2: Success rate
|
||||
if success_rate >= 99.0:
|
||||
print(f"✅ Success Rate: {success_rate:.2f}% (>= 99%)")
|
||||
passed_checks += 1
|
||||
else:
|
||||
print(f"❌ Success Rate: {success_rate:.2f}% (< 99%)")
|
||||
|
||||
# Check 3: No performance degradation
|
||||
if abs(throughput_trend['degradation_pct']) < 10:
|
||||
print(f"✅ Performance Stable: {throughput_trend['degradation_pct']:+.1f}% change (< 10%)")
|
||||
passed_checks += 1
|
||||
else:
|
||||
print(f"❌ Performance Degraded: {throughput_trend['degradation_pct']:+.1f}% change (>= 10%)")
|
||||
|
||||
# Check 4: No memory leak
|
||||
if memory_trend['first_half_avg'] == 0 or abs(memory_trend['degradation_pct']) < 20:
|
||||
print(f"✅ No Memory Leak: {memory_trend['degradation_pct']:+.1f}% change (< 20%)")
|
||||
passed_checks += 1
|
||||
else:
|
||||
print(f"❌ Memory Leak Detected: {memory_trend['degradation_pct']:+.1f}% change (>= 20%)")
|
||||
|
||||
# Check 5: Service health
|
||||
try:
|
||||
health_response = requests.get("http://localhost:8081/health", timeout=5)
|
||||
if health_response.status_code == 200:
|
||||
print(f"✅ Service Health: Healthy post-test")
|
||||
passed_checks += 1
|
||||
else:
|
||||
print(f"❌ Service Health: Unhealthy post-test")
|
||||
except:
|
||||
print(f"❌ Service Health: Unreachable post-test")
|
||||
|
||||
print(f"\n📊 OVERALL: {passed_checks}/{total_checks} checks passed\n")
|
||||
|
||||
if passed_checks >= 4:
|
||||
print("🎉 TEST PASSED - PRODUCTION READY")
|
||||
verdict = "PASS"
|
||||
elif passed_checks >= 3:
|
||||
print("⚠️ TEST PASSED WITH WARNINGS - Monitor in production")
|
||||
verdict = "PASS_WITH_WARNINGS"
|
||||
else:
|
||||
print("❌ TEST FAILED - Not ready for sustained load")
|
||||
verdict = "FAIL"
|
||||
|
||||
print("="*70 + "\n")
|
||||
|
||||
# Save detailed report
|
||||
report = {
|
||||
'test_config': {
|
||||
'duration_secs': test_duration_secs,
|
||||
'num_clients': num_clients,
|
||||
'target_orders_per_sec': target_orders_per_sec,
|
||||
'symbols': symbols
|
||||
},
|
||||
'performance': {
|
||||
'duration': duration,
|
||||
'total_orders': stats['total'],
|
||||
'successful': stats['successful'],
|
||||
'failed': stats['failed'],
|
||||
'success_rate_pct': success_rate,
|
||||
'throughput_per_sec': throughput,
|
||||
'orders_per_minute': orders_per_minute
|
||||
},
|
||||
'latency': {
|
||||
'min_ms': stats['min_lat'],
|
||||
'p50_ms': stats['p50'],
|
||||
'p95_ms': stats['p95'],
|
||||
'p99_ms': stats['p99'],
|
||||
'max_ms': stats['max_lat']
|
||||
},
|
||||
'trends': {
|
||||
'throughput': throughput_trend,
|
||||
'latency_p99': latency_trend,
|
||||
'memory': memory_trend
|
||||
},
|
||||
'time_series': time_series.data_points,
|
||||
'verdict': verdict,
|
||||
'checks_passed': f"{passed_checks}/{total_checks}"
|
||||
}
|
||||
|
||||
with open('/home/jgrusewski/Work/foxhunt/sustained_load_test_results.json', 'w') as f:
|
||||
json.dump(report, f, indent=2)
|
||||
|
||||
print("💾 Detailed results saved to: sustained_load_test_results.json\n")
|
||||
|
||||
return verdict, report
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
verdict, report = asyncio.run(run_sustained_load_test())
|
||||
sys.exit(0 if verdict == "PASS" else 1)
|
||||
except KeyboardInterrupt:
|
||||
print("\n⚠️ Test interrupted by user")
|
||||
sys.exit(2)
|
||||
except Exception as e:
|
||||
print(f"\n❌ Test failed with error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
Reference in New Issue
Block a user