gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: clean up examples, update ML binaries and risk tests

Browse Source 
- Delete 14 unused example files (-3,543 lines): config, adaptive-strategy,
  data, storage, trading_engine, api_gateway, backtesting, trading_service, chaos
- Update ML training/eval binaries: improved CLI args, completion tracking,
  CUDA test cleanup, hyperopt enhancements
- Fix KAN network and TFT module adjustments
- Update risk test assertions for consistency
- Fix backtesting repositories and promotion manager
- Update .serena project config and Cargo dependencies

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-27 01:33:18 +01:00
parent 1de1769c4f
commit afd85b2f8f
46 changed files with 403 additions and 3544 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
testing/integration/chaos/examples/mod.rs
View File

@@ -1,5 +0,0 @@
//! Chaos Engineering Examples Module
pub mod usage_examples;
// DO NOT RE-EXPORT - Use explicit imports at usage sites

484
testing/integration/chaos/examples/usage_examples.rs
View File

@@ -1,484 +0,0 @@
//! Chaos Engineering Usage Examples
//!
//! This file demonstrates how to use the Foxhunt chaos engineering framework
//! for testing system resilience and recovery capabilities.
use anyhow::Result;
use std::path::PathBuf;
use std::time::Duration;
use uuid::Uuid;
// Import our chaos engineering modules
use super::super::chaos_framework::{ChaosExperiment, ChaosOrchestrator, FailureType, Signal};
use super::super::ml_training_chaos::{MLChaosConfig, MLTrainingChaosTests, ModelType};
use crate::chaos::nightly_chaos_runner::{NightlyChaosConfig, NightlyChaosRunner, ChaosJobEvent};
/// Example 1: Simple ML Training Service Kill/Restart Test
///
/// This example demonstrates how to test the MLTrainingService's ability
/// to recover from process termination and resume training from checkpoints.
pub async fn example_ml_service_kill_test() -> Result<()> {
println!("🧪 Example 1: ML Training Service Kill/Restart Test");
// Create chaos orchestrator
let orchestrator = ChaosOrchestrator::new(1);
// Define the experiment
let experiment = ChaosExperiment {
id: Uuid::new_v4(),
name: "MLTrainingService SIGTERM Recovery Test".to_string(),
description: "Test ML service recovery from graceful termination".to_string(),
target_service: "ml_training_service".to_string(),
failure_type: FailureType::ProcessKill {
signal: Signal::SIGTERM,
delay_before_restart_ms: 2000, // 2 second delay
},
duration: Duration::from_secs(5), // Quick failure
recovery_timeout: Duration::from_secs(30),
max_recovery_time_ms: 100, // HFT requirement: sub-100ms
enabled: true,
};
// Register and execute the experiment
orchestrator.register_experiment(experiment.clone()).await?;
let result = orchestrator.execute_experiment(experiment.id).await?;
println!("📊 Result: {:?}", result.status);
if let Some(recovery_time) = result.recovery_time_ms {
println!("⏱️ Recovery time: {}ms", recovery_time);
if recovery_time <= 100 {
println!("✅ Recovery time meets HFT requirements");
} else {
println!("⚠️ Recovery time exceeds HFT requirements");
}
}
println!(
"🔧 Checkpoint integrity: {}",
if result.checkpoint_integrity {
"✅ Valid"
} else {
"❌ Corrupted"
}
);
Ok(())
}
/// Example 2: Comprehensive ML Chaos Test Suite
///
/// This example runs the full ML chaos test suite across all supported models
/// with different failure scenarios and generates a comprehensive report.
pub async fn example_comprehensive_ml_chaos_suite() -> Result<()> {
println!("🧪 Example 2: Comprehensive ML Chaos Test Suite");
// Configure ML chaos testing
let ml_config = MLChaosConfig {
ml_service_endpoint: "http://localhost:8080".to_string(),
checkpoint_base_path: PathBuf::from("/tmp/chaos_checkpoints"),
model_types: vec![
ModelType::TLOB, // Ultra-low latency transformer
ModelType::DQN, // Deep Q-learning for strategy optimization
ModelType::MAMBA2, // State space model for sequence modeling
],
training_timeout_secs: 300,
max_recovery_time_ms: 100, // HFT requirement
gpu_memory_threshold_mb: 4096, // 4GB for testing
};
// Create ML chaos test suite
let ml_chaos = MLTrainingChaosTests::new(ml_config);
println!("🚀 Starting ML chaos test suite...");
let results = ml_chaos.run_ml_chaos_suite().await?;
println!("📈 Test Results:");
println!(" Total tests: {}", results.len());
let successful = results
.iter()
.filter(|r| r.training_loss_continuity)
.count();
let failed = results.len() - successful;
println!(
" Successful: {} ({}%)",
successful,
(successful * 100) / results.len()
);
println!(
" Failed: {} ({}%)",
failed,
(failed * 100) / results.len()
);
// Generate and display report
let report = ml_chaos.generate_chaos_report(&results).await?;
println!("\n📋 Detailed Report:");
println!("{}", report);
Ok(())
}
/// Example 3: Memory Pressure Test with Performance Monitoring
///
/// This example demonstrates how to test system behavior under memory pressure
/// while monitoring performance metrics and recovery times.
pub async fn example_memory_pressure_test() -> Result<()> {
println!("🧪 Example 3: Memory Pressure Test with Performance Monitoring");
let orchestrator = ChaosOrchestrator::new(1);
// Create memory pressure experiment
let experiment = ChaosExperiment {
id: Uuid::new_v4(),
name: "High Memory Pressure Test".to_string(),
description: "Test system behavior under 4GB memory pressure".to_string(),
target_service: "ml_training_service".to_string(),
failure_type: FailureType::MemoryPressure {
target_mb: 4096, // 4GB pressure
duration_ms: 30000, // 30 seconds
},
duration: Duration::from_secs(35),
recovery_timeout: Duration::from_secs(60),
max_recovery_time_ms: 200, // Relaxed for memory pressure
enabled: true,
};
println!("💾 Applying 4GB memory pressure for 30 seconds...");
orchestrator.register_experiment(experiment.clone()).await?;
let result = orchestrator.execute_experiment(experiment.id).await?;
println!("📊 Memory Pressure Test Results:");
println!(" Status: {:?}", result.status);
if let Some(recovery_time) = result.recovery_time_ms {
println!(" Recovery time: {}ms", recovery_time);
// Check performance regression
if let Some(regression) = result.performance_regression {
println!(" Performance impact:");
println!(
" Before: {}ns P99 latency",
regression.latency_p99_before_ns
);
println!(
" After: {}ns P99 latency",
regression.latency_p99_after_ns
);
println!(" Regression: {:.1}%", regression.regression_percent);
}
}
Ok(())
}
/// Example 4: Network Partition Resilience Test
///
/// This example tests the system's ability to handle network partitions
/// affecting critical services like PostgreSQL and Redis.
pub async fn example_network_partition_test() -> Result<()> {
println!("🧪 Example 4: Network Partition Resilience Test");
let orchestrator = ChaosOrchestrator::new(1);
let experiment = ChaosExperiment {
id: Uuid::new_v4(),
name: "Database Network Partition Test".to_string(),
description: "Test resilience to database connection failures".to_string(),
target_service: "ml_training_service".to_string(),
failure_type: FailureType::NetworkPartition {
target_ports: vec![5432, 6379], // PostgreSQL, Redis
duration_ms: 15000, // 15 seconds
},
duration: Duration::from_secs(20),
recovery_timeout: Duration::from_secs(45),
max_recovery_time_ms: 100,
enabled: true,
};
println!("🌐 Creating network partition affecting PostgreSQL and Redis...");
orchestrator.register_experiment(experiment.clone()).await?;
let result = orchestrator.execute_experiment(experiment.id).await?;
println!("📡 Network Partition Test Results:");
println!(" Status: {:?}", result.status);
match result.status {
super::super::chaos_framework::ChaosStatus::Succeeded => {
println!(" ✅ System successfully handled network partition");
}
super::super::chaos_framework::ChaosStatus::RecoveryTimeout => {
println!(" ⏰ Recovery took longer than expected");
}
super::super::chaos_framework::ChaosStatus::Failed => {
println!(" ❌ System failed to recover from network partition");
}
_ => {
println!(" ❓ Unexpected test status");
}
}
Ok(())
}
/// Example 5: GPU Resource Exhaustion Test
///
/// This example tests ML model training behavior when GPU resources
/// are exhausted, simulating scenarios where multiple models compete
/// for limited GPU memory.
pub async fn example_gpu_exhaustion_test() -> Result<()> {
println!("🧪 Example 5: GPU Resource Exhaustion Test");
let orchestrator = ChaosOrchestrator::new(1);
let experiment = ChaosExperiment {
id: Uuid::new_v4(),
name: "GPU Memory Exhaustion Test".to_string(),
description: "Test ML training behavior under GPU memory pressure".to_string(),
target_service: "ml_training_service".to_string(),
failure_type: FailureType::GpuResourceExhaustion {
memory_fill_percent: 95, // Fill 95% of GPU memory
duration_ms: 25000, // 25 seconds
},
duration: Duration::from_secs(30),
recovery_timeout: Duration::from_secs(90), // GPU recovery can be slower
max_recovery_time_ms: 200, // Relaxed for GPU operations
enabled: true,
};
println!("🎮 Exhausting 95% of GPU memory for 25 seconds...");
orchestrator.register_experiment(experiment.clone()).await?;
let result = orchestrator.execute_experiment(experiment.id).await?;
println!("🎯 GPU Exhaustion Test Results:");
println!(" Status: {:?}", result.status);
println!(
" Checkpoint integrity: {}",
if result.checkpoint_integrity {
"✅"
} else {
"❌"
}
);
// GPU-specific analysis would check:
// - Model training continuation after GPU memory cleared
// - Checkpoint consistency during GPU pressure
// - Training loss continuity
// - GPU memory leak detection
Ok(())
}
/// Example 6: Automated Nightly Chaos Testing
///
/// This example sets up automated nightly chaos testing with proper
/// scheduling, notification, and reporting.
pub async fn example_nightly_chaos_automation() -> Result<()> {
println!("🧪 Example 6: Automated Nightly Chaos Testing Setup");
// Configure nightly chaos testing
let config = NightlyChaosConfig {
enabled: true,
schedule_time: chrono::NaiveTime::from_hms_opt(2, 0, 0).unwrap(), // 2 AM
timezone: "UTC".to_string(),
max_duration_hours: 3,
notification_webhook: Some("https://hooks.slack.com/services/YOUR/WEBHOOK".to_string()),
report_storage_path: PathBuf::from("./nightly_chaos_reports"),
ml_chaos_config: MLChaosConfig {
ml_service_endpoint: "http://localhost:8080".to_string(),
checkpoint_base_path: PathBuf::from("/production/ml_checkpoints"),
model_types: vec![
ModelType::TLOB, // Critical for order book analysis
ModelType::DQN, // Risk management
ModelType::MAMBA2, // Market prediction
],
training_timeout_secs: 600, // 10 minutes for production
max_recovery_time_ms: 100,
gpu_memory_threshold_mb: 16384, // 16GB production GPU
},
exclude_weekends: true, // Skip weekends for production safety
retry_on_failure: true,
max_retries: 2,
};
println!("📅 Setting up nightly chaos testing at 2:00 AM UTC...");
println!(" Excluding weekends: {}", config.exclude_weekends);
println!(" Max duration: {} hours", config.max_duration_hours);
println!(" Report storage: {:?}", config.report_storage_path);
// Create and configure the runner
let runner = NightlyChaosRunner::new(config);
// Subscribe to events for monitoring
let mut event_receiver = runner.subscribe_events();
// Start event monitoring in background
let _event_handler = tokio::spawn(async move {
while let Ok(event) = event_receiver.recv().await {
match event {
ChaosJobEvent::JobScheduled { id, scheduled_time } => {
println!("📅 Chaos job {} scheduled for {}", id, scheduled_time);
}
ChaosJobEvent::JobStarted { id } => {
println!("🚀 Chaos job {} started", id);
}
ChaosJobEvent::JobCompleted { id, summary } => {
println!("✅ Chaos job {} completed", id);
println!(
" Average recovery time: {:.1}ms",
summary.average_recovery_time_ms
);
println!(" SLA violations: {}", summary.sla_violations);
println!(" Checkpoint failures: {}", summary.checkpoint_failures);
}
ChaosJobEvent::AlertTriggered {
message,
severity,
} => {
println!("🚨 Alert ({:?}): {}", severity, message);
}
_ => {}
}
}
});
println!("🔄 Nightly chaos automation is configured and ready");
println!(" Use `runner.start().await` to begin scheduled testing");
// In production, you would call runner.start().await here
// For this example, we just show the setup
Ok(())
}
/// Example 7: CI/CD Integration - Quick Chaos Validation
///
/// This example shows how to integrate chaos testing into CI/CD pipelines
/// with quick validation tests that can run in a few minutes.
pub async fn example_ci_cd_quick_validation() -> Result<()> {
println!("🧪 Example 7: CI/CD Quick Chaos Validation");
// Quick config for CI/CD - shorter timeouts, fewer models
let quick_ml_config = MLChaosConfig {
ml_service_endpoint: "http://localhost:8080".to_string(),
checkpoint_base_path: PathBuf::from("/tmp/ci_checkpoints"),
model_types: vec![ModelType::TLOB], // Just test TLOB for speed
training_timeout_secs: 60, // 1 minute max
max_recovery_time_ms: 100,
gpu_memory_threshold_mb: 2048, // Lower for CI environment
};
println!("⚡ Running quick chaos validation for CI/CD...");
println!(" Target: Sub-2 minute execution time");
println!(" Models: TLOB only (fastest)");
println!(" Recovery requirement: < 100ms");
let ml_chaos = MLTrainingChaosTests::new(quick_ml_config);
// Run a single representative test
let orchestrator = ChaosOrchestrator::new(1);
let quick_experiment = ChaosExperiment {
id: Uuid::new_v4(),
name: "CI Quick Recovery Test".to_string(),
description: "Fast recovery test for CI pipeline".to_string(),
target_service: "ml_training_service".to_string(),
failure_type: FailureType::ProcessKill {
signal: Signal::SIGTERM,
delay_before_restart_ms: 1000, // Quick restart
},
duration: Duration::from_secs(2), // Very quick failure
recovery_timeout: Duration::from_secs(15), // Quick recovery
max_recovery_time_ms: 100,
enabled: true,
};
let start_time = std::time::Instant::now();
orchestrator
.register_experiment(quick_experiment.clone())
.await?;
let result = orchestrator.execute_experiment(quick_experiment.id).await?;
let total_time = start_time.elapsed();
println!("⏱️ Total execution time: {:.1}s", total_time.as_secs_f64());
println!("📊 Quick validation result: {:?}", result.status);
match result.status {
super::super::chaos_framework::ChaosStatus::Succeeded => {
println!("✅ CI/CD chaos validation PASSED");
std::process::exit(0);
}
_ => {
println!("❌ CI/CD chaos validation FAILED");
if let Some(recovery_time) = result.recovery_time_ms {
println!(" Recovery time: {}ms", recovery_time);
}
println!(" Checkpoint integrity: {}", result.checkpoint_integrity);
std::process::exit(1);
}
}
}
/// Run all examples
pub async fn run_all_examples() -> Result<()> {
println!("🚀 Running Foxhunt Chaos Engineering Examples\n");
// Example 1: Basic ML service kill test
example_ml_service_kill_test().await?;
println!();
// Example 2: Comprehensive test suite
example_comprehensive_ml_chaos_suite().await?;
println!();
// Example 3: Memory pressure test
example_memory_pressure_test().await?;
println!();
// Example 4: Network partition test
example_network_partition_test().await?;
println!();
// Example 5: GPU exhaustion test
example_gpu_exhaustion_test().await?;
println!();
// Example 6: Nightly automation setup
example_nightly_chaos_automation().await?;
println!();
// Example 7: CI/CD integration
// example_ci_cd_quick_validation().await?; // Skip in demo to avoid exit
println!("🎉 All chaos engineering examples completed successfully!");
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_example_runs() {
// Test that examples can be set up without actual service calls
let ml_config = MLChaosConfig {
ml_service_endpoint: "http://localhost:8080".to_string(),
checkpoint_base_path: PathBuf::from("/tmp/test"),
model_types: vec![ModelType::TLOB],
training_timeout_secs: 60,
max_recovery_time_ms: 100,
gpu_memory_threshold_mb: 2048,
};
let _ml_chaos = MLTrainingChaosTests::new(ml_config);
assert!(true); // Basic creation test
}
}