foxhunt/testing/integration/chaos/failure_injection_tests.rs

//! Chaos Engineering Tests for Foxhunt HFT System
//! 
//! Tests system resilience under failure conditions:
//! - Service failure scenarios and automatic recovery
//! - Network partitioning and reconnection handling
//! - Database failures and data consistency validation
//! - Model rollback and fallback mechanism testing
//! - Resource exhaustion and recovery scenarios

use anyhow::Result;
use std::time::{Duration, Instant};
use tokio::time::{sleep, timeout};
use std::process::Command;

use crate::harness::{TestHarness, TestResult};
use crate::harness::grpc_clients::*;
use crate::harness::fixtures::{TestFixtures, TestTrade, TestModel};

/// Chaos engineering test suite for system resilience validation
pub struct ChaosEngineeringTests {
    harness: TestHarness,
}

impl ChaosEngineeringTests {
    pub async fn new() -> Result<Self> {
        let harness = TestHarness::new().await?;
        Ok(Self { harness })
    }

    /// Run all chaos engineering tests
    pub async fn run_all_tests(&mut self) -> Result<Vec<TestResult>> {
        let mut results = Vec::new();

        // Setup test environment
        self.harness.setup().await?;

        // Service Failure Tests
        results.push(self.test_ml_training_service_failure().await?);
        results.push(self.test_trading_service_failure().await?);
        results.push(self.test_tli_service_failure().await?);

        // Network Failure Tests
        results.push(self.test_network_partition_recovery().await?);
        results.push(self.test_intermittent_network_failures().await?);
        results.push(self.test_connection_timeout_handling().await?);

        // Database Failure Tests
        results.push(self.test_database_connection_failure().await?);
        results.push(self.test_database_transaction_rollback().await?);
        results.push(self.test_data_consistency_under_failure().await?);

        // Resource Exhaustion Tests
        results.push(self.test_memory_exhaustion_recovery().await?);
        results.push(self.test_cpu_overload_handling().await?);
        results.push(self.test_gpu_resource_contention().await?);

        // Model and Training Failure Tests
        results.push(self.test_model_corruption_handling().await?);
        results.push(self.test_training_job_crash_recovery().await?);
        results.push(self.test_model_rollback_under_failure().await?);

        // Cascade Failure Tests
        results.push(self.test_cascade_failure_containment().await?);
        results.push(self.test_circuit_breaker_functionality().await?);

        // Cleanup
        self.harness.cleanup().await?;

        Ok(results)
    }

    /// Test ML Training Service failure and recovery
    async fn test_ml_training_service_failure(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("ml_training_service_failure", |harness| async move {
            println!("Testing ML Training Service failure scenarios...");

            // Start a training job
            let training_request = StartMLTrainingRequest {
                model_name: "failure_test_model".to_string(),
                dataset_id: "failure_test_dataset".to_string(),
                hyperparameters: vec![
                    ("learning_rate".to_string(), "0.001".to_string()),
                    ("batch_size".to_string(), "32".to_string()),
                    ("epochs".to_string(), "100".to_string()), // Long training
                ].into_iter().collect(),
                auto_deploy: false,
            };

            let response = harness.grpc_clients.tli_client
                .start_ml_training(training_request).await?;

            assert!(response.success, "Training should start successfully");
            let job_id = response.job_id.clone();

            // Wait for training to start
            sleep(Duration::from_secs(5)).await;

            // Verify training is running
            let status = harness.grpc_clients.tli_client
                .get_ml_training_status(job_id.clone()).await?;
            println!("Training status before failure: {}", status.status);

            // Simulate ML Training Service failure
            println!("Simulating ML Training Service failure...");
            // In a real environment, this would kill the service process
            // For testing, we simulate the failure by expecting connection errors
            
            // Wait a bit to simulate service downtime
            sleep(Duration::from_secs(3)).await;

            // Test service recovery - the service should automatically restart
            println!("Testing service recovery...");
            let recovery_timeout = Duration::from_secs(60);
            
            let recovery_result = timeout(recovery_timeout, async {
                loop {
                    match harness.grpc_clients.tli_client.get_ml_training_status(job_id.clone()).await {
                        Ok(status) => {
                            println!("Service recovered, training status: {}", status.status);
                            return Ok::<(), anyhow::Error>(());
                        },
                        Err(_) => {
                            // Service still down, continue waiting
                            sleep(Duration::from_secs(2)).await;
                        }
                    }
                }
            }).await;

            match recovery_result {
                Ok(_) => {
                    println!("✅ ML Training Service recovered successfully");
                    
                    // Verify training job state after recovery
                    let post_recovery_status = harness.grpc_clients.tli_client
                        .get_ml_training_status(job_id.clone()).await?;
                    
                    // The job should either be resumed or marked as failed with clear error
                    assert!(!post_recovery_status.status.is_empty(), 
                           "Training status should be available after recovery");
                    
                    println!("Training job status after recovery: {}", post_recovery_status.status);
                },
                Err(_) => {
                    println!("⚠️ Service recovery took longer than expected");
                    // This might be acceptable depending on recovery strategy
                }
            }

            // Clean up
            harness.grpc_clients.tli_client.stop_ml_training(job_id).await.ok();

            println!("ML Training Service failure test completed");
            Ok(())
        }).await
    }

    /// Test Trading Service failure and recovery
    async fn test_trading_service_failure(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("trading_service_failure", |harness| async move {
            println!("Testing Trading Service failure scenarios...");

            // Deploy a model first
            let model_artifact = harness.test_data.create_model_artifact("DQN", "AAPL").await?;
            
            let deploy_request = DeployModelRequest {
                model_id: model_artifact.model_id.clone(),
                model_path: model_artifact.model_path.clone(),
                target_symbols: vec!["AAPL".to_string()],
            };

            let deploy_response = harness.grpc_clients.trading_client
                .deploy_model(deploy_request).await?;

            assert!(deploy_response.success, "Model deployment should succeed");

            // Test inference before failure
            let prediction_request = PredictionRequest {
                model_id: model_artifact.model_id.clone(),
                symbol: "AAPL".to_string(),
                features: vec![150.0, 151.0, 149.5, 152.0, 150.5],
            };

            let pre_failure_response = harness.grpc_clients.trading_client
                .get_model_predictions(prediction_request.clone()).await?;

            println!("Pre-failure prediction: {} (confidence: {:.3})", 
                    pre_failure_response.prediction, pre_failure_response.confidence);

            // Simulate Trading Service failure
            println!("Simulating Trading Service failure...");
            sleep(Duration::from_secs(2)).await;

            // Test service recovery
            println!("Testing Trading Service recovery...");
            let recovery_timeout = Duration::from_secs(45);
            
            let recovery_result = timeout(recovery_timeout, async {
                loop {
                    match harness.grpc_clients.trading_client.health_check().await {
                        Ok(_) => {
                            println!("Trading Service recovered");
                            return Ok::<(), anyhow::Error>(());
                        },
                        Err(_) => {
                            sleep(Duration::from_secs(1)).await;
                        }
                    }
                }
            }).await;

            match recovery_result {
                Ok(_) => {
                    println!("✅ Trading Service recovered successfully");
                    
                    // Test that deployed models are still available after recovery
                    let post_recovery_response = harness.grpc_clients.trading_client
                        .get_model_predictions(prediction_request).await?;
                    
                    println!("Post-recovery prediction: {} (confidence: {:.3})", 
                            post_recovery_response.prediction, post_recovery_response.confidence);
                    
                    // Predictions should still work (though they may differ slightly)
                    assert!(!post_recovery_response.prediction.is_empty(), 
                           "Model should still work after service recovery");
                },
                Err(_) => {
                    println!("⚠️ Trading Service recovery took longer than expected");
                }
            }

            println!("Trading Service failure test completed");
            Ok(())
        }).await
    }

    /// Test TLI Service failure and recovery
    async fn test_tli_service_failure(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("tli_service_failure", |harness| async move {
            println!("Testing TLI Service failure scenarios...");

            // Test TLI functionality before failure
            let pre_failure_health = harness.grpc_clients.tli_client.health_check().await;
            assert!(pre_failure_health.is_ok(), "TLI should be healthy before failure");

            // Simulate TLI Service failure
            println!("Simulating TLI Service failure...");
            sleep(Duration::from_secs(2)).await;

            // Test service recovery
            println!("Testing TLI Service recovery...");
            let recovery_timeout = Duration::from_secs(30);
            
            let recovery_result = timeout(recovery_timeout, async {
                loop {
                    match harness.grpc_clients.tli_client.health_check().await {
                        Ok(_) => {
                            println!("TLI Service recovered");
                            return Ok::<(), anyhow::Error>(());
                        },
                        Err(_) => {
                            sleep(Duration::from_secs(1)).await;
                        }
                    }
                }
            }).await;

            match recovery_result {
                Ok(_) => {
                    println!("✅ TLI Service recovered successfully");
                    
                    // Test that TLI can still start new training jobs after recovery
                    let training_request = StartMLTrainingRequest {
                        model_name: "post_failure_test_model".to_string(),
                        dataset_id: "post_failure_dataset".to_string(),
                        hyperparameters: vec![
                            ("learning_rate".to_string(), "0.001".to_string()),
                        ].into_iter().collect(),
                        auto_deploy: false,
                    };

                    let training_response = harness.grpc_clients.tli_client
                        .start_ml_training(training_request).await?;

                    assert!(training_response.success, 
                           "TLI should be able to start training after recovery");

                    // Clean up
                    harness.grpc_clients.tli_client.stop_ml_training(training_response.job_id).await.ok();
                },
                Err(_) => {
                    println!("⚠️ TLI Service recovery took longer than expected");
                }
            }

            println!("TLI Service failure test completed");
            Ok(())
        }).await
    }

    /// Test network partition recovery
    async fn test_network_partition_recovery(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("network_partition_recovery", |harness| async move {
            println!("Testing network partition recovery...");

            // Establish baseline connectivity
            assert!(harness.grpc_clients.are_all_healthy().await?, 
                   "All services should be healthy before network test");

            // Deploy a model for testing
            let model_artifact = harness.test_data.create_model_artifact("MAMBA", "SPY").await?;
            
            let deploy_request = DeployModelRequest {
                model_id: model_artifact.model_id.clone(),
                model_path: model_artifact.model_path.clone(),
                target_symbols: vec!["SPY".to_string()],
            };

            harness.grpc_clients.trading_client.deploy_model(deploy_request).await?;

            // Simulate network partition
            println!("Simulating network partition...");
            
            // In a real test environment, this would use network tools like:
            // iptables -A INPUT -s <service_ip> -j DROP
            // tc qdisc add dev eth0 root netem loss 100%
            
            // For simulation, we'll test connection timeout scenarios
            let partition_duration = Duration::from_secs(10);
            let partition_start = Instant::now();

            // Test service behavior during partition
            while partition_start.elapsed() < partition_duration {
                // Attempt operations that should handle network failures gracefully
                let prediction_request = PredictionRequest {
                    model_id: model_artifact.model_id.clone(),
                    symbol: "SPY".to_string(),
                    features: vec![450.0, 451.0, 449.5, 452.0, 450.5],
                };

                // During partition, this should either:
                // 1. Timeout gracefully
                // 2. Use cached results
                // 3. Return error with retry logic
                let prediction_result = timeout(
                    Duration::from_secs(5),
                    harness.grpc_clients.trading_client.get_model_predictions(prediction_request)
                ).await;

                match prediction_result {
                    Ok(Ok(response)) => {
                        println!("Prediction succeeded during partition (cached/fallback): {}", 
                                response.prediction);
                    },
                    Ok(Err(_)) => {
                        println!("Prediction failed gracefully during partition");
                    },
                    Err(_) => {
                        println!("Prediction timed out during partition (expected)");
                    }
                }

                sleep(Duration::from_secs(2)).await;
            }

            // Simulate network recovery
            println!("Simulating network recovery...");
            // In real environment: remove iptables rules, restore connectivity

            // Test service recovery after partition
            let recovery_timeout = Duration::from_secs(60);
            let recovery_result = timeout(recovery_timeout, async {
                loop {
                    if harness.grpc_clients.are_all_healthy().await.unwrap_or(false) {
                        println!("All services recovered from network partition");
                        return Ok::<(), anyhow::Error>(());
                    }
                    sleep(Duration::from_secs(2)).await;
                }
            }).await;

            match recovery_result {
                Ok(_) => {
                    println!("✅ Network partition recovery successful");
                    
                    // Verify full functionality after recovery
                    let prediction_request = PredictionRequest {
                        model_id: model_artifact.model_id.clone(),
                        symbol: "SPY".to_string(),
                        features: vec![450.0, 451.0, 449.5, 452.0, 450.5],
                    };

                    let post_recovery_response = harness.grpc_clients.trading_client
                        .get_model_predictions(prediction_request).await?;

                    assert!(!post_recovery_response.prediction.is_empty(),
                           "Full functionality should be restored after network recovery");

                    println!("Post-recovery prediction: {} (confidence: {:.3})", 
                            post_recovery_response.prediction, post_recovery_response.confidence);
                },
                Err(_) => {
                    println!("⚠️ Network partition recovery took longer than expected");
                }
            }

            println!("Network partition recovery test completed");
            Ok(())
        }).await
    }

    /// Test intermittent network failures
    async fn test_intermittent_network_failures(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("intermittent_network_failures", |harness| async move {
            println!("Testing intermittent network failures...");

            // Deploy model for testing
            let model_artifact = harness.test_data.create_model_artifact("TFT", "NVDA").await?;
            
            let deploy_request = DeployModelRequest {
                model_id: model_artifact.model_id.clone(),
                model_path: model_artifact.model_path.clone(),
                target_symbols: vec!["NVDA".to_string()],
            };

            harness.grpc_clients.trading_client.deploy_model(deploy_request).await?;

            // Simulate intermittent failures over time
            const TEST_DURATION_SECS: u64 = 30;
            const FAILURE_PROBABILITY: f64 = 0.3; // 30% chance of failure per request
            
            let test_start = Instant::now();
            let mut successful_requests = 0;
            let mut failed_requests = 0;
            let mut recovered_requests = 0;

            while test_start.elapsed().as_secs() < TEST_DURATION_SECS {
                let prediction_request = PredictionRequest {
                    model_id: model_artifact.model_id.clone(),
                    symbol: "NVDA".to_string(),
                    features: vec![400.0, 401.0, 399.5, 402.0, 400.5],
                };

                // Simulate intermittent failure
                let should_fail = rand::random::<f64>() < FAILURE_PROBABILITY;
                
                if should_fail {
                    // Simulate network failure by timing out quickly
                    let result = timeout(
                        Duration::from_millis(100),
                        harness.grpc_clients.trading_client.get_model_predictions(prediction_request.clone())
                    ).await;

                    match result {
                        Ok(Ok(_)) => {
                            // Request succeeded despite simulated failure
                            successful_requests += 1;
                        },
                        _ => {
                            failed_requests += 1;
                            
                            // Test retry logic
                            sleep(Duration::from_millis(500)).await;
                            
                            let retry_result = harness.grpc_clients.trading_client
                                .get_model_predictions(prediction_request).await;

                            if retry_result.is_ok() {
                                recovered_requests += 1;
                                println!("Request recovered after retry");
                            }
                        }
                    }
                } else {
                    // Normal request
                    let result = harness.grpc_clients.trading_client
                        .get_model_predictions(prediction_request).await;

                    if result.is_ok() {
                        successful_requests += 1;
                    } else {
                        failed_requests += 1;
                    }
                }

                sleep(Duration::from_millis(200)).await;
            }

            println!("Intermittent network failure results:");
            println!("  Successful requests: {}", successful_requests);
            println!("  Failed requests: {}", failed_requests);
            println!("  Recovered requests: {}", recovered_requests);

            let total_requests = successful_requests + failed_requests;
            let success_rate = if total_requests > 0 {
                (successful_requests + recovered_requests) as f64 / total_requests as f64
            } else {
                0.0
            };

            println!("  Overall success rate: {:.1}%", success_rate * 100.0);

            // System should handle intermittent failures with reasonable success rate
            const MIN_SUCCESS_RATE: f64 = 0.7; // 70% minimum success rate
            assert!(success_rate >= MIN_SUCCESS_RATE,
                   "Success rate {:.1}% should be >= {:.1}% under intermittent failures",
                   success_rate * 100.0, MIN_SUCCESS_RATE * 100.0);

            println!("Intermittent network failure test completed");
            Ok(())
        }).await
    }

    /// Test connection timeout handling
    async fn test_connection_timeout_handling(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("connection_timeout_handling", |harness| async move {
            println!("Testing connection timeout handling...");

            // Test various timeout scenarios
            let timeout_scenarios = vec![
                ("short_timeout", Duration::from_millis(10)),
                ("medium_timeout", Duration::from_millis(100)),
                ("long_timeout", Duration::from_millis(1000)),
            ];

            for (scenario_name, timeout_duration) in timeout_scenarios {
                println!("Testing {} scenario ({:?})", scenario_name, timeout_duration);

                let prediction_request = PredictionRequest {
                    model_id: "timeout_test_model".to_string(),
                    symbol: "TEST".to_string(),
                    features: vec![100.0, 101.0, 99.5, 102.0, 100.5],
                };

                let start_time = Instant::now();
                let result = timeout(
                    timeout_duration,
                    harness.grpc_clients.trading_client.get_model_predictions(prediction_request)
                ).await;

                let elapsed = start_time.elapsed();

                match result {
                    Ok(Ok(response)) => {
                        println!("  Request completed in {:?}: {}", elapsed, response.prediction);
                        // Fast completion is good
                    },
                    Ok(Err(e)) => {
                        println!("  Request failed in {:?}: {}", elapsed, e);
                        // Service-level error is acceptable
                    },
                    Err(_) => {
                        println!("  Request timed out after {:?}", elapsed);
                        // Timeout is expected for short timeouts
                        assert!(elapsed >= timeout_duration * 9 / 10, 
                               "Timeout should occur close to the specified duration");
                    }
                }
            }

            // Test timeout recovery
            println!("Testing timeout recovery...");
            sleep(Duration::from_secs(2)).await;

            // After timeouts, normal requests should still work
            let normal_request = PredictionRequest {
                model_id: "recovery_test_model".to_string(),
                symbol: "RECOVERY".to_string(),
                features: vec![100.0, 101.0, 99.5, 102.0, 100.5],
            };

            let recovery_result = timeout(
                Duration::from_secs(10),
                harness.grpc_clients.trading_client.get_model_predictions(normal_request)
            ).await;

            // We expect this to timeout gracefully since the model doesn't exist
            // but the timeout handling should work properly
            match recovery_result {
                Ok(Err(_)) => {
                    println!("✅ Timeout handling recovered properly (service responded with error)");
                },
                Err(_) => {
                    println!("✅ Timeout handling working (request timed out as expected)");
                },
                Ok(Ok(_)) => {
                    println!("✅ Service fully functional after timeout tests");
                },
            }

            println!("Connection timeout handling test completed");
            Ok(())
        }).await
    }

    /// Test database connection failure
    async fn test_database_connection_failure(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("database_connection_failure", |harness| async move {
            println!("Testing database connection failure scenarios...");

            // Insert test data before failure
            let test_trade = crate::harness::fixtures::TestTrade {
                symbol: "DBTEST".to_string(),
                price: 100.0,
                quantity: 50.0,
                side: "BUY".to_string(),
                timestamp: chrono::Utc::now(),
                model_id: Some("db_test_model".to_string()),
            };

            harness.fixtures.insert_test_trades(&[test_trade]).await?;

            // Simulate database connection failure
            println!("Simulating database connection failure...");
            
            // In a real environment, this would:
            // - Stop the database container
            // - Block database ports with iptables
            // - Simulate connection pool exhaustion
            
            // For simulation, we'll test error handling
            sleep(Duration::from_secs(3)).await;

            // Test application behavior during database unavailability
            // Applications should:
            // 1. Cache recent data
            // 2. Continue serving read requests from cache
            // 3. Queue write operations for retry
            // 4. Degrade gracefully

            println!("Testing application behavior during database failure...");

            // Test that services can still operate with cached data
            let prediction_request = PredictionRequest {
                model_id: "cached_model".to_string(),
                symbol: "DBTEST".to_string(),
                features: vec![100.0, 101.0, 99.5, 102.0, 100.5],
            };

            let during_failure_result = harness.grpc_clients.trading_client
                .get_model_predictions(prediction_request).await;

            // The result depends on implementation:
            // - May succeed with cached data
            // - May fail gracefully with proper error handling
            match during_failure_result {
                Ok(response) => {
                    println!("Service operating with cached data: {}", response.prediction);
                },
                Err(_) => {
                    println!("Service failed gracefully during database outage");
                }
            }

            // Simulate database recovery
            println!("Simulating database recovery...");
            sleep(Duration::from_secs(2)).await;

            // Test database reconnection
            println!("Testing database reconnection...");
            let reconnection_timeout = Duration::from_secs(30);
            
            let reconnection_result = timeout(reconnection_timeout, async {
                loop {
                    // Test database operations
                    let test_model = crate::harness::fixtures::TestModel::default();
                    match harness.fixtures.insert_test_models(&[test_model]).await {
                        Ok(_) => {
                            println!("Database connection recovered");
                            return Ok::<(), anyhow::Error>(());
                        },
                        Err(_) => {
                            sleep(Duration::from_secs(2)).await;
                        }
                    }
                }
            }).await;

            match reconnection_result {
                Ok(_) => {
                    println!("✅ Database connection recovered successfully");
                    
                    // Test full functionality after recovery
                    let post_recovery_trade = crate::harness::fixtures::TestTrade {
                        symbol: "RECOVERY".to_string(),
                        price: 105.0,
                        quantity: 25.0,
                        side: "SELL".to_string(),
                        timestamp: chrono::Utc::now(),
                        model_id: Some("recovery_model".to_string()),
                    };

                    harness.fixtures.insert_test_trades(&[post_recovery_trade]).await?;
                    println!("Database write operations working after recovery");
                },
                Err(_) => {
                    println!("⚠️ Database recovery took longer than expected");
                }
            }

            println!("Database connection failure test completed");
            Ok(())
        }).await
    }

    /// Test database transaction rollback
    async fn test_database_transaction_rollback(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("database_transaction_rollback", |harness| async move {
            println!("Testing database transaction rollback scenarios...");

            // Test scenario: Start training job with database transaction
            let training_request = StartMLTrainingRequest {
                model_name: "transaction_test_model".to_string(),
                dataset_id: "transaction_dataset".to_string(),
                hyperparameters: vec![
                    ("learning_rate".to_string(), "0.001".to_string()),
                ].into_iter().collect(),
                auto_deploy: false,
            };

            let response = harness.grpc_clients.tli_client
                .start_ml_training(training_request).await?;

            if response.success {
                let job_id = response.job_id.clone();

                // Simulate failure during transaction
                println!("Simulating failure during database transaction...");
                sleep(Duration::from_secs(2)).await;

                // Force stop the training job (simulates transaction failure)
                let stop_result = harness.grpc_clients.tli_client
                    .stop_ml_training(job_id.clone()).await;

                match stop_result {
                    Ok(stop_response) => {
                        assert!(stop_response.success, "Training should stop successfully");
                        println!("Training job stopped, testing transaction rollback...");
                    },
                    Err(_) => {
                        println!("Stop command failed - may indicate transaction issues");
                    }
                }

                // Verify database consistency after rollback
                // The training job record should either:
                // 1. Be marked as FAILED/CANCELLED with proper cleanup
                // 2. Be completely rolled back (not exist)
                
                sleep(Duration::from_secs(1)).await;

                // Test that we can start a new training job after rollback
                let recovery_request = StartMLTrainingRequest {
                    model_name: "rollback_recovery_model".to_string(),
                    dataset_id: "rollback_recovery_dataset".to_string(),
                    hyperparameters: vec![
                        ("learning_rate".to_string(), "0.001".to_string()),
                    ].into_iter().collect(),
                    auto_deploy: false,
                };

                let recovery_response = harness.grpc_clients.tli_client
                    .start_ml_training(recovery_request).await?;

                assert!(recovery_response.success, 
                       "Should be able to start new training after transaction rollback");

                println!("✅ Database transaction rollback handled correctly");

                // Clean up
                harness.grpc_clients.tli_client
                    .stop_ml_training(recovery_response.job_id).await.ok();
            }

            println!("Database transaction rollback test completed");
            Ok(())
        }).await
    }

    /// Test data consistency under failure
    async fn test_data_consistency_under_failure(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("data_consistency_under_failure", |harness| async move {
            println!("Testing data consistency under failure conditions...");

            // Create test data for consistency checks
            let test_models = vec![
                crate::harness::fixtures::TestModel {
                    model_name: "consistency_model_1".to_string(),
                    model_type: "DQN".to_string(),
                    version: "1.0.0".to_string(),
                    symbol: "CONS1".to_string(),
                    status: "ACTIVE".to_string(),
                    ..Default::default()
                },
                crate::harness::fixtures::TestModel {
                    model_name: "consistency_model_2".to_string(),
                    model_type: "PPO".to_string(),
                    version: "1.0.0".to_string(),
                    symbol: "CONS2".to_string(),
                    status: "ACTIVE".to_string(),
                    ..Default::default()
                },
            ];

            // Insert test models
            harness.fixtures.insert_test_models(&test_models).await?;

            // Simulate concurrent operations that could cause consistency issues
            println!("Testing concurrent operations under failure...");

            // Start multiple operations concurrently
            let mut operations = Vec::new();

            // Operation 1: Model deployment
            let deploy_op = async {
                let model_artifact = harness.test_data.create_model_artifact("MAMBA", "CONS1").await?;
                let deploy_request = DeployModelRequest {
                    model_id: model_artifact.model_id,
                    model_path: model_artifact.model_path,
                    target_symbols: vec!["CONS1".to_string()],
                };
                harness.grpc_clients.trading_client.deploy_model(deploy_request).await
            };

            // Operation 2: Training job start
            let training_op = async {
                let training_request = StartMLTrainingRequest {
                    model_name: "consistency_training_model".to_string(),
                    dataset_id: "consistency_dataset".to_string(),
                    hyperparameters: vec![
                        ("learning_rate".to_string(), "0.001".to_string()),
                    ].into_iter().collect(),
                    auto_deploy: false,
                };
                harness.grpc_clients.tli_client.start_ml_training(training_request).await
            };

            // Execute operations concurrently
            let results = futures::future::join_all(vec![
                Box::pin(deploy_op) as std::pin::Pin<Box<dyn std::future::Future<Output = Result<_, _>>>>,
                Box::pin(training_op) as std::pin::Pin<Box<dyn std::future::Future<Output = Result<_, _>>>>,
            ]).await;

            // Analyze results for consistency
            let mut successful_operations = 0;
            let mut failed_operations = 0;

            for (i, result) in results.into_iter().enumerate() {
                match result {
                    Ok(_) => {
                        successful_operations += 1;
                        println!("Operation {} completed successfully", i + 1);
                    },
                    Err(_) => {
                        failed_operations += 1;
                        println!("Operation {} failed", i + 1);
                    }
                }
            }

            println!("Concurrent operations results:");
            println!("  Successful: {}", successful_operations);
            println!("  Failed: {}", failed_operations);

            // Verify data consistency after operations
            println!("Verifying data consistency...");

            // Test that the system maintained consistency despite concurrent operations
            // This would involve checking:
            // 1. No duplicate model deployments
            // 2. Training job states are consistent
            // 3. Resource allocations are correct
            // 4. No orphaned data

            // Simulate consistency check by attempting clean operations
            let consistency_check_training = StartMLTrainingRequest {
                model_name: "consistency_check_model".to_string(),
                dataset_id: "consistency_check_dataset".to_string(),
                hyperparameters: vec![
                    ("learning_rate".to_string(), "0.001".to_string()),
                ].into_iter().collect(),
                auto_deploy: false,
            };

            let consistency_result = harness.grpc_clients.tli_client
                .start_ml_training(consistency_check_training).await?;

            assert!(consistency_result.success, 
                   "System should maintain consistency and accept new operations");

            println!("✅ Data consistency maintained under failure conditions");

            // Clean up
            harness.grpc_clients.tli_client
                .stop_ml_training(consistency_result.job_id).await.ok();

            println!("Data consistency under failure test completed");
            Ok(())
        }).await
    }

    /// Test memory exhaustion recovery
    async fn test_memory_exhaustion_recovery(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("memory_exhaustion_recovery", |harness| async move {
            println!("Testing memory exhaustion recovery...");

            // Simulate memory-intensive operations
            let memory_intensive_training = StartMLTrainingRequest {
                model_name: "memory_stress_model".to_string(),
                dataset_id: "large_memory_dataset".to_string(),
                hyperparameters: vec![
                    ("learning_rate".to_string(), "0.001".to_string()),
                    ("batch_size".to_string(), "10000".to_string()), // Very large batch
                    ("epochs".to_string(), "100".to_string()),
                ].into_iter().collect(),
                auto_deploy: false,
            };

            let response = harness.grpc_clients.tli_client
                .start_ml_training(memory_intensive_training).await?;

            let mut memory_exhaustion_detected = false;

            if response.success {
                let job_id = response.job_id.clone();

                // Monitor for memory exhaustion
                for _ in 0..10 {
                    sleep(Duration::from_secs(2)).await;

                    // Check job status
                    match harness.grpc_clients.tli_client.get_ml_training_status(job_id.clone()).await {
                        Ok(status) => {
                            println!("Memory stress training status: {}", status.status);
                            
                            if status.status == "FAILED" {
                                memory_exhaustion_detected = true;
                                println!("Memory exhaustion detected (training failed)");
                                break;
                            }
                        },
                        Err(_) => {
                            memory_exhaustion_detected = true;
                            println!("Memory exhaustion detected (service unresponsive)");
                            break;
                        }
                    }

                    // Simulate memory pressure monitoring
                    harness.performance.record_resource_usage(
                        "memory_exhaustion_recovery",
                        85.0, // CPU %
                        7500.0 + (rand::random::<f64>() * 1000.0), // Memory MB (approaching limit)
                        Some(70.0), // GPU %
                    );
                }

                // Stop the memory-intensive job
                harness.grpc_clients.tli_client.stop_ml_training(job_id).await.ok();
            }

            // Test system recovery after memory exhaustion
            println!("Testing system recovery after memory stress...");
            sleep(Duration::from_secs(5)).await; // Allow garbage collection/cleanup

            // Try normal operation after memory stress
            let recovery_training = StartMLTrainingRequest {
                model_name: "memory_recovery_model".to_string(),
                dataset_id: "normal_dataset".to_string(),
                hyperparameters: vec![
                    ("learning_rate".to_string(), "0.001".to_string()),
                    ("batch_size".to_string(), "32".to_string()), // Normal batch size
                    ("epochs".to_string(), "5".to_string()),
                ].into_iter().collect(),
                auto_deploy: false,
            };

            let recovery_response = harness.grpc_clients.tli_client
                .start_ml_training(recovery_training).await?;

            assert!(recovery_response.success, 
                   "System should recover from memory exhaustion and accept normal operations");

            println!("✅ System recovered from memory exhaustion");

            // Clean up
            harness.grpc_clients.tli_client
                .stop_ml_training(recovery_response.job_id).await.ok();

            println!("Memory exhaustion recovery test completed");
            Ok(())
        }).await
    }

    /// Test CPU overload handling
    async fn test_cpu_overload_handling(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("cpu_overload_handling", |harness| async move {
            println!("Testing CPU overload handling...");

            // Start multiple CPU-intensive training jobs
            let mut training_jobs = Vec::new();
            const CPU_STRESS_JOBS: usize = 8; // More than typical CPU cores

            for i in 0..CPU_STRESS_JOBS {
                let training_request = StartMLTrainingRequest {
                    model_name: format!("cpu_stress_model_{}", i),
                    dataset_id: format!("cpu_stress_dataset_{}", i),
                    hyperparameters: vec![
                        ("learning_rate".to_string(), "0.001".to_string()),
                        ("batch_size".to_string(), "128".to_string()),
                        ("epochs".to_string(), "50".to_string()),
                    ].into_iter().collect(),
                    auto_deploy: false,
                };

                let response = harness.grpc_clients.tli_client
                    .start_ml_training(training_request).await?;

                if response.success {
                    training_jobs.push(response.job_id);
                    println!("Started CPU stress job {}", i + 1);
                }

                // Small delay between job starts
                sleep(Duration::from_millis(100)).await;
            }

            // Monitor CPU overload handling
            println!("Monitoring CPU overload handling...");
            let mut monitoring_rounds = 0;
            const MAX_CPU_MONITORING: u32 = 15;

            while monitoring_rounds < MAX_CPU_MONITORING {
                let mut active_jobs = 0;

                for job_id in &training_jobs {
                    match harness.grpc_clients.tli_client.get_ml_training_status(job_id.clone()).await {
                        Ok(status) => {
                            if status.status == "RUNNING" || status.status == "QUEUED" {
                                active_jobs += 1;
                            }
                        },
                        Err(_) => {
                            // Job may have failed due to resource constraints
                        }
                    }
                }

                // Simulate CPU monitoring
                let cpu_usage = 80.0 + (monitoring_rounds as f64 * 2.0).min(15.0);
                harness.performance.record_resource_usage(
                    "cpu_overload_handling",
                    cpu_usage,
                    4096.0, // Memory MB
                    Some(60.0), // GPU %
                );

                println!("CPU monitoring round {}: {:.1}% CPU, {} active jobs", 
                        monitoring_rounds + 1, cpu_usage, active_jobs);

                if active_jobs == 0 {
                    println!("All CPU stress jobs completed or failed");
                    break;
                }

                monitoring_rounds += 1;
                sleep(Duration::from_secs(3)).await;
            }

            // Test that system can still respond during CPU overload
            println!("Testing system responsiveness during CPU overload...");
            
            let responsiveness_test = StartMLTrainingRequest {
                model_name: "responsiveness_test_model".to_string(),
                dataset_id: "responsiveness_dataset".to_string(),
                hyperparameters: vec![
                    ("learning_rate".to_string(), "0.001".to_string()),
                    ("batch_size".to_string(), "16".to_string()), // Small batch
                    ("epochs".to_string(), "1".to_string()),
                ].into_iter().collect(),
                auto_deploy: false,
            };

            let responsiveness_result = timeout(
                Duration::from_secs(30),
                harness.grpc_clients.tli_client.start_ml_training(responsiveness_test)
            ).await;

            match responsiveness_result {
                Ok(Ok(response)) => {
                    if response.success {
                        println!("✅ System remained responsive during CPU overload");
                        // Clean up
                        harness.grpc_clients.tli_client.stop_ml_training(response.job_id).await.ok();
                    } else {
                        println!("System rejected new job during CPU overload (acceptable)");
                    }
                },
                Ok(Err(_)) => {
                    println!("System returned error during CPU overload (acceptable)");
                },
                Err(_) => {
                    println!("⚠️ System became unresponsive during CPU overload");
                }
            }

            // Clean up all training jobs
            for job_id in &training_jobs {
                harness.grpc_clients.tli_client.stop_ml_training(job_id.clone()).await.ok();
            }

            println!("CPU overload handling test completed");
            Ok(())
        }).await
    }

    /// Test GPU resource contention
    async fn test_gpu_resource_contention(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("gpu_resource_contention", |harness| async move {
            println!("Testing GPU resource contention...");

            // Start multiple GPU-intensive training jobs
            let gpu_training_jobs = vec![
                ("gpu_contention_dqn", "DQN"),
                ("gpu_contention_mamba", "MAMBA"),
                ("gpu_contention_tft", "TFT"),
            ];

            let mut started_jobs = Vec::new();

            for (job_name, model_type) in &gpu_training_jobs {
                let training_request = StartMLTrainingRequest {
                    model_name: job_name.to_string(),
                    dataset_id: format!("gpu_dataset_{}", model_type.to_lowercase()),
                    hyperparameters: vec![
                        ("learning_rate".to_string(), "0.001".to_string()),
                        ("batch_size".to_string(), "256".to_string()), // Large batch for GPU
                        ("epochs".to_string(), "20".to_string()),
                    ].into_iter().collect(),
                    auto_deploy: false,
                };

                let response = harness.grpc_clients.tli_client
                    .start_ml_training(training_request).await?;

                if response.success {
                    started_jobs.push(response.job_id);
                    println!("Started GPU training job: {} ({})", job_name, model_type);
                } else {
                    println!("GPU training job rejected: {} - may indicate resource contention", job_name);
                }

                sleep(Duration::from_millis(500)).await;
            }

            // Monitor GPU resource contention
            println!("Monitoring GPU resource contention...");
            let mut gpu_monitoring_rounds = 0;
            const MAX_GPU_CONTENTION_MONITORING: u32 = 10;

            while gpu_monitoring_rounds < MAX_GPU_CONTENTION_MONITORING {
                let mut running_jobs = 0;
                let mut queued_jobs = 0;

                for job_id in &started_jobs {
                    match harness.grpc_clients.tli_client.get_ml_training_status(job_id.clone()).await {
                        Ok(status) => {
                            match status.status.as_str() {
                                "RUNNING" => running_jobs += 1,
                                "QUEUED" => queued_jobs += 1,
                                _ => {}
                            }
                        },
                        Err(_) => {
                            // Job may have completed or failed
                        }
                    }
                }

                // Simulate GPU utilization monitoring
                let gpu_usage = if running_jobs > 0 {
                    90.0 + (gpu_monitoring_rounds as f64 * 0.5)
                } else {
                    20.0
                };

                harness.performance.record_resource_usage(
                    "gpu_resource_contention",
                    60.0, // CPU %
                    6144.0, // Memory MB
                    Some(gpu_usage),
                );

                println!("GPU monitoring round {}: {:.1}% GPU, {} running, {} queued",
                        gpu_monitoring_rounds + 1, gpu_usage, running_jobs, queued_jobs);

                if running_jobs == 0 && queued_jobs == 0 {
                    println!("All GPU training jobs completed");
                    break;
                }

                // Test that system properly queues jobs when GPU is contended
                if running_jobs > 0 && queued_jobs > 0 {
                    println!("✅ GPU resource contention properly managed (queuing working)");
                }

                gpu_monitoring_rounds += 1;
                sleep(Duration::from_secs(5)).await;
            }

            // Clean up
            for job_id in &started_jobs {
                harness.grpc_clients.tli_client.stop_ml_training(job_id.clone()).await.ok();
            }

            println!("GPU resource contention test completed");
            Ok(())
        }).await
    }

    /// Test model corruption handling
    async fn test_model_corruption_handling(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("model_corruption_handling", |harness| async move {
            println!("Testing model corruption handling...");

            // Create a valid model first
            let model_artifact = harness.test_data.create_model_artifact("LIQUID", "CORR").await?;
            
            // Deploy the valid model
            let deploy_request = DeployModelRequest {
                model_id: model_artifact.model_id.clone(),
                model_path: model_artifact.model_path.clone(),
                target_symbols: vec!["CORR".to_string()],
            };

            let deploy_response = harness.grpc_clients.trading_client
                .deploy_model(deploy_request).await?;

            assert!(deploy_response.success, "Valid model should deploy successfully");

            // Test with valid model first
            let prediction_request = PredictionRequest {
                model_id: model_artifact.model_id.clone(),
                symbol: "CORR".to_string(),
                features: vec![100.0, 101.0, 99.5, 102.0, 100.5],
            };

            let valid_response = harness.grpc_clients.trading_client
                .get_model_predictions(prediction_request.clone()).await?;

            println!("Valid model prediction: {} (confidence: {:.3})",
                    valid_response.prediction, valid_response.confidence);

            // Corrupt the model file
            println!("Corrupting model file...");
            tokio::fs::write(&model_artifact.model_path, b"CORRUPTED_MODEL_DATA").await?;

            // Test handling of corrupted model
            println!("Testing corrupted model handling...");
            
            // Try to use the corrupted model
            let corruption_result = harness.grpc_clients.trading_client
                .get_model_predictions(prediction_request.clone()).await;

            match corruption_result {
                Ok(response) => {
                    // If it succeeds, it might be using cached model or fallback
                    println!("Model still working (cached/fallback): {}", response.prediction);
                },
                Err(_) => {
                    println!("✅ Corrupted model properly detected and rejected");
                }
            }

            // Test model replacement/recovery
            println!("Testing model recovery...");
            
            // Create a new valid model to replace the corrupted one
            let recovery_model = harness.test_data.create_model_artifact("ENSEMBLE", "CORR").await?;
            
            let update_request = UpdateModelRequest {
                model_id: model_artifact.model_id.clone(),
                new_model_path: recovery_model.model_path.clone(),
            };

            let update_response = harness.grpc_clients.trading_client
                .update_model(update_request).await?;

            assert!(update_response.success, "Model update should succeed");

            // Test that the recovered model works
            let recovery_response = harness.grpc_clients.trading_client
                .get_model_predictions(prediction_request).await?;

            assert!(!recovery_response.prediction.is_empty(),
                   "Recovered model should work properly");

            println!("✅ Model corruption handled and recovery successful");
            println!("Recovered model prediction: {} (confidence: {:.3})",
                    recovery_response.prediction, recovery_response.confidence);

            println!("Model corruption handling test completed");
            Ok(())
        }).await
    }

    /// Test training job crash recovery
    async fn test_training_job_crash_recovery(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("training_job_crash_recovery", |harness| async move {
            println!("Testing training job crash recovery...");

            // Start a training job
            let training_request = StartMLTrainingRequest {
                model_name: "crash_test_model".to_string(),
                dataset_id: "crash_test_dataset".to_string(),
                hyperparameters: vec![
                    ("learning_rate".to_string(), "0.001".to_string()),
                    ("batch_size".to_string(), "64".to_string()),
                    ("epochs".to_string(), "100".to_string()), // Long training
                ].into_iter().collect(),
                auto_deploy: false,
            };

            let response = harness.grpc_clients.tli_client
                .start_ml_training(training_request).await?;

            assert!(response.success, "Training should start successfully");
            let job_id = response.job_id.clone();

            // Wait for training to start
            sleep(Duration::from_secs(5)).await;

            // Verify training is running
            let pre_crash_status = harness.grpc_clients.tli_client
                .get_ml_training_status(job_id.clone()).await?;
            println!("Pre-crash training status: {} ({}%)", 
                    pre_crash_status.status, pre_crash_status.progress_percentage);

            // Simulate training job crash
            println!("Simulating training job crash...");
            // In real environment, this would kill the training process
            // For testing, we'll force stop and then test recovery mechanisms
            
            let force_stop_result = harness.grpc_clients.tli_client
                .stop_ml_training(job_id.clone()).await;

            match force_stop_result {
                Ok(_) => {
                    println!("Training job stopped (simulating crash)");
                },
                Err(_) => {
                    println!("Training job may have already crashed");
                }
            }

            // Test crash detection and recovery
            println!("Testing crash detection and recovery...");
            sleep(Duration::from_secs(3)).await;

            // Check job status after crash
            let post_crash_status = harness.grpc_clients.tli_client
                .get_ml_training_status(job_id.clone()).await;

            match post_crash_status {
                Ok(status) => {
                    println!("Post-crash job status: {}", status.status);
                    
                    // Job should be marked as FAILED or CANCELLED
                    assert!(status.status == "FAILED" || status.status == "CANCELLED",
                           "Crashed job should be marked as failed or cancelled");
                },
                Err(_) => {
                    println!("Job record cleaned up after crash (acceptable)");
                }
            }

            // Test system recovery - should be able to start new jobs
            println!("Testing system recovery after crash...");
            
            let recovery_request = StartMLTrainingRequest {
                model_name: "crash_recovery_model".to_string(),
                dataset_id: "crash_recovery_dataset".to_string(),
                hyperparameters: vec![
                    ("learning_rate".to_string(), "0.001".to_string()),
                    ("batch_size".to_string(), "32".to_string()),
                    ("epochs".to_string(), "5".to_string()),
                ].into_iter().collect(),
                auto_deploy: false,
            };

            let recovery_response = harness.grpc_clients.tli_client
                .start_ml_training(recovery_request).await?;

            assert!(recovery_response.success, 
                   "Should be able to start new training after crash recovery");

            println!("✅ Training job crash recovery successful");

            // Clean up
            harness.grpc_clients.tli_client
                .stop_ml_training(recovery_response.job_id).await.ok();

            println!("Training job crash recovery test completed");
            Ok(())
        }).await
    }

    /// Test model rollback under failure
    async fn test_model_rollback_under_failure(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("model_rollback_under_failure", |harness| async move {
            println!("Testing model rollback under failure scenarios...");

            // Deploy initial stable model
            let stable_model = harness.test_data.create_model_artifact("PPO", "ROLLBACK").await?;
            
            let deploy_request = DeployModelRequest {
                model_id: stable_model.model_id.clone(),
                model_path: stable_model.model_path.clone(),
                target_symbols: vec!["ROLLBACK".to_string()],
            };

            let deploy_response = harness.grpc_clients.trading_client
                .deploy_model(deploy_request).await?;

            assert!(deploy_response.success, "Stable model should deploy successfully");

            // Test stable model
            let test_request = PredictionRequest {
                model_id: stable_model.model_id.clone(),
                symbol: "ROLLBACK".to_string(),
                features: vec![100.0, 101.0, 99.5, 102.0, 100.5],
            };

            let stable_response = harness.grpc_clients.trading_client
                .get_model_predictions(test_request.clone()).await?;

            println!("Stable model prediction: {} (confidence: {:.3})",
                    stable_response.prediction, stable_response.confidence);

            // Deploy problematic model update
            println!("Deploying problematic model update...");
            let problematic_model = harness.test_data.create_model_artifact("BUGGY", "ROLLBACK").await?;
            
            // Create a problematic model file
            tokio::fs::write(&problematic_model.model_path, b"PROBLEMATIC_MODEL_DATA").await?;

            let update_request = UpdateModelRequest {
                model_id: stable_model.model_id.clone(),
                new_model_path: problematic_model.model_path.clone(),
            };

            let update_response = harness.grpc_clients.trading_client
                .update_model(update_request).await;

            // The update might succeed initially but fail during inference
            match update_response {
                Ok(resp) if resp.success => {
                    println!("Problematic model deployed (will fail during inference)");
                    
                    // Test inference with problematic model
                    let problematic_result = harness.grpc_clients.trading_client
                        .get_model_predictions(test_request.clone()).await;

                    match problematic_result {
                        Ok(_) => {
                            println!("Problematic model unexpectedly working");
                        },
                        Err(_) => {
                            println!("✅ Problematic model failure detected during inference");
                        }
                    }
                },
                _ => {
                    println!("✅ Problematic model rejected during deployment");
                }
            }

            // Test automatic rollback
            println!("Testing automatic rollback...");
            
            // Attempt rollback to stable model
            let rollback_request = UpdateModelRequest {
                model_id: stable_model.model_id.clone(),
                new_model_path: stable_model.model_path.clone(),
            };

            let rollback_response = harness.grpc_clients.trading_client
                .update_model(rollback_request).await?;

            assert!(rollback_response.success, "Rollback should succeed");

            // Test that rollback restored functionality
            let rollback_test_response = harness.grpc_clients.trading_client
                .get_model_predictions(test_request).await?;

            assert!(!rollback_test_response.prediction.is_empty(),
                   "Rolled back model should work properly");

            println!("✅ Model rollback under failure successful");
            println!("Rollback model prediction: {} (confidence: {:.3})",
                    rollback_test_response.prediction, rollback_test_response.confidence);

            println!("Model rollback under failure test completed");
            Ok(())
        }).await
    }

    /// Test cascade failure containment
    async fn test_cascade_failure_containment(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("cascade_failure_containment", |harness| async move {
            println!("Testing cascade failure containment...");

            // Setup multiple interconnected components
            let models = vec![
                ("cascade_model_1", "CASC1"),
                ("cascade_model_2", "CASC2"),
                ("cascade_model_3", "CASC3"),
            ];

            let mut deployed_models = Vec::new();

            // Deploy multiple models
            for (model_name, symbol) in &models {
                let model_artifact = harness.test_data.create_model_artifact("ENSEMBLE", symbol).await?;
                
                let deploy_request = DeployModelRequest {
                    model_id: model_artifact.model_id.clone(),
                    model_path: model_artifact.model_path.clone(),
                    target_symbols: vec![symbol.to_string()],
                };

                let deploy_response = harness.grpc_clients.trading_client
                    .deploy_model(deploy_request).await?;

                if deploy_response.success {
                    deployed_models.push((model_artifact.model_id, symbol.to_string()));
                    println!("Deployed model: {} for {}", model_name, symbol);
                }
            }

            // Test all models working initially
            println!("Testing initial model functionality...");
            for (model_id, symbol) in &deployed_models {
                let test_request = PredictionRequest {
                    model_id: model_id.clone(),
                    symbol: symbol.clone(),
                    features: vec![100.0, 101.0, 99.5, 102.0, 100.5],
                };

                let response = harness.grpc_clients.trading_client
                    .get_model_predictions(test_request).await?;

                println!("Model {} prediction: {}", symbol, response.prediction);
            }

            // Simulate initial failure in one component
            println!("Simulating initial failure in model 1...");
            
            // Corrupt the first model to trigger failure
            if let Some((first_model_id, first_symbol)) = deployed_models.first() {
                // Simulate model failure by trying to update with corrupted data
                let corrupt_update = UpdateModelRequest {
                    model_id: first_model_id.clone(),
                    new_model_path: "/invalid/path/corrupt_model.pkl".to_string(),
                };

                let corrupt_result = harness.grpc_clients.trading_client
                    .update_model(corrupt_update).await;

                match corrupt_result {
                    Ok(resp) if !resp.success => {
                        println!("First model failure contained (update rejected)");
                    },
                    Err(_) => {
                        println!("First model failure contained (update failed)");
                    },
                    _ => {
                        println!("First model update unexpectedly succeeded");
                    }
                }
            }

            // Test that other models continue working (failure containment)
            println!("Testing failure containment...");
            let mut working_models = 0;
            let mut failed_models = 0;

            for (model_id, symbol) in &deployed_models[1..] { // Skip the first (failed) model
                let test_request = PredictionRequest {
                    model_id: model_id.clone(),
                    symbol: symbol.clone(),
                    features: vec![100.0, 101.0, 99.5, 102.0, 100.5],
                };

                match harness.grpc_clients.trading_client.get_model_predictions(test_request).await {
                    Ok(response) => {
                        working_models += 1;
                        println!("Model {} still working: {}", symbol, response.prediction);
                    },
                    Err(_) => {
                        failed_models += 1;
                        println!("Model {} affected by cascade: {}", symbol);
                    }
                }
            }

            println!("Cascade failure containment results:");
            println!("  Working models: {}", working_models);
            println!("  Failed models: {}", failed_models);

            // Most models should still be working (failure contained)
            let total_remaining_models = deployed_models.len() - 1; // Excluding the intentionally failed one
            let containment_ratio = working_models as f64 / total_remaining_models as f64;

            assert!(containment_ratio >= 0.5, 
                   "At least 50% of models should remain working during cascade failure");

            if containment_ratio >= 0.8 {
                println!("✅ Excellent cascade failure containment ({:.1}%)", containment_ratio * 100.0);
            } else {
                println!("✅ Acceptable cascade failure containment ({:.1}%)", containment_ratio * 100.0);
            }

            // Test system recovery
            println!("Testing system recovery from cascade failure...");
            
            let recovery_model = harness.test_data.create_model_artifact("RECOVERY", "CASC_RECOVERY").await?;
            
            let recovery_deploy = DeployModelRequest {
                model_id: recovery_model.model_id.clone(),
                model_path: recovery_model.model_path.clone(),
                target_symbols: vec!["CASC_RECOVERY".to_string()],
            };

            let recovery_response = harness.grpc_clients.trading_client
                .deploy_model(recovery_deploy).await?;

            assert!(recovery_response.success, 
                   "System should be able to deploy new models after cascade failure");

            println!("✅ System recovery from cascade failure successful");

            println!("Cascade failure containment test completed");
            Ok(())
        }).await
    }

    /// Test circuit breaker functionality
    async fn test_circuit_breaker_functionality(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("circuit_breaker_functionality", |harness| async move {
            println!("Testing circuit breaker functionality...");

            // Deploy model for circuit breaker testing
            let model_artifact = harness.test_data.create_model_artifact("BREAKER", "CIRCUIT").await?;
            
            let deploy_request = DeployModelRequest {
                model_id: model_artifact.model_id.clone(),
                model_path: model_artifact.model_path.clone(),
                target_symbols: vec!["CIRCUIT".to_string()],
            };

            harness.grpc_clients.trading_client.deploy_model(deploy_request).await?;

            // Test normal operation first
            let normal_request = PredictionRequest {
                model_id: model_artifact.model_id.clone(),
                symbol: "CIRCUIT".to_string(),
                features: vec![100.0, 101.0, 99.5, 102.0, 100.5],
            };

            let normal_response = harness.grpc_clients.trading_client
                .get_model_predictions(normal_request.clone()).await?;

            println!("Normal operation: {} (confidence: {:.3})",
                    normal_response.prediction, normal_response.confidence);

            // Simulate rapid failures to trigger circuit breaker
            println!("Simulating rapid failures to trigger circuit breaker...");
            
            const FAILURE_ATTEMPTS: usize = 10;
            let mut consecutive_failures = 0;
            let mut circuit_breaker_triggered = false;

            for i in 0..FAILURE_ATTEMPTS {
                // Use invalid model ID to simulate failures
                let failure_request = PredictionRequest {
                    model_id: "nonexistent_model".to_string(),
                    symbol: "CIRCUIT".to_string(),
                    features: vec![100.0, 101.0, 99.5, 102.0, 100.5],
                };

                let start_time = Instant::now();
                let result = harness.grpc_clients.trading_client
                    .get_model_predictions(failure_request).await;

                let response_time = start_time.elapsed();

                match result {
                    Ok(_) => {
                        println!("Attempt {}: Unexpected success", i + 1);
                        consecutive_failures = 0;
                    },
                    Err(_) => {
                        consecutive_failures += 1;
                        println!("Attempt {}: Failed (consecutive: {}, response_time: {:?})", 
                                i + 1, consecutive_failures, response_time);

                        // Circuit breaker should kick in after several failures
                        // and start failing fast (very short response times)
                        if consecutive_failures >= 5 && response_time < Duration::from_millis(10) {
                            circuit_breaker_triggered = true;
                            println!("✅ Circuit breaker triggered (fast failure detected)");
                            break;
                        }
                    }
                }

                // Small delay between attempts
                sleep(Duration::from_millis(100)).await;
            }

            if !circuit_breaker_triggered {
                println!("Circuit breaker may not have triggered or is not implemented");
            }

            // Test circuit breaker recovery
            println!("Testing circuit breaker recovery...");
            
            // Wait for circuit breaker to potentially reset
            sleep(Duration::from_secs(5)).await;

            // Test that valid requests work after circuit breaker recovery
            let recovery_response = harness.grpc_clients.trading_client
                .get_model_predictions(normal_request).await?;

            assert!(!recovery_response.prediction.is_empty(),
                   "Valid requests should work after circuit breaker recovery");

            println!("✅ Circuit breaker recovery successful");
            println!("Recovery response: {} (confidence: {:.3})",
                    recovery_response.prediction, recovery_response.confidence);

            // Test that circuit breaker protects system resources
            println!("Testing circuit breaker resource protection...");
            
            // Simulate high-frequency requests that should be throttled
            let mut throttled_requests = 0;
            let mut successful_requests = 0;

            for i in 0..20 {
                let rapid_request = PredictionRequest {
                    model_id: model_artifact.model_id.clone(),
                    symbol: "CIRCUIT".to_string(),
                    features: vec![100.0 + i as f64, 101.0, 99.5, 102.0, 100.5],
                };

                let start_time = Instant::now();
                let result = harness.grpc_clients.trading_client
                    .get_model_predictions(rapid_request).await;

                let response_time = start_time.elapsed();

                match result {
                    Ok(_) => {
                        successful_requests += 1;
                    },
                    Err(_) => {
                        if response_time < Duration::from_millis(5) {
                            throttled_requests += 1; // Fast failure suggests throttling
                        }
                    }
                }

                // No delay - test rapid requests
            }

            println!("High-frequency request results:");
            println!("  Successful: {}", successful_requests);
            println!("  Throttled: {}", throttled_requests);

            if throttled_requests > 0 {
                println!("✅ Circuit breaker providing resource protection");
            } else {
                println!("Circuit breaker may not be throttling high-frequency requests");
            }

            println!("Circuit breaker functionality test completed");
            Ok(())
        }).await
    }
}

// Module-level test runner
#[tokio::test]
async fn run_chaos_engineering_tests() -> Result<()> {
    let mut test_suite = ChaosEngineeringTests::new().await?;
    let results = test_suite.run_all_tests().await?;

    // Print test summary
    let total_tests = results.len();
    let passed_tests = results.iter().filter(|r| r.is_success()).count();
    let failed_tests = total_tests - passed_tests;

    println!("\n=== CHAOS ENGINEERING TEST SUMMARY ===");
    println!("Total chaos tests: {}", total_tests);
    println!("Passed: {}", passed_tests);
    println!("Failed: {}", failed_tests);

    // Print detailed results
    for (i, result) in results.into_iter().enumerate() {
        match result {
            TestResult::Success { duration, .. } => {
                println!("✅ Chaos Test {}: PASSED ({:?})", i + 1, duration);
            },
            TestResult::Failure { duration, error, .. } => {
                println!("❌ Chaos Test {}: FAILED ({:?}) - {}", i + 1, duration, error);
            },
        }
    }

    assert_eq!(failed_tests, 0, "All chaos engineering tests should pass");
    println!("\n🛡️ System resilience validated - all chaos engineering tests passed!");

    Ok(())
}