foxhunt/tests/integration/performance_regression_tests.rs

//! Performance Regression Tests for Foxhunt HFT System
//! 
//! Validates that performance meets HFT requirements and detects regressions:
//! - Sub-microsecond latency requirements
//! - High-throughput ML training scenarios
//! - Resource utilization monitoring
//! - Performance baseline comparison

use anyhow::Result;
use std::time::{Duration, Instant};
use tokio::time::sleep;
use criterion::black_box;

use crate::harness::{TestHarness, TestResult};
use crate::harness::grpc_clients::*;
use crate::harness::performance::{PerformanceMonitor, RegressionResult};

/// Performance regression test suite for HFT requirements
pub struct PerformanceRegressionTests {
    harness: TestHarness,
    baseline_path: String,
}

impl PerformanceRegressionTests {
    pub async fn new() -> Result<Self> {
        let harness = TestHarness::new().await?;
        Ok(Self { 
            harness,
            baseline_path: "/tmp/test_baselines/performance_baseline.json".to_string(),
        })
    }

    /// Run all performance regression tests
    pub async fn run_all_tests(&mut self) -> Result<Vec<TestResult>> {
        let mut results = Vec::new();

        // Setup test environment
        self.harness.setup().await?;

        // Load existing baseline if available
        self.harness.performance = self.harness.performance.load_baseline(&self.baseline_path).await.unwrap_or(self.harness.performance);

        // Core Performance Tests
        results.push(self.test_ml_inference_latency_regression().await?);
        results.push(self.test_trading_signal_latency_regression().await?);
        results.push(self.test_order_execution_latency_regression().await?);
        
        // Throughput Tests
        results.push(self.test_ml_training_throughput_regression().await?);
        results.push(self.test_market_data_processing_throughput().await?);
        results.push(self.test_concurrent_inference_throughput().await?);

        // Scalability Tests
        results.push(self.test_multi_model_scalability().await?);
        results.push(self.test_high_volume_training_scalability().await?);

        // Resource Utilization Tests
        results.push(self.test_memory_usage_regression().await?);
        results.push(self.test_cpu_utilization_regression().await?);
        results.push(self.test_gpu_utilization_regression().await?);

        // Save new baseline
        self.harness.performance.save_baseline(&self.baseline_path).await?;

        // Cleanup
        self.harness.cleanup().await?;

        Ok(results)
    }

    /// Test ML inference latency regression (critical for HFT)
    async fn test_ml_inference_latency_regression(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("ml_inference_latency_regression", |harness| async move {
            println!("Testing ML inference latency regression...");

            // Deploy multiple models for comprehensive testing
            let models = vec![
                ("DQN", "AAPL"),
                ("MAMBA", "TSLA"), 
                ("TFT", "SPY"),
                ("LIQUID", "NVDA"),
            ];

            let mut deployed_models = Vec::new();
            
            for (model_type, symbol) in &models {
                let model_artifact = harness.test_data.create_model_artifact(model_type, symbol).await?;
                
                let deploy_request = DeployModelRequest {
                    model_id: model_artifact.model_id.clone(),
                    model_path: model_artifact.model_path.clone(),
                    target_symbols: vec![symbol.to_string()],
                };

                harness.grpc_clients.trading_client.deploy_model(deploy_request).await?;
                deployed_models.push((model_artifact.model_id, symbol.to_string()));
            }

            // Warm-up phase
            println!("Warming up models...");
            for (model_id, symbol) in &deployed_models {
                for _ in 0..10 {
                    let request = PredictionRequest {
                        model_id: model_id.clone(),
                        symbol: symbol.clone(),
                        features: vec![100.0, 101.0, 99.5, 102.0, 100.5],
                    };
                    harness.grpc_clients.trading_client.get_model_predictions(request).await?;
                }
            }

            // Performance measurement phase
            const INFERENCE_ITERATIONS: usize = 10000;
            println!("Running {} inference iterations for latency measurement", INFERENCE_ITERATIONS);

            for (model_id, symbol) in &deployed_models {
                let mut latencies = Vec::with_capacity(INFERENCE_ITERATIONS);

                for i in 0..INFERENCE_ITERATIONS {
                    let start_time = Instant::now();

                    let request = PredictionRequest {
                        model_id: model_id.clone(),
                        symbol: symbol.clone(),
                        features: vec![
                            100.0 + (i as f64 * 0.01),
                            101.0 + (i as f64 * 0.01),
                            99.5 + (i as f64 * 0.01),
                            102.0 + (i as f64 * 0.01),
                            100.5 + (i as f64 * 0.01),
                        ],
                    };

                    let _response = harness.grpc_clients.trading_client
                        .get_model_predictions(black_box(request)).await?;

                    let latency = start_time.elapsed();
                    latencies.push(latency);

                    harness.performance.record_latency(
                        "ml_inference_latency_regression", 
                        &format!("{}_inference", model_id), 
                        latency
                    );

                    // Prevent overwhelming the system
                    if i % 1000 == 0 && i > 0 {
                        sleep(Duration::from_millis(1)).await;
                    }
                }

                // Analyze latency statistics
                latencies.sort();
                let p50 = latencies[latencies.len() / 2];
                let p95 = latencies[(latencies.len() * 95) / 100];
                let p99 = latencies[(latencies.len() * 99) / 100];

                println!("Model {} ({}) latency statistics:", model_id, symbol);
                println!("  P50: {:?} ({} ns)", p50, p50.as_nanos());
                println!("  P95: {:?} ({} ns)", p95, p95.as_nanos());
                println!("  P99: {:?} ({} ns)", p99, p99.as_nanos());

                // HFT latency requirements (very strict)
                const MAX_P50_LATENCY_NS: u128 = 100_000; // 100μs
                const MAX_P95_LATENCY_NS: u128 = 500_000; // 500μs
                const MAX_P99_LATENCY_NS: u128 = 1_000_000; // 1ms

                assert!(p50.as_nanos() <= MAX_P50_LATENCY_NS, 
                       "P50 latency regression: {} ns > {} ns", p50.as_nanos(), MAX_P50_LATENCY_NS);

                assert!(p95.as_nanos() <= MAX_P95_LATENCY_NS,
                       "P95 latency regression: {} ns > {} ns", p95.as_nanos(), MAX_P95_LATENCY_NS);

                assert!(p99.as_nanos() <= MAX_P99_LATENCY_NS,
                       "P99 latency regression: {} ns > {} ns", p99.as_nanos(), MAX_P99_LATENCY_NS);
            }

            // Check for regression against baseline
            let regression_result = harness.performance.check_regression("ml_inference_latency_regression");
            match regression_result {
                RegressionResult::LatencyRegression { operation, increase_percent } => {
                    if increase_percent > 5.0 { // 5% threshold
                        return Err(anyhow::anyhow!(
                            "Significant latency regression detected in {}: {:.2}% increase",
                            operation, increase_percent
                        ));
                    } else {
                        println!("Minor latency increase in {}: {:.2}%", operation, increase_percent);
                    }
                },
                RegressionResult::NoRegression => {
                    println!("✅ No latency regression detected");
                },
                RegressionResult::NoBaseline => {
                    println!("No baseline available - establishing new baseline");
                },
                _ => {}
            }

            println!("ML inference latency regression test completed");
            Ok(())
        }).await
    }

    /// Test trading signal generation latency
    async fn test_trading_signal_latency_regression(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("trading_signal_latency_regression", |harness| async move {
            println!("Testing trading signal generation latency...");

            // Deploy ensemble model for signal generation
            let model_artifact = harness.test_data.create_model_artifact("ENSEMBLE", "SPY").await?;
            
            let deploy_request = DeployModelRequest {
                model_id: model_artifact.model_id.clone(),
                model_path: model_artifact.model_path.clone(),
                target_symbols: vec!["SPY".to_string()],
            };

            harness.grpc_clients.trading_client.deploy_model(deploy_request).await?;

            // Test signal generation under realistic market conditions
            const SIGNAL_ITERATIONS: usize = 5000;
            println!("Generating {} trading signals for latency measurement", SIGNAL_ITERATIONS);

            let mut end_to_end_latencies = Vec::with_capacity(SIGNAL_ITERATIONS);

            for i in 0..SIGNAL_ITERATIONS {
                let start_time = Instant::now();

                // Simulate real market data features
                let market_features = vec![
                    450.0 + (i as f64 * 0.001), // Price
                    451.2 + (i as f64 * 0.001), // High  
                    449.8 + (i as f64 * 0.001), // Low
                    50_000_000.0 + (i as f64 * 100.0), // Volume
                    0.02 + (i as f64 * 0.00001), // Volatility
                ];

                // Phase 1: Feature processing latency
                let feature_processing_start = Instant::now();
                let processed_features = black_box(market_features.clone());
                let feature_processing_latency = feature_processing_start.elapsed();

                // Phase 2: Model inference latency
                let inference_start = Instant::now();
                let prediction_request = PredictionRequest {
                    model_id: model_artifact.model_id.clone(),
                    symbol: "SPY".to_string(),
                    features: processed_features,
                };

                let prediction_response = harness.grpc_clients.trading_client
                    .get_model_predictions(prediction_request).await?;
                let inference_latency = inference_start.elapsed();

                // Phase 3: Signal generation latency
                let signal_start = Instant::now();
                let signal_strength = black_box(prediction_response.signal_strength);
                let confidence = black_box(prediction_response.confidence);
                let trading_signal = if confidence > 0.7 && signal_strength.abs() > 0.5 {
                    if signal_strength > 0.0 { "BUY" } else { "SELL" }
                } else {
                    "HOLD"
                };
                let signal_generation_latency = signal_start.elapsed();

                let total_latency = start_time.elapsed();
                end_to_end_latencies.push(total_latency);

                // Record individual phase latencies
                harness.performance.record_latency("trading_signal_latency_regression", "feature_processing", feature_processing_latency);
                harness.performance.record_latency("trading_signal_latency_regression", "model_inference", inference_latency);
                harness.performance.record_latency("trading_signal_latency_regression", "signal_generation", signal_generation_latency);
                harness.performance.record_latency("trading_signal_latency_regression", "end_to_end_signal", total_latency);

                // Simulate realistic market timing
                if i % 100 == 0 && i > 0 {
                    sleep(Duration::from_micros(100)).await; // 100μs between bursts
                }
            }

            // Analyze end-to-end latencies
            end_to_end_latencies.sort();
            let p50 = end_to_end_latencies[end_to_end_latencies.len() / 2];
            let p95 = end_to_end_latencies[(end_to_end_latencies.len() * 95) / 100];
            let p99 = end_to_end_latencies[(end_to_end_latencies.len() * 99) / 100];

            println!("End-to-end signal generation latency:");
            println!("  P50: {:?} ({} μs)", p50, p50.as_micros());
            println!("  P95: {:?} ({} μs)", p95, p95.as_micros());
            println!("  P99: {:?} ({} μs)", p99, p99.as_micros());

            // HFT signal generation requirements
            const MAX_P50_SIGNAL_LATENCY_US: u128 = 50; // 50μs
            const MAX_P95_SIGNAL_LATENCY_US: u128 = 200; // 200μs
            const MAX_P99_SIGNAL_LATENCY_US: u128 = 500; // 500μs

            assert!(p50.as_micros() <= MAX_P50_SIGNAL_LATENCY_US,
                   "Signal generation P50 latency regression: {} μs > {} μs",
                   p50.as_micros(), MAX_P50_SIGNAL_LATENCY_US);

            assert!(p95.as_micros() <= MAX_P95_SIGNAL_LATENCY_US,
                   "Signal generation P95 latency regression: {} μs > {} μs", 
                   p95.as_micros(), MAX_P95_SIGNAL_LATENCY_US);

            println!("Trading signal latency regression test completed");
            Ok(())
        }).await
    }

    /// Test order execution latency
    async fn test_order_execution_latency_regression(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("order_execution_latency_regression", |harness| async move {
            println!("Testing order execution latency...");

            const ORDER_ITERATIONS: usize = 1000;
            let mut execution_latencies = Vec::with_capacity(ORDER_ITERATIONS);

            for i in 0..ORDER_ITERATIONS {
                let start_time = Instant::now();

                // Simulate order execution pipeline
                // Phase 1: Order validation
                let validation_start = Instant::now();
                let order_valid = black_box(true); // Simulate validation
                let validation_latency = validation_start.elapsed();

                // Phase 2: Risk check
                let risk_check_start = Instant::now();
                let risk_approved = black_box(true); // Simulate risk approval
                let risk_check_latency = risk_check_start.elapsed();

                // Phase 3: Order submission
                let submission_start = Instant::now();
                let order_submitted = black_box(true); // Simulate submission
                let submission_latency = submission_start.elapsed();

                let total_execution_latency = start_time.elapsed();
                execution_latencies.push(total_execution_latency);

                // Record phase latencies
                harness.performance.record_latency("order_execution_latency_regression", "order_validation", validation_latency);
                harness.performance.record_latency("order_execution_latency_regression", "risk_check", risk_check_latency);
                harness.performance.record_latency("order_execution_latency_regression", "order_submission", submission_latency);
                harness.performance.record_latency("order_execution_latency_regression", "total_execution", total_execution_latency);

                // Simulate realistic order flow timing
                if i % 50 == 0 && i > 0 {
                    sleep(Duration::from_micros(10)).await;
                }
            }

            // Analyze execution latencies
            execution_latencies.sort();
            let p50 = execution_latencies[execution_latencies.len() / 2];
            let p95 = execution_latencies[(execution_latencies.len() * 95) / 100];
            let p99 = execution_latencies[(execution_latencies.len() * 99) / 100];

            println!("Order execution latency:");
            println!("  P50: {:?} ({} μs)", p50, p50.as_micros());
            println!("  P95: {:?} ({} μs)", p95, p95.as_micros());
            println!("  P99: {:?} ({} μs)", p99, p99.as_micros());

            // Ultra-low latency requirements for order execution
            const MAX_P50_EXECUTION_LATENCY_US: u128 = 10; // 10μs
            const MAX_P95_EXECUTION_LATENCY_US: u128 = 50; // 50μs
            const MAX_P99_EXECUTION_LATENCY_US: u128 = 100; // 100μs

            assert!(p50.as_micros() <= MAX_P50_EXECUTION_LATENCY_US,
                   "Order execution P50 latency regression: {} μs > {} μs",
                   p50.as_micros(), MAX_P50_EXECUTION_LATENCY_US);

            println!("Order execution latency regression test completed");
            Ok(())
        }).await
    }

    /// Test ML training throughput regression
    async fn test_ml_training_throughput_regression(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("ml_training_throughput_regression", |harness| async move {
            println!("Testing ML training throughput regression...");

            // Start multiple concurrent training jobs
            let training_jobs = vec![
                ("throughput_test_dqn", "DQN"),
                ("throughput_test_ppo", "PPO"),
                ("throughput_test_mamba", "MAMBA"),
            ];

            let mut started_jobs = Vec::new();
            let overall_start = Instant::now();

            for (job_name, model_type) in &training_jobs {
                let start_time = Instant::now();

                let training_request = StartMLTrainingRequest {
                    model_name: job_name.to_string(),
                    dataset_id: "throughput_test_dataset".to_string(),
                    hyperparameters: vec![
                        ("learning_rate".to_string(), "0.001".to_string()),
                        ("batch_size".to_string(), "64".to_string()),
                        ("epochs".to_string(), "10".to_string()), // Short for throughput test
                    ].into_iter().collect(),
                    auto_deploy: false,
                };

                let response = harness.grpc_clients.tli_client
                    .start_ml_training(training_request).await?;

                let startup_latency = start_time.elapsed();
                harness.performance.record_latency("ml_training_throughput_regression", "job_startup", startup_latency);

                if response.success {
                    started_jobs.push(response.job_id);
                    println!("Started training job: {} ({})", job_name, model_type);
                }
            }

            // Monitor training throughput
            let mut throughput_measurements = 0;
            const MAX_THROUGHPUT_CHECKS: u32 = 20;

            while throughput_measurements < MAX_THROUGHPUT_CHECKS {
                let check_start = Instant::now();
                let mut active_jobs = 0;

                for job_id in &started_jobs {
                    match harness.grpc_clients.tli_client.get_ml_training_status(job_id.clone()).await {
                        Ok(status) => {
                            if status.status == "RUNNING" {
                                active_jobs += 1;
                                
                                // Record training progress as throughput metric
                                if status.current_epoch > 0 {
                                    let epochs_per_minute = status.current_epoch as f64 / 
                                        (check_start.elapsed().as_secs() as f64 / 60.0);
                                    
                                    harness.performance.record_throughput(
                                        "ml_training_throughput_regression",
                                        "epochs_per_minute",
                                        status.current_epoch as u64,
                                        check_start.elapsed(),
                                    );
                                }
                            }
                        },
                        Err(_) => {
                            // Job may have completed or failed
                        }
                    }
                }

                if active_jobs == 0 {
                    println!("All training jobs completed");
                    break;
                }

                throughput_measurements += 1;
                sleep(Duration::from_secs(5)).await;
            }

            let total_duration = overall_start.elapsed();
            
            // Record overall throughput metrics
            harness.performance.record_throughput(
                "ml_training_throughput_regression",
                "concurrent_jobs",
                started_jobs.len() as u64,
                total_duration,
            );

            // Clean up any remaining jobs
            for job_id in &started_jobs {
                harness.grpc_clients.tli_client.stop_ml_training(job_id.clone()).await.ok();
            }

            println!("Processed {} concurrent training jobs in {:?}", 
                    started_jobs.len(), total_duration);

            // Check throughput regression
            let regression_result = harness.performance.check_regression("ml_training_throughput_regression");
            match regression_result {
                RegressionResult::ThroughputRegression { operation, decrease_percent } => {
                    if decrease_percent > 10.0 { // 10% threshold
                        return Err(anyhow::anyhow!(
                            "Significant throughput regression in {}: {:.2}% decrease",
                            operation, decrease_percent
                        ));
                    }
                },
                RegressionResult::NoRegression => {
                    println!("✅ No throughput regression detected");
                },
                _ => {}
            }

            println!("ML training throughput regression test completed");
            Ok(())
        }).await
    }

    /// Test market data processing throughput
    async fn test_market_data_processing_throughput(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("market_data_processing_throughput", |harness| async move {
            println!("Testing market data processing throughput...");

            const MARKET_DATA_POINTS: usize = 100_000;
            let start_time = Instant::now();

            // Generate large volume of market data
            let mut processed_count = 0;

            for i in 0..MARKET_DATA_POINTS {
                // Simulate market data processing
                let market_tick = black_box((
                    450.0 + (i as f64 * 0.001), // Price
                    50_000 + i, // Volume
                    chrono::Utc::now().timestamp_nanos(), // Timestamp
                ));

                // Simulate processing operations
                let _processed = black_box(market_tick.0 * market_tick.1 as f64);
                processed_count += 1;

                // Record progress periodically
                if i % 10_000 == 0 && i > 0 {
                    harness.performance.record_throughput(
                        "market_data_processing_throughput",
                        "data_points_processed",
                        processed_count,
                        start_time.elapsed(),
                    );
                }
            }

            let total_duration = start_time.elapsed();
            let throughput = processed_count as f64 / total_duration.as_secs_f64();

            println!("Processed {} market data points in {:?}", processed_count, total_duration);
            println!("Throughput: {:.0} points/second", throughput);

            // Record final throughput
            harness.performance.record_throughput(
                "market_data_processing_throughput",
                "final_throughput",
                processed_count as u64,
                total_duration,
            );

            // Minimum throughput requirement
            const MIN_THROUGHPUT_PPS: f64 = 50_000.0; // 50K points per second
            assert!(throughput >= MIN_THROUGHPUT_PPS,
                   "Market data throughput regression: {:.0} < {:.0} points/second",
                   throughput, MIN_THROUGHPUT_PPS);

            println!("Market data processing throughput test completed");
            Ok(())
        }).await
    }

    /// Test concurrent inference throughput
    async fn test_concurrent_inference_throughput(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("concurrent_inference_throughput", |harness| async move {
            println!("Testing concurrent inference throughput...");

            // Deploy model for concurrent testing
            let model_artifact = harness.test_data.create_model_artifact("ENSEMBLE", "QQQ").await?;
            
            let deploy_request = DeployModelRequest {
                model_id: model_artifact.model_id.clone(),
                model_path: model_artifact.model_path.clone(),
                target_symbols: vec!["QQQ".to_string()],
            };

            harness.grpc_clients.trading_client.deploy_model(deploy_request).await?;

            // Test concurrent inference load
            const CONCURRENT_REQUESTS: usize = 1000;
            const BATCH_SIZE: usize = 100;
            
            let start_time = Instant::now();
            let mut total_inferences = 0;

            for batch in 0..(CONCURRENT_REQUESTS / BATCH_SIZE) {
                let batch_start = Instant::now();
                let mut batch_futures = Vec::new();

                // Create batch of concurrent requests
                for i in 0..BATCH_SIZE {
                    let request = PredictionRequest {
                        model_id: model_artifact.model_id.clone(),
                        symbol: "QQQ".to_string(),
                        features: vec![
                            350.0 + ((batch * BATCH_SIZE + i) as f64 * 0.01),
                            351.0,
                            349.5,
                            352.0,
                            350.5,
                        ],
                    };

                    let future = harness.grpc_clients.trading_client
                        .get_model_predictions(request);
                    batch_futures.push(future);
                }

                // Wait for all requests in batch to complete
                let results = futures::future::try_join_all(batch_futures).await?;
                
                let batch_duration = batch_start.elapsed();
                let batch_throughput = BATCH_SIZE as f64 / batch_duration.as_secs_f64();
                
                harness.performance.record_throughput(
                    "concurrent_inference_throughput",
                    "batch_inference",
                    BATCH_SIZE as u64,
                    batch_duration,
                );

                total_inferences += results.len();

                println!("Batch {} completed: {} inferences in {:?} ({:.0} inf/sec)",
                        batch + 1, BATCH_SIZE, batch_duration, batch_throughput);

                // Small delay between batches
                sleep(Duration::from_millis(10)).await;
            }

            let total_duration = start_time.elapsed();
            let overall_throughput = total_inferences as f64 / total_duration.as_secs_f64();

            println!("Total concurrent inferences: {} in {:?}", total_inferences, total_duration);
            println!("Overall throughput: {:.0} inferences/second", overall_throughput);

            // Record final throughput metrics
            harness.performance.record_throughput(
                "concurrent_inference_throughput",
                "overall_throughput",
                total_inferences as u64,
                total_duration,
            );

            // Minimum concurrent inference throughput
            const MIN_INFERENCE_THROUGHPUT: f64 = 1000.0; // 1K inferences per second
            assert!(overall_throughput >= MIN_INFERENCE_THROUGHPUT,
                   "Concurrent inference throughput regression: {:.0} < {:.0} inf/sec",
                   overall_throughput, MIN_INFERENCE_THROUGHPUT);

            println!("Concurrent inference throughput test completed");
            Ok(())
        }).await
    }

    /// Test multi-model scalability
    async fn test_multi_model_scalability(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("multi_model_scalability", |harness| async move {
            println!("Testing multi-model scalability...");

            let model_types = vec!["DQN", "PPO", "MAMBA", "TFT", "LIQUID"];
            let symbols = vec!["AAPL", "MSFT", "GOOGL", "TSLA", "NVDA"];
            
            let mut deployed_models = Vec::new();
            let deployment_start = Instant::now();

            // Deploy multiple models
            for model_type in &model_types {
                for symbol in &symbols {
                    let model_artifact = harness.test_data.create_model_artifact(model_type, symbol).await?;
                    
                    let deploy_request = DeployModelRequest {
                        model_id: model_artifact.model_id.clone(),
                        model_path: model_artifact.model_path.clone(),
                        target_symbols: vec![symbol.to_string()],
                    };

                    let deploy_response = harness.grpc_clients.trading_client
                        .deploy_model(deploy_request).await?;

                    if deploy_response.success {
                        deployed_models.push((model_artifact.model_id, symbol.to_string()));
                    }
                }
            }

            let deployment_duration = deployment_start.elapsed();
            println!("Deployed {} models in {:?}", deployed_models.len(), deployment_duration);

            // Test inference performance with all models
            const INFERENCE_ROUNDS: usize = 100;
            let inference_start = Instant::now();

            for round in 0..INFERENCE_ROUNDS {
                let round_start = Instant::now();
                let mut round_futures = Vec::new();

                // Make predictions with all models
                for (model_id, symbol) in &deployed_models {
                    let request = PredictionRequest {
                        model_id: model_id.clone(),
                        symbol: symbol.clone(),
                        features: vec![
                            100.0 + (round as f64),
                            101.0,
                            99.5,
                            102.0,
                            100.5,
                        ],
                    };

                    let future = harness.grpc_clients.trading_client
                        .get_model_predictions(request);
                    round_futures.push(future);
                }

                // Wait for all predictions
                let _results = futures::future::try_join_all(round_futures).await?;
                
                let round_duration = round_start.elapsed();
                harness.performance.record_latency(
                    "multi_model_scalability",
                    "all_models_inference",
                    round_duration,
                );

                if round % 10 == 0 {
                    println!("Round {} completed: {} models in {:?}",
                            round + 1, deployed_models.len(), round_duration);
                }
            }

            let total_inference_duration = inference_start.elapsed();
            let total_predictions = deployed_models.len() * INFERENCE_ROUNDS;
            let prediction_throughput = total_predictions as f64 / total_inference_duration.as_secs_f64();

            println!("Multi-model scalability results:");
            println!("  Models deployed: {}", deployed_models.len());
            println!("  Total predictions: {}", total_predictions);
            println!("  Total time: {:?}", total_inference_duration);
            println!("  Throughput: {:.0} predictions/second", prediction_throughput);

            // Record scalability metrics
            harness.performance.record_throughput(
                "multi_model_scalability",
                "model_deployment",
                deployed_models.len() as u64,
                deployment_duration,
            );

            harness.performance.record_throughput(
                "multi_model_scalability",
                "multi_model_inference",
                total_predictions as u64,
                total_inference_duration,
            );

            // Scalability requirements
            const MIN_MODELS_SUPPORTED: usize = 20;
            const MIN_MULTI_MODEL_THROUGHPUT: f64 = 500.0;

            assert!(deployed_models.len() >= MIN_MODELS_SUPPORTED,
                   "Multi-model scalability regression: {} < {} models",
                   deployed_models.len(), MIN_MODELS_SUPPORTED);

            assert!(prediction_throughput >= MIN_MULTI_MODEL_THROUGHPUT,
                   "Multi-model throughput regression: {:.0} < {:.0} pred/sec",
                   prediction_throughput, MIN_MULTI_MODEL_THROUGHPUT);

            println!("Multi-model scalability test completed");
            Ok(())
        }).await
    }

    /// Test high-volume training scalability
    async fn test_high_volume_training_scalability(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("high_volume_training_scalability", |harness| async move {
            println!("Testing high-volume training scalability...");

            // Start multiple training jobs with large datasets
            const MAX_CONCURRENT_JOBS: usize = 5;
            let mut training_jobs = Vec::new();
            let start_time = Instant::now();

            for i in 0..MAX_CONCURRENT_JOBS {
                let training_request = StartMLTrainingRequest {
                    model_name: format!("scalability_test_model_{}", i),
                    dataset_id: format!("large_dataset_{}", i),
                    hyperparameters: vec![
                        ("learning_rate".to_string(), "0.001".to_string()),
                        ("batch_size".to_string(), "128".to_string()), // Larger batch size
                        ("epochs".to_string(), "20".to_string()),
                    ].into_iter().collect(),
                    auto_deploy: false,
                };

                let response = harness.grpc_clients.tli_client
                    .start_ml_training(training_request).await?;

                if response.success {
                    training_jobs.push(response.job_id);
                    println!("Started high-volume training job {}", i + 1);
                }

                // Small delay between job starts
                sleep(Duration::from_millis(100)).await;
            }

            // Monitor training progress and resource utilization
            let mut monitoring_rounds = 0;
            const MAX_MONITORING_ROUNDS: u32 = 30;

            while monitoring_rounds < MAX_MONITORING_ROUNDS {
                let monitor_start = Instant::now();
                let mut active_jobs = 0;
                let mut total_progress = 0.0;

                for job_id in &training_jobs {
                    match harness.grpc_clients.tli_client.get_ml_training_status(job_id.clone()).await {
                        Ok(status) => {
                            if status.status == "RUNNING" || status.status == "QUEUED" {
                                active_jobs += 1;
                                total_progress += status.progress_percentage;
                            }
                        },
                        Err(_) => {
                            // Job may have completed
                        }
                    }
                }

                // Simulate resource monitoring
                harness.performance.record_resource_usage(
                    "high_volume_training_scalability",
                    85.0 + (monitoring_rounds as f64 * 0.5), // CPU %
                    16384.0 + (monitoring_rounds as f64 * 100.0), // Memory MB
                    Some(90.0 + (monitoring_rounds as f64 * 0.2)), // GPU %
                );

                if active_jobs == 0 {
                    println!("All high-volume training jobs completed");
                    break;
                }

                let avg_progress = if active_jobs > 0 { total_progress / active_jobs as f64 } else { 0.0 };
                println!("Monitoring round {}: {} active jobs, avg progress: {:.1}%",
                        monitoring_rounds + 1, active_jobs, avg_progress);

                monitoring_rounds += 1;
                sleep(Duration::from_secs(10)).await;
            }

            let total_duration = start_time.elapsed();

            // Clean up jobs
            for job_id in &training_jobs {
                harness.grpc_clients.tli_client.stop_ml_training(job_id.clone()).await.ok();
            }

            // Record scalability metrics
            harness.performance.record_throughput(
                "high_volume_training_scalability",
                "concurrent_training_jobs",
                training_jobs.len() as u64,
                total_duration,
            );

            println!("High-volume training scalability results:");
            println!("  Concurrent jobs started: {}", training_jobs.len());
            println!("  Total monitoring duration: {:?}", total_duration);

            // Scalability requirements
            assert!(training_jobs.len() >= 3,
                   "High-volume training scalability: should support at least 3 concurrent jobs");

            println!("High-volume training scalability test completed");
            Ok(())
        }).await
    }

    /// Test memory usage regression
    async fn test_memory_usage_regression(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("memory_usage_regression", |harness| async move {
            println!("Testing memory usage regression...");

            // Baseline memory measurement
            let initial_memory = 1024.0; // MB - simulate initial memory usage

            // Deploy models and monitor memory usage
            let model_types = vec!["DQN", "MAMBA", "TFT"];
            let mut deployed_models = Vec::new();

            for (i, model_type) in model_types.iter().enumerate() {
                let model_artifact = harness.test_data.create_model_artifact(model_type, "AAPL").await?;
                
                let deploy_request = DeployModelRequest {
                    model_id: model_artifact.model_id.clone(),
                    model_path: model_artifact.model_path.clone(),
                    target_symbols: vec!["AAPL".to_string()],
                };

                harness.grpc_clients.trading_client.deploy_model(deploy_request).await?;
                deployed_models.push(model_artifact.model_id);

                // Simulate memory usage increase
                let current_memory = initial_memory + ((i + 1) as f64 * 256.0);
                harness.performance.record_resource_usage(
                    "memory_usage_regression",
                    50.0, // CPU %
                    current_memory,
                    Some(30.0), // GPU %
                );

                println!("Deployed model {}: estimated memory usage {:.0} MB", 
                        model_type, current_memory);
            }

            // Memory stress test with inference load
            const INFERENCE_LOAD: usize = 1000;
            let mut peak_memory = initial_memory;

            for i in 0..INFERENCE_LOAD {
                for model_id in &deployed_models {
                    let request = PredictionRequest {
                        model_id: model_id.clone(),
                        symbol: "AAPL".to_string(),
                        features: vec![150.0 + (i as f64 * 0.01), 151.0, 149.5, 152.0, 150.5],
                    };

                    let _response = harness.grpc_clients.trading_client
                        .get_model_predictions(request).await?;
                }

                // Simulate memory usage fluctuation
                let current_memory = initial_memory + 768.0 + (i as f64 * 0.1);
                peak_memory = peak_memory.max(current_memory);

                if i % 100 == 0 {
                    harness.performance.record_resource_usage(
                        "memory_usage_regression",
                        60.0 + (i as f64 * 0.01),
                        current_memory,
                        Some(40.0),
                    );
                }
            }

            println!("Memory usage analysis:");
            println!("  Initial memory: {:.0} MB", initial_memory);
            println!("  Peak memory: {:.0} MB", peak_memory);
            println!("  Memory increase: {:.0} MB", peak_memory - initial_memory);

            // Memory regression thresholds
            const MAX_MEMORY_INCREASE_MB: f64 = 2048.0; // 2GB increase limit
            const MAX_PEAK_MEMORY_MB: f64 = 8192.0; // 8GB peak limit

            assert!(peak_memory - initial_memory <= MAX_MEMORY_INCREASE_MB,
                   "Memory usage regression: {:.0} MB increase > {:.0} MB limit",
                   peak_memory - initial_memory, MAX_MEMORY_INCREASE_MB);

            assert!(peak_memory <= MAX_PEAK_MEMORY_MB,
                   "Peak memory regression: {:.0} MB > {:.0} MB limit",
                   peak_memory, MAX_PEAK_MEMORY_MB);

            println!("Memory usage regression test completed");
            Ok(())
        }).await
    }

    /// Test CPU utilization regression
    async fn test_cpu_utilization_regression(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("cpu_utilization_regression", |harness| async move {
            println!("Testing CPU utilization regression...");

            // CPU stress test with computational load
            const COMPUTATION_ROUNDS: usize = 100;
            let mut cpu_measurements = Vec::new();

            for round in 0..COMPUTATION_ROUNDS {
                let computation_start = Instant::now();

                // Simulate CPU-intensive operations
                let mut result = 0.0;
                for i in 0..10000 {
                    result += black_box((i as f64).sin() * (i as f64).cos());
                }

                let computation_time = computation_start.elapsed();
                
                // Simulate CPU usage percentage based on computation time
                let cpu_usage = 30.0 + (computation_time.as_micros() as f64 / 1000.0).min(60.0);
                cpu_measurements.push(cpu_usage);

                harness.performance.record_resource_usage(
                    "cpu_utilization_regression",
                    cpu_usage,
                    2048.0, // Memory MB
                    Some(20.0), // GPU %
                );

                if round % 10 == 0 {
                    println!("CPU stress round {}: {:.1}% utilization", round + 1, cpu_usage);
                }
            }

            // Analyze CPU utilization
            let avg_cpu = cpu_measurements.iter().sum::<f64>() / cpu_measurements.len() as f64;
            let max_cpu = cpu_measurements.iter().fold(0.0, |a, &b| a.max(b));

            println!("CPU utilization analysis:");
            println!("  Average CPU usage: {:.1}%", avg_cpu);
            println!("  Peak CPU usage: {:.1}%", max_cpu);

            // CPU utilization thresholds
            const MAX_AVG_CPU_USAGE: f64 = 80.0; // 80% average
            const MAX_PEAK_CPU_USAGE: f64 = 95.0; // 95% peak

            assert!(avg_cpu <= MAX_AVG_CPU_USAGE,
                   "CPU utilization regression: {:.1}% avg > {:.1}% limit",
                   avg_cpu, MAX_AVG_CPU_USAGE);

            assert!(max_cpu <= MAX_PEAK_CPU_USAGE,
                   "Peak CPU utilization regression: {:.1}% > {:.1}% limit",
                   max_cpu, MAX_PEAK_CPU_USAGE);

            println!("CPU utilization regression test completed");
            Ok(())
        }).await
    }

    /// Test GPU utilization regression
    async fn test_gpu_utilization_regression(&mut self) -> Result<TestResult> {
        self.harness.execute_scenario("gpu_utilization_regression", |harness| async move {
            println!("Testing GPU utilization regression...");

            // GPU stress test with ML training simulation
            let training_request = StartMLTrainingRequest {
                model_name: "gpu_stress_test_model".to_string(),
                dataset_id: "gpu_stress_dataset".to_string(),
                hyperparameters: vec![
                    ("learning_rate".to_string(), "0.001".to_string()),
                    ("batch_size".to_string(), "256".to_string()), // Large batch for GPU stress
                    ("epochs".to_string(), "5".to_string()),
                ].into_iter().collect(),
                auto_deploy: false,
            };

            let response = harness.grpc_clients.tli_client
                .start_ml_training(training_request).await?;

            if response.success {
                let job_id = response.job_id;

                // Monitor GPU utilization during training
                let mut gpu_measurements = Vec::new();
                let mut monitoring_rounds = 0;
                const MAX_GPU_MONITORING: u32 = 20;

                while monitoring_rounds < MAX_GPU_MONITORING {
                    let status = harness.grpc_clients.tli_client
                        .get_ml_training_status(job_id.clone()).await?;

                    if status.status == "FAILED" || status.status == "COMPLETED" {
                        break;
                    }

                    // Simulate GPU utilization based on training progress
                    let gpu_usage = if status.status == "RUNNING" {
                        70.0 + (status.progress_percentage * 0.2) + (monitoring_rounds as f64 * 0.5)
                    } else {
                        20.0 // Idle
                    };

                    gpu_measurements.push(gpu_usage);

                    harness.performance.record_resource_usage(
                        "gpu_utilization_regression",
                        50.0, // CPU %
                        4096.0, // Memory MB
                        Some(gpu_usage),
                    );

                    println!("GPU monitoring round {}: {:.1}% utilization (training: {}%)",
                            monitoring_rounds + 1, gpu_usage, status.progress_percentage);

                    monitoring_rounds += 1;
                    sleep(Duration::from_secs(3)).await;
                }

                // Stop training
                harness.grpc_clients.tli_client.stop_ml_training(job_id).await.ok();

                // Analyze GPU utilization
                if !gpu_measurements.is_empty() {
                    let avg_gpu = gpu_measurements.iter().sum::<f64>() / gpu_measurements.len() as f64;
                    let max_gpu = gpu_measurements.iter().fold(0.0, |a, &b| a.max(b));

                    println!("GPU utilization analysis:");
                    println!("  Average GPU usage: {:.1}%", avg_gpu);
                    println!("  Peak GPU usage: {:.1}%", max_gpu);

                    // GPU utilization thresholds
                    const MAX_AVG_GPU_USAGE: f64 = 90.0; // 90% average during training
                    const MAX_PEAK_GPU_USAGE: f64 = 100.0; // 100% peak (acceptable)

                    assert!(max_gpu <= MAX_PEAK_GPU_USAGE,
                           "Peak GPU utilization regression: {:.1}% > {:.1}% limit",
                           max_gpu, MAX_PEAK_GPU_USAGE);

                    // Note: We don't assert on average GPU usage being too high during training
                    // as high GPU usage is expected and desirable during ML training
                    if avg_gpu > MAX_AVG_GPU_USAGE {
                        println!("Warning: High average GPU usage {:.1}%, but this may be expected during training", avg_gpu);
                    }
                } else {
                    println!("No GPU measurements recorded - training may have completed quickly");
                }
            } else {
                println!("GPU stress training failed to start - this may indicate resource constraints");
            }

            println!("GPU utilization regression test completed");
            Ok(())
        }).await
    }
}

// Module-level test runner
#[tokio::test]
async fn run_performance_regression_tests() -> Result<()> {
    let mut test_suite = PerformanceRegressionTests::new().await?;
    let results = test_suite.run_all_tests().await?;

    // Print test summary
    let total_tests = results.len();
    let passed_tests = results.iter().filter(|r| r.is_success()).count();
    let failed_tests = total_tests - passed_tests;

    println!("\n=== PERFORMANCE REGRESSION TEST SUMMARY ===");
    println!("Total tests: {}", total_tests);
    println!("Passed: {}", passed_tests);
    println!("Failed: {}", failed_tests);

    for (i, result) in results.iter().enumerate() {
        match result {
            TestResult::Success { duration, .. } => {
                println!("✅ Performance Test {}: PASSED ({:?})", i + 1, duration);
            },
            TestResult::Failure { duration, error, .. } => {
                println!("❌ Performance Test {}: FAILED ({:?}) - {}", i + 1, duration, error);
            },
        }
    }

    assert_eq!(failed_tests, 0, "All performance regression tests should pass");
    println!("\n🚀 All performance requirements validated - no regressions detected!");

    Ok(())
}