foxhunt/tli/tests/integration/error_handling_tests.rs

//! Error handling and resilience integration tests for TLI system
//!
//! This module tests various error scenarios including service unavailability,
//! network timeouts, database connection failures, and invalid data handling.

use std::sync::{Arc, Mutex};
use std::time::Duration;
use tokio::time::{sleep, timeout};
use uuid::Uuid;
use wiremock::{
    matchers::{method, path},
    Mock, MockServer, ResponseTemplate,
};

use crate::integration::{TestConfig, TestUtilities};
use crate::mocks::grpc_server::{FailureMode, MockBacktestingServer, MockTradingServer};
use tli::client::{BacktestingClient, TliClientBuilder, TradingClient};
use tli::error::{ErrorCode, ErrorSeverity, TliError};
use tli::prelude::*;

/// Error handling test suite
pub struct ErrorHandlingTests {
    config: TestConfig,
    trading_client: Option<TradingClient>,
    backtesting_client: Option<BacktestingClient>,
    mock_servers: Vec<Box<dyn MockServer>>,
    wiremock_servers: Vec<MockServer>,
}

impl ErrorHandlingTests {
    pub fn new(config: TestConfig) -> Self {
        Self {
            config,
            trading_client: None,
            backtesting_client: None,
            mock_servers: Vec::new(),
            wiremock_servers: Vec::new(),
        }
    }

    /// Setup test environment with controllable failure modes
    pub async fn setup(&mut self) -> TliResult<()> {
        tracing::info!("Setting up error handling test environment");

        // Start controllable mock servers
        let mut trading_server = MockTradingServer::new(self.config.mock_server_port)?;
        let trading_port = trading_server.start()?;
        self.mock_servers.push(Box::new(trading_server));

        let mut backtesting_server =
            MockBacktestingServer::new(self.config.mock_server_port + 100)?;
        let backtesting_port = backtesting_server.start()?;
        self.mock_servers.push(Box::new(backtesting_server));

        // Wait for services to be ready
        sleep(Duration::from_millis(300)).await;

        // Create TLI client suite with retry configuration
        let client_suite = TliClientBuilder::new()
            .with_service_endpoint(
                "trading_service".to_string(),
                format!("http://localhost:{}", trading_port),
            )
            .with_service_endpoint(
                "backtesting_service".to_string(),
                format!("http://localhost:{}", backtesting_port),
            )
            .with_trading_config(create_resilient_trading_config())
            .with_backtesting_config(create_resilient_backtesting_config())
            .build()
            .await?;

        self.trading_client = client_suite.trading_client;
        self.backtesting_client = client_suite.backtesting_client;

        tracing::info!("Error handling test environment setup complete");
        Ok(())
    }

    /// Setup test environment for network failure scenarios
    pub async fn setup_with_network_failures(&mut self) -> TliResult<()> {
        tracing::info!("Setting up network failure test environment");

        // Start WireMock servers to simulate network issues
        let trading_mock = MockServer::start().await;
        let backtesting_mock = MockServer::start().await;

        let trading_port = trading_mock.address().port();
        let backtesting_port = backtesting_mock.address().port();

        self.wiremock_servers.push(trading_mock);
        self.wiremock_servers.push(backtesting_mock);

        // Create clients pointing to WireMock servers
        let client_suite = TliClientBuilder::new()
            .with_service_endpoint(
                "trading_service".to_string(),
                format!("http://localhost:{}", trading_port),
            )
            .with_service_endpoint(
                "backtesting_service".to_string(),
                format!("http://localhost:{}", backtesting_port),
            )
            .with_trading_config(create_resilient_trading_config())
            .with_backtesting_config(create_resilient_backtesting_config())
            .build()
            .await?;

        self.trading_client = client_suite.trading_client;
        self.backtesting_client = client_suite.backtesting_client;

        tracing::info!("Network failure test environment setup complete");
        Ok(())
    }

    /// Cleanup test environment
    pub async fn teardown(&mut self) -> TliResult<()> {
        tracing::info!("Tearing down error handling test environment");

        // Shutdown clients
        if let Some(client) = self.trading_client.take() {
            client.shutdown().await;
        }
        if let Some(client) = self.backtesting_client.take() {
            client.shutdown().await;
        }

        // Stop mock servers
        for server in &mut self.mock_servers {
            let _ = server.stop();
        }
        self.mock_servers.clear();

        // WireMock servers are automatically cleaned up when dropped
        self.wiremock_servers.clear();

        tracing::info!("Error handling test environment teardown complete");
        Ok(())
    }

    /// Configure mock server to simulate specific failure mode
    pub async fn configure_failure_mode(
        &mut self,
        service: &str,
        failure_mode: FailureMode,
    ) -> TliResult<()> {
        for server in &mut self.mock_servers {
            if server.get_service_name() == service {
                server.set_failure_mode(failure_mode)?;
                break;
            }
        }
        Ok(())
    }
}

fn create_resilient_trading_config() -> TradingClientConfig {
    TradingClientConfig {
        service_name: "trading_service".to_string(),
        request_timeout: Duration::from_secs(5),
        order_validation: OrderValidationConfig {
            enable_pre_trade_checks: true,
            max_order_value: 1000000.0,
            require_confirmation: false,
        },
        risk_management: RiskManagementConfig {
            enable_position_limits: true,
            max_position_size: 10000.0,
            max_daily_loss: 50000.0,
        },
        market_data: MarketDataConfig {
            subscription_timeout: Duration::from_secs(30),
            reconnect_interval: Duration::from_secs(5),
            max_reconnect_attempts: 5,
        },
        monitoring: MonitoringConfig {
            enable_health_checks: true,
            health_check_interval: Duration::from_secs(10),
            enable_circuit_breaker: true,
            circuit_breaker_threshold: 5,
        },
        event_streaming: EventStreamConfig {
            buffer_size: 1000,
            reconnect_policy: ReconnectPolicy::ExponentialBackoff,
            max_reconnect_attempts: 10,
        },
    }
}

fn create_resilient_backtesting_config() -> BacktestingClientConfig {
    BacktestingClientConfig {
        service_name: "backtesting_service".to_string(),
        request_timeout: Duration::from_secs(30),
        long_running_timeout: Duration::from_secs(300),
        retry_config: RetryConfig {
            max_attempts: 3,
            initial_delay: Duration::from_millis(100),
            max_delay: Duration::from_secs(10),
            backoff_multiplier: 2.0,
        },
    }
}

/// Service unavailability tests
#[cfg(test)]
mod service_unavailability_tests {
    use super::*;

    #[tokio::test]
    async fn test_trading_service_unavailable() {
        let mut test_env = ErrorHandlingTests::new(TestConfig::default());
        test_env
            .setup()
            .await
            .expect("Failed to setup test environment");

        // Configure trading service to be unavailable
        test_env
            .configure_failure_mode("trading_service", FailureMode::ServiceUnavailable)
            .await
            .expect("Failed to configure failure mode");

        let trading_client = test_env
            .trading_client
            .as_ref()
            .expect("Trading client not available");

        // Attempt order submission
        let order_request = SubmitOrderRequest {
            symbol: TestUtilities::generate_test_symbol("AAPL"),
            side: OrderSide::Buy as i32,
            order_type: OrderType::Market as i32,
            quantity: 100.0,
            price: Some(150.0),
            stop_price: None,
            time_in_force: "DAY".to_string(),
            client_order_id: Uuid::new_v4().to_string(),
        };

        let result = trading_client.submit_order(order_request).await;

        assert!(
            result.is_err(),
            "Order submission should fail when service unavailable"
        );

        let error = result.unwrap_err();
        match error {
            TliError::ServiceUnavailable(_) => {
                tracing::info!("Correctly identified service unavailable error");
            }
            _ => panic!("Expected ServiceUnavailable error, got: {:?}", error),
        }

        test_env
            .teardown()
            .await
            .expect("Failed to teardown test environment");
    }

    #[tokio::test]
    async fn test_service_recovery_after_failure() {
        let mut test_env = ErrorHandlingTests::new(TestConfig::default());
        test_env
            .setup()
            .await
            .expect("Failed to setup test environment");

        let trading_client = test_env
            .trading_client
            .as_ref()
            .expect("Trading client not available");

        // First, make service unavailable
        test_env
            .configure_failure_mode("trading_service", FailureMode::ServiceUnavailable)
            .await
            .expect("Failed to configure failure mode");

        let order_request = SubmitOrderRequest {
            symbol: TestUtilities::generate_test_symbol("AAPL"),
            side: OrderSide::Buy as i32,
            order_type: OrderType::Market as i32,
            quantity: 100.0,
            price: Some(150.0),
            stop_price: None,
            time_in_force: "DAY".to_string(),
            client_order_id: Uuid::new_v4().to_string(),
        };

        // Verify service is failing
        let result = trading_client.submit_order(order_request.clone()).await;
        assert!(result.is_err(), "Service should be failing");

        // Restore service
        test_env
            .configure_failure_mode("trading_service", FailureMode::None)
            .await
            .expect("Failed to restore service");

        // Wait for circuit breaker to reset
        sleep(Duration::from_secs(1)).await;

        // Verify service recovery
        let recovery_result = trading_client.submit_order(order_request).await;
        assert!(
            recovery_result.is_ok(),
            "Service should recover after restoration"
        );

        test_env
            .teardown()
            .await
            .expect("Failed to teardown test environment");
    }

    #[tokio::test]
    async fn test_partial_service_failure() {
        let mut test_env = ErrorHandlingTests::new(TestConfig::default());
        test_env
            .setup()
            .await
            .expect("Failed to setup test environment");

        // Configure only backtesting service to fail
        test_env
            .configure_failure_mode("backtesting_service", FailureMode::ServiceUnavailable)
            .await
            .expect("Failed to configure failure mode");

        let trading_client = test_env
            .trading_client
            .as_ref()
            .expect("Trading client not available");
        let backtesting_client = test_env
            .backtesting_client
            .as_ref()
            .expect("Backtesting client not available");

        // Trading service should still work
        let order_request = SubmitOrderRequest {
            symbol: TestUtilities::generate_test_symbol("AAPL"),
            side: OrderSide::Buy as i32,
            order_type: OrderType::Market as i32,
            quantity: 100.0,
            price: Some(150.0),
            stop_price: None,
            time_in_force: "DAY".to_string(),
            client_order_id: Uuid::new_v4().to_string(),
        };

        let trading_result = trading_client.submit_order(order_request).await;
        assert!(trading_result.is_ok(), "Trading service should still work");

        // Backtesting service should fail
        let backtest_request = CreateBacktestRequest {
            name: "Test Backtest".to_string(),
            strategy_id: "test_strategy".to_string(),
            start_date: "2024-01-01".to_string(),
            end_date: "2024-01-31".to_string(),
            initial_capital: 100000.0,
            symbols: vec!["AAPL".to_string()],
            parameters: HashMap::new(),
        };

        let backtest_result = backtesting_client.create_backtest(backtest_request).await;
        assert!(backtest_result.is_err(), "Backtesting service should fail");

        test_env
            .teardown()
            .await
            .expect("Failed to teardown test environment");
    }
}

/// Network timeout and connection tests
#[cfg(test)]
mod network_timeout_tests {
    use super::*;

    #[tokio::test]
    async fn test_connection_timeout_handling() {
        let mut test_env = ErrorHandlingTests::new(TestConfig::default());
        test_env
            .setup_with_network_failures()
            .await
            .expect("Failed to setup test environment");

        // Configure mock to not respond (simulate timeout)
        let trading_mock = &test_env.wiremock_servers[0];
        Mock::given(method("POST"))
            .and(path("/trading.TradingService/SubmitOrder"))
            .respond_with(ResponseTemplate::new(200).set_delay(Duration::from_secs(10))) // Longer than client timeout
            .mount(trading_mock)
            .await;

        let trading_client = test_env
            .trading_client
            .as_ref()
            .expect("Trading client not available");

        let order_request = SubmitOrderRequest {
            symbol: TestUtilities::generate_test_symbol("AAPL"),
            side: OrderSide::Buy as i32,
            order_type: OrderType::Market as i32,
            quantity: 100.0,
            price: Some(150.0),
            stop_price: None,
            time_in_force: "DAY".to_string(),
            client_order_id: Uuid::new_v4().to_string(),
        };

        let start_time = std::time::Instant::now();
        let result = trading_client.submit_order(order_request).await;
        let elapsed = start_time.elapsed();

        assert!(result.is_err(), "Request should timeout");
        assert!(
            elapsed < Duration::from_secs(8),
            "Should timeout within client timeout period"
        );

        let error = result.unwrap_err();
        match error {
            TliError::Timeout(_) => {
                tracing::info!("Correctly identified timeout error");
            }
            _ => panic!("Expected Timeout error, got: {:?}", error),
        }

        test_env
            .teardown()
            .await
            .expect("Failed to teardown test environment");
    }

    #[tokio::test]
    async fn test_connection_refused_handling() {
        let mut test_env = ErrorHandlingTests::new(TestConfig::default());

        // Create clients pointing to non-existent services
        let client_suite = TliClientBuilder::new()
            .with_service_endpoint(
                "trading_service".to_string(),
                "http://localhost:99999".to_string(),
            ) // Invalid port
            .with_service_endpoint(
                "backtesting_service".to_string(),
                "http://localhost:99998".to_string(),
            ) // Invalid port
            .with_trading_config(create_resilient_trading_config())
            .with_backtesting_config(create_resilient_backtesting_config())
            .build()
            .await
            .expect("Client creation should succeed");

        test_env.trading_client = client_suite.trading_client;

        let trading_client = test_env
            .trading_client
            .as_ref()
            .expect("Trading client not available");

        let order_request = SubmitOrderRequest {
            symbol: TestUtilities::generate_test_symbol("AAPL"),
            side: OrderSide::Buy as i32,
            order_type: OrderType::Market as i32,
            quantity: 100.0,
            price: Some(150.0),
            stop_price: None,
            time_in_force: "DAY".to_string(),
            client_order_id: Uuid::new_v4().to_string(),
        };

        let result = trading_client.submit_order(order_request).await;

        assert!(result.is_err(), "Connection should be refused");

        let error = result.unwrap_err();
        match error {
            TliError::Connection(_) => {
                tracing::info!("Correctly identified connection error");
            }
            _ => panic!("Expected Connection error, got: {:?}", error),
        }

        test_env
            .teardown()
            .await
            .expect("Failed to teardown test environment");
    }

    #[tokio::test]
    async fn test_intermittent_network_failures() {
        let mut test_env = ErrorHandlingTests::new(TestConfig::default());
        test_env
            .setup_with_network_failures()
            .await
            .expect("Failed to setup test environment");

        let trading_mock = &test_env.wiremock_servers[0];

        // Configure mock to fail 50% of the time
        let success_counter = Arc::new(Mutex::new(0));
        let counter_clone = success_counter.clone();

        Mock::given(method("POST"))
            .and(path("/trading.TradingService/SubmitOrder"))
            .respond_with(move |_req| {
                let mut counter = counter_clone.lock().unwrap();
                *counter += 1;
                if *counter % 2 == 0 {
                    ResponseTemplate::new(200).set_body_json(serde_json::json!({
                        "success": true,
                        "order_id": "test_order_123",
                        "message": "Order submitted successfully",
                        "timestamp_unix_nanos": 1234567890000000000i64
                    }))
                } else {
                    ResponseTemplate::new(500).set_body("Internal Server Error")
                }
            })
            .mount(trading_mock)
            .await;

        let trading_client = test_env
            .trading_client
            .as_ref()
            .expect("Trading client not available");

        // Attempt multiple requests
        let mut successful_requests = 0;
        let mut failed_requests = 0;
        let total_requests = 10;

        for i in 0..total_requests {
            let order_request = SubmitOrderRequest {
                symbol: TestUtilities::generate_test_symbol("AAPL"),
                side: OrderSide::Buy as i32,
                order_type: OrderType::Market as i32,
                quantity: 100.0,
                price: Some(150.0),
                stop_price: None,
                time_in_force: "DAY".to_string(),
                client_order_id: format!("order_{}", i),
            };

            match trading_client.submit_order(order_request).await {
                Ok(_) => successful_requests += 1,
                Err(_) => failed_requests += 1,
            }
        }

        tracing::info!(
            "Intermittent failure test: {} successful, {} failed",
            successful_requests,
            failed_requests
        );

        // Should have both successes and failures
        assert!(
            successful_requests > 0,
            "Should have some successful requests"
        );
        assert!(failed_requests > 0, "Should have some failed requests");

        test_env
            .teardown()
            .await
            .expect("Failed to teardown test environment");
    }
}

/// Database connection failure tests
#[cfg(test)]
mod database_failure_tests {
    use super::*;

    #[tokio::test]
    async fn test_database_connection_failure() {
        let mut test_env = ErrorHandlingTests::new(TestConfig::default());
        test_env
            .setup()
            .await
            .expect("Failed to setup test environment");

        // Configure database failure
        test_env
            .configure_failure_mode("trading_service", FailureMode::DatabaseError)
            .await
            .expect("Failed to configure database failure");

        let trading_client = test_env
            .trading_client
            .as_ref()
            .expect("Trading client not available");

        // Attempt to get positions (requires database access)
        let position_request = GetPositionsRequest {
            account_id: Some("test_account".to_string()),
            symbol_filter: None,
            include_zero_positions: false,
        };

        let result = trading_client.get_positions(position_request).await;

        assert!(result.is_err(), "Database operation should fail");

        let error = result.unwrap_err();
        match error {
            TliError::Database(_) => {
                tracing::info!("Correctly identified database error");
            }
            _ => panic!("Expected Database error, got: {:?}", error),
        }

        test_env
            .teardown()
            .await
            .expect("Failed to teardown test environment");
    }

    #[tokio::test]
    async fn test_database_transaction_rollback() {
        let mut test_env = ErrorHandlingTests::new(TestConfig::default());
        test_env
            .setup()
            .await
            .expect("Failed to setup test environment");

        // Configure intermittent database failures
        test_env
            .configure_failure_mode("trading_service", FailureMode::TransactionFailure)
            .await
            .expect("Failed to configure transaction failure");

        let trading_client = test_env
            .trading_client
            .as_ref()
            .expect("Trading client not available");

        // Attempt batch order submission (requires transactions)
        let batch_orders = vec![
            SubmitOrderRequest {
                symbol: TestUtilities::generate_test_symbol("AAPL"),
                side: OrderSide::Buy as i32,
                order_type: OrderType::Market as i32,
                quantity: 100.0,
                price: Some(150.0),
                stop_price: None,
                time_in_force: "DAY".to_string(),
                client_order_id: Uuid::new_v4().to_string(),
            },
            SubmitOrderRequest {
                symbol: TestUtilities::generate_test_symbol("GOOGL"),
                side: OrderSide::Buy as i32,
                order_type: OrderType::Market as i32,
                quantity: 50.0,
                price: Some(2800.0),
                stop_price: None,
                time_in_force: "DAY".to_string(),
                client_order_id: Uuid::new_v4().to_string(),
            },
        ];

        let batch_request = SubmitBatchOrdersRequest {
            orders: batch_orders,
            all_or_none: true, // Requires transaction
            max_acceptable_failures: 0,
        };

        let result = trading_client.submit_batch_orders(batch_request).await;

        assert!(
            result.is_err(),
            "Batch operation should fail due to transaction failure"
        );

        let error = result.unwrap_err();
        match error {
            TliError::Database(_) | TliError::TransactionFailure(_) => {
                tracing::info!("Correctly identified transaction failure");
            }
            _ => panic!(
                "Expected Database or TransactionFailure error, got: {:?}",
                error
            ),
        }

        test_env
            .teardown()
            .await
            .expect("Failed to teardown test environment");
    }

    #[tokio::test]
    async fn test_database_recovery_mechanisms() {
        let mut test_env = ErrorHandlingTests::new(TestConfig::default());
        test_env
            .setup()
            .await
            .expect("Failed to setup test environment");

        let trading_client = test_env
            .trading_client
            .as_ref()
            .expect("Trading client not available");

        // First, cause database failure
        test_env
            .configure_failure_mode("trading_service", FailureMode::DatabaseError)
            .await
            .expect("Failed to configure database failure");

        let position_request = GetPositionsRequest {
            account_id: Some("test_account".to_string()),
            symbol_filter: None,
            include_zero_positions: false,
        };

        // Verify failure
        let result = trading_client.get_positions(position_request.clone()).await;
        assert!(result.is_err(), "Database operation should fail");

        // Restore database
        test_env
            .configure_failure_mode("trading_service", FailureMode::None)
            .await
            .expect("Failed to restore database");

        // Wait for recovery
        sleep(Duration::from_secs(1)).await;

        // Verify recovery
        let recovery_result = trading_client.get_positions(position_request).await;
        assert!(
            recovery_result.is_ok(),
            "Database operation should succeed after recovery"
        );

        test_env
            .teardown()
            .await
            .expect("Failed to teardown test environment");
    }
}

/// Invalid data handling tests
#[cfg(test)]
mod invalid_data_tests {
    use super::*;

    #[tokio::test]
    async fn test_invalid_order_data_handling() {
        let mut test_env = ErrorHandlingTests::new(TestConfig::default());
        test_env
            .setup()
            .await
            .expect("Failed to setup test environment");

        let trading_client = test_env
            .trading_client
            .as_ref()
            .expect("Trading client not available");

        // Test invalid symbol
        let invalid_symbol_request = SubmitOrderRequest {
            symbol: "".to_string(), // Empty symbol
            side: OrderSide::Buy as i32,
            order_type: OrderType::Market as i32,
            quantity: 100.0,
            price: Some(150.0),
            stop_price: None,
            time_in_force: "DAY".to_string(),
            client_order_id: Uuid::new_v4().to_string(),
        };

        let result = trading_client.submit_order(invalid_symbol_request).await;
        assert!(result.is_err(), "Order with empty symbol should fail");

        let error = result.unwrap_err();
        match error {
            TliError::InvalidInput(_) | TliError::Validation(_) => {
                tracing::info!("Correctly identified invalid symbol error");
            }
            _ => panic!(
                "Expected InvalidInput or Validation error, got: {:?}",
                error
            ),
        }

        // Test invalid quantity
        let invalid_quantity_request = SubmitOrderRequest {
            symbol: TestUtilities::generate_test_symbol("AAPL"),
            side: OrderSide::Buy as i32,
            order_type: OrderType::Market as i32,
            quantity: -100.0, // Negative quantity
            price: Some(150.0),
            stop_price: None,
            time_in_force: "DAY".to_string(),
            client_order_id: Uuid::new_v4().to_string(),
        };

        let result = trading_client.submit_order(invalid_quantity_request).await;
        assert!(result.is_err(), "Order with negative quantity should fail");

        // Test invalid price
        let invalid_price_request = SubmitOrderRequest {
            symbol: TestUtilities::generate_test_symbol("AAPL"),
            side: OrderSide::Buy as i32,
            order_type: OrderType::Limit as i32,
            quantity: 100.0,
            price: Some(-10.0), // Negative price
            stop_price: None,
            time_in_force: "DAY".to_string(),
            client_order_id: Uuid::new_v4().to_string(),
        };

        let result = trading_client.submit_order(invalid_price_request).await;
        assert!(result.is_err(), "Order with negative price should fail");

        test_env
            .teardown()
            .await
            .expect("Failed to teardown test environment");
    }

    #[tokio::test]
    async fn test_malformed_configuration_data() {
        let mut test_env = ErrorHandlingTests::new(TestConfig::default());
        test_env
            .setup()
            .await
            .expect("Failed to setup test environment");

        let trading_client = test_env
            .trading_client
            .as_ref()
            .expect("Trading client not available");

        // Test invalid configuration values
        let invalid_configs = vec![
            Configuration {
                id: Uuid::new_v4().to_string(),
                config_type: ConfigurationType::TradingLimits as i32,
                key: "max_position_size".to_string(),
                value: "not_a_number".to_string(), // Invalid numeric value
                account_id: Some("test_account".to_string()),
                last_updated: chrono::Utc::now().timestamp(),
            },
            Configuration {
                id: Uuid::new_v4().to_string(),
                config_type: ConfigurationType::TradingLimits as i32,
                key: "".to_string(), // Empty key
                value: "1000".to_string(),
                account_id: Some("test_account".to_string()),
                last_updated: chrono::Utc::now().timestamp(),
            },
        ];

        let update_request = UpdateConfigurationRequest {
            configurations: invalid_configs,
            validate_before_update: true,
        };

        let result = trading_client.update_configuration(update_request).await;

        // Should either fail validation or return validation errors
        match result {
            Ok(response) => {
                assert!(
                    !response.success || !response.validation_errors.is_empty(),
                    "Invalid configuration should fail validation"
                );
            }
            Err(error) => match error {
                TliError::InvalidInput(_) | TliError::Validation(_) => {
                    tracing::info!("Correctly identified invalid configuration error");
                }
                _ => panic!(
                    "Expected InvalidInput or Validation error, got: {:?}",
                    error
                ),
            },
        }

        test_env
            .teardown()
            .await
            .expect("Failed to teardown test environment");
    }

    #[tokio::test]
    async fn test_corrupted_backtest_data() {
        let mut test_env = ErrorHandlingTests::new(TestConfig::default());
        test_env
            .setup()
            .await
            .expect("Failed to setup test environment");

        let backtesting_client = test_env
            .backtesting_client
            .as_ref()
            .expect("Backtesting client not available");

        // Test invalid date format
        let invalid_date_request = CreateBacktestRequest {
            name: "Invalid Date Test".to_string(),
            strategy_id: "test_strategy".to_string(),
            start_date: "invalid-date-format".to_string(), // Invalid date
            end_date: "2024-01-31".to_string(),
            initial_capital: 100000.0,
            symbols: vec!["AAPL".to_string()],
            parameters: HashMap::new(),
        };

        let result = backtesting_client
            .create_backtest(invalid_date_request)
            .await;
        assert!(result.is_err(), "Backtest with invalid date should fail");

        // Test invalid initial capital
        let invalid_capital_request = CreateBacktestRequest {
            name: "Invalid Capital Test".to_string(),
            strategy_id: "test_strategy".to_string(),
            start_date: "2024-01-01".to_string(),
            end_date: "2024-01-31".to_string(),
            initial_capital: -1000.0, // Negative capital
            symbols: vec!["AAPL".to_string()],
            parameters: HashMap::new(),
        };

        let result = backtesting_client
            .create_backtest(invalid_capital_request)
            .await;
        assert!(
            result.is_err(),
            "Backtest with negative capital should fail"
        );

        // Test empty symbols list
        let empty_symbols_request = CreateBacktestRequest {
            name: "Empty Symbols Test".to_string(),
            strategy_id: "test_strategy".to_string(),
            start_date: "2024-01-01".to_string(),
            end_date: "2024-01-31".to_string(),
            initial_capital: 100000.0,
            symbols: vec![], // Empty symbols
            parameters: HashMap::new(),
        };

        let result = backtesting_client
            .create_backtest(empty_symbols_request)
            .await;
        assert!(result.is_err(), "Backtest with empty symbols should fail");

        test_env
            .teardown()
            .await
            .expect("Failed to teardown test environment");
    }
}

/// Circuit breaker and rate limiting tests
#[cfg(test)]
mod resilience_pattern_tests {
    use super::*;

    #[tokio::test]
    async fn test_circuit_breaker_activation() {
        let mut test_env = ErrorHandlingTests::new(TestConfig::default());
        test_env
            .setup()
            .await
            .expect("Failed to setup test environment");

        // Configure service to fail consistently
        test_env
            .configure_failure_mode("trading_service", FailureMode::InternalError)
            .await
            .expect("Failed to configure failure mode");

        let trading_client = test_env
            .trading_client
            .as_ref()
            .expect("Trading client not available");

        let order_request = SubmitOrderRequest {
            symbol: TestUtilities::generate_test_symbol("AAPL"),
            side: OrderSide::Buy as i32,
            order_type: OrderType::Market as i32,
            quantity: 100.0,
            price: Some(150.0),
            stop_price: None,
            time_in_force: "DAY".to_string(),
            client_order_id: Uuid::new_v4().to_string(),
        };

        // Make multiple requests to trigger circuit breaker
        let mut consecutive_failures = 0;
        let max_attempts = 10;

        for attempt in 1..=max_attempts {
            let result = trading_client.submit_order(order_request.clone()).await;

            if result.is_err() {
                consecutive_failures += 1;
                tracing::info!("Attempt {}: Request failed", attempt);

                // Check if we're getting circuit breaker errors
                let error = result.unwrap_err();
                match error {
                    TliError::CircuitBreakerOpen(_) => {
                        tracing::info!(
                            "Circuit breaker activated after {} failures",
                            consecutive_failures
                        );
                        break;
                    }
                    _ => {
                        // Continue with other types of errors
                    }
                }
            } else {
                consecutive_failures = 0;
            }

            sleep(Duration::from_millis(100)).await;
        }

        assert!(
            consecutive_failures >= 3,
            "Should have multiple consecutive failures"
        );

        test_env
            .teardown()
            .await
            .expect("Failed to teardown test environment");
    }

    #[tokio::test]
    async fn test_retry_with_exponential_backoff() {
        let mut test_env = ErrorHandlingTests::new(TestConfig::default());
        test_env
            .setup()
            .await
            .expect("Failed to setup test environment");

        // Configure intermittent failures
        test_env
            .configure_failure_mode("backtesting_service", FailureMode::IntermittentFailure)
            .await
            .expect("Failed to configure failure mode");

        let backtesting_client = test_env
            .backtesting_client
            .as_ref()
            .expect("Backtesting client not available");

        let backtest_request = CreateBacktestRequest {
            name: "Retry Test Backtest".to_string(),
            strategy_id: "test_strategy".to_string(),
            start_date: "2024-01-01".to_string(),
            end_date: "2024-01-31".to_string(),
            initial_capital: 100000.0,
            symbols: vec!["AAPL".to_string()],
            parameters: HashMap::new(),
        };

        let start_time = std::time::Instant::now();
        let result = backtesting_client.create_backtest(backtest_request).await;
        let elapsed = start_time.elapsed();

        // The client should eventually succeed due to retry logic
        // or fail after exhausting retries
        match result {
            Ok(_) => {
                tracing::info!("Request succeeded after retries in {:?}", elapsed);
                assert!(
                    elapsed > Duration::from_millis(100),
                    "Should have taken some time due to retries"
                );
            }
            Err(error) => {
                tracing::info!("Request failed after retries: {:?}", error);
                assert!(
                    elapsed > Duration::from_millis(500),
                    "Should have taken time due to multiple retry attempts"
                );
            }
        }

        test_env
            .teardown()
            .await
            .expect("Failed to teardown test environment");
    }

    #[tokio::test]
    async fn test_rate_limiting_handling() {
        let mut test_env = ErrorHandlingTests::new(TestConfig::default());
        test_env
            .setup()
            .await
            .expect("Failed to setup test environment");

        // Configure rate limiting
        test_env
            .configure_failure_mode("trading_service", FailureMode::RateLimited)
            .await
            .expect("Failed to configure rate limiting");

        let trading_client = test_env
            .trading_client
            .as_ref()
            .expect("Trading client not available");

        // Rapidly submit multiple orders
        let mut rate_limited_count = 0;
        let total_requests = 20;

        for i in 0..total_requests {
            let order_request = SubmitOrderRequest {
                symbol: TestUtilities::generate_test_symbol("AAPL"),
                side: OrderSide::Buy as i32,
                order_type: OrderType::Market as i32,
                quantity: 100.0,
                price: Some(150.0),
                stop_price: None,
                time_in_force: "DAY".to_string(),
                client_order_id: format!("rapid_order_{}", i),
            };

            let result = trading_client.submit_order(order_request).await;

            if let Err(error) = result {
                match error {
                    TliError::RateLimited(_) => {
                        rate_limited_count += 1;
                        tracing::info!("Request {} rate limited", i);
                    }
                    _ => {
                        tracing::info!("Request {} failed with other error: {:?}", i, error);
                    }
                }
            }

            // Small delay between requests
            sleep(Duration::from_millis(10)).await;
        }

        assert!(
            rate_limited_count > 0,
            "Should have encountered rate limiting"
        );
        tracing::info!(
            "Rate limited {} out of {} requests",
            rate_limited_count,
            total_requests
        );

        test_env
            .teardown()
            .await
            .expect("Failed to teardown test environment");
    }
}

/// Graceful degradation tests
#[cfg(test)]
mod graceful_degradation_tests {
    use super::*;

    #[tokio::test]
    async fn test_fallback_to_cached_data() {
        let mut test_env = ErrorHandlingTests::new(TestConfig::default());
        test_env
            .setup()
            .await
            .expect("Failed to setup test environment");

        let trading_client = test_env
            .trading_client
            .as_ref()
            .expect("Trading client not available");

        // First, get positions while service is working (populate cache)
        let position_request = GetPositionsRequest {
            account_id: Some("test_account".to_string()),
            symbol_filter: None,
            include_zero_positions: false,
        };

        let initial_result = trading_client.get_positions(position_request.clone()).await;
        assert!(initial_result.is_ok(), "Initial request should succeed");

        // Now configure service to fail
        test_env
            .configure_failure_mode("trading_service", FailureMode::ServiceUnavailable)
            .await
            .expect("Failed to configure failure mode");

        // Subsequent request should fall back to cached data
        let cached_result = trading_client.get_positions(position_request).await;

        match cached_result {
            Ok(response) => {
                // Should indicate data is from cache
                assert!(
                    response.is_cached.unwrap_or(false),
                    "Response should indicate cached data"
                );
                tracing::info!("Successfully fell back to cached data");
            }
            Err(error) => {
                // Some implementations might not have caching
                tracing::info!("Cache fallback not available: {:?}", error);
            }
        }

        test_env
            .teardown()
            .await
            .expect("Failed to teardown test environment");
    }

    #[tokio::test]
    async fn test_reduced_functionality_mode() {
        let mut test_env = ErrorHandlingTests::new(TestConfig::default());
        test_env
            .setup()
            .await
            .expect("Failed to setup test environment");

        // Configure partial service degradation
        test_env
            .configure_failure_mode("trading_service", FailureMode::PartialDegradation)
            .await
            .expect("Failed to configure degradation");

        let trading_client = test_env
            .trading_client
            .as_ref()
            .expect("Trading client not available");

        // Basic order submission should still work
        let order_request = SubmitOrderRequest {
            symbol: TestUtilities::generate_test_symbol("AAPL"),
            side: OrderSide::Buy as i32,
            order_type: OrderType::Market as i32,
            quantity: 100.0,
            price: Some(150.0),
            stop_price: None,
            time_in_force: "DAY".to_string(),
            client_order_id: Uuid::new_v4().to_string(),
        };

        let order_result = trading_client.submit_order(order_request).await;
        assert!(
            order_result.is_ok(),
            "Basic order submission should work in degraded mode"
        );

        // Advanced features might be disabled
        let analytics_request = PortfolioAnalyticsRequest {
            account_id: "test_account".to_string(),
            calculation_date: chrono::Utc::now().timestamp(),
            include_realized_pnl: true,
            include_unrealized_pnl: true,
            include_risk_metrics: true,
        };

        let analytics_result = trading_client
            .get_portfolio_analytics(analytics_request)
            .await;

        match analytics_result {
            Ok(response) => {
                // Should indicate reduced functionality
                assert!(
                    response.warnings.is_some(),
                    "Should have warnings about reduced functionality"
                );
            }
            Err(error) => {
                // Advanced features disabled in degraded mode
                match error {
                    TliError::FeatureUnavailable(_) => {
                        tracing::info!("Advanced features correctly disabled in degraded mode");
                    }
                    _ => panic!("Unexpected error in degraded mode: {:?}", error),
                }
            }
        }

        test_env
            .teardown()
            .await
            .expect("Failed to teardown test environment");
    }
}