//! Error handling and resilience integration tests for TLI system //! //! This module tests various error scenarios including service unavailability, //! network timeouts, database connection failures, and invalid data handling. use std::sync::{Arc, Mutex}; use std::time::Duration; use tokio::time::{sleep, timeout}; use uuid::Uuid; use wiremock::{ matchers::{method, path}, Mock, MockServer, ResponseTemplate, }; use crate::integration::{TestConfig, TestUtilities}; use crate::mocks::grpc_server::{FailureMode, MockBacktestingServer, MockTradingServer}; use tli::client::{BacktestingClient, TliClientBuilder, TradingClient}; use tli::error::{ErrorCode, ErrorSeverity, TliError}; use tli::prelude::*; /// Error handling test suite pub struct ErrorHandlingTests { config: TestConfig, trading_client: Option, backtesting_client: Option, mock_servers: Vec>, wiremock_servers: Vec, } impl ErrorHandlingTests { pub fn new(config: TestConfig) -> Self { Self { config, trading_client: None, backtesting_client: None, mock_servers: Vec::new(), wiremock_servers: Vec::new(), } } /// Setup test environment with controllable failure modes pub async fn setup(&mut self) -> TliResult<()> { tracing::info!("Setting up error handling test environment"); // Start controllable mock servers let mut trading_server = MockTradingServer::new(self.config.mock_server_port)?; let trading_port = trading_server.start()?; self.mock_servers.push(Box::new(trading_server)); let mut backtesting_server = MockBacktestingServer::new(self.config.mock_server_port + 100)?; let backtesting_port = backtesting_server.start()?; self.mock_servers.push(Box::new(backtesting_server)); // Wait for services to be ready sleep(Duration::from_millis(300)).await; // Create TLI client suite with retry configuration let client_suite = TliClientBuilder::new() .with_service_endpoint( "trading_service".to_string(), format!("http://localhost:{}", trading_port), ) .with_service_endpoint( "backtesting_service".to_string(), format!("http://localhost:{}", backtesting_port), ) .with_trading_config(create_resilient_trading_config()) .with_backtesting_config(create_resilient_backtesting_config()) .build() .await?; self.trading_client = client_suite.trading_client; self.backtesting_client = client_suite.backtesting_client; tracing::info!("Error handling test environment setup complete"); Ok(()) } /// Setup test environment for network failure scenarios pub async fn setup_with_network_failures(&mut self) -> TliResult<()> { tracing::info!("Setting up network failure test environment"); // Start WireMock servers to simulate network issues let trading_mock = MockServer::start().await; let backtesting_mock = MockServer::start().await; let trading_port = trading_mock.address().port(); let backtesting_port = backtesting_mock.address().port(); self.wiremock_servers.push(trading_mock); self.wiremock_servers.push(backtesting_mock); // Create clients pointing to WireMock servers let client_suite = TliClientBuilder::new() .with_service_endpoint( "trading_service".to_string(), format!("http://localhost:{}", trading_port), ) .with_service_endpoint( "backtesting_service".to_string(), format!("http://localhost:{}", backtesting_port), ) .with_trading_config(create_resilient_trading_config()) .with_backtesting_config(create_resilient_backtesting_config()) .build() .await?; self.trading_client = client_suite.trading_client; self.backtesting_client = client_suite.backtesting_client; tracing::info!("Network failure test environment setup complete"); Ok(()) } /// Cleanup test environment pub async fn teardown(&mut self) -> TliResult<()> { tracing::info!("Tearing down error handling test environment"); // Shutdown clients if let Some(client) = self.trading_client.take() { client.shutdown().await; } if let Some(client) = self.backtesting_client.take() { client.shutdown().await; } // Stop mock servers for server in &mut self.mock_servers { let _ = server.stop(); } self.mock_servers.clear(); // WireMock servers are automatically cleaned up when dropped self.wiremock_servers.clear(); tracing::info!("Error handling test environment teardown complete"); Ok(()) } /// Configure mock server to simulate specific failure mode pub async fn configure_failure_mode( &mut self, service: &str, failure_mode: FailureMode, ) -> TliResult<()> { for server in &mut self.mock_servers { if server.get_service_name() == service { server.set_failure_mode(failure_mode)?; break; } } Ok(()) } } fn create_resilient_trading_config() -> TradingClientConfig { TradingClientConfig { service_name: "trading_service".to_string(), request_timeout: Duration::from_secs(5), order_validation: OrderValidationConfig { enable_pre_trade_checks: true, max_order_value: 1000000.0, require_confirmation: false, }, risk_management: RiskManagementConfig { enable_position_limits: true, max_position_size: 10000.0, max_daily_loss: 50000.0, }, market_data: MarketDataConfig { subscription_timeout: Duration::from_secs(30), reconnect_interval: Duration::from_secs(5), max_reconnect_attempts: 5, }, monitoring: MonitoringConfig { enable_health_checks: true, health_check_interval: Duration::from_secs(10), enable_circuit_breaker: true, circuit_breaker_threshold: 5, }, event_streaming: EventStreamConfig { buffer_size: 1000, reconnect_policy: ReconnectPolicy::ExponentialBackoff, max_reconnect_attempts: 10, }, } } fn create_resilient_backtesting_config() -> BacktestingClientConfig { BacktestingClientConfig { service_name: "backtesting_service".to_string(), request_timeout: Duration::from_secs(30), long_running_timeout: Duration::from_secs(300), retry_config: RetryConfig { max_attempts: 3, initial_delay: Duration::from_millis(100), max_delay: Duration::from_secs(10), backoff_multiplier: 2.0, }, } } /// Service unavailability tests #[cfg(test)] mod service_unavailability_tests { use super::*; #[tokio::test] async fn test_trading_service_unavailable() { let mut test_env = ErrorHandlingTests::new(TestConfig::default()); test_env .setup() .await .expect("Failed to setup test environment"); // Configure trading service to be unavailable test_env .configure_failure_mode("trading_service", FailureMode::ServiceUnavailable) .await .expect("Failed to configure failure mode"); let trading_client = test_env .trading_client .as_ref() .expect("Trading client not available"); // Attempt order submission let order_request = SubmitOrderRequest { symbol: TestUtilities::generate_test_symbol("AAPL"), side: OrderSide::Buy as i32, order_type: OrderType::Market as i32, quantity: 100.0, price: Some(150.0), stop_price: None, time_in_force: "DAY".to_string(), client_order_id: Uuid::new_v4().to_string(), }; let result = trading_client.submit_order(order_request).await; assert!( result.is_err(), "Order submission should fail when service unavailable" ); let error = result.unwrap_err(); match error { TliError::ServiceUnavailable(_) => { tracing::info!("Correctly identified service unavailable error"); } _ => panic!("Expected ServiceUnavailable error, got: {:?}", error), } test_env .teardown() .await .expect("Failed to teardown test environment"); } #[tokio::test] async fn test_service_recovery_after_failure() { let mut test_env = ErrorHandlingTests::new(TestConfig::default()); test_env .setup() .await .expect("Failed to setup test environment"); let trading_client = test_env .trading_client .as_ref() .expect("Trading client not available"); // First, make service unavailable test_env .configure_failure_mode("trading_service", FailureMode::ServiceUnavailable) .await .expect("Failed to configure failure mode"); let order_request = SubmitOrderRequest { symbol: TestUtilities::generate_test_symbol("AAPL"), side: OrderSide::Buy as i32, order_type: OrderType::Market as i32, quantity: 100.0, price: Some(150.0), stop_price: None, time_in_force: "DAY".to_string(), client_order_id: Uuid::new_v4().to_string(), }; // Verify service is failing let result = trading_client.submit_order(order_request.clone()).await; assert!(result.is_err(), "Service should be failing"); // Restore service test_env .configure_failure_mode("trading_service", FailureMode::None) .await .expect("Failed to restore service"); // Wait for circuit breaker to reset sleep(Duration::from_secs(1)).await; // Verify service recovery let recovery_result = trading_client.submit_order(order_request).await; assert!( recovery_result.is_ok(), "Service should recover after restoration" ); test_env .teardown() .await .expect("Failed to teardown test environment"); } #[tokio::test] async fn test_partial_service_failure() { let mut test_env = ErrorHandlingTests::new(TestConfig::default()); test_env .setup() .await .expect("Failed to setup test environment"); // Configure only backtesting service to fail test_env .configure_failure_mode("backtesting_service", FailureMode::ServiceUnavailable) .await .expect("Failed to configure failure mode"); let trading_client = test_env .trading_client .as_ref() .expect("Trading client not available"); let backtesting_client = test_env .backtesting_client .as_ref() .expect("Backtesting client not available"); // Trading service should still work let order_request = SubmitOrderRequest { symbol: TestUtilities::generate_test_symbol("AAPL"), side: OrderSide::Buy as i32, order_type: OrderType::Market as i32, quantity: 100.0, price: Some(150.0), stop_price: None, time_in_force: "DAY".to_string(), client_order_id: Uuid::new_v4().to_string(), }; let trading_result = trading_client.submit_order(order_request).await; assert!(trading_result.is_ok(), "Trading service should still work"); // Backtesting service should fail let backtest_request = CreateBacktestRequest { name: "Test Backtest".to_string(), strategy_id: "test_strategy".to_string(), start_date: "2024-01-01".to_string(), end_date: "2024-01-31".to_string(), initial_capital: 100000.0, symbols: vec!["AAPL".to_string()], parameters: HashMap::new(), }; let backtest_result = backtesting_client.create_backtest(backtest_request).await; assert!(backtest_result.is_err(), "Backtesting service should fail"); test_env .teardown() .await .expect("Failed to teardown test environment"); } } /// Network timeout and connection tests #[cfg(test)] mod network_timeout_tests { use super::*; #[tokio::test] async fn test_connection_timeout_handling() { let mut test_env = ErrorHandlingTests::new(TestConfig::default()); test_env .setup_with_network_failures() .await .expect("Failed to setup test environment"); // Configure mock to not respond (simulate timeout) let trading_mock = &test_env.wiremock_servers[0]; Mock::given(method("POST")) .and(path("/trading.TradingService/SubmitOrder")) .respond_with(ResponseTemplate::new(200).set_delay(Duration::from_secs(10))) // Longer than client timeout .mount(trading_mock) .await; let trading_client = test_env .trading_client .as_ref() .expect("Trading client not available"); let order_request = SubmitOrderRequest { symbol: TestUtilities::generate_test_symbol("AAPL"), side: OrderSide::Buy as i32, order_type: OrderType::Market as i32, quantity: 100.0, price: Some(150.0), stop_price: None, time_in_force: "DAY".to_string(), client_order_id: Uuid::new_v4().to_string(), }; let start_time = std::time::Instant::now(); let result = trading_client.submit_order(order_request).await; let elapsed = start_time.elapsed(); assert!(result.is_err(), "Request should timeout"); assert!( elapsed < Duration::from_secs(8), "Should timeout within client timeout period" ); let error = result.unwrap_err(); match error { TliError::Timeout(_) => { tracing::info!("Correctly identified timeout error"); } _ => panic!("Expected Timeout error, got: {:?}", error), } test_env .teardown() .await .expect("Failed to teardown test environment"); } #[tokio::test] async fn test_connection_refused_handling() { let mut test_env = ErrorHandlingTests::new(TestConfig::default()); // Create clients pointing to non-existent services let client_suite = TliClientBuilder::new() .with_service_endpoint( "trading_service".to_string(), "http://localhost:99999".to_string(), ) // Invalid port .with_service_endpoint( "backtesting_service".to_string(), "http://localhost:99998".to_string(), ) // Invalid port .with_trading_config(create_resilient_trading_config()) .with_backtesting_config(create_resilient_backtesting_config()) .build() .await .expect("Client creation should succeed"); test_env.trading_client = client_suite.trading_client; let trading_client = test_env .trading_client .as_ref() .expect("Trading client not available"); let order_request = SubmitOrderRequest { symbol: TestUtilities::generate_test_symbol("AAPL"), side: OrderSide::Buy as i32, order_type: OrderType::Market as i32, quantity: 100.0, price: Some(150.0), stop_price: None, time_in_force: "DAY".to_string(), client_order_id: Uuid::new_v4().to_string(), }; let result = trading_client.submit_order(order_request).await; assert!(result.is_err(), "Connection should be refused"); let error = result.unwrap_err(); match error { TliError::Connection(_) => { tracing::info!("Correctly identified connection error"); } _ => panic!("Expected Connection error, got: {:?}", error), } test_env .teardown() .await .expect("Failed to teardown test environment"); } #[tokio::test] async fn test_intermittent_network_failures() { let mut test_env = ErrorHandlingTests::new(TestConfig::default()); test_env .setup_with_network_failures() .await .expect("Failed to setup test environment"); let trading_mock = &test_env.wiremock_servers[0]; // Configure mock to fail 50% of the time let success_counter = Arc::new(Mutex::new(0)); let counter_clone = success_counter.clone(); Mock::given(method("POST")) .and(path("/trading.TradingService/SubmitOrder")) .respond_with(move |_req| { let mut counter = counter_clone.lock().unwrap(); *counter += 1; if *counter % 2 == 0 { ResponseTemplate::new(200).set_body_json(serde_json::json!({ "success": true, "order_id": "test_order_123", "message": "Order submitted successfully", "timestamp_unix_nanos": 1234567890000000000i64 })) } else { ResponseTemplate::new(500).set_body("Internal Server Error") } }) .mount(trading_mock) .await; let trading_client = test_env .trading_client .as_ref() .expect("Trading client not available"); // Attempt multiple requests let mut successful_requests = 0; let mut failed_requests = 0; let total_requests = 10; for i in 0..total_requests { let order_request = SubmitOrderRequest { symbol: TestUtilities::generate_test_symbol("AAPL"), side: OrderSide::Buy as i32, order_type: OrderType::Market as i32, quantity: 100.0, price: Some(150.0), stop_price: None, time_in_force: "DAY".to_string(), client_order_id: format!("order_{}", i), }; match trading_client.submit_order(order_request).await { Ok(_) => successful_requests += 1, Err(_) => failed_requests += 1, } } tracing::info!( "Intermittent failure test: {} successful, {} failed", successful_requests, failed_requests ); // Should have both successes and failures assert!( successful_requests > 0, "Should have some successful requests" ); assert!(failed_requests > 0, "Should have some failed requests"); test_env .teardown() .await .expect("Failed to teardown test environment"); } } /// Database connection failure tests #[cfg(test)] mod database_failure_tests { use super::*; #[tokio::test] async fn test_database_connection_failure() { let mut test_env = ErrorHandlingTests::new(TestConfig::default()); test_env .setup() .await .expect("Failed to setup test environment"); // Configure database failure test_env .configure_failure_mode("trading_service", FailureMode::DatabaseError) .await .expect("Failed to configure database failure"); let trading_client = test_env .trading_client .as_ref() .expect("Trading client not available"); // Attempt to get positions (requires database access) let position_request = GetPositionsRequest { account_id: Some("test_account".to_string()), symbol_filter: None, include_zero_positions: false, }; let result = trading_client.get_positions(position_request).await; assert!(result.is_err(), "Database operation should fail"); let error = result.unwrap_err(); match error { TliError::Database(_) => { tracing::info!("Correctly identified database error"); } _ => panic!("Expected Database error, got: {:?}", error), } test_env .teardown() .await .expect("Failed to teardown test environment"); } #[tokio::test] async fn test_database_transaction_rollback() { let mut test_env = ErrorHandlingTests::new(TestConfig::default()); test_env .setup() .await .expect("Failed to setup test environment"); // Configure intermittent database failures test_env .configure_failure_mode("trading_service", FailureMode::TransactionFailure) .await .expect("Failed to configure transaction failure"); let trading_client = test_env .trading_client .as_ref() .expect("Trading client not available"); // Attempt batch order submission (requires transactions) let batch_orders = vec![ SubmitOrderRequest { symbol: TestUtilities::generate_test_symbol("AAPL"), side: OrderSide::Buy as i32, order_type: OrderType::Market as i32, quantity: 100.0, price: Some(150.0), stop_price: None, time_in_force: "DAY".to_string(), client_order_id: Uuid::new_v4().to_string(), }, SubmitOrderRequest { symbol: TestUtilities::generate_test_symbol("GOOGL"), side: OrderSide::Buy as i32, order_type: OrderType::Market as i32, quantity: 50.0, price: Some(2800.0), stop_price: None, time_in_force: "DAY".to_string(), client_order_id: Uuid::new_v4().to_string(), }, ]; let batch_request = SubmitBatchOrdersRequest { orders: batch_orders, all_or_none: true, // Requires transaction max_acceptable_failures: 0, }; let result = trading_client.submit_batch_orders(batch_request).await; assert!( result.is_err(), "Batch operation should fail due to transaction failure" ); let error = result.unwrap_err(); match error { TliError::Database(_) | TliError::TransactionFailure(_) => { tracing::info!("Correctly identified transaction failure"); } _ => panic!( "Expected Database or TransactionFailure error, got: {:?}", error ), } test_env .teardown() .await .expect("Failed to teardown test environment"); } #[tokio::test] async fn test_database_recovery_mechanisms() { let mut test_env = ErrorHandlingTests::new(TestConfig::default()); test_env .setup() .await .expect("Failed to setup test environment"); let trading_client = test_env .trading_client .as_ref() .expect("Trading client not available"); // First, cause database failure test_env .configure_failure_mode("trading_service", FailureMode::DatabaseError) .await .expect("Failed to configure database failure"); let position_request = GetPositionsRequest { account_id: Some("test_account".to_string()), symbol_filter: None, include_zero_positions: false, }; // Verify failure let result = trading_client.get_positions(position_request.clone()).await; assert!(result.is_err(), "Database operation should fail"); // Restore database test_env .configure_failure_mode("trading_service", FailureMode::None) .await .expect("Failed to restore database"); // Wait for recovery sleep(Duration::from_secs(1)).await; // Verify recovery let recovery_result = trading_client.get_positions(position_request).await; assert!( recovery_result.is_ok(), "Database operation should succeed after recovery" ); test_env .teardown() .await .expect("Failed to teardown test environment"); } } /// Invalid data handling tests #[cfg(test)] mod invalid_data_tests { use super::*; #[tokio::test] async fn test_invalid_order_data_handling() { let mut test_env = ErrorHandlingTests::new(TestConfig::default()); test_env .setup() .await .expect("Failed to setup test environment"); let trading_client = test_env .trading_client .as_ref() .expect("Trading client not available"); // Test invalid symbol let invalid_symbol_request = SubmitOrderRequest { symbol: "".to_string(), // Empty symbol side: OrderSide::Buy as i32, order_type: OrderType::Market as i32, quantity: 100.0, price: Some(150.0), stop_price: None, time_in_force: "DAY".to_string(), client_order_id: Uuid::new_v4().to_string(), }; let result = trading_client.submit_order(invalid_symbol_request).await; assert!(result.is_err(), "Order with empty symbol should fail"); let error = result.unwrap_err(); match error { TliError::InvalidInput(_) | TliError::Validation(_) => { tracing::info!("Correctly identified invalid symbol error"); } _ => panic!( "Expected InvalidInput or Validation error, got: {:?}", error ), } // Test invalid quantity let invalid_quantity_request = SubmitOrderRequest { symbol: TestUtilities::generate_test_symbol("AAPL"), side: OrderSide::Buy as i32, order_type: OrderType::Market as i32, quantity: -100.0, // Negative quantity price: Some(150.0), stop_price: None, time_in_force: "DAY".to_string(), client_order_id: Uuid::new_v4().to_string(), }; let result = trading_client.submit_order(invalid_quantity_request).await; assert!(result.is_err(), "Order with negative quantity should fail"); // Test invalid price let invalid_price_request = SubmitOrderRequest { symbol: TestUtilities::generate_test_symbol("AAPL"), side: OrderSide::Buy as i32, order_type: OrderType::Limit as i32, quantity: 100.0, price: Some(-10.0), // Negative price stop_price: None, time_in_force: "DAY".to_string(), client_order_id: Uuid::new_v4().to_string(), }; let result = trading_client.submit_order(invalid_price_request).await; assert!(result.is_err(), "Order with negative price should fail"); test_env .teardown() .await .expect("Failed to teardown test environment"); } #[tokio::test] async fn test_malformed_configuration_data() { let mut test_env = ErrorHandlingTests::new(TestConfig::default()); test_env .setup() .await .expect("Failed to setup test environment"); let trading_client = test_env .trading_client .as_ref() .expect("Trading client not available"); // Test invalid configuration values let invalid_configs = vec![ Configuration { id: Uuid::new_v4().to_string(), config_type: ConfigurationType::TradingLimits as i32, key: "max_position_size".to_string(), value: "not_a_number".to_string(), // Invalid numeric value account_id: Some("test_account".to_string()), last_updated: chrono::Utc::now().timestamp(), }, Configuration { id: Uuid::new_v4().to_string(), config_type: ConfigurationType::TradingLimits as i32, key: "".to_string(), // Empty key value: "1000".to_string(), account_id: Some("test_account".to_string()), last_updated: chrono::Utc::now().timestamp(), }, ]; let update_request = UpdateConfigurationRequest { configurations: invalid_configs, validate_before_update: true, }; let result = trading_client.update_configuration(update_request).await; // Should either fail validation or return validation errors match result { Ok(response) => { assert!( !response.success || !response.validation_errors.is_empty(), "Invalid configuration should fail validation" ); } Err(error) => match error { TliError::InvalidInput(_) | TliError::Validation(_) => { tracing::info!("Correctly identified invalid configuration error"); } _ => panic!( "Expected InvalidInput or Validation error, got: {:?}", error ), }, } test_env .teardown() .await .expect("Failed to teardown test environment"); } #[tokio::test] async fn test_corrupted_backtest_data() { let mut test_env = ErrorHandlingTests::new(TestConfig::default()); test_env .setup() .await .expect("Failed to setup test environment"); let backtesting_client = test_env .backtesting_client .as_ref() .expect("Backtesting client not available"); // Test invalid date format let invalid_date_request = CreateBacktestRequest { name: "Invalid Date Test".to_string(), strategy_id: "test_strategy".to_string(), start_date: "invalid-date-format".to_string(), // Invalid date end_date: "2024-01-31".to_string(), initial_capital: 100000.0, symbols: vec!["AAPL".to_string()], parameters: HashMap::new(), }; let result = backtesting_client .create_backtest(invalid_date_request) .await; assert!(result.is_err(), "Backtest with invalid date should fail"); // Test invalid initial capital let invalid_capital_request = CreateBacktestRequest { name: "Invalid Capital Test".to_string(), strategy_id: "test_strategy".to_string(), start_date: "2024-01-01".to_string(), end_date: "2024-01-31".to_string(), initial_capital: -1000.0, // Negative capital symbols: vec!["AAPL".to_string()], parameters: HashMap::new(), }; let result = backtesting_client .create_backtest(invalid_capital_request) .await; assert!( result.is_err(), "Backtest with negative capital should fail" ); // Test empty symbols list let empty_symbols_request = CreateBacktestRequest { name: "Empty Symbols Test".to_string(), strategy_id: "test_strategy".to_string(), start_date: "2024-01-01".to_string(), end_date: "2024-01-31".to_string(), initial_capital: 100000.0, symbols: vec![], // Empty symbols parameters: HashMap::new(), }; let result = backtesting_client .create_backtest(empty_symbols_request) .await; assert!(result.is_err(), "Backtest with empty symbols should fail"); test_env .teardown() .await .expect("Failed to teardown test environment"); } } /// Circuit breaker and rate limiting tests #[cfg(test)] mod resilience_pattern_tests { use super::*; #[tokio::test] async fn test_circuit_breaker_activation() { let mut test_env = ErrorHandlingTests::new(TestConfig::default()); test_env .setup() .await .expect("Failed to setup test environment"); // Configure service to fail consistently test_env .configure_failure_mode("trading_service", FailureMode::InternalError) .await .expect("Failed to configure failure mode"); let trading_client = test_env .trading_client .as_ref() .expect("Trading client not available"); let order_request = SubmitOrderRequest { symbol: TestUtilities::generate_test_symbol("AAPL"), side: OrderSide::Buy as i32, order_type: OrderType::Market as i32, quantity: 100.0, price: Some(150.0), stop_price: None, time_in_force: "DAY".to_string(), client_order_id: Uuid::new_v4().to_string(), }; // Make multiple requests to trigger circuit breaker let mut consecutive_failures = 0; let max_attempts = 10; for attempt in 1..=max_attempts { let result = trading_client.submit_order(order_request.clone()).await; if result.is_err() { consecutive_failures += 1; tracing::info!("Attempt {}: Request failed", attempt); // Check if we're getting circuit breaker errors let error = result.unwrap_err(); match error { TliError::CircuitBreakerOpen(_) => { tracing::info!( "Circuit breaker activated after {} failures", consecutive_failures ); break; } _ => { // Continue with other types of errors } } } else { consecutive_failures = 0; } sleep(Duration::from_millis(100)).await; } assert!( consecutive_failures >= 3, "Should have multiple consecutive failures" ); test_env .teardown() .await .expect("Failed to teardown test environment"); } #[tokio::test] async fn test_retry_with_exponential_backoff() { let mut test_env = ErrorHandlingTests::new(TestConfig::default()); test_env .setup() .await .expect("Failed to setup test environment"); // Configure intermittent failures test_env .configure_failure_mode("backtesting_service", FailureMode::IntermittentFailure) .await .expect("Failed to configure failure mode"); let backtesting_client = test_env .backtesting_client .as_ref() .expect("Backtesting client not available"); let backtest_request = CreateBacktestRequest { name: "Retry Test Backtest".to_string(), strategy_id: "test_strategy".to_string(), start_date: "2024-01-01".to_string(), end_date: "2024-01-31".to_string(), initial_capital: 100000.0, symbols: vec!["AAPL".to_string()], parameters: HashMap::new(), }; let start_time = std::time::Instant::now(); let result = backtesting_client.create_backtest(backtest_request).await; let elapsed = start_time.elapsed(); // The client should eventually succeed due to retry logic // or fail after exhausting retries match result { Ok(_) => { tracing::info!("Request succeeded after retries in {:?}", elapsed); assert!( elapsed > Duration::from_millis(100), "Should have taken some time due to retries" ); } Err(error) => { tracing::info!("Request failed after retries: {:?}", error); assert!( elapsed > Duration::from_millis(500), "Should have taken time due to multiple retry attempts" ); } } test_env .teardown() .await .expect("Failed to teardown test environment"); } #[tokio::test] async fn test_rate_limiting_handling() { let mut test_env = ErrorHandlingTests::new(TestConfig::default()); test_env .setup() .await .expect("Failed to setup test environment"); // Configure rate limiting test_env .configure_failure_mode("trading_service", FailureMode::RateLimited) .await .expect("Failed to configure rate limiting"); let trading_client = test_env .trading_client .as_ref() .expect("Trading client not available"); // Rapidly submit multiple orders let mut rate_limited_count = 0; let total_requests = 20; for i in 0..total_requests { let order_request = SubmitOrderRequest { symbol: TestUtilities::generate_test_symbol("AAPL"), side: OrderSide::Buy as i32, order_type: OrderType::Market as i32, quantity: 100.0, price: Some(150.0), stop_price: None, time_in_force: "DAY".to_string(), client_order_id: format!("rapid_order_{}", i), }; let result = trading_client.submit_order(order_request).await; if let Err(error) = result { match error { TliError::RateLimited(_) => { rate_limited_count += 1; tracing::info!("Request {} rate limited", i); } _ => { tracing::info!("Request {} failed with other error: {:?}", i, error); } } } // Small delay between requests sleep(Duration::from_millis(10)).await; } assert!( rate_limited_count > 0, "Should have encountered rate limiting" ); tracing::info!( "Rate limited {} out of {} requests", rate_limited_count, total_requests ); test_env .teardown() .await .expect("Failed to teardown test environment"); } } /// Graceful degradation tests #[cfg(test)] mod graceful_degradation_tests { use super::*; #[tokio::test] async fn test_fallback_to_cached_data() { let mut test_env = ErrorHandlingTests::new(TestConfig::default()); test_env .setup() .await .expect("Failed to setup test environment"); let trading_client = test_env .trading_client .as_ref() .expect("Trading client not available"); // First, get positions while service is working (populate cache) let position_request = GetPositionsRequest { account_id: Some("test_account".to_string()), symbol_filter: None, include_zero_positions: false, }; let initial_result = trading_client.get_positions(position_request.clone()).await; assert!(initial_result.is_ok(), "Initial request should succeed"); // Now configure service to fail test_env .configure_failure_mode("trading_service", FailureMode::ServiceUnavailable) .await .expect("Failed to configure failure mode"); // Subsequent request should fall back to cached data let cached_result = trading_client.get_positions(position_request).await; match cached_result { Ok(response) => { // Should indicate data is from cache assert!( response.is_cached.unwrap_or(false), "Response should indicate cached data" ); tracing::info!("Successfully fell back to cached data"); } Err(error) => { // Some implementations might not have caching tracing::info!("Cache fallback not available: {:?}", error); } } test_env .teardown() .await .expect("Failed to teardown test environment"); } #[tokio::test] async fn test_reduced_functionality_mode() { let mut test_env = ErrorHandlingTests::new(TestConfig::default()); test_env .setup() .await .expect("Failed to setup test environment"); // Configure partial service degradation test_env .configure_failure_mode("trading_service", FailureMode::PartialDegradation) .await .expect("Failed to configure degradation"); let trading_client = test_env .trading_client .as_ref() .expect("Trading client not available"); // Basic order submission should still work let order_request = SubmitOrderRequest { symbol: TestUtilities::generate_test_symbol("AAPL"), side: OrderSide::Buy as i32, order_type: OrderType::Market as i32, quantity: 100.0, price: Some(150.0), stop_price: None, time_in_force: "DAY".to_string(), client_order_id: Uuid::new_v4().to_string(), }; let order_result = trading_client.submit_order(order_request).await; assert!( order_result.is_ok(), "Basic order submission should work in degraded mode" ); // Advanced features might be disabled let analytics_request = PortfolioAnalyticsRequest { account_id: "test_account".to_string(), calculation_date: chrono::Utc::now().timestamp(), include_realized_pnl: true, include_unrealized_pnl: true, include_risk_metrics: true, }; let analytics_result = trading_client .get_portfolio_analytics(analytics_request) .await; match analytics_result { Ok(response) => { // Should indicate reduced functionality assert!( response.warnings.is_some(), "Should have warnings about reduced functionality" ); } Err(error) => { // Advanced features disabled in degraded mode match error { TliError::FeatureUnavailable(_) => { tracing::info!("Advanced features correctly disabled in degraded mode"); } _ => panic!("Unexpected error in degraded mode: {:?}", error), } } } test_env .teardown() .await .expect("Failed to teardown test environment"); } }