foxhunt/testing/service-integration/tests/service_health_resilience_e2e.rs

#![cfg(feature = "__integration_tests")]
#![allow(unexpected_cfgs)]
//! End-to-End Service Health and Resilience Tests
//!
//! This test suite validates:
//! - Service health checks and status monitoring
//! - Graceful degradation when optional services unavailable
//! - Circuit breaker validation
//! - Timeout and retry handling
//! - Service discovery and failover
//!
//! Test Count: 15 tests
//! Coverage: Full service health and resilience validation

mod common;

use anyhow::Result;
use common::auth_helpers::{create_auth_interceptor, get_api_addr, TestAuthConfig};
use std::time::Duration as StdDuration;
use tokio::time::timeout;
use tonic::{transport::Channel, Code, Request};
use uuid::Uuid;

// Generated proto code
pub mod trading {
    tonic::include_proto!("trading");
}

use trading::{
    backtesting_service_client::BacktestingServiceClient,
    trading_service_client::TradingServiceClient, GetSystemStatusRequest, OrderSide, OrderType,
    StartBacktestRequest, SubmitOrderRequest, SystemStatus,
};

// ============================================================================
// SECTION 1: SYSTEM HEALTH MONITORING E2E TESTS (5 tests)
// ============================================================================

#[tokio::test]
async fn test_e2e_system_health_all_services() -> Result<()> {
    println!("\n=== E2E Test: System Health - All Services ===");

    let api_addr = get_api_addr();
    let channel = Channel::from_shared(api_addr)?.connect().await?;

    let config = TestAuthConfig::admin().with_user_id("health_monitor");
    let interceptor = create_auth_interceptor(config)?;
    let mut client = TradingServiceClient::with_interceptor(channel, interceptor);

    let request = Request::new(GetSystemStatusRequest {
        service_names: vec![], // Empty = all services
    });

    let response = client.get_system_status(request).await?;
    let system_status = response.into_inner();

    println!("✓ System status retrieved");
    println!("  Overall Status: {:?}", system_status.overall_status);
    println!("  Services Monitored: {}", system_status.services.len());

    for service in &system_status.services {
        println!("  - {}: {:?}", service.name, service.status);
        if !service.message.is_empty() {
            println!("    Message: {}", service.message);
        }
    }

    // At least API Gateway should be healthy
    let api_healthy = system_status
        .services
        .iter()
        .any(|s| s.name.contains("gateway") && s.status == SystemStatus::Healthy as i32);

    assert!(api_healthy, "API should be healthy");

    Ok(())
}

#[tokio::test]
async fn test_e2e_system_health_specific_service() -> Result<()> {
    println!("\n=== E2E Test: System Health - Specific Service ===");

    let api_addr = get_api_addr();
    let channel = Channel::from_shared(api_addr)?.connect().await?;

    let config = TestAuthConfig::admin().with_user_id("health_monitor");
    let interceptor = create_auth_interceptor(config)?;
    let mut client = TradingServiceClient::with_interceptor(channel, interceptor);

    let request = Request::new(GetSystemStatusRequest {
        service_names: vec!["trading_service".to_string()],
    });

    let response = client.get_system_status(request).await?;
    let system_status = response.into_inner();

    println!("✓ Trading service status retrieved");

    if let Some(trading_service) = system_status.services.first() {
        println!("  Service: {}", trading_service.name);
        println!("  Status: {:?}", trading_service.status);
        println!("  Last Check: {}", trading_service.last_check_unix_nanos);

        assert_eq!(trading_service.name, "trading_service");
    }

    Ok(())
}

#[tokio::test]
async fn test_e2e_health_check_interval() -> Result<()> {
    println!("\n=== E2E Test: Health Check Update Interval ===");

    let api_addr = get_api_addr();
    let channel = Channel::from_shared(api_addr)?.connect().await?;

    let config = TestAuthConfig::admin().with_user_id("health_monitor");
    let interceptor = create_auth_interceptor(config)?;
    let mut client = TradingServiceClient::with_interceptor(channel, interceptor);

    // Get initial health status
    let request1 = Request::new(GetSystemStatusRequest {
        service_names: vec![],
    });

    let response1 = client.get_system_status(request1).await?;
    let initial_timestamp = response1.into_inner().timestamp_unix_nanos;

    println!("✓ Initial health check: {}", initial_timestamp);

    // Wait and check again
    tokio::time::sleep(StdDuration::from_secs(2)).await;

    let request2 = Request::new(GetSystemStatusRequest {
        service_names: vec![],
    });

    let response2 = client.get_system_status(request2).await?;
    let updated_timestamp = response2.into_inner().timestamp_unix_nanos;

    println!("✓ Updated health check: {}", updated_timestamp);

    assert!(
        updated_timestamp > initial_timestamp,
        "Health check should update"
    );

    Ok(())
}

#[tokio::test]
async fn test_e2e_health_status_transitions() -> Result<()> {
    println!("\n=== E2E Test: Health Status Transitions Monitoring ===");

    let api_addr = get_api_addr();
    let channel = Channel::from_shared(api_addr)?.connect().await?;

    let config = TestAuthConfig::admin().with_user_id("health_monitor");
    let interceptor = create_auth_interceptor(config)?;
    let mut client = TradingServiceClient::with_interceptor(channel, interceptor);

    // Subscribe to system status changes
    let request = Request::new(trading::SubscribeSystemStatusRequest {
        service_names: vec![], // Monitor all services
    });

    let mut stream = client.subscribe_system_status(request).await?.into_inner();

    println!("✓ System status stream established");

    // Monitor status changes for a short period
    let mut events_received = 0;
    while let Ok(event_result) = timeout(StdDuration::from_secs(5), stream.message()).await {
        if let Ok(Some(event)) = event_result {
            events_received += 1;
            println!(
                "  Status Event #{}: {} - {:?} → {:?}",
                events_received, event.service_name, event.previous_status, event.status
            );

            if events_received >= 3 {
                break;
            }
        }
    }

    println!("✓ Monitored {} status change events", events_received);

    Ok(())
}

#[tokio::test]
async fn test_e2e_degraded_service_detection() -> Result<()> {
    println!("\n=== E2E Test: Degraded Service Detection ===");

    let api_addr = get_api_addr();
    let channel = Channel::from_shared(api_addr)?.connect().await?;

    let config = TestAuthConfig::admin().with_user_id("health_monitor");
    let interceptor = create_auth_interceptor(config)?;
    let mut client = TradingServiceClient::with_interceptor(channel, interceptor);

    let request = Request::new(GetSystemStatusRequest {
        service_names: vec![],
    });

    let response = client.get_system_status(request).await?;
    let system_status = response.into_inner();

    println!("✓ Checking for degraded services");

    let degraded_services: Vec<_> = system_status
        .services
        .iter()
        .filter(|s| s.status == SystemStatus::Degraded as i32)
        .collect();

    println!("  Degraded Services: {}", degraded_services.len());

    for service in degraded_services {
        println!("  - {}: {}", service.name, service.message);
    }

    // System should handle degraded services gracefully
    assert!(
        system_status.overall_status != SystemStatus::Critical as i32,
        "Overall system should not be critical"
    );

    Ok(())
}

// ============================================================================
// SECTION 2: GRACEFUL DEGRADATION E2E TESTS (5 tests)
// ============================================================================

#[tokio::test]
async fn test_e2e_trading_service_available_backtesting_optional() -> Result<()> {
    println!("\n=== E2E Test: Trading Service Available, Backtesting Optional ===");

    let api_addr = get_api_addr();
    let channel = Channel::from_shared(api_addr)?.connect().await?;

    let config = TestAuthConfig::trader().with_user_id("test_trader");
    let interceptor = create_auth_interceptor(config)?;
    let mut trading_client = TradingServiceClient::with_interceptor(channel.clone(), interceptor);

    // Trading service should work (core functionality)
    let order_request = Request::new(SubmitOrderRequest {
        symbol: "BTC/USD".to_string(),
        side: OrderSide::Buy as i32,
        order_type: OrderType::Market as i32,
        quantity: 0.01,
        price: None,
        stop_price: None,
        time_in_force: "IOC".to_string(),
        client_order_id: Uuid::new_v4().to_string(),
    });

    let order_result = trading_client.submit_order(order_request).await;

    assert!(
        order_result.is_ok(),
        "Core trading functionality should work"
    );
    println!("✓ Core trading service operational");

    // Backtesting might be unavailable (graceful degradation)
    let config2 = TestAuthConfig::trader().with_user_id("test_trader");
    let interceptor2 = create_auth_interceptor(config2)?;
    let mut backtesting_client = BacktestingServiceClient::with_interceptor(channel, interceptor2);

    let backtest_request = Request::new(StartBacktestRequest {
        strategy_name: "test".to_string(),
        symbols: vec!["BTC/USD".to_string()],
        start_date_unix_nanos: 0,
        end_date_unix_nanos: 0,
        initial_capital: 10000.0,
        parameters: std::collections::HashMap::new(),
        save_results: false,
        description: "Optional service test".to_string(),
    });

    let backtest_result = backtesting_client.start_backtest(backtest_request).await;

    if backtest_result.is_ok() {
        println!("✓ Backtesting service also available");
    } else {
        println!("✓ Backtesting service unavailable - graceful degradation");
        println!("  Core trading continues to work");
    }

    Ok(())
}

#[tokio::test]
async fn test_e2e_partial_service_failure_handling() -> Result<()> {
    println!("\n=== E2E Test: Partial Service Failure Handling ===");

    let api_addr = get_api_addr();
    let channel = Channel::from_shared(api_addr)?.connect().await?;

    let config = TestAuthConfig::admin().with_user_id("resilience_tester");
    let interceptor = create_auth_interceptor(config)?;
    let mut client = TradingServiceClient::with_interceptor(channel, interceptor);

    // Check which services are available
    let status_request = Request::new(GetSystemStatusRequest {
        service_names: vec![],
    });

    let status_response = client.get_system_status(status_request).await?;
    let system_status = status_response.into_inner();

    let healthy_services = system_status
        .services
        .iter()
        .filter(|s| s.status == SystemStatus::Healthy as i32)
        .count();

    let total_services = system_status.services.len();

    println!("✓ Service availability check");
    println!("  Healthy: {}/{}", healthy_services, total_services);
    println!("  Overall Status: {:?}", system_status.overall_status);

    // System should remain operational with partial failures
    assert!(
        system_status.overall_status != SystemStatus::Unknown as i32,
        "System should report known status"
    );

    Ok(())
}

#[tokio::test]
async fn test_e2e_circuit_breaker_validation() -> Result<()> {
    println!("\n=== E2E Test: Circuit Breaker Validation ===");

    let api_addr = get_api_addr();
    let channel = Channel::from_shared(api_addr)?.connect().await?;

    let config = TestAuthConfig::trader().with_user_id("test_trader");
    let interceptor = create_auth_interceptor(config)?;
    let mut client = TradingServiceClient::with_interceptor(channel, interceptor);

    // Submit multiple rapid requests to potentially trigger circuit breaker
    let mut success_count = 0;
    let mut circuit_breaker_triggered = false;

    for i in 0..20 {
        let request = Request::new(SubmitOrderRequest {
            symbol: "BTC/USD".to_string(),
            side: OrderSide::Buy as i32,
            order_type: OrderType::Market as i32,
            quantity: 0.001,
            price: None,
            stop_price: None,
            time_in_force: "IOC".to_string(),
            client_order_id: Uuid::new_v4().to_string(),
        });

        match client.submit_order(request).await {
            Ok(_) => success_count += 1,
            Err(status) => {
                if status.code() == Code::Unavailable || status.message().contains("circuit") {
                    circuit_breaker_triggered = true;
                    println!("  Circuit breaker triggered at request #{}", i + 1);
                    break;
                }
            },
        }

        tokio::time::sleep(StdDuration::from_millis(50)).await;
    }

    println!("✓ Circuit breaker test completed");
    println!("  Successful requests: {}", success_count);
    println!("  Circuit breaker triggered: {}", circuit_breaker_triggered);

    // Either all succeeded (circuit breaker not needed) or it was triggered (working correctly)
    assert!(
        success_count > 0 || circuit_breaker_triggered,
        "Should have some successful requests or trigger circuit breaker"
    );

    Ok(())
}

#[tokio::test]
async fn test_e2e_timeout_handling() -> Result<()> {
    println!("\n=== E2E Test: Request Timeout Handling ===");

    let api_addr = get_api_addr();
    let channel = Channel::from_shared(api_addr)?
        .timeout(StdDuration::from_millis(100)) // Very short timeout
        .connect()
        .await?;

    let config = TestAuthConfig::trader().with_user_id("test_trader");
    let interceptor = create_auth_interceptor(config)?;
    let mut client = TradingServiceClient::with_interceptor(channel, interceptor);

    let request = Request::new(trading::GetPositionsRequest { symbol: None });

    // Request with very short timeout
    let result = timeout(StdDuration::from_millis(200), client.get_positions(request)).await;

    match result {
        Ok(Ok(_)) => {
            println!("✓ Request completed within timeout");
        },
        Ok(Err(status)) => {
            println!("✓ Request failed gracefully: {:?}", status.code());
        },
        Err(_) => {
            println!("✓ Timeout handled correctly");
        },
    }

    Ok(())
}

#[tokio::test]
async fn test_e2e_retry_logic_validation() -> Result<()> {
    println!("\n=== E2E Test: Retry Logic Validation ===");

    let api_addr = get_api_addr();
    let channel = Channel::from_shared(api_addr)?.connect().await?;

    let config = TestAuthConfig::trader().with_user_id("test_trader");
    let interceptor = create_auth_interceptor(config)?;
    let mut client = TradingServiceClient::with_interceptor(channel, interceptor);

    // Simulate retryable operation
    let mut retry_count = 0;
    let max_retries = 3;
    let mut last_error = None;

    while retry_count < max_retries {
        let request = Request::new(SubmitOrderRequest {
            symbol: "BTC/USD".to_string(),
            side: OrderSide::Buy as i32,
            order_type: OrderType::Market as i32,
            quantity: 0.001,
            price: None,
            stop_price: None,
            time_in_force: "IOC".to_string(),
            client_order_id: Uuid::new_v4().to_string(),
        });

        match client.submit_order(request).await {
            Ok(response) => {
                println!("✓ Request succeeded on retry #{}", retry_count + 1);
                println!("  Order ID: {}", response.into_inner().order_id);
                break;
            },
            Err(err) => {
                retry_count += 1;
                last_error = Some(err);
                println!("  Retry #{} failed", retry_count);

                if retry_count < max_retries {
                    // Exponential backoff
                    let backoff = StdDuration::from_millis(100 * 2_u64.pow(retry_count as u32));
                    tokio::time::sleep(backoff).await;
                }
            },
        }
    }

    if retry_count == max_retries {
        println!("✓ Max retries reached");
        if let Some(err) = last_error {
            println!("  Final error: {:?}", err.code());
        }
    }

    Ok(())
}

// ============================================================================
// SECTION 3: SERVICE DISCOVERY AND FAILOVER E2E TESTS (5 tests)
// ============================================================================

#[tokio::test]
async fn test_e2e_api_routing() -> Result<()> {
    println!("\n=== E2E Test: API Gateway Routing Validation ===");

    let api_addr = get_api_addr();
    let channel = Channel::from_shared(api_addr)?.connect().await?;

    let config = TestAuthConfig::trader().with_user_id("routing_tester");
    let interceptor = create_auth_interceptor(config)?;
    let mut client = TradingServiceClient::with_interceptor(channel, interceptor);

    // Test multiple routing paths
    let tests = vec![(
        "Account Info",
        trading::GetAccountInfoRequest {
            account_id: "test_account".to_string(),
        },
    )];

    for (name, request) in tests {
        let result = client.get_account_info(Request::new(request)).await;

        match result {
            Ok(_) => println!("  ✓ {} routed successfully", name),
            Err(err) => println!("  ✗ {} routing failed: {:?}", name, err.code()),
        }
    }

    println!("✓ Routing validation completed");

    Ok(())
}

#[tokio::test]
async fn test_e2e_service_discovery() -> Result<()> {
    println!("\n=== E2E Test: Service Discovery ===");

    let api_addr = get_api_addr();
    let channel = Channel::from_shared(api_addr)?.connect().await?;

    let config = TestAuthConfig::admin().with_user_id("discovery_tester");
    let interceptor = create_auth_interceptor(config)?;
    let mut client = TradingServiceClient::with_interceptor(channel, interceptor);

    let request = Request::new(GetSystemStatusRequest {
        service_names: vec![],
    });

    let response = client.get_system_status(request).await?;
    let system_status = response.into_inner();

    println!("✓ Discovered services:");

    for service in &system_status.services {
        println!("  - {}", service.name);

        for (key, value) in &service.details {
            println!("    {}: {}", key, value);
        }
    }

    assert!(
        !system_status.services.is_empty(),
        "Should discover at least one service"
    );

    Ok(())
}

#[tokio::test]
async fn test_e2e_concurrent_service_requests() -> Result<()> {
    println!("\n=== E2E Test: Concurrent Service Requests ===");

    let api_addr = get_api_addr();
    let mut handles = vec![];

    // Spawn 10 concurrent requests
    for i in 0..10 {
        let addr = api_addr.clone();
        let handle = tokio::spawn(async move {
            let channel = Channel::from_shared(addr).unwrap().connect().await.unwrap();

            let config = TestAuthConfig::trader().with_user_id("concurrent_tester");
            let interceptor = create_auth_interceptor(config).unwrap();
            let mut client = TradingServiceClient::with_interceptor(channel, interceptor);

            let request = Request::new(trading::GetAccountInfoRequest {
                account_id: format!("test_account_{}", i),
            });

            client.get_account_info(request).await
        });

        handles.push(handle);
    }

    let results = futures::future::join_all(handles).await;
    let successful = results
        .iter()
        .filter(|r| matches!(r, Ok(Ok(_))))
        .count();

    println!("✓ Concurrent requests completed");
    println!("  Total: 10, Successful: {}", successful);

    assert!(
        successful >= 8,
        "At least 80% of concurrent requests should succeed"
    );

    Ok(())
}

#[tokio::test]
async fn test_e2e_load_balancing_verification() -> Result<()> {
    println!("\n=== E2E Test: Load Balancing Verification ===");

    let api_addr = get_api_addr();
    let channel = Channel::from_shared(api_addr)?.connect().await?;

    let config = TestAuthConfig::trader().with_user_id("load_tester");
    let interceptor = create_auth_interceptor(config)?;
    let mut client = TradingServiceClient::with_interceptor(channel, interceptor);

    // Send multiple requests and track response times
    let mut response_times = vec![];

    for _ in 0..20 {
        let start = std::time::Instant::now();

        let request = Request::new(trading::GetAccountInfoRequest {
            account_id: "load_test_account".to_string(),
        });

        if client.get_account_info(request).await.is_ok() {
            response_times.push(start.elapsed());
        }

        tokio::time::sleep(StdDuration::from_millis(50)).await;
    }

    if !response_times.is_empty() {
        let avg_time = response_times.iter().sum::<StdDuration>() / response_times.len() as u32;
        let max_time = response_times.iter().max().unwrap();
        let min_time = response_times.iter().min().unwrap();

        println!("✓ Load balancing metrics:");
        println!("  Average Response Time: {:?}", avg_time);
        println!("  Min Response Time: {:?}", min_time);
        println!("  Max Response Time: {:?}", max_time);
        println!("  Response Time Variance: {:?}", *max_time - *min_time);
    }

    Ok(())
}

#[tokio::test]
async fn test_e2e_service_failover() -> Result<()> {
    println!("\n=== E2E Test: Service Failover Behavior ===");

    let api_addr = get_api_addr();
    let channel = Channel::from_shared(api_addr)?.connect().await?;

    let config = TestAuthConfig::admin().with_user_id("failover_tester");
    let interceptor = create_auth_interceptor(config)?;
    let mut client = TradingServiceClient::with_interceptor(channel, interceptor);

    // Monitor system status for failover indicators
    let request = Request::new(GetSystemStatusRequest {
        service_names: vec![],
    });

    let response = client.get_system_status(request).await?;
    let system_status = response.into_inner();

    // Check for failover or backup service configurations
    let has_redundancy = system_status.services.len() > 1;

    println!("✓ Failover configuration check");
    println!("  Services Available: {}", system_status.services.len());
    println!("  Redundancy Detected: {}", has_redundancy);

    for service in &system_status.services {
        if let Some(failover_config) = service.details.get("failover_enabled") {
            println!("  - {}: Failover {}", service.name, failover_config);
        }
    }

    Ok(())
}