foxhunt/services/api_gateway/tests/routing_edge_cases.rs

//! Request Routing and Backend Failure Edge Case Tests
//!
//! This test suite focuses on request routing scenarios including:
//!
//! 1. Backend Service Failures:
//!    - Connection refused
//!    - Service timeout
//!    - Network errors
//!    - Circuit breaker activation
//!
//! 2. Load Balancing:
//!    - Round-robin distribution
//!    - Failover to healthy backends
//!    - Sticky sessions
//!
//! 3. Service Discovery:
//!    - Backend registration
//!    - Health check integration
//!    - Dynamic endpoint updates
//!
//! 4. Timeout Handling:
//!    - Request timeout
//!    - Connection timeout
//!    - Streaming timeout

use anyhow::Result;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::time::timeout;
use tonic::transport::Endpoint;
use tonic::Status;

// ============================================================================
// Backend Connection Tests
// ============================================================================

#[tokio::test]
async fn test_backend_connection_refused() -> Result<()> {
    println!("\n=== Test: Backend Connection Refused ===");

    // Try to connect to non-existent backend
    let endpoint = Endpoint::from_static("http://localhost:59999") // Non-existent port
        .connect_timeout(Duration::from_millis(100))
        .timeout(Duration::from_millis(100));

    let result = endpoint.connect().await;

    assert!(
        result.is_err(),
        "Connection to non-existent backend should fail"
    );

    if let Err(e) = result {
        println!("✓ Connection refused as expected: {}", e);
    }

    Ok(())
}

#[tokio::test]
async fn test_backend_connection_timeout() -> Result<()> {
    println!("\n=== Test: Backend Connection Timeout ===");

    // Use a non-routable IP (192.0.2.0 is TEST-NET-1 from RFC 5737)
    let endpoint = Endpoint::from_static("http://192.0.2.1:50000")
        .connect_timeout(Duration::from_millis(50))
        .timeout(Duration::from_millis(50));

    let start = Instant::now();
    let result = endpoint.connect().await;
    let elapsed = start.elapsed();

    assert!(result.is_err(), "Connection should timeout");
    assert!(
        elapsed < Duration::from_millis(200),
        "Timeout should be enforced"
    );

    println!("✓ Connection timeout after {:?}", elapsed);

    Ok(())
}

#[tokio::test]
async fn test_invalid_endpoint_url() -> Result<()> {
    println!("\n=== Test: Invalid Endpoint URL ===");

    // Try invalid URL format
    let result = Endpoint::from_shared("not-a-valid-url");

    assert!(result.is_err(), "Invalid URL should be rejected");

    if let Err(e) = result {
        println!("✓ Invalid URL rejected: {}", e);
    }

    Ok(())
}

#[tokio::test]
async fn test_missing_scheme_in_url() -> Result<()> {
    println!("\n=== Test: Missing Scheme in URL ===");

    // URL without http:// or https://
    let result = Endpoint::from_shared("localhost:50000");

    assert!(result.is_err(), "URL without scheme should be rejected");

    if let Err(e) = result {
        println!("✓ URL without scheme rejected: {}", e);
    }

    Ok(())
}

// ============================================================================
// Request Timeout Tests
// ============================================================================

#[tokio::test]
async fn test_request_timeout_enforced() -> Result<()> {
    println!("\n=== Test: Request Timeout Enforced ===");

    // Simulate long-running operation
    let operation = async {
        tokio::time::sleep(Duration::from_millis(500)).await;
        Ok::<_, anyhow::Error>("completed")
    };

    // Apply 100ms timeout
    let start = Instant::now();
    let result = timeout(Duration::from_millis(100), operation).await;
    let elapsed = start.elapsed();

    assert!(result.is_err(), "Request should timeout");
    assert!(
        elapsed < Duration::from_millis(200),
        "Timeout should be enforced quickly"
    );

    println!("✓ Request timeout enforced after {:?}", elapsed);

    Ok(())
}

#[tokio::test]
async fn test_streaming_timeout() -> Result<()> {
    println!("\n=== Test: Streaming Timeout ===");

    // Simulate slow streaming response
    let stream_operation = async {
        tokio::time::sleep(Duration::from_millis(500)).await;
        Ok::<_, anyhow::Error>("stream chunk")
    };

    // Apply timeout
    let start = Instant::now();
    let result = timeout(Duration::from_millis(100), stream_operation).await;
    let elapsed = start.elapsed();

    assert!(result.is_err(), "Streaming should timeout");
    println!("✓ Streaming timeout after {:?}", elapsed);

    Ok(())
}

// ============================================================================
// Circuit Breaker Tests
// ============================================================================

#[tokio::test]
async fn test_circuit_breaker_opens_after_failures() -> Result<()> {
    println!("\n=== Test: Circuit Breaker Opens After Failures ===");

    let failure_threshold = 5;
    let failure_count = Arc::new(AtomicUsize::new(0));
    let circuit_open = Arc::new(AtomicUsize::new(0)); // 0=closed, 1=open

    // Simulate consecutive failures
    for i in 1..=10 {
        // Simulate backend call failure
        let count = failure_count.fetch_add(1, Ordering::SeqCst) + 1;

        if count >= failure_threshold && circuit_open.load(Ordering::SeqCst) == 0 {
            circuit_open.store(1, Ordering::SeqCst);
            println!("  ✓ Circuit breaker opened after {} failures", count);
        }

        if circuit_open.load(Ordering::SeqCst) == 1 {
            println!("  Request #{}: Circuit OPEN - fail fast", i);
        } else {
            println!("  Request #{}: Circuit CLOSED - attempting", i);
        }
    }

    let final_count = failure_count.load(Ordering::SeqCst);
    let is_open = circuit_open.load(Ordering::SeqCst) == 1;

    assert!(is_open, "Circuit breaker should be open");
    assert!(
        final_count >= failure_threshold,
        "Should have recorded all failures"
    );

    println!(
        "✓ Circuit breaker correctly opened after {} failures",
        final_count
    );

    Ok(())
}

#[tokio::test]
async fn test_circuit_breaker_half_open_state() -> Result<()> {
    println!("\n=== Test: Circuit Breaker Half-Open State ===");

    let circuit_state = Arc::new(AtomicUsize::new(1)); // 0=closed, 1=open, 2=half-open

    // Simulate circuit opening
    println!("  Circuit state: OPEN");

    // Wait for timeout (simulate)
    tokio::time::sleep(Duration::from_millis(50)).await;

    // Transition to half-open
    circuit_state.store(2, Ordering::SeqCst);
    println!("  Circuit state: HALF-OPEN (testing with probe request)");

    // Simulate successful probe request
    tokio::time::sleep(Duration::from_millis(10)).await;
    let success = true; // Simulated success

    if success {
        circuit_state.store(0, Ordering::SeqCst); // Close circuit
        println!("  ✓ Probe succeeded - Circuit state: CLOSED");
    } else {
        circuit_state.store(1, Ordering::SeqCst); // Reopen circuit
        println!("  Probe failed - Circuit state: OPEN");
    }

    assert_eq!(
        circuit_state.load(Ordering::SeqCst),
        0,
        "Circuit should be closed after successful probe"
    );

    Ok(())
}

#[tokio::test]
async fn test_circuit_breaker_reset_after_success() -> Result<()> {
    println!("\n=== Test: Circuit Breaker Reset After Success ===");

    let failure_count = Arc::new(AtomicUsize::new(0));
    let success_count = Arc::new(AtomicUsize::new(0));

    // Simulate failures
    for _ in 0..3 {
        failure_count.fetch_add(1, Ordering::SeqCst);
    }

    println!("  Failures: {}", failure_count.load(Ordering::SeqCst));

    // Simulate success
    success_count.fetch_add(1, Ordering::SeqCst);
    failure_count.store(0, Ordering::SeqCst); // Reset on success

    println!(
        "  Success - failure count reset: {}",
        failure_count.load(Ordering::SeqCst)
    );

    assert_eq!(
        failure_count.load(Ordering::SeqCst),
        0,
        "Failure count should reset"
    );
    assert_eq!(
        success_count.load(Ordering::SeqCst),
        1,
        "Success should be recorded"
    );

    println!("✓ Circuit breaker reset successfully");

    Ok(())
}

// ============================================================================
// Load Balancing Tests
// ============================================================================

#[tokio::test]
async fn test_round_robin_distribution() -> Result<()> {
    println!("\n=== Test: Round-Robin Load Distribution ===");

    let backends = vec!["backend-1", "backend-2", "backend-3"];
    let current_index = Arc::new(AtomicUsize::new(0));
    let request_counts = Arc::new([
        AtomicUsize::new(0),
        AtomicUsize::new(0),
        AtomicUsize::new(0),
    ]);

    // Simulate 15 requests with round-robin
    for _ in 0..15 {
        let index = current_index.fetch_add(1, Ordering::SeqCst) % backends.len();
        request_counts[index].fetch_add(1, Ordering::SeqCst);
    }

    // Check distribution
    let counts: Vec<usize> = request_counts
        .iter()
        .map(|c| c.load(Ordering::SeqCst))
        .collect();

    println!("\n  Request Distribution:");
    for (i, count) in counts.iter().enumerate() {
        println!("  {} -> {} requests", backends[i], count);
    }

    // Each backend should get 5 requests
    for count in &counts {
        assert_eq!(*count, 5, "Each backend should get equal requests");
    }

    println!("✓ Round-robin distribution working correctly");

    Ok(())
}

#[tokio::test]
async fn test_failover_to_healthy_backend() -> Result<()> {
    println!("\n=== Test: Failover to Healthy Backend ===");

    struct Backend {
        name: String,
        healthy: bool,
    }

    let backends = vec![
        Backend {
            name: "backend-1".to_string(),
            healthy: false,
        }, // Unhealthy
        Backend {
            name: "backend-2".to_string(),
            healthy: true,
        }, // Healthy
        Backend {
            name: "backend-3".to_string(),
            healthy: false,
        }, // Unhealthy
    ];

    // Select first healthy backend
    let selected = backends.iter().find(|b| b.healthy);

    assert!(selected.is_some(), "Should find healthy backend");

    if let Some(backend) = selected {
        println!("✓ Failover selected: {}", backend.name);
        assert_eq!(backend.name, "backend-2");
    }

    Ok(())
}

#[tokio::test]
async fn test_all_backends_unhealthy() -> Result<()> {
    println!("\n=== Test: All Backends Unhealthy ===");

    struct Backend {
        name: String,
        healthy: bool,
    }

    let backends = vec![
        Backend {
            name: "backend-1".to_string(),
            healthy: false,
        },
        Backend {
            name: "backend-2".to_string(),
            healthy: false,
        },
        Backend {
            name: "backend-3".to_string(),
            healthy: false,
        },
    ];

    // Try to find healthy backend
    let selected = backends.iter().find(|b| b.healthy);

    assert!(selected.is_none(), "Should not find any healthy backend");
    println!("✓ Correctly detected no healthy backends");

    Ok(())
}

// ============================================================================
// Health Check Tests
// ============================================================================

#[tokio::test]
async fn test_health_check_marks_unhealthy_on_failure() -> Result<()> {
    println!("\n=== Test: Health Check Marks Unhealthy on Failure ===");

    let is_healthy = Arc::new(AtomicUsize::new(1)); // 1=healthy, 0=unhealthy

    // Simulate health check failure
    let health_check_result = Err::<(), _>("Connection failed");

    if health_check_result.is_err() {
        is_healthy.store(0, Ordering::SeqCst);
        println!("  ✓ Backend marked unhealthy");
    }

    assert_eq!(
        is_healthy.load(Ordering::SeqCst),
        0,
        "Backend should be unhealthy"
    );

    Ok(())
}

#[tokio::test]
async fn test_health_check_recovers_on_success() -> Result<()> {
    println!("\n=== Test: Health Check Recovers on Success ===");

    let is_healthy = Arc::new(AtomicUsize::new(0)); // Start unhealthy

    println!("  Backend initially: UNHEALTHY");

    // Simulate successful health check
    let health_check_result = Ok::<(), &str>(());

    if health_check_result.is_ok() {
        is_healthy.store(1, Ordering::SeqCst);
        println!("  ✓ Backend marked healthy");
    }

    assert_eq!(
        is_healthy.load(Ordering::SeqCst),
        1,
        "Backend should be healthy"
    );

    Ok(())
}

#[tokio::test]
async fn test_health_check_interval_respected() -> Result<()> {
    println!("\n=== Test: Health Check Interval Respected ===");

    let last_check = Arc::new(AtomicUsize::new(0));
    let check_interval_ms = 100;

    // First check
    let now = Instant::now();
    last_check.store(now.elapsed().as_millis() as usize, Ordering::SeqCst);
    println!("  First health check");

    // Try immediate second check - should be skipped
    let elapsed_since_last = now.elapsed().as_millis() as usize - last_check.load(Ordering::SeqCst);

    if elapsed_since_last < check_interval_ms {
        println!("  Second check skipped (interval not elapsed)");
    }

    // Wait for interval
    tokio::time::sleep(Duration::from_millis(check_interval_ms as u64 + 10)).await;

    // Now check should proceed
    let elapsed_since_last = now.elapsed().as_millis() as usize - last_check.load(Ordering::SeqCst);

    if elapsed_since_last >= check_interval_ms {
        println!("  Third check executed (interval elapsed)");
        last_check.store(now.elapsed().as_millis() as usize, Ordering::SeqCst);
    }

    println!("✓ Health check interval correctly enforced");

    Ok(())
}

// ============================================================================
// Endpoint Configuration Tests
// ============================================================================

#[tokio::test]
async fn test_tcp_keepalive_configuration() -> Result<()> {
    println!("\n=== Test: TCP Keepalive Configuration ===");

    let endpoint = Endpoint::from_static("http://localhost:50000")
        .tcp_keepalive(Some(Duration::from_secs(60)));

    println!("✓ TCP keepalive configured: 60s");

    // Endpoint configured successfully
    assert!(true);

    Ok(())
}

#[tokio::test]
async fn test_http2_keepalive_configuration() -> Result<()> {
    println!("\n=== Test: HTTP/2 Keepalive Configuration ===");

    let endpoint = Endpoint::from_static("http://localhost:50000")
        .http2_keep_alive_interval(Duration::from_secs(30));

    println!("✓ HTTP/2 keepalive configured: 30s");

    assert!(true);

    Ok(())
}

#[tokio::test]
async fn test_multiple_endpoint_configurations() -> Result<()> {
    println!("\n=== Test: Multiple Endpoint Configurations ===");

    let endpoint = Endpoint::from_static("http://localhost:50000")
        .connect_timeout(Duration::from_millis(5000))
        .timeout(Duration::from_millis(30000))
        .tcp_keepalive(Some(Duration::from_secs(60)))
        .http2_keep_alive_interval(Duration::from_secs(30));

    println!("✓ Endpoint configured with:");
    println!("  - Connect timeout: 5000ms");
    println!("  - Request timeout: 30000ms");
    println!("  - TCP keepalive: 60s");
    println!("  - HTTP/2 keepalive: 30s");

    assert!(true);

    Ok(())
}

// ============================================================================
// Error Handling Tests
// ============================================================================

#[tokio::test]
async fn test_status_code_mapping() -> Result<()> {
    println!("\n=== Test: Status Code Mapping ===");

    // Test various error scenarios
    let errors = vec![
        (tonic::Code::Unavailable, "Service unavailable"),
        (tonic::Code::DeadlineExceeded, "Request timeout"),
        (tonic::Code::Internal, "Internal error"),
        (tonic::Code::Unauthenticated, "Authentication failed"),
        (tonic::Code::PermissionDenied, "Permission denied"),
    ];

    for (code, message) in errors {
        let status = Status::new(code, message);
        println!("  {} -> {}", code, status.message());
        assert_eq!(status.code(), code);
    }

    println!("✓ Status code mapping correct");

    Ok(())
}

#[tokio::test]
async fn test_metadata_propagation() -> Result<()> {
    println!("\n=== Test: Metadata Propagation ===");

    use tonic::metadata::{MetadataMap, MetadataValue};

    let mut metadata = MetadataMap::new();
    metadata.insert("x-request-id", MetadataValue::try_from("req-123")?);
    metadata.insert("x-user-id", MetadataValue::try_from("user-456")?);

    // Verify metadata
    assert_eq!(
        metadata.get("x-request-id").unwrap(),
        &MetadataValue::try_from("req-123")?
    );
    assert_eq!(
        metadata.get("x-user-id").unwrap(),
        &MetadataValue::try_from("user-456")?
    );

    println!("✓ Metadata propagation working");

    Ok(())
}