foxhunt/services/api/tests/real_backend_integration_test.rs

//! Real Backend Integration Tests for API Gateway
//!
//! Tests that verify the API Gateway correctly proxies requests to real backend services:
//! - Trading Service (port 50052)
//! - Backtesting Service (port 50053)
//! - ML Training Service (port 50054)
//!
//! All tests use REAL gRPC communication with REAL services (no mocks).

#[path = "common/mod.rs"]
mod common;

use anyhow::Result;
use common::{cleanup_redis, generate_test_token, wait_for_redis};
use std::time::Duration;
use tonic::transport::Channel;
use tonic::Request;

const REDIS_URL: &str = "redis://localhost:6379";
const API_GATEWAY_URL: &str = "http://localhost:50051";
const TRADING_SERVICE_URL: &str = "http://localhost:50052";
const BACKTESTING_SERVICE_URL: &str = "http://localhost:50053";
const ML_TRAINING_SERVICE_URL: &str = "http://localhost:50054";

// Import proto definitions - we'll use tli's generated proto that includes all service definitions
// API Gateway tests need to use tli's proto definitions since they're testing the client-facing interface
use fxt::proto::health::{health_client::HealthClient, HealthCheckRequest};

// We use the standard health client for all service health checks

// ML Training proto from api
use api::ml_training::{
    ml_training_service_client::MlTrainingServiceClient, HealthCheckRequest as MlHealthRequest,
};

/// Helper to wait for a service to be ready
async fn wait_for_service_ready(url: &str, service_name: &str) -> Result<()> {
    let max_attempts = 30;
    let retry_delay = Duration::from_millis(500);

    for attempt in 1..=max_attempts {
        match tokio::net::TcpStream::connect(url.trim_start_matches("http://")).await {
            Ok(_) => {
                println!("✓ {} is ready (attempt {})", service_name, attempt);
                return Ok(());
            },
            Err(_) if attempt < max_attempts => {
                tokio::time::sleep(retry_delay).await;
            },
            Err(e) => {
                return Err(anyhow::anyhow!(
                    "{} not ready after {} attempts: {}",
                    service_name,
                    max_attempts,
                    e
                ));
            },
        }
    }

    unreachable!()
}

/// Setup test environment (Redis, wait for services)
async fn setup_test_environment() -> Result<()> {
    // Wait for Redis
    wait_for_redis(REDIS_URL, 50).await?;
    cleanup_redis(REDIS_URL).await?;

    // Wait for all backend services
    wait_for_service_ready("localhost:50051", "API Gateway").await?;
    wait_for_service_ready("localhost:50052", "Trading Service").await?;
    wait_for_service_ready("localhost:50053", "Backtesting Service").await?;
    wait_for_service_ready("localhost:50054", "ML Training Service").await?;

    println!("✓ All services are ready for testing");
    Ok(())
}

// ============================================================================
// TRADING SERVICE INTEGRATION TESTS
// ============================================================================

#[tokio::test]
async fn test_trading_service_direct_connection() -> Result<()> {
    println!("\n=== Test: Trading Service Direct Connection (No Proxy) ===");

    setup_test_environment().await?;

    // Connect directly to Trading Service (bypass API Gateway)
    let channel = Channel::from_static(TRADING_SERVICE_URL)
        .connect()
        .await
        .map_err(|e| anyhow::anyhow!("Failed to connect to Trading Service: {}", e))?;

    let mut client = HealthClient::new(channel);

    // Call health check
    let request = Request::new(HealthCheckRequest {
        service: "trading".to_string(),
    });
    let response = client
        .check(request)
        .await
        .map_err(|e| anyhow::anyhow!("Trading Service health check failed: {}", e))?;

    let health = response.into_inner();
    use fxt::proto::health::ServingStatus;
    let status_str = match ServingStatus::try_from(health.status) {
        Ok(ServingStatus::Serving) => "serving",
        Ok(ServingStatus::NotServing) => "not_serving",
        Ok(ServingStatus::Unknown) | Ok(ServingStatus::ServiceUnknown) | Err(_) => "unknown",
    };
    println!("✓ Trading Service health: {}", status_str);
    assert_eq!(
        health.status, ServingStatus::Serving as i32,
        "Trading Service should be serving"
    );

    Ok(())
}

#[tokio::test]
async fn test_trading_service_via_api_proxy() -> Result<()> {
    println!("\n=== Test: Trading Service via API Gateway Proxy ===");

    setup_test_environment().await?;

    // Generate valid JWT token
    let (token, _jti) = generate_test_token(
        "test_user",
        vec!["trader".to_string()],
        vec!["api.access".to_string(), "trading.submit".to_string()],
        3600,
    )?;

    // Connect to API Gateway (which proxies to Trading Service)
    let channel = Channel::from_static(API_GATEWAY_URL)
        .connect()
        .await
        .map_err(|e| anyhow::anyhow!("Failed to connect to API Gateway: {}", e))?;

    let mut client = HealthClient::new(channel);

    // Call health check through API Gateway proxy
    let mut request = Request::new(HealthCheckRequest {
        service: "trading".to_string(),
    });
    request.metadata_mut().insert(
        "authorization",
        format!("Bearer {}", token).parse().expect("INVARIANT: Valid parse input"),
    );

    let start = std::time::Instant::now();
    let response = client
        .check(request)
        .await
        .map_err(|e| anyhow::anyhow!("Trading Service health check via proxy failed: {}", e))?;
    let elapsed = start.elapsed();

    let health = response.into_inner();
    use fxt::proto::health::ServingStatus;
    let status_str = match ServingStatus::try_from(health.status) {
        Ok(ServingStatus::Serving) => "serving",
        Ok(ServingStatus::NotServing) => "not_serving",
        Ok(ServingStatus::Unknown) | Ok(ServingStatus::ServiceUnknown) | Err(_) => "unknown",
    };
    println!("✓ Trading Service health via proxy: {}", status_str);
    println!("  Proxy latency: {:?} (target: <1ms)", elapsed);

    assert_eq!(
        health.status, ServingStatus::Serving as i32,
        "Trading Service should be serving"
    );
    assert!(
        elapsed < Duration::from_millis(50),
        "Proxy latency should be <50ms, got {:?}",
        elapsed
    );

    Ok(())
}

#[tokio::test]
async fn test_trading_service_proxy_requires_auth() -> Result<()> {
    println!("\n=== Test: Trading Service Proxy Requires Authentication ===");

    setup_test_environment().await?;

    // Connect to API Gateway WITHOUT auth token
    let channel = Channel::from_static(API_GATEWAY_URL)
        .connect()
        .await
        .map_err(|e| anyhow::anyhow!("Failed to connect to API Gateway: {}", e))?;

    let mut client = HealthClient::new(channel);

    // Call health check without auth header
    let request = Request::new(HealthCheckRequest {
        service: "trading".to_string(),
    });
    let result = client.check(request).await;

    // Should fail with UNAUTHENTICATED
    assert!(
        result.is_err(),
        "Request without auth should be rejected by API Gateway"
    );

    let error = result.unwrap_err();
    println!("✓ Correctly rejected: {:?}", error.code());
    assert_eq!(
        error.code(),
        tonic::Code::Unauthenticated,
        "Should return UNAUTHENTICATED"
    );

    Ok(())
}

// ============================================================================
// BACKTESTING SERVICE INTEGRATION TESTS
// ============================================================================

#[tokio::test]
async fn test_backtesting_service_direct_connection() -> Result<()> {
    println!("\n=== Test: Backtesting Service Direct Connection (No Proxy) ===");

    setup_test_environment().await?;

    // Connect directly to Backtesting Service (bypass API Gateway)
    let channel = Channel::from_static(BACKTESTING_SERVICE_URL)
        .connect()
        .await
        .map_err(|e| anyhow::anyhow!("Failed to connect to Backtesting Service: {}", e))?;

    let mut client = HealthClient::new(channel);

    // Call health check
    let request = Request::new(HealthCheckRequest {
        service: "backtesting".to_string(),
    });
    let response = client
        .check(request)
        .await
        .map_err(|e| anyhow::anyhow!("Backtesting Service health check failed: {}", e))?;

    let health = response.into_inner();
    use fxt::proto::health::ServingStatus;
    let status_str = match ServingStatus::try_from(health.status) {
        Ok(ServingStatus::Serving) => "serving",
        Ok(ServingStatus::NotServing) => "not_serving",
        Ok(ServingStatus::Unknown) | Ok(ServingStatus::ServiceUnknown) | Err(_) => "unknown",
    };
    println!("✓ Backtesting Service health: {}", status_str);
    assert_eq!(
        health.status, ServingStatus::Serving as i32,
        "Backtesting Service should be serving"
    );

    Ok(())
}

#[tokio::test]
async fn test_backtesting_service_via_api_proxy() -> Result<()> {
    println!("\n=== Test: Backtesting Service via API Gateway Proxy ===");

    setup_test_environment().await?;

    // Generate valid JWT token
    let (token, _jti) = generate_test_token(
        "test_user",
        vec!["trader".to_string()],
        vec!["api.access".to_string(), "backtesting.run".to_string()],
        3600,
    )?;

    // Connect to API Gateway (which proxies to Backtesting Service)
    let channel = Channel::from_static(API_GATEWAY_URL)
        .connect()
        .await
        .map_err(|e| anyhow::anyhow!("Failed to connect to API Gateway: {}", e))?;

    let mut client = HealthClient::new(channel);

    // Call health check through API Gateway proxy
    let mut request = Request::new(HealthCheckRequest {
        service: "backtesting".to_string(),
    });
    request.metadata_mut().insert(
        "authorization",
        format!("Bearer {}", token).parse().expect("INVARIANT: Valid parse input"),
    );

    let start = std::time::Instant::now();
    let response = client
        .check(request)
        .await
        .map_err(|e| anyhow::anyhow!("Backtesting Service health check via proxy failed: {}", e))?;
    let elapsed = start.elapsed();

    let health = response.into_inner();
    use fxt::proto::health::ServingStatus;
    let status_str = match ServingStatus::try_from(health.status) {
        Ok(ServingStatus::Serving) => "serving",
        Ok(ServingStatus::NotServing) => "not_serving",
        Ok(ServingStatus::Unknown) | Ok(ServingStatus::ServiceUnknown) | Err(_) => "unknown",
    };
    println!("✓ Backtesting Service health via proxy: {}", status_str);
    println!("  Proxy latency: {:?} (target: <1ms)", elapsed);

    assert_eq!(
        health.status, ServingStatus::Serving as i32,
        "Backtesting Service should be serving"
    );
    assert!(
        elapsed < Duration::from_millis(50),
        "Proxy latency should be <50ms, got {:?}",
        elapsed
    );

    Ok(())
}

#[tokio::test]
async fn test_backtesting_service_proxy_requires_auth() -> Result<()> {
    println!("\n=== Test: Backtesting Service Proxy Requires Authentication ===");

    setup_test_environment().await?;

    // Connect to API Gateway WITHOUT auth token
    let channel = Channel::from_static(API_GATEWAY_URL)
        .connect()
        .await
        .map_err(|e| anyhow::anyhow!("Failed to connect to API Gateway: {}", e))?;

    let mut client = HealthClient::new(channel);

    // Call health check without auth header
    let request = Request::new(HealthCheckRequest {
        service: "backtesting".to_string(),
    });
    let result = client.check(request).await;

    // Should fail with UNAUTHENTICATED
    assert!(
        result.is_err(),
        "Request without auth should be rejected by API Gateway"
    );

    let error = result.unwrap_err();
    println!("✓ Correctly rejected: {:?}", error.code());
    assert_eq!(
        error.code(),
        tonic::Code::Unauthenticated,
        "Should return UNAUTHENTICATED"
    );

    Ok(())
}

// ============================================================================
// ML TRAINING SERVICE INTEGRATION TESTS
// ============================================================================

#[tokio::test]
async fn test_ml_training_service_direct_connection() -> Result<()> {
    println!("\n=== Test: ML Training Service Direct Connection (No Proxy) ===");

    setup_test_environment().await?;

    // Connect directly to ML Training Service (bypass API Gateway)
    let channel = Channel::from_static(ML_TRAINING_SERVICE_URL)
        .connect()
        .await
        .map_err(|e| anyhow::anyhow!("Failed to connect to ML Training Service: {}", e))?;

    let mut client = MlTrainingServiceClient::new(channel);

    // Call health check
    let request = Request::new(MlHealthRequest {});
    let response = client
        .health_check(request)
        .await
        .map_err(|e| anyhow::anyhow!("ML Training Service health check failed: {}", e))?;

    let health = response.into_inner();
    println!("✓ ML Training Service health: {}", if health.healthy { "healthy" } else { "unhealthy" });
    assert!(
        health.healthy,
        "ML Training Service should be healthy: {}",
        health.message
    );

    Ok(())
}

#[tokio::test]
async fn test_ml_training_service_via_api_proxy() -> Result<()> {
    println!("\n=== Test: ML Training Service via API Gateway Proxy ===");

    setup_test_environment().await?;

    // Generate valid JWT token
    let (token, _jti) = generate_test_token(
        "test_user",
        vec!["admin".to_string()],
        vec!["api.access".to_string(), "ml.train".to_string()],
        3600,
    )?;

    // Connect to API Gateway (which proxies to ML Training Service)
    let channel = Channel::from_static(API_GATEWAY_URL)
        .connect()
        .await
        .map_err(|e| anyhow::anyhow!("Failed to connect to API Gateway: {}", e))?;

    let mut client = MlTrainingServiceClient::new(channel);

    // Call health check through API Gateway proxy
    let mut request = Request::new(MlHealthRequest {});
    request.metadata_mut().insert(
        "authorization",
        format!("Bearer {}", token).parse().expect("INVARIANT: Valid parse input"),
    );

    let start = std::time::Instant::now();
    let response = client
        .health_check(request)
        .await
        .map_err(|e| anyhow::anyhow!("ML Training Service health check via proxy failed: {}", e))?;
    let elapsed = start.elapsed();

    let health = response.into_inner();
    println!("✓ ML Training Service health via proxy: {}", if health.healthy { "healthy" } else { "unhealthy" });
    println!("  Proxy latency: {:?} (target: <1ms)", elapsed);

    assert!(
        health.healthy,
        "ML Training Service should be healthy: {}",
        health.message
    );
    assert!(
        elapsed < Duration::from_millis(50),
        "Proxy latency should be <50ms, got {:?}",
        elapsed
    );

    Ok(())
}

#[tokio::test]
async fn test_ml_training_service_proxy_requires_auth() -> Result<()> {
    println!("\n=== Test: ML Training Service Proxy Requires Authentication ===");

    setup_test_environment().await?;

    // Connect to API Gateway WITHOUT auth token
    let channel = Channel::from_static(API_GATEWAY_URL)
        .connect()
        .await
        .map_err(|e| anyhow::anyhow!("Failed to connect to API Gateway: {}", e))?;

    let mut client = MlTrainingServiceClient::new(channel);

    // Call health check without auth header
    let request = Request::new(MlHealthRequest {});
    let result = client.health_check(request).await;

    // Should fail with UNAUTHENTICATED
    assert!(
        result.is_err(),
        "Request without auth should be rejected by API Gateway"
    );

    let error = result.unwrap_err();
    println!("✓ Correctly rejected: {:?}", error.code());
    assert_eq!(
        error.code(),
        tonic::Code::Unauthenticated,
        "Should return UNAUTHENTICATED"
    );

    Ok(())
}

// ============================================================================
// MULTI-SERVICE INTEGRATION TESTS
// ============================================================================

#[tokio::test]
async fn test_api_routes_to_all_backend_services() -> Result<()> {
    println!("\n=== Test: API Gateway Routes to All Backend Services ===");

    setup_test_environment().await?;

    // Generate valid JWT token
    let (token, _jti) = generate_test_token(
        "test_user",
        vec!["admin".to_string(), "trader".to_string()],
        vec![
            "api.access".to_string(),
            "trading.submit".to_string(),
            "backtesting.run".to_string(),
            "ml.train".to_string(),
        ],
        3600,
    )?;

    // Connect to API Gateway once
    let channel = Channel::from_static(API_GATEWAY_URL)
        .connect()
        .await
        .map_err(|e| anyhow::anyhow!("Failed to connect to API Gateway: {}", e))?;

    // Test Trading Service routing
    {
        let mut client = HealthClient::new(channel.clone());
        let mut request = Request::new(HealthCheckRequest {
            service: "trading".to_string(),
        });
        request.metadata_mut().insert(
            "authorization",
            format!("Bearer {}", token).parse().expect("INVARIANT: Valid parse input"),
        );

        let response = client.check(request).await?;
        use fxt::proto::health::ServingStatus;
        let health = response.into_inner();
        let status_str = match ServingStatus::try_from(health.status) {
            Ok(ServingStatus::Serving) => "serving",
            Ok(ServingStatus::NotServing) => "not_serving",
            Ok(ServingStatus::Unknown) | Ok(ServingStatus::ServiceUnknown) | Err(_) => "unknown",
        };
        println!("  ✓ Trading Service: {}", status_str);
    }

    // Test Backtesting Service routing
    {
        let mut client = HealthClient::new(channel.clone());
        let mut request = Request::new(HealthCheckRequest {
            service: "backtesting".to_string(),
        });
        request.metadata_mut().insert(
            "authorization",
            format!("Bearer {}", token).parse().expect("INVARIANT: Valid parse input"),
        );

        let response = client.check(request).await?;
        use fxt::proto::health::ServingStatus;
        let health = response.into_inner();
        let status_str = match ServingStatus::try_from(health.status) {
            Ok(ServingStatus::Serving) => "serving",
            Ok(ServingStatus::NotServing) => "not_serving",
            Ok(ServingStatus::Unknown) | Ok(ServingStatus::ServiceUnknown) | Err(_) => "unknown",
        };
        println!("  ✓ Backtesting Service: {}", status_str);
    }

    // Test ML Training Service routing
    {
        let mut client = MlTrainingServiceClient::new(channel.clone());
        let mut request = Request::new(MlHealthRequest {});
        request.metadata_mut().insert(
            "authorization",
            format!("Bearer {}", token).parse().expect("INVARIANT: Valid parse input"),
        );

        let response = client.health_check(request).await?;
        let health = response.into_inner();
        println!("  ✓ ML Training Service: {}", if health.healthy { "healthy" } else { "unhealthy" });
    }

    println!("✓ API Gateway successfully routes to all 3 backend services");

    Ok(())
}

#[tokio::test]
async fn test_api_proxy_latency_across_services() -> Result<()> {
    println!("\n=== Test: API Gateway Proxy Latency Across All Services ===");

    setup_test_environment().await?;

    // Generate valid JWT token
    let (token, _jti) = generate_test_token(
        "test_user",
        vec!["admin".to_string(), "trader".to_string()],
        vec![
            "api.access".to_string(),
            "trading.submit".to_string(),
            "backtesting.run".to_string(),
            "ml.train".to_string(),
        ],
        3600,
    )?;

    let channel = Channel::from_static(API_GATEWAY_URL).connect().await?;

    let mut latencies = Vec::new();

    // Measure Trading Service latency (10 samples)
    for _ in 0..10 {
        let mut client = HealthClient::new(channel.clone());
        let mut request = Request::new(HealthCheckRequest {
            service: "trading".to_string(),
        });
        request.metadata_mut().insert(
            "authorization",
            format!("Bearer {}", token).parse().expect("INVARIANT: Valid parse input"),
        );

        let start = std::time::Instant::now();
        let _ = client.check(request).await?;
        latencies.push(start.elapsed());
    }

    // Calculate P50, P95, P99
    latencies.sort();
    let p50 = latencies[latencies.len() / 2];
    let p95 = latencies[latencies.len() * 95 / 100];
    let p99 = latencies[latencies.len() * 99 / 100];

    println!("\n  Proxy Latency Statistics:");
    println!("  ├─ P50: {:?}", p50);
    println!("  ├─ P95: {:?}", p95);
    println!("  └─ P99: {:?}", p99);

    assert!(
        p99 < Duration::from_millis(50),
        "P99 latency should be <50ms, got {:?}",
        p99
    );
    println!("✓ Proxy latency within acceptable bounds");

    Ok(())
}