foxhunt/services/broker_gateway_service/tests/error_recovery_tests.rs

#![allow(
    clippy::tests_outside_test_module,
    clippy::unwrap_used,
    clippy::expect_used,
    clippy::indexing_slicing,
    clippy::str_to_string,
    clippy::string_to_string,
    clippy::useless_format,
    clippy::items_after_test_module,
    clippy::too_many_arguments,
    clippy::doc_markdown,
    clippy::shadow_unrelated,
    clippy::use_debug,
    clippy::needless_as_bytes,
    clippy::missing_const_for_fn,
    unused_imports,
    unused_variables,
    unused_mut,
    dead_code,
)]
//! Comprehensive Error Recovery Tests for Broker Gateway Service
//!
//! Tests cover:
//! 1. Network timeout during order submission (retry 3x, then fail)
//! 2. FIX session disconnect (auto-reconnect within 30s)
//! 3. Sequence number gap (trigger resend request)
//! 4. Order rejection → retry with reduced quantity
//! 5. Circuit breaker activation (5 timeouts → OPEN → HALF_OPEN)
//! 6. Database connection loss (queue orders, replay on reconnect)
//! 7. Concurrent order failures (ensure thread safety)

use broker_gateway_service::error_handler::{
    CircuitBreaker, CircuitBreakerConfig, CircuitBreakerState, DeadLetterEntry, DeadLetterQueue,
    ErrorHandler, ErrorRecoveryStrategy,
};
use broker_gateway_service::recovery::{
    HealthMonitor, HealthStatusLevel, OrderRecovery, Position, PositionRecovery,
    SessionRecovery,
};
use serial_test::serial;
use sqlx::PgPool;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::RwLock;

/// Database URL for integration tests
const DATABASE_URL: &str = "postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt";

/// Session ID for testing
const TEST_SESSION_ID: &str = "FOXHUNT_TEST-CQG_TEST";

/// Helper function to get database pool
async fn get_test_db_pool() -> PgPool {
    PgPool::connect(DATABASE_URL)
        .await
        .expect("Failed to connect to test database")
}

/// Helper function to cleanup test data
async fn cleanup_test_data(pool: &PgPool) {
    // Delete test orders
    sqlx::query("DELETE FROM broker_orders WHERE account_id LIKE 'TEST_%'")
        .execute(pool)
        .await
        .ok();

    // Delete test sessions
    sqlx::query("DELETE FROM broker_sessions WHERE session_id LIKE '%TEST%'")
        .execute(pool)
        .await
        .ok();

    // Delete test positions
    sqlx::query("DELETE FROM broker_positions WHERE account_id LIKE 'TEST_%'")
        .execute(pool)
        .await
        .ok();
}

/// Test 1: Network timeout during order submission (retry 3x, then fail)
#[tokio::test]
#[serial]
async fn test_network_timeout_retry_then_fail() {
    let handler = ErrorHandler::new();

    // Simulate network timeout error
    let attempt_count = Arc::new(std::sync::atomic::AtomicU32::new(0));
    let count_clone = Arc::clone(&attempt_count);
    let operation = move || -> Result<(), String> {
        let current = count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
        if current < 2 {
            Err("Network timeout".to_string())
        } else {
            Ok(())
        }
    };

    // This should retry 2 times before succeeding
    let result = handler
        .retry_with_backoff(operation, "submit_order")
        .await;

    assert!(result.is_ok());
    assert_eq!(attempt_count.load(std::sync::atomic::Ordering::SeqCst), 3);

    // Test failure after exhausting retries
    let fail_count = Arc::new(std::sync::atomic::AtomicU32::new(0));
    let fail_clone = Arc::clone(&fail_count);
    let failing_operation = move || -> Result<(), String> {
        fail_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
        Err("Network timeout".to_string())
    };

    let result = handler
        .retry_with_backoff(failing_operation, "submit_order")
        .await;

    assert!(result.is_err());
    assert_eq!(fail_count.load(std::sync::atomic::Ordering::SeqCst), 3);

    // Verify error sent to dead letter queue
    handler
        .send_to_dlq(
            "test-order-1".to_string(),
            "Network timeout after 3 retries".to_string(),
            3,
        )
        .await;

    let dlq = handler.dead_letter_queue();
    assert_eq!(dlq.len().await, 1);

    let entries = dlq.get_all().await;
    assert_eq!(entries[0].client_order_id, "test-order-1");
    assert_eq!(entries[0].retry_attempts, 3);
}

/// Test 2: FIX session disconnect (auto-reconnect within 30s)
#[tokio::test]
#[serial]
async fn test_fix_session_auto_reconnect() {
    let pool = get_test_db_pool().await;
    cleanup_test_data(&pool).await;

    let session_state = Arc::new(RwLock::new(
        broker_gateway_service::proto::broker_gateway::SessionState::Active,
    ));

    let session_recovery = SessionRecovery::new(pool.clone(), session_state.clone(), TEST_SESSION_ID.to_string());

    // Initialize session
    let session_info = session_recovery.recover_session().await.unwrap();
    assert_eq!(session_info.sender_seq_num, 1);
    assert_eq!(session_info.target_seq_num, 1);

    // Simulate disconnect
    session_recovery.disconnect().await.unwrap();
    let state = *session_state.read().await;
    assert_eq!(
        state,
        broker_gateway_service::proto::broker_gateway::SessionState::Disconnected
    );

    // Simulate auto-reconnect
    let reconnect_start = std::time::Instant::now();
    session_recovery.reconnect().await.unwrap();
    let reconnect_duration = reconnect_start.elapsed();

    // Verify reconnection succeeded within 30 seconds
    assert!(reconnect_duration < Duration::from_secs(30));

    let state = *session_state.read().await;
    assert_eq!(
        state,
        broker_gateway_service::proto::broker_gateway::SessionState::Active
    );

    // Cleanup
    cleanup_test_data(&pool).await;
    pool.close().await;
}

/// Test 3: Sequence number gap (trigger resend request)
#[tokio::test]
#[serial]
async fn test_sequence_number_gap_detection() {
    let pool = get_test_db_pool().await;
    cleanup_test_data(&pool).await;

    let session_state = Arc::new(RwLock::new(
        broker_gateway_service::proto::broker_gateway::SessionState::Active,
    ));

    let session_recovery = SessionRecovery::new(pool.clone(), session_state.clone(), TEST_SESSION_ID.to_string());

    // Initialize session
    session_recovery.recover_session().await.unwrap();

    // Update sequence numbers normally
    session_recovery
        .update_sequence_numbers(10, 10)
        .await
        .unwrap();

    // Recover session again to verify persistence
    let session_info = session_recovery.recover_session().await.unwrap();
    assert_eq!(session_info.sender_seq_num, 10);
    assert_eq!(session_info.target_seq_num, 10);

    // Simulate sequence number gap (target_seq_num jumps from 10 to 15)
    // In production, this would trigger a resend request (FIX Tag 35=2)
    let expected_seq = session_info.target_seq_num;
    let received_seq = 15_i64;
    let gap = received_seq - expected_seq;

    assert!(gap > 0, "Sequence number gap detected: {}", gap);
    assert_eq!(gap, 5); // Gap of 5 messages

    // Cleanup
    cleanup_test_data(&pool).await;
    pool.close().await;
}

/// Test 4: Order rejection → retry with reduced quantity
#[tokio::test]
#[serial]
async fn test_order_rejection_retry_reduced_quantity() {
    let pool = get_test_db_pool().await;
    cleanup_test_data(&pool).await;

    // Insert test order
    sqlx::query(
        r#"
        INSERT INTO broker_orders
        (client_order_id, account_id, symbol, side, order_type, quantity, status, submitted_at, created_at, updated_at)
        VALUES ('test-order-reject-1', 'TEST_ACCOUNT', 'ES', 'BUY', 'MARKET', 100, 'PENDING_SUBMIT', NOW(), NOW(), NOW())
        "#
    )
    .execute(&pool)
    .await
    .unwrap();

    let order_recovery = OrderRecovery::new(pool.clone());

    // Recover unsent orders
    let unsent_orders = order_recovery.recover_unsent_orders().await.unwrap();
    assert_eq!(unsent_orders.len(), 1);
    assert_eq!(unsent_orders[0].quantity, rust_decimal::Decimal::new(100, 0));

    // Simulate rejection due to invalid quantity
    let error_msg = "Invalid quantity: exceeds margin requirements";
    let handler = ErrorHandler::new();
    let strategy = handler.classify_error(error_msg);

    // Validation errors should fail fast (don't retry)
    assert_eq!(strategy, ErrorRecoveryStrategy::FailFast);

    // In production, we would reduce quantity and retry
    // For this test, we'll mark it as failed
    order_recovery
        .mark_order_failed("test-order-reject-1", error_msg)
        .await
        .unwrap();

    // Verify order is marked as REJECTED
    let result: (String, Option<String>) = sqlx::query_as(
        r#"
        SELECT status, metadata->>'reject_reason' as reject_reason
        FROM broker_orders
        WHERE client_order_id = 'test-order-reject-1'
        "#
    )
    .fetch_one(&pool)
    .await
    .unwrap();

    assert_eq!(result.0, "REJECTED");
    assert_eq!(result.1.unwrap(), error_msg);

    // Cleanup
    cleanup_test_data(&pool).await;
    pool.close().await;
}

/// Test 5: Circuit breaker activation (5 timeouts → OPEN → HALF_OPEN)
#[tokio::test]
async fn test_circuit_breaker_state_transitions() {
    let config = CircuitBreakerConfig {
        failure_threshold: 5,
        success_threshold: 1,
        timeout: Duration::from_millis(100),
    };
    let cb = CircuitBreaker::new("test_broker", config);

    // Initially CLOSED
    assert_eq!(cb.state().await, CircuitBreakerState::Closed);
    assert!(cb.can_execute().await);

    // Record 4 failures (below threshold)
    for _ in 0..4 {
        cb.record_failure().await;
    }

    assert_eq!(cb.state().await, CircuitBreakerState::Closed);
    assert!(cb.can_execute().await);

    // 5th failure should OPEN circuit
    cb.record_failure().await;
    assert_eq!(cb.state().await, CircuitBreakerState::Open);
    assert!(!cb.can_execute().await);

    // Wait for circuit breaker timeout
    tokio::time::sleep(Duration::from_millis(150)).await;

    // Manually reset to simulate timeout (in production, this happens automatically)
    cb.reset().await;
    assert_eq!(cb.state().await, CircuitBreakerState::Closed);
    assert!(cb.can_execute().await);
}

/// Test 6: Database connection loss (queue orders, replay on reconnect)
#[tokio::test]
#[serial]
async fn test_database_reconnect_replay_orders() {
    let pool = get_test_db_pool().await;
    cleanup_test_data(&pool).await;

    // Insert 3 test orders in PENDING_SUBMIT state
    for i in 1..=3 {
        sqlx::query(
            r#"
            INSERT INTO broker_orders
            (client_order_id, account_id, symbol, side, order_type, quantity, status, submitted_at, created_at, updated_at)
            VALUES ($1, 'TEST_ACCOUNT', 'ES', 'BUY', 'MARKET', 10, 'PENDING_SUBMIT', NOW(), NOW(), NOW())
            "#,
        )
        .bind(format!("test-order-replay-{}", i))
        .execute(&pool)
        .await
        .unwrap();
    }

    let order_recovery = OrderRecovery::new(pool.clone());

    // Simulate database reconnection and order replay
    let unsent_orders = order_recovery.recover_unsent_orders().await.unwrap();
    assert_eq!(unsent_orders.len(), 3);

    // Mark first two orders as submitted
    order_recovery
        .mark_order_submitted("test-order-replay-1")
        .await
        .unwrap();
    order_recovery
        .mark_order_submitted("test-order-replay-2")
        .await
        .unwrap();

    // Verify only 1 order remains PENDING_SUBMIT
    let unsent_orders = order_recovery.recover_unsent_orders().await.unwrap();
    assert_eq!(unsent_orders.len(), 1);
    assert_eq!(unsent_orders[0].client_order_id, "test-order-replay-3");

    // Cleanup
    cleanup_test_data(&pool).await;
    pool.close().await;
}

/// Test 7: Concurrent order failures (ensure thread safety)
#[tokio::test]
async fn test_concurrent_failures_thread_safety() {
    let handler = Arc::new(ErrorHandler::new());

    // Create entries for DLQ
    let mut entries = vec![];
    for i in 0..10 {
        entries.push(DeadLetterEntry {
            client_order_id: format!("concurrent-order-{}", i),
            error: "Concurrent failure test".to_string(),
            timestamp: std::time::Instant::now(),
            retry_attempts: 1,
        });
    }

    // Test concurrent DLQ add operations
    let mut add_tasks = vec![];
    for entry in entries {
        let handler_clone = Arc::clone(&handler);
        let task = tokio::spawn(async move {
            handler_clone.dead_letter_queue().add(entry).await;
        });
        add_tasks.push(task);
    }

    // Wait for all adds to complete
    futures::future::join_all(add_tasks).await;

    // Verify all 10 entries added
    let dlq = handler.dead_letter_queue();
    assert_eq!(dlq.len().await, 10);

    // Test concurrent removal operations
    let mut remove_tasks = vec![];
    for i in 0..5 {
        let handler_clone = Arc::clone(&handler);
        let task = tokio::spawn(async move {
            handler_clone
                .dead_letter_queue()
                .remove(&format!("concurrent-order-{}", i))
                .await
        });
        remove_tasks.push(task);
    }

    let remove_results: Vec<bool> = futures::future::join_all(remove_tasks)
        .await
        .into_iter()
        .map(|r| r.unwrap())
        .collect();

    // Verify all 5 removals succeeded
    assert_eq!(remove_results.iter().filter(|&&r| r).count(), 5);

    // Verify 5 entries remain
    assert_eq!(dlq.len().await, 5);

    // Test concurrent circuit breaker failure recording
    let cb = handler.circuit_breaker();
    let mut cb_tasks = vec![];
    for _ in 0..10 {
        let handler_clone = Arc::clone(&handler);
        let task = tokio::spawn(async move {
            handler_clone.circuit_breaker().record_failure().await;
        });
        cb_tasks.push(task);
    }

    // Wait for all failures to be recorded
    futures::future::join_all(cb_tasks).await;

    // Verify circuit breaker opened after 5+ failures
    assert_eq!(cb.state().await, CircuitBreakerState::Open);
}

/// Test: Health monitor returns DEGRADED when session disconnected
#[tokio::test]
#[serial]
async fn test_health_monitor_degraded_state() {
    let pool = get_test_db_pool().await;
    cleanup_test_data(&pool).await;

    let session_state = Arc::new(RwLock::new(
        broker_gateway_service::proto::broker_gateway::SessionState::Disconnected,
    ));

    let health_monitor = HealthMonitor::new(session_state.clone(), pool.clone());

    // Check health with disconnected session
    let health = health_monitor.check_health().await;
    assert_eq!(health.level, HealthStatusLevel::Degraded);
    assert!(health.database_healthy);
    assert!(!health.session_connected);

    // Simulate reconnection
    {
        let mut state = session_state.write().await;
        *state = broker_gateway_service::proto::broker_gateway::SessionState::Active;
    }

    // Check health with active session
    let health = health_monitor.check_health().await;
    assert_eq!(health.level, HealthStatusLevel::Healthy);
    assert!(health.database_healthy);
    assert!(health.session_connected);

    // Cleanup
    cleanup_test_data(&pool).await;
    pool.close().await;
}

/// Test: Position recovery and reconciliation
#[tokio::test]
#[serial]
async fn test_position_recovery_reconciliation() {
    let pool = get_test_db_pool().await;
    cleanup_test_data(&pool).await;

    let position_recovery = PositionRecovery::new(pool.clone());

    // Insert test positions
    let test_positions = vec![
        Position {
            symbol: "ES".to_string(),
            quantity: rust_decimal::Decimal::new(10, 0),
            avg_entry_price: Some(rust_decimal::Decimal::new(5000, 0)),
            market_value: Some(rust_decimal::Decimal::new(50000, 0)),
            unrealized_pnl: Some(rust_decimal::Decimal::new(500, 0)),
            realized_pnl: Some(rust_decimal::Decimal::new(0, 0)),
            last_updated: chrono::Utc::now(),
        },
        Position {
            symbol: "NQ".to_string(),
            quantity: rust_decimal::Decimal::new(-5, 0), // Short position
            avg_entry_price: Some(rust_decimal::Decimal::new(15000, 0)),
            market_value: Some(rust_decimal::Decimal::new(-75000, 0)),
            unrealized_pnl: Some(rust_decimal::Decimal::new(-250, 0)),
            realized_pnl: Some(rust_decimal::Decimal::new(0, 0)),
            last_updated: chrono::Utc::now(),
        },
    ];

    for position in &test_positions {
        position_recovery
            .update_position("TEST_ACCOUNT", position)
            .await
            .unwrap();
    }

    // Reconcile positions
    let reconciled = position_recovery
        .reconcile_positions("TEST_ACCOUNT")
        .await
        .unwrap();

    assert_eq!(reconciled.len(), 2);

    // Verify ES position (long)
    let es_position = reconciled.iter().find(|p| p.symbol == "ES").unwrap();
    assert_eq!(es_position.quantity, rust_decimal::Decimal::new(10, 0));
    assert!(es_position.quantity > rust_decimal::Decimal::ZERO);

    // Verify NQ position (short)
    let nq_position = reconciled.iter().find(|p| p.symbol == "NQ").unwrap();
    assert_eq!(nq_position.quantity, rust_decimal::Decimal::new(-5, 0));
    assert!(nq_position.quantity < rust_decimal::Decimal::ZERO);

    // Cleanup
    cleanup_test_data(&pool).await;
    pool.close().await;
}

/// Test: Error classification logic
#[test]
fn test_error_classification_strategies() {
    let handler = ErrorHandler::new();

    // Test validation errors (FailFast)
    assert_eq!(
        handler.classify_error("Invalid order quantity"),
        ErrorRecoveryStrategy::FailFast
    );
    assert_eq!(
        handler.classify_error("Validation failed: missing symbol"),
        ErrorRecoveryStrategy::FailFast
    );

    // Test network errors (Retry)
    assert_eq!(
        handler.classify_error("Connection timeout"),
        ErrorRecoveryStrategy::Retry
    );
    assert_eq!(
        handler.classify_error("Network unavailable"),
        ErrorRecoveryStrategy::Retry
    );

    // Test database errors (CircuitBreak)
    assert_eq!(
        handler.classify_error("Database connection failed"),
        ErrorRecoveryStrategy::CircuitBreak
    );
    assert_eq!(
        handler.classify_error("PostgreSQL error: deadlock detected"),
        ErrorRecoveryStrategy::CircuitBreak
    );
}

/// Test: Exponential backoff calculations
#[test]
fn test_exponential_backoff_progression() {
    let handler = ErrorHandler::new();

    // Test exponential progression: 100ms → 200ms → 400ms → 800ms → 1600ms
    let delays: Vec<Duration> = (0..5).map(|i| handler.calculate_backoff(i)).collect();

    assert_eq!(delays[0], Duration::from_millis(100));
    assert_eq!(delays[1], Duration::from_millis(200));
    assert_eq!(delays[2], Duration::from_millis(400));
    assert_eq!(delays[3], Duration::from_millis(800));
    assert_eq!(delays[4], Duration::from_millis(1_600));

    // Test capping at 25.6 seconds
    assert_eq!(
        handler.calculate_backoff(20),
        Duration::from_millis(25_600)
    );
}

/// Test: Dead letter queue overflow behavior
#[tokio::test]
async fn test_dead_letter_queue_overflow() {
    let dlq = DeadLetterQueue::new();

    // Add entries up to MAX_DLQ_SIZE (10,000)
    // For testing, we'll add 100 entries to avoid excessive memory usage
    for i in 0..100 {
        let entry = DeadLetterEntry {
            client_order_id: format!("overflow-test-{}", i),
            error: "Test overflow".to_string(),
            timestamp: std::time::Instant::now(),
            retry_attempts: 1,
        };

        dlq.add(entry).await;
    }

    assert_eq!(dlq.len().await, 100);

    // Clear all entries
    dlq.clear().await;
    assert!(dlq.is_empty().await);
}