foxhunt/services/ml_training_service/tests/advanced_streaming_tests.rs

#![allow(
    dead_code,
    unused_variables,
    unused_mut,
    clippy::unwrap_used,
    clippy::expect_used,
    clippy::indexing_slicing,
    clippy::redundant_pattern_matching
)]
//! Advanced gRPC Streaming Tests
//!
//! Tests cover:
//! - Slow consumer handling (backpressure)
//! - Network interruption recovery
//! - Multiple concurrent streams (16+ jobs)
//! - Delta streaming bandwidth validation
//! - Stream cleanup and resource management

use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::{broadcast, mpsc, RwLock};
use tokio::time::{sleep, timeout};
use uuid::Uuid;

// ============================================================================
// MOCK TYPES (simplified for unit testing)
// ============================================================================

#[derive(Clone, Debug)]
struct StreamUpdate {
    job_id: Uuid,
    epoch: u32,
    progress: f64,
    message: String,
}

struct StreamManager {
    active_streams: Arc<RwLock<HashMap<Uuid, broadcast::Sender<StreamUpdate>>>>,
    message_count: Arc<RwLock<u64>>,
}

impl StreamManager {
    fn new() -> Self {
        Self {
            active_streams: Arc::new(RwLock::new(HashMap::new())),
            message_count: Arc::new(RwLock::new(0)),
        }
    }

    async fn create_stream(&self, job_id: Uuid) -> broadcast::Receiver<StreamUpdate> {
        let (tx, rx) = broadcast::channel(1000); // Large buffer for backpressure test
        self.active_streams.write().await.insert(job_id, tx);
        rx
    }

    async fn send_update(&self, update: StreamUpdate) -> usize {
        let job_id = update.job_id;
        let streams = self.active_streams.read().await;

        if let Some(tx) = streams.get(&job_id) {
            let receiver_count = tx.receiver_count();
            let _ = tx.send(update);
            *self.message_count.write().await += 1;
            receiver_count
        } else {
            0
        }
    }

    async fn close_stream(&self, job_id: Uuid) {
        self.active_streams.write().await.remove(&job_id);
    }

    async fn get_message_count(&self) -> u64 {
        *self.message_count.read().await
    }

    async fn active_stream_count(&self) -> usize {
        self.active_streams.read().await.len()
    }
}

// ============================================================================
// SLOW CONSUMER HANDLING (BACKPRESSURE)
// ============================================================================

#[tokio::test]
async fn test_slow_consumer_with_fast_producer() {
    let manager = StreamManager::new();
    let job_id = Uuid::new_v4();
    let mut rx = manager.create_stream(job_id).await;

    // Fast producer: send 100 updates rapidly
    let manager_clone = Arc::new(manager);
    let producer = {
        let manager = Arc::clone(&manager_clone);
        tokio::spawn(async move {
            for i in 0..100 {
                manager
                    .send_update(StreamUpdate {
                        job_id,
                        epoch: i,
                        progress: i as f64,
                        message: format!("Update {}", i),
                    })
                    .await;
                // No delay - fire hose
            }
        })
    };

    // Slow consumer: process with delays
    let consumer = tokio::spawn(async move {
        let mut received = 0;
        while let Ok(update) = timeout(Duration::from_millis(500), rx.recv()).await {
            if update.is_ok() {
                received += 1;
                sleep(Duration::from_millis(10)).await; // Slow processing
            } else {
                break;
            }
        }
        received
    });

    producer.await.unwrap();
    let received = consumer.await.unwrap();

    // Some messages may be dropped due to buffer overflow
    // But consumer should handle gracefully
    println!("Received {} out of 100 messages", received);
    assert!(received > 0, "Consumer received no messages");
}

#[tokio::test]
async fn test_backpressure_with_bounded_buffer() {
    let (tx, mut rx) = mpsc::channel::<StreamUpdate>(10); // Small buffer

    // Producer sends 100 messages
    let producer = tokio::spawn(async move {
        let job_id = Uuid::new_v4();
        let mut sent = 0;
        let start = Instant::now();

        for i in 0..100 {
            let update = StreamUpdate {
                job_id,
                epoch: i,
                progress: i as f64,
                message: format!("Update {}", i),
            };

            // This will block when buffer is full (backpressure)
            if tx.send(update).await.is_ok() {
                sent += 1;
            } else {
                break;
            }
        }

        (sent, start.elapsed())
    });

    // Slow consumer
    tokio::spawn(async move {
        while let Some(_update) = rx.recv().await {
            sleep(Duration::from_millis(5)).await;
        }
    });

    let (sent, elapsed) = producer.await.unwrap();

    println!("Sent {} messages in {:?}", sent, elapsed);
    assert_eq!(sent, 100, "All messages should be sent with backpressure");
    assert!(
        elapsed > Duration::from_millis(450),
        "Backpressure should slow down producer"
    );
}

#[tokio::test]
async fn test_channel_overflow_handling() {
    let (tx, _rx) = broadcast::channel::<StreamUpdate>(5); // Very small buffer

    let job_id = Uuid::new_v4();

    // Send many messages without consumer (overflow)
    for i in 0..20 {
        let update = StreamUpdate {
            job_id,
            epoch: i,
            progress: i as f64,
            message: format!("Overflow {}", i),
        };
        let _ = tx.send(update); // May fail, that's okay
    }

    // Now subscribe and try to receive
    let mut rx2 = tx.subscribe();

    // Should receive only recent messages (old ones dropped)
    let mut received = 0;
    while timeout(Duration::from_millis(10), rx2.recv())
        .await
        .is_ok()
    {
        received += 1;
    }

    // Should receive less than 20 due to overflow
    println!("Received {} messages after overflow", received);
    assert!(received < 20);
}

// ============================================================================
// NETWORK INTERRUPTION RECOVERY
// ============================================================================

#[tokio::test]
async fn test_reconnection_after_disconnect() {
    let manager = Arc::new(StreamManager::new());
    let job_id = Uuid::new_v4();

    // First connection
    let mut rx1 = manager.create_stream(job_id).await;

    // Send some updates
    for i in 0..10 {
        manager
            .send_update(StreamUpdate {
                job_id,
                epoch: i,
                progress: i as f64 * 10.0,
                message: format!("Update {}", i),
            })
            .await;
    }

    // Simulate disconnect (drop receiver)
    drop(rx1);

    // Reconnect (new receiver)
    let mut rx2 = manager.create_stream(job_id).await;

    // Send more updates
    for i in 10..20 {
        manager
            .send_update(StreamUpdate {
                job_id,
                epoch: i,
                progress: i as f64 * 10.0,
                message: format!("Update {}", i),
            })
            .await;
    }

    // New receiver should get new messages
    let mut received = 0;
    while let Ok(Ok(_)) = timeout(Duration::from_millis(10), rx2.recv()).await {
        received += 1;
    }

    assert!(received > 0, "Reconnected stream should receive messages");
}

#[tokio::test]
async fn test_graceful_stream_cleanup() {
    let manager = Arc::new(StreamManager::new());
    let job_id = Uuid::new_v4();

    let rx = manager.create_stream(job_id).await;
    assert_eq!(manager.active_stream_count().await, 1);

    // Close stream
    manager.close_stream(job_id).await;
    assert_eq!(manager.active_stream_count().await, 0);

    // Sending to closed stream should fail gracefully
    let receivers = manager
        .send_update(StreamUpdate {
            job_id,
            epoch: 1,
            progress: 10.0,
            message: "Should not be received".to_string(),
        })
        .await;

    assert_eq!(receivers, 0);
    drop(rx);
}

// ============================================================================
// MULTIPLE CONCURRENT STREAMS (16+ JOBS)
// ============================================================================

#[tokio::test]
async fn test_16_concurrent_streams() {
    let manager = Arc::new(StreamManager::new());
    let mut handles = vec![];
    let job_ids: Vec<Uuid> = (0..16).map(|_| Uuid::new_v4()).collect();

    // Create 16 receivers
    let mut receivers = vec![];
    for job_id in &job_ids {
        receivers.push(manager.create_stream(*job_id).await);
    }

    // Spawn 16 producers
    for (i, job_id) in job_ids.iter().enumerate() {
        let manager_clone = Arc::clone(&manager);
        let job_id = *job_id;

        let handle = tokio::spawn(async move {
            for epoch in 0..50 {
                manager_clone
                    .send_update(StreamUpdate {
                        job_id,
                        epoch,
                        progress: epoch as f64 * 2.0,
                        message: format!("Job {} - Epoch {}", i, epoch),
                    })
                    .await;
                sleep(Duration::from_micros(100)).await;
            }
        });
        handles.push(handle);
    }

    // Wait for all producers
    for handle in handles {
        handle.await.unwrap();
    }

    // Total messages: 16 jobs × 50 epochs = 800
    let total = manager.get_message_count().await;
    assert_eq!(total, 800);

    drop(receivers);
}

#[tokio::test]
async fn test_100_concurrent_streams_stress() {
    let manager = Arc::new(StreamManager::new());
    let job_count = 100;

    // Create streams
    let job_ids: Vec<Uuid> = (0..job_count).map(|_| Uuid::new_v4()).collect();
    let mut _receivers = Vec::new();
    for id in &job_ids {
        _receivers.push(manager.create_stream(*id).await);
    }

    assert_eq!(manager.active_stream_count().await, job_count);

    // Send updates to all jobs concurrently
    let mut handles = vec![];
    for job_id in &job_ids {
        let manager_clone = Arc::clone(&manager);
        let job_id = *job_id;

        let handle = tokio::spawn(async move {
            for epoch in 0..10 {
                manager_clone
                    .send_update(StreamUpdate {
                        job_id,
                        epoch,
                        progress: epoch as f64 * 10.0,
                        message: format!("Epoch {}", epoch),
                    })
                    .await;
            }
        });
        handles.push(handle);
    }

    let start = Instant::now();
    for handle in handles {
        handle.await.unwrap();
    }
    let elapsed = start.elapsed();

    // 100 jobs × 10 messages = 1000 total
    assert_eq!(manager.get_message_count().await, 1000);

    println!("100 streams, 1000 messages: {:?}", elapsed);
    assert!(elapsed < Duration::from_secs(2), "Streaming too slow");
}

// ============================================================================
// DELTA STREAMING BANDWIDTH VALIDATION
// ============================================================================

#[tokio::test]
async fn test_message_throughput_single_stream() {
    let manager = Arc::new(StreamManager::new());
    let job_id = Uuid::new_v4();
    let mut rx = manager.create_stream(job_id).await;

    let message_count = 10_000;

    // Send 10K messages as fast as possible
    let sender = {
        let manager = Arc::clone(&manager);
        tokio::spawn(async move {
            let start = Instant::now();
            for i in 0..message_count {
                manager
                    .send_update(StreamUpdate {
                        job_id,
                        epoch: i,
                        progress: (i as f64 / message_count as f64) * 100.0,
                        message: format!("Progress {}", i),
                    })
                    .await;
            }
            start.elapsed()
        })
    };

    // Receive all messages
    let receiver = tokio::spawn(async move {
        let mut received = 0;
        let start = Instant::now();
        while let Ok(Ok(_)) = timeout(Duration::from_secs(5), rx.recv()).await {
            received += 1;
            if received >= message_count {
                break;
            }
        }
        (received, start.elapsed())
    });

    let send_time = sender.await.unwrap();
    let (received, recv_time) = receiver.await.unwrap();

    println!(
        "10K messages: sent in {:?}, received {} in {:?}",
        send_time, received, recv_time
    );

    // Target: >1000 messages/sec
    let msgs_per_sec = received as f64 / recv_time.as_secs_f64();
    println!("Throughput: {:.0} msg/sec", msgs_per_sec);
    assert!(msgs_per_sec > 1000.0, "Throughput too low: {:.0} msg/sec", msgs_per_sec);
}

#[tokio::test]
async fn test_latency_distribution() {
    let manager = Arc::new(StreamManager::new());
    let job_id = Uuid::new_v4();
    let mut rx = manager.create_stream(job_id).await;

    let mut latencies = vec![];
    let iterations = 1000;

    for i in 0..iterations {
        let send_start = Instant::now();

        manager
            .send_update(StreamUpdate {
                job_id,
                epoch: i,
                progress: i as f64,
                message: format!("Latency test {}", i),
            })
            .await;

        // Measure time to receive
        if let Ok(Ok(_)) = timeout(Duration::from_millis(100), rx.recv()).await {
            latencies.push(send_start.elapsed());
        }
    }

    // Calculate percentiles
    latencies.sort();
    let p50 = latencies[latencies.len() / 2];
    let p95 = latencies[latencies.len() * 95 / 100];
    let p99 = latencies[latencies.len() * 99 / 100];

    println!(
        "Latency P50: {:?}, P95: {:?}, P99: {:?}",
        p50, p95, p99
    );

    // Target: P99 < 1ms
    assert!(p99 < Duration::from_millis(10), "P99 latency too high: {:?}", p99);
}

// ============================================================================
// RESOURCE CLEANUP
// ============================================================================

#[tokio::test]
async fn test_memory_leak_prevention() {
    let manager = Arc::new(StreamManager::new());

    // Create and destroy 1000 streams
    for _ in 0..1000 {
        let job_id = Uuid::new_v4();
        let rx = manager.create_stream(job_id).await;
        drop(rx);
        manager.close_stream(job_id).await;
    }

    // Should have no active streams
    assert_eq!(manager.active_stream_count().await, 0);
}

#[tokio::test]
async fn test_orphaned_stream_cleanup() {
    let manager = Arc::new(StreamManager::new());
    let job_ids: Vec<Uuid> = (0..10).map(|_| Uuid::new_v4()).collect();

    // Create streams but don't read from them
    let mut receivers = Vec::new();
    for id in &job_ids {
        receivers.push(manager.create_stream(*id).await);
    }

    // Send messages
    for job_id in &job_ids {
        for i in 0..100 {
            manager
                .send_update(StreamUpdate {
                    job_id: *job_id,
                    epoch: i,
                    progress: i as f64,
                    message: format!("Orphaned {}", i),
                })
                .await;
        }
    }

    // Drop all receivers (simulate client disconnect)
    drop(receivers);

    // Cleanup
    for job_id in &job_ids {
        manager.close_stream(*job_id).await;
    }

    assert_eq!(manager.active_stream_count().await, 0);
}

// ============================================================================
// EDGE CASES
// ============================================================================

#[tokio::test]
async fn test_zero_buffer_channel() {
    // Rendezvous channel (0 buffer) - requires synchronous handoff
    let (tx, mut rx) = mpsc::channel::<StreamUpdate>(1); // Minimal buffer

    let job_id = Uuid::new_v4();

    // Spawn sender
    let sender = tokio::spawn(async move {
        for i in 0..10 {
            let update = StreamUpdate {
                job_id,
                epoch: i,
                progress: i as f64 * 10.0,
                message: format!("Sync {}", i),
            };
            tx.send(update).await.unwrap();
        }
    });

    // Receiver
    let receiver = tokio::spawn(async move {
        let mut count = 0;
        while let Some(_) = rx.recv().await {
            count += 1;
            if count >= 10 {
                break;
            }
        }
        count
    });

    sender.await.unwrap();
    let received = receiver.await.unwrap();

    assert_eq!(received, 10);
}

#[tokio::test]
async fn test_rapid_subscribe_unsubscribe() {
    let manager = Arc::new(StreamManager::new());
    let job_id = Uuid::new_v4();

    // Rapidly subscribe and unsubscribe
    for _ in 0..100 {
        let rx = manager.create_stream(job_id).await;
        manager
            .send_update(StreamUpdate {
                job_id,
                epoch: 1,
                progress: 10.0,
                message: "Rapid test".to_string(),
            })
            .await;
        drop(rx);
    }

    // Should handle without issues
    assert!(manager.get_message_count().await >= 100);
}

#[tokio::test]
async fn test_concurrent_send_and_close() {
    let manager = Arc::new(StreamManager::new());
    let job_id = Uuid::new_v4();
    let _rx = manager.create_stream(job_id).await;

    // Concurrent sends and close
    let sender = {
        let manager = Arc::clone(&manager);
        tokio::spawn(async move {
            for i in 0..100 {
                manager
                    .send_update(StreamUpdate {
                        job_id,
                        epoch: i,
                        progress: i as f64,
                        message: format!("Concurrent {}", i),
                    })
                    .await;
                sleep(Duration::from_micros(10)).await;
            }
        })
    };

    // Close after short delay
    sleep(Duration::from_millis(5)).await;
    manager.close_stream(job_id).await;

    sender.await.unwrap();

    // Should not crash
    assert!(manager.active_stream_count().await == 0);
}