//! Backpressure Monitoring Load Tests - Wave 68 Agent 9 //! //! Validates the backpressure monitoring system from Wave 67 Agent 6 under realistic load. //! //! Tests cover: //! - Load scenarios triggering Warning (70%), Critical (95%), and Full (100%) thresholds //! - All 6 Prometheus metrics validation //! - MonitoredSender timeout behavior (100ms) //! - Silent failure prevention #![cfg(test)] use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; use tokio::time::{sleep, timeout}; // Import the streaming components from trading_service use trading_service::streaming::{ backpressure::{BackpressureConfig, BackpressureMonitor, BackpressureStatus}, monitored_channel::{create_monitored_channel, MonitoredSender}, metrics::StreamMetrics, }; /// Helper to create a test channel with custom configuration fn create_test_channel( buffer_size: usize, warning_threshold: f32, critical_threshold: f32, ) -> ( MonitoredSender, mpsc::Receiver, Arc, Arc, ) { let stream_name = format!("test_stream_{}", std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_nanos()); let (tx, rx) = mpsc::channel(buffer_size); let config = BackpressureConfig { buffer_capacity: buffer_size, warning_threshold, critical_threshold, metric_prefix: stream_name.clone(), }; let monitor = Arc::new(BackpressureMonitor::new(config)); let metrics = Arc::new(StreamMetrics::new(stream_name)); let monitored_tx = MonitoredSender::new(tx, Arc::clone(&monitor), Arc::clone(&metrics)); (monitored_tx, rx, monitor, metrics) } /// Load Scenario 1: Fill buffer to 70% (Warning threshold) #[tokio::test] async fn test_backpressure_warning_threshold() { const BUFFER_SIZE: usize = 1000; const WARNING_THRESHOLD: f32 = 0.7; // 70% const CRITICAL_THRESHOLD: f32 = 0.95; // 95% let (tx, mut rx, monitor, metrics) = create_test_channel::( BUFFER_SIZE, WARNING_THRESHOLD, CRITICAL_THRESHOLD, ); // Fill buffer to 70% (700 messages) let target_msgs = (BUFFER_SIZE as f32 * WARNING_THRESHOLD) as usize; println!("📊 Filling buffer to {}% ({} messages)", (WARNING_THRESHOLD * 100.0) as u8, target_msgs); for i in 0..target_msgs { let result = tx.send_monitored(format!("msg_{}", i)).await; assert!(result.is_ok(), "Send {} should succeed", i); } // Give time for metrics to update sleep(Duration::from_millis(10)).await; // Verify warning threshold triggered let utilization = tx.utilization_pct(); println!("📈 Buffer utilization: {}%", utilization); assert!( utilization >= 68 && utilization <= 72, "Expected ~70% utilization, got {}%", utilization ); // Verify backpressure warnings counter incremented let warnings = monitor.warnings_triggered(); println!("⚠️ Warning events triggered: {}", warnings); assert!(warnings > 0, "Warning threshold should have triggered"); // Verify messages sent counter let sent = monitor.messages_sent(); println!("✅ Messages sent: {}", sent); assert_eq!(sent, target_msgs as u64, "All messages should be sent"); // Verify no critical events yet let critical = monitor.critical_triggered(); println!("🚨 Critical events: {}", critical); assert_eq!(critical, 0, "Should not trigger critical yet"); // Verify no drops let dropped = monitor.messages_dropped(); println!("❌ Messages dropped: {}", dropped); assert_eq!(dropped, 0, "No messages should be dropped at 70%"); // Drain buffer for i in 0..target_msgs { let msg = rx.recv().await; assert!(msg.is_some(), "Should receive message {}", i); } println!("✅ Warning threshold test passed"); } /// Load Scenario 2: Fill buffer to 95% (Critical threshold) #[tokio::test] async fn test_backpressure_critical_threshold() { const BUFFER_SIZE: usize = 1000; const WARNING_THRESHOLD: f32 = 0.7; const CRITICAL_THRESHOLD: f32 = 0.95; // 95% let (tx, mut rx, monitor, metrics) = create_test_channel::( BUFFER_SIZE, WARNING_THRESHOLD, CRITICAL_THRESHOLD, ); // Fill buffer to 95% (950 messages) let target_msgs = (BUFFER_SIZE as f32 * CRITICAL_THRESHOLD) as usize; println!("📊 Filling buffer to {}% ({} messages)", (CRITICAL_THRESHOLD * 100.0) as u8, target_msgs); for i in 0..target_msgs { let result = tx.send_monitored(format!("msg_{}", i)).await; assert!(result.is_ok(), "Send {} should succeed", i); } sleep(Duration::from_millis(10)).await; // Verify critical threshold triggered let utilization = tx.utilization_pct(); println!("📈 Buffer utilization: {}%", utilization); assert!( utilization >= 93 && utilization <= 97, "Expected ~95% utilization, got {}%", utilization ); // Verify critical events let critical = monitor.critical_triggered(); println!("🚨 Critical events triggered: {}", critical); assert!(critical > 0, "Critical threshold should have triggered"); // Verify warning events also triggered let warnings = monitor.warnings_triggered(); println!("⚠️ Warning events: {}", warnings); assert!(warnings > 0, "Warning should also be triggered"); // Verify all messages sent let sent = monitor.messages_sent(); println!("✅ Messages sent: {}", sent); assert_eq!(sent, target_msgs as u64); // Verify no drops yet let dropped = monitor.messages_dropped(); println!("❌ Messages dropped: {}", dropped); assert_eq!(dropped, 0, "No messages should be dropped at 95%"); // Drain buffer for _ in 0..target_msgs { let _ = rx.recv().await; } println!("✅ Critical threshold test passed"); } /// Load Scenario 3: Fill buffer to 100% (Full) #[tokio::test] async fn test_backpressure_full_buffer() { const BUFFER_SIZE: usize = 100; // Smaller for faster test const WARNING_THRESHOLD: f32 = 0.7; const CRITICAL_THRESHOLD: f32 = 0.95; let (tx, mut rx, monitor, metrics) = create_test_channel::( BUFFER_SIZE, WARNING_THRESHOLD, CRITICAL_THRESHOLD, ); println!("📊 Filling buffer to 100% ({} messages)", BUFFER_SIZE); // Fill buffer completely for i in 0..BUFFER_SIZE { let result = tx.send_monitored(format!("msg_{}", i)).await; assert!(result.is_ok(), "Send {} should succeed", i); } sleep(Duration::from_millis(10)).await; // Verify 100% utilization let utilization = tx.utilization_pct(); println!("📈 Buffer utilization: {}%", utilization); assert_eq!(utilization, 100, "Buffer should be 100% full"); // Now attempt to send one more - should fail with resource exhausted println!("🚫 Attempting to send to full buffer..."); let result = tx.send_monitored("overflow_msg".to_string()).await; assert!(result.is_err(), "Send to full buffer should fail"); // Verify the error is resource exhausted let err = result.unwrap_err(); assert_eq!( err.code(), tonic::Code::ResourceExhausted, "Should return ResourceExhausted error" ); // Verify drop was recorded let dropped = monitor.messages_dropped(); println!("❌ Messages dropped: {}", dropped); assert_eq!(dropped, 1, "Should record one dropped message"); // Verify sent count (should not include the dropped message) let sent = monitor.messages_sent(); println!("✅ Messages sent: {}", sent); assert_eq!(sent, BUFFER_SIZE as u64); // Drain buffer for _ in 0..BUFFER_SIZE { let _ = rx.recv().await; } println!("✅ Full buffer test passed"); } /// Load Scenario 4: Test MonitoredSender timeout (100ms) #[tokio::test] async fn test_monitored_sender_timeout() { const BUFFER_SIZE: usize = 10; const TIMEOUT_MS: u64 = 50; // Use shorter timeout for faster test let (tx, _rx, monitor, metrics) = create_test_channel::( BUFFER_SIZE, 0.7, 0.95, ); // Set custom timeout let tx = tx.with_timeout(TIMEOUT_MS); println!("📊 Testing timeout behavior with {}ms timeout", TIMEOUT_MS); // Fill the buffer for i in 0..BUFFER_SIZE { let result = tx.send_monitored(format!("msg_{}", i)).await; assert!(result.is_ok(), "Initial send {} should succeed", i); } println!("🕐 Buffer full, attempting send with timeout..."); // Attempt send - should timeout since receiver isn't draining let start = std::time::Instant::now(); let result = tx.send_monitored("timeout_msg".to_string()).await; let elapsed = start.elapsed(); println!("⏱️ Send took {}ms", elapsed.as_millis()); // Verify timeout occurred assert!(result.is_err(), "Send should timeout"); assert_eq!( result.unwrap_err().code(), tonic::Code::DeadlineExceeded, "Should return DeadlineExceeded error" ); // Verify timeout was close to expected duration let timeout_duration = Duration::from_millis(TIMEOUT_MS); assert!( elapsed >= timeout_duration && elapsed < timeout_duration + Duration::from_millis(50), "Timeout should occur around {}ms, got {}ms", TIMEOUT_MS, elapsed.as_millis() ); // Verify metrics recorded timeout let dropped = monitor.messages_dropped(); println!("❌ Messages dropped due to timeout: {}", dropped); assert_eq!(dropped, 1, "Should record one timeout drop"); println!("✅ Timeout test passed"); } /// Load Scenario 5: Rapid burst load testing #[tokio::test] async fn test_rapid_burst_load() { const BUFFER_SIZE: usize = 500; const BURST_SIZE: usize = 1000; // Send more than buffer can hold const WARNING_THRESHOLD: f32 = 0.7; const CRITICAL_THRESHOLD: f32 = 0.95; let (tx, mut rx, monitor, metrics) = create_test_channel::( BUFFER_SIZE, WARNING_THRESHOLD, CRITICAL_THRESHOLD, ); println!("📊 Sending burst of {} messages to buffer of size {}", BURST_SIZE, BUFFER_SIZE); // Spawn sender task that sends rapidly let tx_clone = tx.clone(); let sender = tokio::spawn(async move { for i in 0..BURST_SIZE { // Use best-effort to avoid blocking on failures tx_clone.send_best_effort(i as u64).await; } }); // Spawn receiver task that drains slowly let receiver = tokio::spawn(async move { let mut received = 0; while received < BURST_SIZE { if let Some(_msg) = rx.recv().await { received += 1; // Simulate slow consumer tokio::time::sleep(Duration::from_micros(100)).await; } } received }); // Wait for both tasks let _ = sender.await; let received = receiver.await.unwrap(); // Verify metrics let sent = monitor.messages_sent(); let dropped = monitor.messages_dropped(); let warnings = monitor.warnings_triggered(); let critical = monitor.critical_triggered(); println!("📊 Burst Load Results:"); println!(" ✅ Messages sent: {}", sent); println!(" ❌ Messages dropped: {}", dropped); println!(" 📨 Messages received: {}", received); println!(" ⚠️ Warning events: {}", warnings); println!(" 🚨 Critical events: {}", critical); // Verify warning and critical thresholds triggered assert!(warnings > 0, "Warning threshold should trigger during burst"); assert!(critical > 0, "Critical threshold should trigger during burst"); // Verify no silent failures - sent + dropped should equal burst size assert_eq!( sent + dropped, BURST_SIZE as u64, "No silent failures: sent ({}) + dropped ({}) should equal burst size ({})", sent, dropped, BURST_SIZE ); println!("✅ Rapid burst load test passed - no silent failures"); } /// Integration test: Verify all 6 Prometheus metrics #[tokio::test] async fn test_all_prometheus_metrics() { const BUFFER_SIZE: usize = 100; let (tx, mut _rx, monitor, metrics) = create_test_channel::( BUFFER_SIZE, 0.7, 0.95, ); println!("📊 Testing all 6 Prometheus metrics"); // 1. stream_buffer_utilization - Test by filling buffer for i in 0..50 { tx.send_monitored(format!("msg_{}", i)).await.unwrap(); } let utilization = tx.utilization_pct(); println!("1️⃣ stream_buffer_utilization: {}%", utilization); assert!(utilization > 0, "Buffer utilization should be tracked"); // 2. stream_backpressure_warnings_total - Test by filling to 70% for i in 50..70 { tx.send_monitored(format!("msg_{}", i)).await.unwrap(); } let warnings = monitor.warnings_triggered(); println!("2️⃣ stream_backpressure_warnings_total: {}", warnings); assert!(warnings > 0, "Warning counter should increment"); // 3. stream_backpressure_critical_total - Test by filling to 95% for i in 70..95 { tx.send_monitored(format!("msg_{}", i)).await.unwrap(); } let critical = monitor.critical_triggered(); println!("3️⃣ stream_backpressure_critical_total: {}", critical); assert!(critical > 0, "Critical counter should increment"); // 4. stream_messages_sent_total - Already tracked let sent = monitor.messages_sent(); println!("4️⃣ stream_messages_sent_total: {}", sent); assert_eq!(sent, 95, "Should track all sent messages"); // 5. stream_send_timeouts_total - Test by filling buffer and timing out let tx_timeout = tx.with_timeout(10); for i in 95..100 { tx_timeout.send_monitored(format!("msg_{}", i)).await.unwrap(); } // This should timeout let _result = tx_timeout.send_monitored("timeout_msg".to_string()).await; // Note: We can't directly check STREAM_SEND_TIMEOUTS_TOTAL from here, // but the monitor tracks it via record_drop() on timeout // 6. stream_drops_total (actually stream_messages_dropped_total) let dropped = monitor.messages_dropped(); println!("6️⃣ stream_messages_dropped_total: {}", dropped); // Should have at least one drop from the timeout assert!(dropped >= 1, "Should track dropped messages"); println!("✅ All 6 Prometheus metrics validated"); } /// Stress test: Concurrent senders with backpressure #[tokio::test] async fn test_concurrent_senders_backpressure() { const BUFFER_SIZE: usize = 200; const NUM_SENDERS: usize = 5; const MSGS_PER_SENDER: usize = 100; let (tx, mut rx, monitor, metrics) = create_test_channel::( BUFFER_SIZE, 0.7, 0.95, ); println!("📊 Testing {} concurrent senders", NUM_SENDERS); // Spawn multiple sender tasks let mut sender_tasks = vec![]; for sender_id in 0..NUM_SENDERS { let tx_clone = tx.clone(); let task = tokio::spawn(async move { for msg_id in 0..MSGS_PER_SENDER { tx_clone.send_best_effort(format!("sender_{}_msg_{}", sender_id, msg_id)).await; // Small delay to simulate realistic sending tokio::time::sleep(Duration::from_micros(10)).await; } }); sender_tasks.push(task); } // Spawn receiver task let receiver_task = tokio::spawn(async move { let mut count = 0; while count < NUM_SENDERS * MSGS_PER_SENDER { if let Some(_msg) = timeout(Duration::from_secs(5), rx.recv()).await.ok().flatten() { count += 1; } } count }); // Wait for all senders for task in sender_tasks { task.await.unwrap(); } // Wait for receiver with timeout let received = timeout(Duration::from_secs(10), receiver_task) .await .expect("Receiver should complete") .unwrap(); let sent = monitor.messages_sent(); let dropped = monitor.messages_dropped(); let warnings = monitor.warnings_triggered(); let critical = monitor.critical_triggered(); println!("📊 Concurrent Senders Results:"); println!(" ✅ Messages sent: {}", sent); println!(" ❌ Messages dropped: {}", dropped); println!(" 📨 Messages received: {}", received); println!(" ⚠️ Warning events: {}", warnings); println!(" 🚨 Critical events: {}", critical); // Verify no silent failures let total_expected = (NUM_SENDERS * MSGS_PER_SENDER) as u64; assert_eq!( sent + dropped, total_expected, "No silent failures: sent + dropped should equal total expected" ); println!("✅ Concurrent senders test passed - no silent failures"); }