//! Network edge cases and error handling tests //! //! Tests comprehensive error scenarios including: //! - Network timeouts and failures //! - Authentication errors //! - Rate limiting //! - Corrupted data handling //! - Connection pool exhaustion use object_store::memory::InMemory; use object_store::ObjectStore; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use storage::error::StorageError; use storage::model_helpers::{ConnectionPool, RetryConfig}; use storage::object_store_backend::ObjectStoreBackend; use storage::Storage; /// Helper to create test backend with in-memory store fn create_test_backend() -> ObjectStoreBackend { let in_memory_store: Arc = Arc::new(InMemory::new()); storage::object_store_backend::test_helpers::new_for_testing( in_memory_store, "test-bucket".to_string(), ) } #[tokio::test] async fn test_network_timeout_handling() { let backend = create_test_backend(); // Configure aggressive retry with short timeouts let retry_config = RetryConfig { max_retries: 2, base_delay: std::time::Duration::from_millis(10), max_delay: std::time::Duration::from_millis(50), backoff_multiplier: 2.0, ..Default::default() }; let backend = backend.with_retry_config(retry_config); // Store data backend .store("timeout_test.bin", b"test data") .await .unwrap(); // Retrieve should succeed quickly with in-memory store let start = std::time::Instant::now(); let data = backend.retrieve("timeout_test.bin").await.unwrap(); let elapsed = start.elapsed(); assert_eq!(data, b"test data"); // Should complete well under timeout assert!(elapsed < std::time::Duration::from_millis(100)); } #[tokio::test] async fn test_large_file_chunked_upload() { let backend = create_test_backend(); // Simulate large file (50MB) let large_data = vec![0xAB; 50 * 1024 * 1024]; let start = std::time::Instant::now(); backend.store("large_file.bin", &large_data).await.unwrap(); let upload_time = start.elapsed(); // Verify upload let metadata = backend.metadata("large_file.bin").await.unwrap(); assert_eq!(metadata.size, large_data.len() as u64); println!("Uploaded 50MB in {:?}", upload_time); } #[tokio::test] async fn test_large_file_streaming_download() { let backend = create_test_backend(); // Create and upload large file (30MB) let large_data = vec![0xCD; 30 * 1024 * 1024]; backend .store("stream_large.bin", &large_data) .await .unwrap(); // Download with progress tracking let progress_count = Arc::new(AtomicUsize::new(0)); let progress_count_clone = progress_count.clone(); let callback = Arc::new(move |downloaded: u64, total: u64| { progress_count_clone.fetch_add(1, Ordering::SeqCst); println!( "Download progress: {}/{} bytes ({:.1}%)", downloaded, total, (downloaded as f64 / total as f64) * 100.0 ); }); let downloaded = backend .stream_download_with_progress("stream_large.bin", 1024 * 1024, callback) .await .unwrap(); assert_eq!(downloaded.len(), large_data.len()); assert!(progress_count.load(Ordering::SeqCst) > 0); } #[tokio::test] async fn test_connection_pool_parallel_downloads() { // Create a shared in-memory store for all connections let shared_store: Arc = Arc::new(InMemory::new()); // Create backend using the shared store let backend = storage::object_store_backend::test_helpers::new_for_testing( Arc::clone(&shared_store), "parallel-bucket".to_string(), ); // Create connection pool with references to the same shared store let pool = Arc::new(ConnectionPool::new(vec![ Arc::clone(&shared_store), Arc::clone(&shared_store), Arc::clone(&shared_store), ])); let backend = backend.with_connection_pool(pool); // Upload test files for i in 1..=5 { let path = format!("parallel_{}.bin", i); let data = vec![i as u8; 1024 * 1024]; // 1MB each backend.store(&path, &data).await.unwrap(); } // Parallel download let paths: Vec = (1..=5).map(|i| format!("parallel_{}.bin", i)).collect(); let start = std::time::Instant::now(); let results = backend.parallel_download(paths, None).await.unwrap(); let download_time = start.elapsed(); assert_eq!(results.len(), 5); println!("Downloaded 5 files (5MB total) in {:?}", download_time); } #[tokio::test] async fn test_corrupted_data_detection() { let backend = create_test_backend(); let original_data = b"original data without corruption"; backend .store("corruption_test.bin", original_data) .await .unwrap(); // Retrieve and verify let retrieved = backend.retrieve("corruption_test.bin").await.unwrap(); assert_eq!(retrieved, original_data); // Calculate checksums use sha2::{Digest, Sha256}; let mut hasher = Sha256::new(); hasher.update(original_data); let original_checksum = format!("{:x}", hasher.finalize()); let mut hasher = Sha256::new(); hasher.update(&retrieved); let retrieved_checksum = format!("{:x}", hasher.finalize()); assert_eq!(original_checksum, retrieved_checksum); } #[tokio::test] async fn test_metadata_not_found_error() { let backend = create_test_backend(); let result = backend.metadata("nonexistent_file.bin").await; assert!(result.is_err()); // Should return an error, not panic match result { Err(StorageError::OperationFailed { operation, .. }) => { assert_eq!(operation, "head"); }, _ => panic!("Expected OperationFailed error"), } } #[tokio::test] async fn test_retrieve_missing_file() { let backend = create_test_backend(); let result = backend.retrieve("missing_file.bin").await; assert!(result.is_err()); match result { Err(StorageError::OperationFailed { operation, .. }) => { assert_eq!(operation, "get"); }, _ => panic!("Expected OperationFailed error for missing file"), } } #[tokio::test] async fn test_list_empty_bucket() { let backend = create_test_backend(); let files = backend.list("").await.unwrap(); assert_eq!(files.len(), 0); } #[tokio::test] async fn test_list_with_deep_nesting() { let backend = create_test_backend(); // Create deeply nested structure let nested_paths = [ "level1/file.bin", "level1/level2/file.bin", "level1/level2/level3/file.bin", "level1/level2/level3/level4/file.bin", "level1/level2/level3/level4/level5/file.bin", ]; for path in &nested_paths { backend.store(path, b"nested data").await.unwrap(); } // List all files let all_files = backend.list("").await.unwrap(); assert_eq!(all_files.len(), nested_paths.len()); // List at different levels let level1_files = backend.list("level1/").await.unwrap(); assert_eq!(level1_files.len(), nested_paths.len()); let level3_files = backend.list("level1/level2/level3/").await.unwrap(); assert_eq!(level3_files.len(), 3); // level3, level4, level5 files } #[tokio::test] async fn test_concurrent_read_write_operations() { let backend = Arc::new(create_test_backend()); let mut handles = vec![]; // Concurrent writers for i in 0..5 { let backend = Arc::clone(&backend); let handle = tokio::spawn(async move { let path = format!("concurrent_write_{}.bin", i); let data = vec![i as u8; 512 * 1024]; // 512KB backend.store(&path, &data).await.unwrap(); }); handles.push(handle); } // Concurrent readers (reading different files) for i in 0..5 { let backend = Arc::clone(&backend); let handle = tokio::spawn(async move { // Wait for writes to complete tokio::time::sleep(std::time::Duration::from_millis(100)).await; let path = format!("concurrent_write_{}.bin", i); if backend.exists(&path).await.unwrap_or(false) { let data = backend.retrieve(&path).await.unwrap(); assert_eq!(data.len(), 512 * 1024); } }); handles.push(handle); } for handle in handles { handle.await.unwrap(); } } #[tokio::test] async fn test_path_sanitization() { let backend = create_test_backend(); // Test various path formats let test_cases = vec![ ("normal/path.bin", true), ("path/with/slashes.bin", true), ("path-with-dashes.bin", true), ("path_with_underscores.bin", true), ("path.with.dots.bin", true), ("UPPERCASE.BIN", true), ("mixed_Case_123.bin", true), ]; for (path, should_succeed) in test_cases { let result = backend.store(path, b"test data").await; if should_succeed { assert!(result.is_ok(), "Failed to store path: {}", path); assert!(backend.exists(path).await.unwrap()); } } } #[tokio::test] async fn test_metadata_etag_tracking() { let backend = create_test_backend(); let path = "etag_test.bin"; let data = b"test data for etag"; backend.store(path, data).await.unwrap(); let metadata = backend.metadata(path).await.unwrap(); assert!(metadata.etag.is_some()); // Store again with different data let new_data = b"updated data for etag"; backend.store(path, new_data).await.unwrap(); let new_metadata = backend.metadata(path).await.unwrap(); assert!(new_metadata.etag.is_some()); // ETags might differ for different content // (behavior depends on object store implementation) } #[tokio::test] async fn test_storage_quota_simulation() { let backend = create_test_backend(); let mut total_size = 0u64; let quota_limit = 100 * 1024 * 1024; // 100MB quota // Upload files until approaching quota for i in 0..20 { let path = format!("quota_test_{}.bin", i); let data = vec![0xFF; 5 * 1024 * 1024]; // 5MB each backend.store(&path, &data).await.unwrap(); total_size += data.len() as u64; if total_size >= quota_limit { break; } } // Verify we stored close to quota assert!(total_size >= quota_limit * 9 / 10); // At least 90% of quota } #[tokio::test] async fn test_delete_and_recreate() { let backend = create_test_backend(); let path = "delete_recreate.bin"; let data1 = b"first version"; let data2 = b"second version after deletion"; // Create backend.store(path, data1).await.unwrap(); assert!(backend.exists(path).await.unwrap()); // Delete backend.delete(path).await.unwrap(); assert!(!backend.exists(path).await.unwrap()); // Recreate with different data backend.store(path, data2).await.unwrap(); let retrieved = backend.retrieve(path).await.unwrap(); assert_eq!(retrieved, data2); } #[tokio::test] async fn test_progress_callback_accuracy() { let backend = create_test_backend(); let data = vec![0xAA; 10 * 1024 * 1024]; // 10MB backend.store("progress_accuracy.bin", &data).await.unwrap(); let total_bytes = Arc::new(AtomicUsize::new(0)); let total_bytes_clone = total_bytes.clone(); let callback = Arc::new(move |downloaded: u64, total: u64| { total_bytes_clone.store(total as usize, Ordering::SeqCst); assert!(downloaded <= total); }); let _ = backend .download_with_progress("progress_accuracy.bin", Some(callback)) .await .unwrap(); // Verify total size reported matches actual size let reported_total = total_bytes.load(Ordering::SeqCst); assert_eq!(reported_total, data.len()); } #[tokio::test] async fn test_exists_performance() { let backend = create_test_backend(); // Store files for i in 0..100 { let path = format!("exists_perf_{}.bin", i); backend.store(&path, b"data").await.unwrap(); } // Benchmark exists checks let start = std::time::Instant::now(); for i in 0..100 { let path = format!("exists_perf_{}.bin", i); assert!(backend.exists(&path).await.unwrap()); } let elapsed = start.elapsed(); println!("100 exists checks completed in {:?}", elapsed); // Should be fast with in-memory store assert!(elapsed < std::time::Duration::from_secs(1)); } #[tokio::test] async fn test_list_performance_large_directory() { let backend = create_test_backend(); // Create 500 files for i in 0..500 { let path = format!("large_dir/file_{}.bin", i); backend.store(&path, b"data").await.unwrap(); } // Benchmark list operation let start = std::time::Instant::now(); let files = backend.list("large_dir/").await.unwrap(); let elapsed = start.elapsed(); assert_eq!(files.len(), 500); println!("Listed 500 files in {:?}", elapsed); } #[tokio::test] async fn test_metadata_performance() { let backend = create_test_backend(); // Store files with different sizes let sizes = [1024, 10240, 102400, 1048576]; // 1KB, 10KB, 100KB, 1MB for (idx, &size) in sizes.iter().enumerate() { let path = format!("metadata_perf_{}.bin", idx); let data = vec![0xFF; size]; backend.store(&path, &data).await.unwrap(); } // Benchmark metadata retrieval let start = std::time::Instant::now(); for (idx, &size) in sizes.iter().enumerate() { let path = format!("metadata_perf_{}.bin", idx); let metadata = backend.metadata(&path).await.unwrap(); assert_eq!(metadata.size, size as u64); } let elapsed = start.elapsed(); println!( "Retrieved metadata for {} files in {:?}", sizes.len(), elapsed ); }