//! Integration tests for MinIO upload functionality //! //! These tests validate: //! 1. Successful upload to MinIO after download //! 2. Retry logic on upload failures //! 3. Metadata tagging //! 4. Upload progress tracking //! //! TDD: These tests are written FIRST and should FAIL until implementation is complete. #[path = "common/upload_types.rs"] mod upload_types; #[path = "common/mock_uploader.rs"] mod mock_uploader; use mock_uploader::{create_test_uploader, create_test_uploader_with_failures}; use sha2::{Digest, Sha256}; use tempfile::TempDir; /// Test: Successfully upload DBN file to MinIO #[tokio::test] async fn test_upload_dbn_file_to_minio() { // Arrange: Create test file let temp_dir = TempDir::new().expect("Failed to create temp dir"); let test_file = temp_dir.path().join("ES.FUT.2024-01-01.dbn"); std::fs::write(&test_file, b"mock DBN data").expect("Failed to write test file"); let uploader = create_test_uploader().await; // Act: Upload file let result = uploader .upload_file( &test_file, "market-data/ES.FUT/2024-01-01.dbn", Some("application/x-dbn".to_string()), ) .await; // Assert: Upload succeeds assert!(result.is_ok(), "Upload should succeed: {:?}", result.err()); let upload_result = result.unwrap(); assert!(!upload_result.object_url.is_empty()); assert!(upload_result.size_bytes > 0); assert!(upload_result.upload_duration_ms > 0); } /// Test: Upload with metadata tagging #[tokio::test] async fn test_upload_with_metadata_tags() { // Arrange let temp_dir = TempDir::new().expect("Failed to create temp dir"); let test_file = temp_dir.path().join("test.dbn"); std::fs::write(&test_file, b"test data").expect("Failed to write test file"); let uploader = create_test_uploader().await; let mut tags = std::collections::HashMap::new(); tags.insert("symbol".to_string(), "ES.FUT".to_string()); tags.insert("date".to_string(), "2024-01-01".to_string()); tags.insert("schema".to_string(), "ohlcv-1m".to_string()); // Act: Upload with tags let result = uploader .upload_file_with_tags(&test_file, "market-data/test.dbn", None, tags) .await; // Assert: Tags are stored assert!(result.is_ok()); // Verify tags can be retrieved let metadata = uploader .get_object_metadata("market-data/test.dbn") .await .expect("Failed to get metadata"); assert_eq!(metadata.tags.get("symbol"), Some(&"ES.FUT".to_string())); assert_eq!(metadata.tags.get("date"), Some(&"2024-01-01".to_string())); } /// Test: Upload with progress callback #[tokio::test] async fn test_upload_with_progress_tracking() { // Arrange: Create large test file let temp_dir = TempDir::new().expect("Failed to create temp dir"); let test_file = temp_dir.path().join("large.dbn"); let large_data = vec![0u8; 10 * 1024 * 1024]; // 10 MB std::fs::write(&test_file, large_data).expect("Failed to write test file"); let uploader = create_test_uploader().await; let progress_updates = std::sync::Arc::new(std::sync::Mutex::new(vec![])); let progress_clone = progress_updates.clone(); // Progress callback let callback = move |bytes_uploaded: u64, total_bytes: u64| { let mut updates = progress_clone.lock().expect("INVARIANT: Lock should not be poisoned"); updates.push((bytes_uploaded, total_bytes)); }; // Act: Upload with progress tracking let result = uploader .upload_file_with_progress(&test_file, "market-data/large.dbn", None, callback) .await; // Assert: Progress was tracked assert!(result.is_ok()); let updates = progress_updates.lock().expect("INVARIANT: Lock should not be poisoned"); assert!(!updates.is_empty(), "Should have progress updates"); // Verify progress increased monotonically for i in 1..updates.len() { assert!( updates[i].0 >= updates[i - 1].0, "Progress should increase monotonically" ); } // Final update should be 100% let last_update = updates.last().expect("INVARIANT: Collection should be non-empty"); assert_eq!(last_update.0, last_update.1, "Should reach 100%"); } /// Test: Retry logic on transient failures #[tokio::test] async fn test_upload_retries_on_transient_failures() { // Arrange: Uploader with simulated transient failures let temp_dir = TempDir::new().expect("Failed to create temp dir"); let test_file = temp_dir.path().join("test.dbn"); std::fs::write(&test_file, b"test data").expect("Failed to write test file"); let uploader = create_test_uploader_with_failures(2).await; // Fail twice, then succeed // Act: Upload (should retry and succeed) let result = uploader .upload_file(&test_file, "market-data/test.dbn", None) .await; // Assert: Eventually succeeds after retries assert!(result.is_ok(), "Should succeed after retries"); let upload_result = result.unwrap(); assert_eq!(upload_result.retry_count, 2, "Should have retried twice"); } /// Test: Upload fails after max retries exceeded #[tokio::test] async fn test_upload_fails_after_max_retries() { // Arrange: Uploader that always fails let temp_dir = TempDir::new().expect("Failed to create temp dir"); let test_file = temp_dir.path().join("test.dbn"); std::fs::write(&test_file, b"test data").expect("Failed to write test file"); let uploader = create_test_uploader_with_failures(10).await; // Always fail // Act: Upload (should fail after max retries) let result = uploader .upload_file(&test_file, "market-data/test.dbn", None) .await; // Assert: Fails with appropriate error assert!(result.is_err(), "Should fail after max retries"); let error = result.unwrap_err(); assert!( error.to_string().contains("max retries exceeded"), "Error should mention max retries: {}", error ); } /// Test: Upload validates file exists before attempting #[tokio::test] async fn test_upload_validates_file_exists() { // Arrange: Non-existent file let temp_dir = TempDir::new().expect("Failed to create temp dir"); let non_existent = temp_dir.path().join("does_not_exist.dbn"); let uploader = create_test_uploader().await; // Act: Try to upload non-existent file let result = uploader .upload_file(&non_existent, "market-data/test.dbn", None) .await; // Assert: Fails with file not found error assert!(result.is_err()); let error = result.unwrap_err(); assert!( error.to_string().contains("file not found") || error.to_string().contains("No such file"), "Error should indicate file not found: {}", error ); } /// Test: Upload calculates checksum for data integrity #[tokio::test] async fn test_upload_calculates_checksum() { // Arrange let temp_dir = TempDir::new().expect("Failed to create temp dir"); let test_file = temp_dir.path().join("test.dbn"); let test_data = b"test data for checksum"; std::fs::write(&test_file, test_data).expect("Failed to write test file"); let uploader = create_test_uploader().await; // Act: Upload file let result = uploader .upload_file(&test_file, "market-data/test.dbn", None) .await .expect("Upload failed"); // Assert: Checksum is calculated and matches assert!(!result.checksum.is_empty(), "Should have checksum"); // Verify checksum matches expected value (SHA256) let mut hasher = Sha256::new(); hasher.update(test_data); let expected_checksum = format!("{:x}", hasher.finalize()); assert_eq!( result.checksum, expected_checksum, "Checksum should match expected value" ); } /// Test: Concurrent uploads work correctly #[tokio::test] async fn test_concurrent_uploads() { // Arrange: Multiple files to upload let temp_dir = TempDir::new().expect("Failed to create temp dir"); let uploader = create_test_uploader().await; let mut upload_tasks = vec![]; for i in 0..5 { let test_file = temp_dir.path().join(format!("test_{}.dbn", i)); std::fs::write(&test_file, format!("test data {}", i).as_bytes()) .expect("Failed to write test file"); let uploader_clone = uploader.clone(); let file_clone = test_file.clone(); // Spawn concurrent upload tasks let task = tokio::spawn(async move { uploader_clone .upload_file(&file_clone, &format!("market-data/test_{}.dbn", i), None) .await }); upload_tasks.push(task); } // Act: Wait for all uploads let results = futures::future::join_all(upload_tasks).await; // Assert: All uploads succeed for result in results { let upload_result = result.expect("Task panicked").expect("Upload failed"); assert!(!upload_result.object_url.is_empty()); } }