//! TDD Tests for DBN File Uploader //! //! These tests are written FIRST to define the expected behavior. //! They should FAIL initially until implementation is complete. use data::dbn_uploader::{DbnMetadata, DbnUploader, DbnUploaderConfig}; use std::path::PathBuf; use std::time::Duration; use tempfile::TempDir; use tokio::fs; use tokio::time::sleep; /// Test 1: File watcher detects new DBN files #[tokio::test] async fn test_file_watcher_detects_new_dbn_files() { let temp_dir = TempDir::new().expect("Failed to create temp dir"); let watch_path = temp_dir.path().to_path_buf(); // Create a test DBN file BEFORE creating uploader let test_file = watch_path.join("ES.FUT_ohlcv-1m_2024-01-02.dbn"); fs::write(&test_file, b"fake dbn data") .await .expect("Failed to write test file"); let config = DbnUploaderConfig { watch_path: watch_path.clone(), bucket_name: "ml-models".to_string(), upload_prefix: "training-data/".to_string(), poll_interval: Duration::from_millis(100), compression_enabled: false, // Disable for this test deduplication_enabled: false, }; let uploader = DbnUploader::new(config).await.expect("Failed to create uploader"); // Manually trigger scan (since start_watching() is blocking and runs forever) // Note: In production, files will be detected by the background watcher loop let detected = uploader.scan_for_testing().await.expect("Failed to scan"); assert_eq!( detected.len(), 1, "Should detect exactly 1 DBN file" ); assert_eq!( detected[0].file_name().unwrap().to_str().unwrap(), "ES.FUT_ohlcv-1m_2024-01-02.dbn" ); } /// Test 2: Compression before upload (gzip) #[tokio::test] async fn test_compression_before_upload() { let temp_dir = TempDir::new().expect("Failed to create temp dir"); let test_file = temp_dir.path().join("test.dbn"); // Create test data (highly compressible) let test_data = b"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; // 36 bytes fs::write(&test_file, test_data) .await .expect("Failed to write test file"); // Compress the file let compressed = DbnUploader::compress_file(&test_file) .await .expect("Failed to compress"); // Compressed size should be smaller assert!( compressed.len() < test_data.len(), "Compressed size {} should be less than original {}", compressed.len(), test_data.len() ); // Verify it's valid gzip by checking magic bytes assert_eq!(compressed[0], 0x1f, "First byte should be 0x1f (gzip magic)"); assert_eq!(compressed[1], 0x8b, "Second byte should be 0x8b (gzip magic)"); } /// Test 3: Deduplication - skip if file already in MinIO #[tokio::test] async fn test_deduplication_skips_existing_files() { let temp_dir = TempDir::new().expect("Failed to create temp dir"); let watch_path = temp_dir.path().to_path_buf(); let config = DbnUploaderConfig { watch_path: watch_path.clone(), bucket_name: "ml-models".to_string(), upload_prefix: "training-data/".to_string(), poll_interval: Duration::from_millis(100), compression_enabled: true, deduplication_enabled: true, }; let uploader = DbnUploader::new(config).await.expect("Failed to create uploader"); let test_file = watch_path.join("duplicate.dbn"); fs::write(&test_file, b"test data") .await .expect("Failed to write test file"); // Simulate file already exists in MinIO let should_upload = uploader .should_upload_file(&test_file) .await .expect("Failed to check deduplication"); // For now, should be true since MinIO is not mocked // In real implementation, this would check MinIO and return false if exists assert!(should_upload, "Should upload since MinIO check is not mocked"); } /// Test 4: Metadata extraction from DBN filename #[tokio::test] async fn test_metadata_extraction_from_filename() { let filename = "ES.FUT_ohlcv-1m_2024-01-02.dbn"; let metadata = DbnMetadata::from_filename(filename).expect("Failed to extract metadata"); assert_eq!(metadata.symbol, "ES.FUT"); assert_eq!(metadata.schema, "ohlcv-1m"); assert_eq!(metadata.date_range, "2024-01-02"); } /// Test 5: Metadata extraction with date range #[tokio::test] async fn test_metadata_extraction_with_date_range() { let filename = "ZN.FUT_ohlcv-1m_2024-01-02_to_2024-01-31.dbn"; let metadata = DbnMetadata::from_filename(filename).expect("Failed to extract metadata"); assert_eq!(metadata.symbol, "ZN.FUT"); assert_eq!(metadata.schema, "ohlcv-1m"); assert_eq!(metadata.date_range, "2024-01-02_to_2024-01-31"); } /// Test 6: Handles invalid filename gracefully #[tokio::test] async fn test_invalid_filename_returns_error() { let invalid_filename = "not_a_valid_dbn_file.txt"; let result = DbnMetadata::from_filename(invalid_filename); assert!(result.is_err(), "Should return error for invalid filename"); } /// Test 7: Only processes .dbn files (ignores .md, .txt, etc) #[tokio::test] async fn test_ignores_non_dbn_files() { let temp_dir = TempDir::new().expect("Failed to create temp dir"); let watch_path = temp_dir.path().to_path_buf(); // Create various files BEFORE creating uploader fs::write(watch_path.join("valid.dbn"), b"dbn data") .await .expect("Failed to write .dbn file"); fs::write(watch_path.join("README.md"), b"markdown") .await .expect("Failed to write .md file"); fs::write(watch_path.join("config.txt"), b"text") .await .expect("Failed to write .txt file"); let config = DbnUploaderConfig { watch_path: watch_path.clone(), bucket_name: "ml-models".to_string(), upload_prefix: "training-data/".to_string(), poll_interval: Duration::from_millis(100), compression_enabled: false, deduplication_enabled: false, }; let uploader = DbnUploader::new(config).await.expect("Failed to create uploader"); // Manually trigger scan let detected_files = uploader.scan_for_testing().await.expect("Failed to scan"); assert_eq!( detected_files.len(), 1, "Should only detect .dbn files, found: {:?}", detected_files ); assert!(detected_files[0] .file_name() .unwrap() .to_str() .unwrap() .ends_with(".dbn")); } /// Test 8: Handles empty watch directory #[tokio::test] async fn test_handles_empty_directory() { let temp_dir = TempDir::new().expect("Failed to create temp dir"); let watch_path = temp_dir.path().to_path_buf(); let config = DbnUploaderConfig { watch_path: watch_path.clone(), bucket_name: "ml-models".to_string(), upload_prefix: "training-data/".to_string(), poll_interval: Duration::from_millis(100), compression_enabled: false, deduplication_enabled: false, }; let uploader = DbnUploader::new(config).await.expect("Failed to create uploader"); sleep(Duration::from_millis(200)).await; let detected_files = uploader.get_detected_files().await; assert_eq!(detected_files.len(), 0, "Should detect no files"); } /// Test 9: Upload generates correct MinIO key #[tokio::test] async fn test_upload_generates_correct_minio_key() { let filename = PathBuf::from("ES.FUT_ohlcv-1m_2024-01-02.dbn"); let prefix = "training-data/"; let key = DbnUploader::generate_upload_key(&filename, prefix); assert_eq!( key, "training-data/ES.FUT_ohlcv-1m_2024-01-02.dbn.gz", "Key should include prefix and .gz extension" ); } /// Test 10: Compressed file size is tracked in metadata #[tokio::test] async fn test_compression_size_tracked() { let temp_dir = TempDir::new().expect("Failed to create temp dir"); let test_file = temp_dir.path().join("test.dbn"); // Use highly compressible data (gzip header is ~20 bytes, so need enough data) let test_data = b"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; // 100 bytes fs::write(&test_file, test_data) .await .expect("Failed to write test file"); let original_size = test_data.len(); let compressed = DbnUploader::compress_file(&test_file) .await .expect("Failed to compress"); let compressed_size = compressed.len(); assert!( compressed_size > 0, "Compressed size should be greater than 0" ); assert!( original_size > compressed_size, "Original size {} should be greater than compressed {}", original_size, compressed_size ); } /// Test 11: Metadata includes file size #[tokio::test] async fn test_metadata_includes_file_size() { let temp_dir = TempDir::new().expect("Failed to create temp dir"); let test_file = temp_dir.path().join("ES.FUT_ohlcv-1m_2024-01-02.dbn"); let test_data = b"test data"; fs::write(&test_file, test_data) .await .expect("Failed to write test file"); let metadata = DbnMetadata::from_file(&test_file) .await .expect("Failed to extract metadata"); assert_eq!(metadata.file_size_bytes, test_data.len() as u64); assert_eq!(metadata.symbol, "ES.FUT"); } /// Test 12: Upload adds correct tags to MinIO metadata #[tokio::test] async fn test_upload_adds_metadata_tags() { let metadata = DbnMetadata { symbol: "ES.FUT".to_string(), schema: "ohlcv-1m".to_string(), date_range: "2024-01-02".to_string(), file_size_bytes: 1024, }; let tags = DbnUploader::generate_metadata_tags(&metadata); assert_eq!(tags.get("symbol"), Some(&"ES.FUT".to_string())); assert_eq!(tags.get("schema"), Some(&"ohlcv-1m".to_string())); assert_eq!(tags.get("date_range"), Some(&"2024-01-02".to_string())); assert_eq!(tags.get("original_size"), Some(&"1024".to_string())); }