//! Storage and parquet persistence edge case tests //! //! Covers disk operations, corruption scenarios, and persistence edge cases //! //! NOTE: Many tests commented out due to API changes: //! - ParquetReader/ParquetWriter removed (now using ParquetMarketDataWriter) //! - CompressionConfig/VersioningConfig renamed to DataCompressionConfig/DataVersioningConfig //! TODO: Rewrite tests to match current APIs #![allow(unused_crate_dependencies)] use chrono::Utc; use config::data_config::{DataCompressionAlgorithm, DataStorageConfig, DataStorageFormat}; use data::error::DataError; // TODO: ParquetReader and ParquetWriter have been removed - use ParquetMarketDataWriter instead // use data::parquet_persistence::{ParquetReader, ParquetWriter}; use data::storage::{EnhancedDatasetMetadata, StorageManager}; use std::collections::HashMap; use std::path::PathBuf; // ============================================================================ // Storage Manager Tests - Edge Cases // ============================================================================ #[tokio::test] async fn test_storage_with_readonly_directory() { // Skip on Windows where readonly handling is different #[cfg(not(target_os = "windows"))] { use std::os::unix::fs::PermissionsExt; let temp_dir = std::env::temp_dir().join("foxhunt_readonly_test"); let _ = std::fs::create_dir_all(&temp_dir); // Make directory readonly let metadata = std::fs::metadata(&temp_dir).unwrap(); let mut permissions = metadata.permissions(); permissions.set_mode(0o444); // readonly std::fs::set_permissions(&temp_dir, permissions).ok(); let config = DataStorageConfig { format: DataStorageFormat::Parquet, compression: config::data_config::DataCompressionConfig { enabled: false, algorithm: DataCompressionAlgorithm::None, level: Some(0), }, path: temp_dir.to_string_lossy().to_string(), base_directory: temp_dir.clone(), partition_by: vec![], versioning: config::data_config::DataVersioningConfig { enabled: false, version_format: "v%Y%m%d_%H%M%S".to_string(), keep_versions: 1, }, retention: config::data_config::DataRetentionConfig::default(), }; let result = StorageManager::new(config).await; // Should fail due to readonly assert!(result.is_err()); // Cleanup - restore permissions let metadata = std::fs::metadata(&temp_dir).unwrap(); let mut permissions = metadata.permissions(); permissions.set_mode(0o755); std::fs::set_permissions(&temp_dir, permissions).ok(); std::fs::remove_dir_all(&temp_dir).ok(); } } #[tokio::test] async fn test_storage_concurrent_writes() { let temp_dir = std::env::temp_dir().join("foxhunt_concurrent_test"); let _ = std::fs::remove_dir_all(&temp_dir); let config = DataStorageConfig { format: DataStorageFormat::Parquet, compression: config::data_config::DataCompressionConfig { enabled: false, algorithm: DataCompressionAlgorithm::None, level: Some(0), }, path: temp_dir.to_string_lossy().to_string(), base_directory: temp_dir.clone(), partition_by: vec![], versioning: config::data_config::DataVersioningConfig { enabled: true, version_format: "v%Y%m%d_%H%M%S".to_string(), keep_versions: 10, }, retention: config::data_config::DataRetentionConfig::default(), }; let storage = StorageManager::new(config).await.unwrap(); // Simulate concurrent writes let handles: Vec<_> = (0..5) .map(|i| { let id = format!("dataset_{}", i); let data = vec![i as u8; 1000]; tokio::spawn(async move { // Simulate write operation Ok::<_, DataError>(()) }) }) .collect(); for handle in handles { assert!(handle.await.is_ok()); } // Cleanup std::fs::remove_dir_all(&temp_dir).ok(); } #[tokio::test] async fn test_storage_version_overflow() { let temp_dir = std::env::temp_dir().join("foxhunt_version_test"); let _ = std::fs::remove_dir_all(&temp_dir); let config = DataStorageConfig { format: DataStorageFormat::Parquet, compression: config::data_config::DataCompressionConfig { enabled: false, algorithm: DataCompressionAlgorithm::None, level: Some(0), }, path: temp_dir.to_string_lossy().to_string(), base_directory: temp_dir.clone(), partition_by: vec![], versioning: config::data_config::DataVersioningConfig { enabled: true, version_format: "v%Y%m%d_%H%M%S".to_string(), keep_versions: 3, // Small limit }, retention: config::data_config::DataRetentionConfig::default(), }; let storage = StorageManager::new(config).await; assert!(storage.is_ok()); // Cleanup std::fs::remove_dir_all(&temp_dir).ok(); } #[test] fn test_metadata_serialization_edge_cases() { let metadata = EnhancedDatasetMetadata { id: "test_dataset".to_string(), version: "v1.0.0".to_string(), created_at: Utc::now(), file_path: PathBuf::from("/path/to/file"), original_size: usize::MAX, compressed_size: 0, compression_ratio: 0.0, format: DataStorageFormat::Parquet, checksum: "a".repeat(64), tags: HashMap::new(), }; let json = serde_json::to_string(&metadata).unwrap(); let deserialized: EnhancedDatasetMetadata = serde_json::from_str(&json).unwrap(); assert_eq!(metadata.id, deserialized.id); assert_eq!(metadata.original_size, deserialized.original_size); } #[test] fn test_metadata_with_special_characters() { let mut tags = HashMap::new(); tags.insert("key with spaces".to_string(), "value".to_string()); tags.insert("unicode_🦀".to_string(), "value_🔥".to_string()); tags.insert("special!@#$%".to_string(), "chars&*()".to_string()); let metadata = EnhancedDatasetMetadata { id: "test/dataset:with:special".to_string(), version: "v1.0.0-alpha+001".to_string(), created_at: Utc::now(), file_path: PathBuf::from("/path/with spaces/file.parquet"), original_size: 1000, compressed_size: 500, compression_ratio: 0.5, format: DataStorageFormat::Parquet, checksum: "abc123".to_string(), tags, }; let json = serde_json::to_string(&metadata).unwrap(); let deserialized: EnhancedDatasetMetadata = serde_json::from_str(&json).unwrap(); assert_eq!(metadata.tags.len(), deserialized.tags.len()); } // ============================================================================ // Compression Tests - Edge Cases // ============================================================================ #[test] fn test_compression_empty_data() { let empty_data: Vec = vec![]; // ZSTD compression let compressed = zstd::encode_all(&empty_data[..], 3); assert!(compressed.is_ok()); let decompressed = zstd::decode_all(&compressed.unwrap()[..]); assert!(decompressed.is_ok()); assert_eq!(decompressed.unwrap(), empty_data); } #[test] fn test_compression_single_byte() { let single_byte: Vec = vec![0xFF]; let compressed = zstd::encode_all(&single_byte[..], 3).unwrap(); let decompressed = zstd::decode_all(&compressed[..]).unwrap(); assert_eq!(single_byte, decompressed); } #[test] fn test_compression_highly_compressible() { let data: Vec = vec![0; 100_000]; // All zeros let compressed = zstd::encode_all(&data[..], 3).unwrap(); let ratio = compressed.len() as f64 / data.len() as f64; // Should achieve very high compression assert!(ratio < 0.01); } #[test] fn test_compression_random_data() { use rand::Rng; let mut rng = rand::thread_rng(); let data: Vec = (0..10_000).map(|_| rng.gen()).collect(); let compressed = zstd::encode_all(&data[..], 3).unwrap(); let ratio = compressed.len() as f64 / data.len() as f64; // Random data shouldn't compress well assert!(ratio > 0.8); } #[test] fn test_compression_max_level() { let data: Vec = vec![1, 2, 3, 4, 5]; // Test maximum compression level (22 for ZSTD) let compressed = zstd::encode_all(&data[..], 22).unwrap(); let decompressed = zstd::decode_all(&compressed[..]).unwrap(); assert_eq!(data, decompressed); } #[test] fn test_compression_negative_level() { let data: Vec = vec![1, 2, 3, 4, 5]; // ZSTD supports negative levels for faster compression let compressed = zstd::encode_all(&data[..], -1).unwrap(); let decompressed = zstd::decode_all(&compressed[..]).unwrap(); assert_eq!(data, decompressed); } // ============================================================================ // Checksum Tests - Edge Cases // ============================================================================ #[test] fn test_checksum_empty_data() { use sha2::{Digest, Sha256}; let empty: Vec = vec![]; let hash = Sha256::digest(&empty); let hex = format!("{:x}", hash); assert_eq!(hex.len(), 64); assert_eq!( hex, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" ); } #[test] fn test_checksum_consistency() { use sha2::{Digest, Sha256}; let data = b"test data"; let hash1 = format!("{:x}", Sha256::digest(data)); let hash2 = format!("{:x}", Sha256::digest(data)); assert_eq!(hash1, hash2); } #[test] fn test_checksum_different_data() { use sha2::{Digest, Sha256}; let data1 = b"test data 1"; let data2 = b"test data 2"; let hash1 = format!("{:x}", Sha256::digest(data1)); let hash2 = format!("{:x}", Sha256::digest(data2)); assert_ne!(hash1, hash2); } #[test] fn test_checksum_large_data() { use sha2::{Digest, Sha256}; let large_data: Vec = vec![0xFF; 10_000_000]; // 10MB let hash = format!("{:x}", Sha256::digest(&large_data)); assert_eq!(hash.len(), 64); } // ============================================================================ // File Path Tests - Edge Cases // ============================================================================ #[test] fn test_path_normalization() { let paths = vec![ PathBuf::from("/path/to/file"), PathBuf::from("/path/to/../to/file"), PathBuf::from("./path/to/file"), PathBuf::from("../path/to/file"), ]; for path in paths { assert!(path.to_string_lossy().len() > 0); } } #[test] fn test_path_with_unicode() { let unicode_paths = vec![ PathBuf::from("/path/to/файл.parquet"), PathBuf::from("/path/to/文件.parquet"), PathBuf::from("/path/to/🦀.parquet"), ]; for path in unicode_paths { assert!(path.to_string_lossy().len() > 0); } } #[test] fn test_path_max_length() { // Test very long path let long_component = "a".repeat(100); let long_path = format!( "/path/{}/{}/{}", long_component, long_component, long_component ); let path = PathBuf::from(long_path); assert!(path.to_string_lossy().len() > 300); } // ============================================================================ // Data Corruption Tests // ============================================================================ #[test] fn test_corrupted_checksum_detection() { use sha2::{Digest, Sha256}; let data = b"original data"; let correct_hash = format!("{:x}", Sha256::digest(data)); let corrupted_data = b"corrupted data"; let corrupted_hash = format!("{:x}", Sha256::digest(corrupted_data)); // Checksums should not match assert_ne!(correct_hash, corrupted_hash); } #[test] fn test_partial_data_corruption() { use sha2::{Digest, Sha256}; let mut data = vec![0u8; 1000]; let original_hash = format!("{:x}", Sha256::digest(&data)); // Corrupt single byte data[500] = 0xFF; let corrupted_hash = format!("{:x}", Sha256::digest(&data)); assert_ne!(original_hash, corrupted_hash); } // ============================================================================ // Versioning Tests - Edge Cases // ============================================================================ #[test] fn test_version_string_parsing() { let versions = vec![ "v1.0.0", "v1.2.3-alpha", "v2.0.0-rc1", "latest", "20231201-snapshot", ]; for version in versions { assert!(!version.is_empty()); assert!(version.len() < 100); } } #[test] fn test_version_comparison() { let versions = vec![ ("v1.0.0", "v1.0.1"), ("v1.9.9", "v2.0.0"), ("v1.0.0-alpha", "v1.0.0"), ]; for (v1, v2) in versions { // Simple string comparison (not semantic versioning) assert!(v1 != v2); } } // ============================================================================ // Memory Tests // ============================================================================ #[test] fn test_large_buffer_allocation() { // Test allocation of large buffers let sizes = vec![1_000, 10_000, 100_000, 1_000_000]; for size in sizes { let buffer: Vec = Vec::with_capacity(size); assert_eq!(buffer.len(), 0); assert!(buffer.capacity() >= size); } } #[test] fn test_buffer_reuse() { let mut buffer: Vec = Vec::with_capacity(10_000); // Use buffer buffer.extend_from_slice(&[0xFF; 5_000]); assert_eq!(buffer.len(), 5_000); // Clear and reuse buffer.clear(); assert_eq!(buffer.len(), 0); assert!(buffer.capacity() >= 5_000); // Reuse without reallocation buffer.extend_from_slice(&[0xAA; 3_000]); assert_eq!(buffer.len(), 3_000); } // ============================================================================ // Concurrency Tests // ============================================================================ #[tokio::test] async fn test_concurrent_reads() { let data = vec![0xFF; 1000]; let handles: Vec<_> = (0..10) .map(|_| { let data_clone = data.clone(); tokio::spawn(async move { // Simulate concurrent read data_clone.len() }) }) .collect(); for handle in handles { let len = handle.await.unwrap(); assert_eq!(len, 1000); } } // ============================================================================ // Error Recovery Tests // ============================================================================ #[test] fn test_io_error_categories() { use std::io::ErrorKind; let error_kinds = vec![ ErrorKind::NotFound, ErrorKind::PermissionDenied, ErrorKind::ConnectionRefused, ErrorKind::ConnectionReset, ErrorKind::ConnectionAborted, ErrorKind::NotConnected, ErrorKind::AddrInUse, ErrorKind::AddrNotAvailable, ErrorKind::BrokenPipe, ErrorKind::AlreadyExists, ErrorKind::WouldBlock, ErrorKind::InvalidInput, ErrorKind::InvalidData, ErrorKind::TimedOut, ErrorKind::WriteZero, ErrorKind::Interrupted, ErrorKind::UnexpectedEof, ]; for kind in error_kinds { let error = std::io::Error::new(kind, "test error"); let data_error: DataError = error.into(); assert!(matches!(data_error, DataError::Io(_))); } } // ============================================================================ // Format Validation Tests // ============================================================================ #[test] fn test_storage_format_extensions() { let format_extensions = vec![ (DataStorageFormat::Parquet, "parquet"), (DataStorageFormat::Arrow, "arrow"), (DataStorageFormat::Json, "json"), (DataStorageFormat::Csv, "csv"), ]; for (format, ext) in format_extensions { let debug_str = format!("{:?}", format); assert!(debug_str.to_lowercase().contains(ext)); } } #[test] fn test_compression_algorithm_names() { let algorithms = vec![ (DataCompressionAlgorithm::ZSTD, "zstd"), (DataCompressionAlgorithm::LZ4, "lz4"), (DataCompressionAlgorithm::GZIP, "gzip"), (DataCompressionAlgorithm::None, "none"), ]; for (algo, name) in algorithms { let debug_str = format!("{:?}", algo); assert!(debug_str.to_lowercase().contains(name)); } }