foxhunt/services/ml_training_service/tests/advanced_data_discovery_tests.rs

#![allow(
    clippy::unwrap_used,
    clippy::expect_used,
    clippy::indexing_slicing,
    clippy::useless_vec
)]
//! Advanced Data File Discovery Tests
//!
//! Tests cover:
//! - Concurrent file discovery (thread safety)
//! - Symlink and junction handling
//! - Network path validation
//! - Large dataset performance (1000+ files)
//! - Error handling and recovery

use ml_training_service::data_file_discovery::{DataFileDiscovery, DataFormat};
use std::fs::{self, File};
use std::io::Write;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Instant;
use tempfile::TempDir;

// ============================================================================
// TEST HELPERS
// ============================================================================

fn create_test_parquet_file(dir: &std::path::Path, name: &str) -> PathBuf {
    let path = dir.join(name);
    let mut file = File::create(&path).unwrap();
    // Write minimal Parquet magic bytes (PAR1)
    file.write_all(b"PAR1").unwrap();
    file.write_all(&[0u8; 100]).unwrap(); // Padding
    file.write_all(b"PAR1").unwrap();
    path
}

fn create_test_dbn_file(dir: &std::path::Path, name: &str) -> PathBuf {
    let path = dir.join(name);
    let mut file = File::create(&path).unwrap();
    // Write minimal DBN header
    file.write_all(b"DBN").unwrap();
    file.write_all(&[1u8; 100]).unwrap(); // Version + padding
    path
}

// ============================================================================
// CONCURRENT FILE DISCOVERY (THREAD SAFETY)
// ============================================================================

#[test]
fn test_concurrent_discovery_same_directory() {
    let temp_dir = TempDir::new().unwrap();
    let base_path = temp_dir.path();

    // Create test files
    for i in 0..10 {
        create_test_parquet_file(base_path, &format!("ASSET{}_180d.parquet", i));
    }

    let discovery = Arc::new(DataFileDiscovery::new(base_path));
    let mut handles = vec![];

    // Spawn 10 threads, each discovering different assets concurrently
    for i in 0..10 {
        let discovery_clone = Arc::clone(&discovery);
        let handle = std::thread::spawn(move || {
            let assets = vec![format!("ASSET{}", i)];
            let result = discovery_clone.discover(&assets);
            assert!(result.is_ok(), "Thread {} failed discovery", i);
            result.unwrap()
        });
        handles.push(handle);
    }

    // Wait for all threads
    for (i, handle) in handles.into_iter().enumerate() {
        let files = handle.join().expect("INVARIANT: Thread should complete successfully");
        assert_eq!(files.len(), 1);
        assert_eq!(files[0].asset, format!("ASSET{}", i));
        assert_eq!(files[0].format, DataFormat::Parquet);
    }
}

#[test]
fn test_concurrent_discovery_shared_assets() {
    let temp_dir = TempDir::new().unwrap();
    let base_path = temp_dir.path();

    // Create test files for 4 assets
    for symbol in &["ES_FUT", "NQ_FUT", "6E_FUT", "ZN_FUT"] {
        create_test_parquet_file(base_path, &format!("{}_180d.parquet", symbol));
    }

    let discovery = Arc::new(DataFileDiscovery::new(base_path));
    let mut handles = vec![];

    // 50 threads all discovering same 4 assets
    for _ in 0..50 {
        let discovery_clone = Arc::clone(&discovery);
        let handle = std::thread::spawn(move || {
            let assets = vec!["ES_FUT", "NQ_FUT", "6E_FUT", "ZN_FUT"]
                .iter()
                .map(|s| s.to_string())
                .collect::<Vec<_>>();
            discovery_clone.discover(&assets)
        });
        handles.push(handle);
    }

    // All should succeed
    for handle in handles {
        let result = handle.join().expect("INVARIANT: Thread should complete successfully");
        assert!(result.is_ok());
        assert_eq!(result.unwrap().len(), 4);
    }
}

#[test]
fn test_concurrent_mixed_success_failure() {
    let temp_dir = TempDir::new().unwrap();
    let base_path = temp_dir.path();

    // Create files for only half the assets
    create_test_parquet_file(base_path, "EXISTS1_180d.parquet");
    create_test_parquet_file(base_path, "EXISTS2_180d.parquet");

    let discovery = Arc::new(DataFileDiscovery::new(base_path));
    let mut handles = vec![];

    // Half the threads will succeed, half will fail
    for i in 0..20 {
        let discovery_clone = Arc::clone(&discovery);
        let handle = std::thread::spawn(move || {
            let asset = if i % 2 == 0 {
                "EXISTS1".to_string()
            } else {
                "MISSING".to_string()
            };
            discovery_clone.discover(&[asset])
        });
        handles.push(handle);
    }

    let mut success_count = 0;
    let mut failure_count = 0;

    for handle in handles {
        match handle.join().expect("INVARIANT: Thread should complete successfully") {
            Ok(_) => success_count += 1,
            Err(_) => failure_count += 1,
        }
    }

    assert_eq!(success_count, 10);
    assert_eq!(failure_count, 10);
}

// ============================================================================
// SYMLINK AND JUNCTION HANDLING
// ============================================================================

#[cfg(unix)]
#[test]
fn test_discover_via_symlink() {
    use std::os::unix::fs::symlink;

    let temp_dir = TempDir::new().unwrap();
    let real_dir = temp_dir.path().join("real_data");
    let link_dir = temp_dir.path().join("link_data");

    fs::create_dir(&real_dir).unwrap();
    create_test_parquet_file(&real_dir, "ES_FUT_180d.parquet");

    // Create symlink to real directory
    symlink(&real_dir, &link_dir).unwrap();

    // Discovery via symlink should work
    let discovery = DataFileDiscovery::new(&link_dir);
    let result = discovery.discover(&["ES_FUT".to_string()]);

    assert!(result.is_ok());
    let files = result.unwrap();
    assert_eq!(files.len(), 1);
    assert_eq!(files[0].format, DataFormat::Parquet);
}

#[cfg(unix)]
#[test]
fn test_symlink_to_individual_files() {
    use std::os::unix::fs::symlink;

    let temp_dir = TempDir::new().unwrap();
    let real_dir = temp_dir.path().join("real");
    let link_dir = temp_dir.path().join("linked");

    fs::create_dir(&real_dir).unwrap();
    fs::create_dir(&link_dir).unwrap();

    let real_file = create_test_parquet_file(&real_dir, "ES_FUT_180d.parquet");
    let link_file = link_dir.join("ES_FUT_180d.parquet");

    // Symlink individual file
    symlink(&real_file, &link_file).unwrap();

    let discovery = DataFileDiscovery::new(&link_dir);
    let result = discovery.discover(&["ES_FUT".to_string()]);

    assert!(result.is_ok());
}

#[cfg(unix)]
#[test]
fn test_broken_symlink_handled_gracefully() {
    use std::os::unix::fs::symlink;

    let temp_dir = TempDir::new().unwrap();
    let link_dir = temp_dir.path();
    let nonexistent = temp_dir.path().join("nonexistent.parquet");

    // Create broken symlink
    symlink(&nonexistent, link_dir.join("ES_FUT_180d.parquet")).unwrap();

    let discovery = DataFileDiscovery::new(link_dir);
    let result = discovery.discover(&["ES_FUT".to_string()]);

    // Should fail gracefully (file doesn't exist)
    assert!(result.is_err());
}

// ============================================================================
// NETWORK PATH VALIDATION (conceptual - actual network tests require setup)
// ============================================================================

#[test]
fn test_nonexistent_directory_error() {
    let discovery = DataFileDiscovery::new("/nonexistent/path/that/does/not/exist");
    let result = discovery.discover(&["ES_FUT".to_string()]);

    // Should fail - directory doesn't exist
    assert!(result.is_err());
}

#[test]
fn test_empty_directory() {
    let temp_dir = TempDir::new().unwrap();
    let discovery = DataFileDiscovery::new(temp_dir.path());
    let result = discovery.discover(&["ES_FUT".to_string()]);

    // Should fail - no files found
    assert!(result.is_err());
    let err_msg = result.unwrap_err().to_string();
    assert!(err_msg.contains("No data file found"));
}

#[test]
fn test_directory_with_wrong_files() {
    let temp_dir = TempDir::new().unwrap();
    let base_path = temp_dir.path();

    // Create files that don't match any pattern
    File::create(base_path.join("random.txt")).unwrap();
    File::create(base_path.join("test.csv")).unwrap();
    File::create(base_path.join("ES_FUT.json")).unwrap();

    let discovery = DataFileDiscovery::new(base_path);
    let result = discovery.discover(&["ES_FUT".to_string()]);

    // Should fail - no valid files
    assert!(result.is_err());
}

// ============================================================================
// LARGE DATASET PERFORMANCE (1000+ FILES)
// ============================================================================

#[test]
fn test_performance_1000_parquet_files() {
    let temp_dir = TempDir::new().unwrap();
    let base_path = temp_dir.path();

    // Create 1000 parquet files
    for i in 0..1000 {
        create_test_parquet_file(base_path, &format!("ASSET{:04}_180d.parquet", i));
    }

    let discovery = DataFileDiscovery::new(base_path);

    // Discover 100 assets
    let assets: Vec<String> = (0..100)
        .map(|i| format!("ASSET{:04}", i))
        .collect();

    let start = Instant::now();
    let result = discovery.discover(&assets);
    let elapsed = start.elapsed();

    assert!(result.is_ok());
    assert_eq!(result.unwrap().len(), 100);

    println!("100 assets from 1000 files: {:?} ({:.2}μs per asset)",
             elapsed, elapsed.as_micros() as f64 / 100.0);

    // Target: <50ms for 100 assets even with 1000 files present
    assert!(elapsed.as_millis() < 100, "Discovery too slow: {:?}", elapsed);
}

#[test]
fn test_performance_mixed_formats_large_scale() {
    let temp_dir = TempDir::new().unwrap();
    let base_path = temp_dir.path();

    // Create 500 parquet and 500 DBN files
    for i in 0..500 {
        create_test_parquet_file(base_path, &format!("P{:04}_180d.parquet", i));
        create_test_dbn_file(base_path, &format!("D{:04}_180d.dbn", i));
    }

    let discovery = DataFileDiscovery::new(base_path);

    // Discover 50 parquet and 50 DBN assets
    let mut assets: Vec<String> = (0..50)
        .map(|i| format!("P{:04}", i))
        .collect();
    assets.extend((0..50).map(|i| format!("D{:04}", i)));

    let start = Instant::now();
    let result = discovery.discover(&assets);
    let elapsed = start.elapsed();

    assert!(result.is_ok());
    let files = result.unwrap();
    assert_eq!(files.len(), 100);

    // Count formats
    let parquet_count = files.iter().filter(|f| f.format == DataFormat::Parquet).count();
    let dbn_count = files.iter().filter(|f| f.format == DataFormat::DBN).count();
    assert_eq!(parquet_count, 50);
    assert_eq!(dbn_count, 50);

    println!("100 mixed format assets: {:?}", elapsed);
    assert!(elapsed.as_millis() < 100);
}

#[test]
fn test_performance_single_asset_cold_cache() {
    let temp_dir = TempDir::new().unwrap();
    let base_path = temp_dir.path();

    // Create 100 files to simulate realistic directory
    for i in 0..100 {
        create_test_parquet_file(base_path, &format!("ASSET{}_180d.parquet", i));
    }

    // Measure time for single asset discovery (cold cache)
    let iterations = 100;
    let mut total_elapsed = std::time::Duration::ZERO;

    for i in 0..iterations {
        let discovery = DataFileDiscovery::new(base_path);
        let assets = vec![format!("ASSET{}", i % 100)];

        let start = Instant::now();
        let _ = discovery.discover(&assets).unwrap();
        total_elapsed += start.elapsed();
    }

    let avg = total_elapsed / iterations;
    println!("Single asset discovery (cold): {:?} average", avg);

    // Target: <1ms per asset even with cold cache
    assert!(avg.as_millis() < 5, "Single discovery too slow: {:?}", avg);
}

// ============================================================================
// FILE FORMAT PREFERENCE
// ============================================================================

#[test]
fn test_parquet_preferred_over_dbn() {
    let temp_dir = TempDir::new().unwrap();
    let base_path = temp_dir.path();

    // Create both Parquet and DBN for same asset
    create_test_parquet_file(base_path, "ES_FUT_180d.parquet");
    create_test_dbn_file(base_path, "ES_FUT_180d.dbn");

    let discovery = DataFileDiscovery::new(base_path);
    let result = discovery.discover(&["ES_FUT".to_string()]).unwrap();

    assert_eq!(result.len(), 1);
    // Should prefer Parquet
    assert_eq!(result[0].format, DataFormat::Parquet);
}

#[test]
fn test_multiple_parquet_variants() {
    let temp_dir = TempDir::new().unwrap();
    let base_path = temp_dir.path();

    // Create multiple Parquet variants
    create_test_parquet_file(base_path, "ES_FUT_180d.parquet");
    create_test_parquet_file(base_path, "ES_FUT_90d.parquet");
    create_test_parquet_file(base_path, "ES_FUT_small.parquet");

    let discovery = DataFileDiscovery::new(base_path);
    let result = discovery.discover(&["ES_FUT".to_string()]).unwrap();

    assert_eq!(result.len(), 1);
    assert_eq!(result[0].format, DataFormat::Parquet);

    // Should discover one of them (preference order: 180d > 90d > small)
    let path_str = result[0].path.to_str().expect("INVARIANT: Path should be valid UTF-8");
    assert!(path_str.contains("ES_FUT"));
    assert!(path_str.ends_with(".parquet"));
}

#[test]
fn test_dbn_fallback_when_no_parquet() {
    let temp_dir = TempDir::new().unwrap();
    let base_path = temp_dir.path();

    // Only create DBN file
    create_test_dbn_file(base_path, "ES_FUT_180d.dbn");

    let discovery = DataFileDiscovery::new(base_path);
    let result = discovery.discover(&["ES_FUT".to_string()]).unwrap();

    assert_eq!(result.len(), 1);
    assert_eq!(result[0].format, DataFormat::DBN);
}

// ============================================================================
// ERROR HANDLING AND RECOVERY
// ============================================================================

#[test]
fn test_partial_discovery_failure() {
    let temp_dir = TempDir::new().unwrap();
    let base_path = temp_dir.path();

    // Create files for only 2 out of 3 assets
    create_test_parquet_file(base_path, "ES_FUT_180d.parquet");
    create_test_parquet_file(base_path, "NQ_FUT_180d.parquet");

    let discovery = DataFileDiscovery::new(base_path);
    let assets = vec!["ES_FUT", "NQ_FUT", "MISSING_ASSET"]
        .iter()
        .map(|s| s.to_string())
        .collect::<Vec<_>>();

    let result = discovery.discover(&assets);

    // Should fail if any asset is missing
    assert!(result.is_err());
    let err_msg = result.unwrap_err().to_string();
    assert!(err_msg.contains("MISSING_ASSET"));
}

#[test]
fn test_case_sensitive_asset_names() {
    let temp_dir = TempDir::new().unwrap();
    let base_path = temp_dir.path();

    create_test_parquet_file(base_path, "ES_FUT_180d.parquet");

    let discovery = DataFileDiscovery::new(base_path);

    // Try different cases
    let lowercase = discovery.discover(&["es_fut".to_string()]);
    let uppercase = discovery.discover(&["ES_FUT".to_string()]);

    // Case sensitivity depends on filesystem (Unix: sensitive, Windows: insensitive)
    // At minimum, exact match should work
    assert!(uppercase.is_ok() || lowercase.is_ok());
}

#[test]
fn test_special_characters_in_directory() {
    let temp_dir = TempDir::new().unwrap();
    let special_dir = temp_dir.path().join("test data (2024)");
    fs::create_dir(&special_dir).unwrap();

    create_test_parquet_file(&special_dir, "ES_FUT_180d.parquet");

    let discovery = DataFileDiscovery::new(&special_dir);
    let result = discovery.discover(&["ES_FUT".to_string()]);

    assert!(result.is_ok());
}

#[test]
fn test_very_long_directory_path() {
    let temp_dir = TempDir::new().unwrap();
    let mut long_path = temp_dir.path().to_path_buf();

    // Create nested directory structure
    for i in 0..10 {
        long_path = long_path.join(format!("level_{}", i));
    }
    fs::create_dir_all(&long_path).unwrap();

    create_test_parquet_file(&long_path, "ES_FUT_180d.parquet");

    let discovery = DataFileDiscovery::new(&long_path);
    let result = discovery.discover(&["ES_FUT".to_string()]);

    assert!(result.is_ok());
}

// ============================================================================
// EDGE CASES
// ============================================================================

#[test]
fn test_empty_asset_list() {
    let temp_dir = TempDir::new().unwrap();
    let discovery = DataFileDiscovery::new(temp_dir.path());

    let result = discovery.discover(&[]);
    assert!(result.is_ok());
    assert_eq!(result.unwrap().len(), 0);
}

#[test]
fn test_duplicate_assets_in_request() {
    let temp_dir = TempDir::new().unwrap();
    let base_path = temp_dir.path();

    create_test_parquet_file(base_path, "ES_FUT_180d.parquet");

    let discovery = DataFileDiscovery::new(base_path);
    let assets = vec!["ES_FUT", "ES_FUT", "ES_FUT"]
        .iter()
        .map(|s| s.to_string())
        .collect::<Vec<_>>();

    let result = discovery.discover(&assets);

    // May return 3 identical entries or deduplicate
    assert!(result.is_ok());
}

#[test]
fn test_asset_name_with_underscores() {
    let temp_dir = TempDir::new().unwrap();
    let base_path = temp_dir.path();

    create_test_parquet_file(base_path, "ES_FUT_180d.parquet");
    create_test_parquet_file(base_path, "NQ_FUT_180d.parquet");

    let discovery = DataFileDiscovery::new(base_path);
    let result = discovery.discover(&["ES_FUT".to_string(), "NQ_FUT".to_string()]);

    assert!(result.is_ok());
    assert_eq!(result.unwrap().len(), 2);
}