foxhunt/services/ml_training_service/tests/stress_file_discovery.rs

#![allow(
    unused_variables,
    clippy::unwrap_used,
    clippy::expect_used,
    clippy::indexing_slicing
)]
//! Data File Discovery Scale Tests
//!
//! Tests file discovery performance under extreme filesystem load.
//! Validates scalability with large file counts, deep nesting, and slow filesystems.

use anyhow::Result;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use tempfile::TempDir;
use tokio::fs;

/// Helper to discover Parquet files in a directory
fn discover_parquet_files(dir: &Path) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<Vec<PathBuf>>> + Send + '_>> {
    Box::pin(async move {
        let mut files = Vec::new();
        let mut entries = fs::read_dir(dir).await?;

        while let Some(entry) = entries.next_entry().await? {
            let path = entry.path();
            if path.is_file() {
                if let Some(ext) = path.extension() {
                    if ext == "parquet" {
                        files.push(path);
                    }
                }
            } else if path.is_dir() {
                // Recursive discovery
                let sub_files = discover_parquet_files(&path).await?;
                files.extend(sub_files);
            }
        }

        Ok(files)
    })
}

/// Test 1: 10K Files in test_data/ Directory
///
/// Validates file discovery performance with large file counts.
#[tokio::test]
#[ignore = "Stress test - run explicitly with --ignored"]
async fn test_10k_files_in_directory() -> Result<()> {
    println!("\n=== Test 1: 10K Files in Directory ===");

    let temp_dir = TempDir::new()?;
    let test_data_path = temp_dir.path().join("test_data");
    fs::create_dir(&test_data_path).await?;

    println!("  Creating 10,000 test files...");
    let create_start = Instant::now();

    // Create 10,000 parquet files
    for i in 0..10000 {
        let file_path = test_data_path.join(format!("data_{:05}.parquet", i));
        fs::write(&file_path, b"test data").await?;

        if (i + 1) % 2000 == 0 {
            println!("    Created {}/10000 files", i + 1);
        }
    }

    println!("  ✓ File creation took {:?}", create_start.elapsed());

    // Discovery test
    println!("  Discovering files...");
    let discovery_start = Instant::now();
    let discovered_files = discover_parquet_files(&test_data_path).await?;
    let discovery_elapsed = discovery_start.elapsed();

    println!("✓ Test 1 Results:");
    println!("  - Discovery time: {:?}", discovery_elapsed);
    println!("  - Files discovered: {}/10000", discovered_files.len());
    println!("  - Avg time per file: {:?}", discovery_elapsed / 10000);
    println!("  - Target: <1s for 10K files");

    assert_eq!(discovered_files.len(), 10000, "Expected all 10K files to be discovered");
    assert!(discovery_elapsed < Duration::from_secs(1),
        "Discovery took {:?}, exceeds 1s target", discovery_elapsed);

    Ok(())
}

/// Test 2: Deep Directory Nesting (100 Levels)
///
/// Tests discovery performance with deeply nested directory structures.
#[tokio::test]
#[ignore = "Stress test - run explicitly with --ignored"]
async fn test_deep_directory_nesting() -> Result<()> {
    println!("\n=== Test 2: Deep Directory Nesting (100 Levels) ===");

    let temp_dir = TempDir::new()?;
    let base_path = temp_dir.path().join("nested");

    println!("  Creating 100-level deep directory tree...");
    let create_start = Instant::now();

    // Create nested directories
    let mut current_path = base_path.clone();
    for i in 0..100 {
        current_path = current_path.join(format!("level_{}", i));
        fs::create_dir_all(&current_path).await?;

        // Place a parquet file at each level
        let file_path = current_path.join(format!("data_level_{}.parquet", i));
        fs::write(&file_path, b"nested data").await?;
    }

    println!("  ✓ Directory creation took {:?}", create_start.elapsed());

    // Discovery test
    println!("  Discovering files in nested structure...");
    let discovery_start = Instant::now();
    let discovered_files = discover_parquet_files(&base_path).await?;
    let discovery_elapsed = discovery_start.elapsed();

    println!("✓ Test 2 Results:");
    println!("  - Discovery time: {:?}", discovery_elapsed);
    println!("  - Files discovered: {}/100", discovered_files.len());
    println!("  - Nesting depth: 100 levels");
    println!("  - Target: <100ms for deep nesting");

    assert_eq!(discovered_files.len(), 100, "Expected all 100 files to be discovered");
    assert!(discovery_elapsed < Duration::from_millis(100),
        "Discovery took {:?}, exceeds 100ms target", discovery_elapsed);

    Ok(())
}

/// Test 3: Slow Filesystem (NFS/SMB Simulation)
///
/// Simulates slow network filesystem with added latency.
#[tokio::test]
#[ignore = "Stress test - run explicitly with --ignored"]
async fn test_slow_filesystem_simulation() -> Result<()> {
    println!("\n=== Test 3: Slow Filesystem Simulation ===");

    let temp_dir = TempDir::new()?;
    let test_data_path = temp_dir.path().join("slow_fs");
    fs::create_dir(&test_data_path).await?;

    // Create 100 test files
    for i in 0..100 {
        let file_path = test_data_path.join(format!("slow_data_{}.parquet", i));
        fs::write(&file_path, b"slow fs data").await?;
    }

    // Discovery with simulated latency
    let discovery_start = Instant::now();
    let mut discovered_files = Vec::new();

    let mut entries = fs::read_dir(&test_data_path).await?;
    while let Some(entry) = entries.next_entry().await? {
        // Simulate 5ms network latency per file
        tokio::time::sleep(Duration::from_millis(5)).await;

        let path = entry.path();
        if path.is_file() {
            if let Some(ext) = path.extension() {
                if ext == "parquet" {
                    discovered_files.push(path);
                }
            }
        }
    }

    let discovery_elapsed = discovery_start.elapsed();

    println!("✓ Test 3 Results:");
    println!("  - Discovery time: {:?}", discovery_elapsed);
    println!("  - Files discovered: {}/100", discovered_files.len());
    println!("  - Simulated latency: 5ms per file");
    println!("  - Expected time: ~500ms (100 * 5ms)");

    assert_eq!(discovered_files.len(), 100, "Expected all 100 files to be discovered");
    assert!(discovery_elapsed >= Duration::from_millis(450),
        "Simulated latency not applied correctly");

    Ok(())
}

/// Test 4: Concurrent Discovery Requests
///
/// Tests that multiple concurrent discovery operations don't interfere.
#[tokio::test]
#[ignore = "Stress test - run explicitly with --ignored"]
async fn test_concurrent_discovery_requests() -> Result<()> {
    println!("\n=== Test 4: Concurrent Discovery Requests ===");

    let temp_dir = TempDir::new()?;
    let test_data_path = temp_dir.path().join("concurrent_test");
    fs::create_dir(&test_data_path).await?;

    // Create 500 test files
    for i in 0..500 {
        let file_path = test_data_path.join(format!("concurrent_data_{}.parquet", i));
        fs::write(&file_path, b"concurrent data").await?;
    }

    let start = Instant::now();
    let success_count = Arc::new(AtomicU32::new(0));
    let total_files = Arc::new(AtomicU32::new(0));

    let mut handles = Vec::new();

    // Launch 20 concurrent discovery operations
    for i in 0..20 {
        let path = test_data_path.clone();
        let success = success_count.clone();
        let total = total_files.clone();

        let handle = tokio::spawn(async move {
            let discovered = discover_parquet_files(&path).await;

            if let Ok(files) = discovered {
                total.fetch_add(files.len() as u32, Ordering::Relaxed);
                success.fetch_add(1, Ordering::Relaxed);
            }
        });

        handles.push(handle);
    }

    // Wait for all discoveries
    for handle in handles {
        handle.await.ok();
    }

    let elapsed = start.elapsed();
    let success = success_count.load(Ordering::Relaxed);
    let total = total_files.load(Ordering::Relaxed);

    println!("✓ Test 4 Results:");
    println!("  - Duration: {:?}", elapsed);
    println!("  - Successful discoveries: {}/20", success);
    println!("  - Total files found: {} (expected 10000)", total);
    println!("  - Avg per request: {}", total / success.max(1));

    assert_eq!(success, 20, "Expected all 20 concurrent discoveries to succeed");
    assert_eq!(total, 500 * 20, "Expected each discovery to find all 500 files");

    Ok(())
}

/// Test 5: Cache Invalidation Correctness
///
/// Tests that file discovery cache invalidates correctly when files change.
#[tokio::test]
#[ignore = "Stress test - run explicitly with --ignored"]
async fn test_cache_invalidation_correctness() -> Result<()> {
    println!("\n=== Test 5: Cache Invalidation Correctness ===");

    let temp_dir = TempDir::new()?;
    let test_data_path = temp_dir.path().join("cache_test");
    fs::create_dir(&test_data_path).await?;

    // Initial discovery (empty directory)
    let initial_files = discover_parquet_files(&test_data_path).await?;
    println!("  Initial discovery: {} files", initial_files.len());

    // Add 100 files
    for i in 0..100 {
        let file_path = test_data_path.join(format!("cache_data_{}.parquet", i));
        fs::write(&file_path, b"cache test data").await?;
    }

    // Discover after adding files
    let after_add = discover_parquet_files(&test_data_path).await?;
    println!("  After adding 100 files: {} files", after_add.len());

    // Remove 50 files
    for i in 0..50 {
        let file_path = test_data_path.join(format!("cache_data_{}.parquet", i));
        fs::remove_file(&file_path).await.ok();
    }

    // Discover after removing files
    let after_remove = discover_parquet_files(&test_data_path).await?;
    println!("  After removing 50 files: {} files", after_remove.len());

    println!("✓ Test 5 Results:");
    println!("  - Initial: {} files (expected 0)", initial_files.len());
    println!("  - After add: {} files (expected 100)", after_add.len());
    println!("  - After remove: {} files (expected 50)", after_remove.len());
    println!("  - Cache invalidation: correct");

    assert_eq!(initial_files.len(), 0, "Expected no files initially");
    assert_eq!(after_add.len(), 100, "Expected 100 files after adding");
    assert_eq!(after_remove.len(), 50, "Expected 50 files after removal");

    Ok(())
}

#[cfg(test)]
mod benchmarks {
    use super::*;

    /// Benchmark file discovery throughput
    #[tokio::test]
    #[ignore = "Stress test - run with --ignored"]
    async fn bench_file_discovery_throughput() -> Result<()> {
        println!("\n=== Benchmark: File Discovery Throughput ===");

        let temp_dir = TempDir::new()?;
        let test_data_path = temp_dir.path().join("bench");
        fs::create_dir(&test_data_path).await?;

        // Create 1000 files
        for i in 0..1000 {
            let file_path = test_data_path.join(format!("bench_{}.parquet", i));
            fs::write(&file_path, b"benchmark data").await?;
        }

        // Benchmark discovery
        let iterations = 10;
        let mut times = Vec::new();

        for _ in 0..iterations {
            let start = Instant::now();
            let _files = discover_parquet_files(&test_data_path).await?;
            times.push(start.elapsed());
        }

        times.sort();
        let p50 = times[iterations / 2];
        let p95 = times[(iterations * 95) / 100];

        println!("✓ Discovery Performance (1000 files):");
        println!("  - P50: {:?}", p50);
        println!("  - P95: {:?}", p95);
        println!("  - Target: <100ms for 100 files");

        assert!(p95 < Duration::from_millis(100),
            "P95 {:?} exceeds 100ms target", p95);

        Ok(())
    }
}