foxhunt/data/tests/dbn_uploader_tests.rs

//! TDD Tests for DBN File Uploader
//!
//! These tests are written FIRST to define the expected behavior.
//! They should FAIL initially until implementation is complete.

use data::dbn_uploader::{DbnMetadata, DbnUploader, DbnUploaderConfig};
use std::path::PathBuf;
use std::time::Duration;
use tempfile::TempDir;
use tokio::fs;
use tokio::time::sleep;

/// Test 1: File watcher detects new DBN files
#[tokio::test]
async fn test_file_watcher_detects_new_dbn_files() {
    let temp_dir = TempDir::new().expect("Failed to create temp dir");
    let watch_path = temp_dir.path().to_path_buf();

    // Create a test DBN file BEFORE creating uploader
    let test_file = watch_path.join("ES.FUT_ohlcv-1m_2024-01-02.dbn");
    fs::write(&test_file, b"fake dbn data")
        .await
        .expect("Failed to write test file");

    let config = DbnUploaderConfig {
        watch_path: watch_path.clone(),
        bucket_name: "ml-models".to_string(),
        upload_prefix: "training-data/".to_string(),
        poll_interval: Duration::from_millis(100),
        compression_enabled: false, // Disable for this test
        deduplication_enabled: false,
    };

    let uploader = DbnUploader::new(config).await.expect("Failed to create uploader");

    // Manually trigger scan (since start_watching() is blocking and runs forever)
    // Note: In production, files will be detected by the background watcher loop
    let detected = uploader.scan_for_testing().await.expect("Failed to scan");

    assert_eq!(
        detected.len(),
        1,
        "Should detect exactly 1 DBN file"
    );
    assert_eq!(
        detected[0].file_name().unwrap().to_str().unwrap(),
        "ES.FUT_ohlcv-1m_2024-01-02.dbn"
    );
}

/// Test 2: Compression before upload (gzip)
#[tokio::test]
async fn test_compression_before_upload() {
    let temp_dir = TempDir::new().expect("Failed to create temp dir");
    let test_file = temp_dir.path().join("test.dbn");

    // Create test data (highly compressible)
    let test_data = b"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; // 36 bytes
    fs::write(&test_file, test_data)
        .await
        .expect("Failed to write test file");

    // Compress the file
    let compressed = DbnUploader::compress_file(&test_file)
        .await
        .expect("Failed to compress");

    // Compressed size should be smaller
    assert!(
        compressed.len() < test_data.len(),
        "Compressed size {} should be less than original {}",
        compressed.len(),
        test_data.len()
    );

    // Verify it's valid gzip by checking magic bytes
    assert_eq!(compressed[0], 0x1f, "First byte should be 0x1f (gzip magic)");
    assert_eq!(compressed[1], 0x8b, "Second byte should be 0x8b (gzip magic)");
}

/// Test 3: Deduplication - skip if file already in MinIO
#[tokio::test]
async fn test_deduplication_skips_existing_files() {
    let temp_dir = TempDir::new().expect("Failed to create temp dir");
    let watch_path = temp_dir.path().to_path_buf();

    let config = DbnUploaderConfig {
        watch_path: watch_path.clone(),
        bucket_name: "ml-models".to_string(),
        upload_prefix: "training-data/".to_string(),
        poll_interval: Duration::from_millis(100),
        compression_enabled: true,
        deduplication_enabled: true,
    };

    let uploader = DbnUploader::new(config).await.expect("Failed to create uploader");

    let test_file = watch_path.join("duplicate.dbn");
    fs::write(&test_file, b"test data")
        .await
        .expect("Failed to write test file");

    // Simulate file already exists in MinIO
    let should_upload = uploader
        .should_upload_file(&test_file)
        .await
        .expect("Failed to check deduplication");

    // For now, should be true since MinIO is not mocked
    // In real implementation, this would check MinIO and return false if exists
    assert!(should_upload, "Should upload since MinIO check is not mocked");
}

/// Test 4: Metadata extraction from DBN filename
#[tokio::test]
async fn test_metadata_extraction_from_filename() {
    let filename = "ES.FUT_ohlcv-1m_2024-01-02.dbn";
    let metadata = DbnMetadata::from_filename(filename).expect("Failed to extract metadata");

    assert_eq!(metadata.symbol, "ES.FUT");
    assert_eq!(metadata.schema, "ohlcv-1m");
    assert_eq!(metadata.date_range, "2024-01-02");
}

/// Test 5: Metadata extraction with date range
#[tokio::test]
async fn test_metadata_extraction_with_date_range() {
    let filename = "ZN.FUT_ohlcv-1m_2024-01-02_to_2024-01-31.dbn";
    let metadata = DbnMetadata::from_filename(filename).expect("Failed to extract metadata");

    assert_eq!(metadata.symbol, "ZN.FUT");
    assert_eq!(metadata.schema, "ohlcv-1m");
    assert_eq!(metadata.date_range, "2024-01-02_to_2024-01-31");
}

/// Test 6: Handles invalid filename gracefully
#[tokio::test]
async fn test_invalid_filename_returns_error() {
    let invalid_filename = "not_a_valid_dbn_file.txt";
    let result = DbnMetadata::from_filename(invalid_filename);

    assert!(result.is_err(), "Should return error for invalid filename");
}

/// Test 7: Only processes .dbn files (ignores .md, .txt, etc)
#[tokio::test]
async fn test_ignores_non_dbn_files() {
    let temp_dir = TempDir::new().expect("Failed to create temp dir");
    let watch_path = temp_dir.path().to_path_buf();

    // Create various files BEFORE creating uploader
    fs::write(watch_path.join("valid.dbn"), b"dbn data")
        .await
        .expect("Failed to write .dbn file");
    fs::write(watch_path.join("README.md"), b"markdown")
        .await
        .expect("Failed to write .md file");
    fs::write(watch_path.join("config.txt"), b"text")
        .await
        .expect("Failed to write .txt file");

    let config = DbnUploaderConfig {
        watch_path: watch_path.clone(),
        bucket_name: "ml-models".to_string(),
        upload_prefix: "training-data/".to_string(),
        poll_interval: Duration::from_millis(100),
        compression_enabled: false,
        deduplication_enabled: false,
    };

    let uploader = DbnUploader::new(config).await.expect("Failed to create uploader");

    // Manually trigger scan
    let detected_files = uploader.scan_for_testing().await.expect("Failed to scan");

    assert_eq!(
        detected_files.len(),
        1,
        "Should only detect .dbn files, found: {:?}",
        detected_files
    );
    assert!(detected_files[0]
        .file_name()
        .unwrap()
        .to_str()
        .unwrap()
        .ends_with(".dbn"));
}

/// Test 8: Handles empty watch directory
#[tokio::test]
async fn test_handles_empty_directory() {
    let temp_dir = TempDir::new().expect("Failed to create temp dir");
    let watch_path = temp_dir.path().to_path_buf();

    let config = DbnUploaderConfig {
        watch_path: watch_path.clone(),
        bucket_name: "ml-models".to_string(),
        upload_prefix: "training-data/".to_string(),
        poll_interval: Duration::from_millis(100),
        compression_enabled: false,
        deduplication_enabled: false,
    };

    let uploader = DbnUploader::new(config).await.expect("Failed to create uploader");

    sleep(Duration::from_millis(200)).await;

    let detected_files = uploader.get_detected_files().await;
    assert_eq!(detected_files.len(), 0, "Should detect no files");
}

/// Test 9: Upload generates correct MinIO key
#[tokio::test]
async fn test_upload_generates_correct_minio_key() {
    let filename = PathBuf::from("ES.FUT_ohlcv-1m_2024-01-02.dbn");
    let prefix = "training-data/";

    let key = DbnUploader::generate_upload_key(&filename, prefix);

    assert_eq!(
        key,
        "training-data/ES.FUT_ohlcv-1m_2024-01-02.dbn.gz",
        "Key should include prefix and .gz extension"
    );
}

/// Test 10: Compressed file size is tracked in metadata
#[tokio::test]
async fn test_compression_size_tracked() {
    let temp_dir = TempDir::new().expect("Failed to create temp dir");
    let test_file = temp_dir.path().join("test.dbn");

    // Use highly compressible data (gzip header is ~20 bytes, so need enough data)
    let test_data = b"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; // 100 bytes
    fs::write(&test_file, test_data)
        .await
        .expect("Failed to write test file");

    let original_size = test_data.len();
    let compressed = DbnUploader::compress_file(&test_file)
        .await
        .expect("Failed to compress");

    let compressed_size = compressed.len();

    assert!(
        compressed_size > 0,
        "Compressed size should be greater than 0"
    );
    assert!(
        original_size > compressed_size,
        "Original size {} should be greater than compressed {}",
        original_size,
        compressed_size
    );
}

/// Test 11: Metadata includes file size
#[tokio::test]
async fn test_metadata_includes_file_size() {
    let temp_dir = TempDir::new().expect("Failed to create temp dir");
    let test_file = temp_dir.path().join("ES.FUT_ohlcv-1m_2024-01-02.dbn");

    let test_data = b"test data";
    fs::write(&test_file, test_data)
        .await
        .expect("Failed to write test file");

    let metadata = DbnMetadata::from_file(&test_file)
        .await
        .expect("Failed to extract metadata");

    assert_eq!(metadata.file_size_bytes, test_data.len() as u64);
    assert_eq!(metadata.symbol, "ES.FUT");
}

/// Test 12: Upload adds correct tags to MinIO metadata
#[tokio::test]
async fn test_upload_adds_metadata_tags() {
    let metadata = DbnMetadata {
        symbol: "ES.FUT".to_string(),
        schema: "ohlcv-1m".to_string(),
        date_range: "2024-01-02".to_string(),
        file_size_bytes: 1024,
    };

    let tags = DbnUploader::generate_metadata_tags(&metadata);

    assert_eq!(tags.get("symbol"), Some(&"ES.FUT".to_string()));
    assert_eq!(tags.get("schema"), Some(&"ohlcv-1m".to_string()));
    assert_eq!(tags.get("date_range"), Some(&"2024-01-02".to_string()));
    assert_eq!(tags.get("original_size"), Some(&"1024".to_string()));
}