foxhunt/crates/data/tests/real_data_integration_tests.rs

//! Comprehensive Integration Tests for Real Parquet Data
//!
//! Tests validate that real BTC/ETH Parquet files from September 2024
//! work correctly with the entire Foxhunt trading system.
//!
//! Test Files:
//! - BTC-USD_30day_2024-09.parquet (871 KB, 41,550 rows)
//! - ETH-USD_30day_2024-09.parquet (801 KB, 42,220 rows)

#![allow(
    clippy::absurd_extreme_comparisons,
    clippy::assertions_on_constants,
    clippy::assertions_on_result_states,
    clippy::clone_on_copy,
    clippy::decimal_literal_representation,
    clippy::doc_lazy_continuation,
    clippy::doc_markdown,
    clippy::double_comparisons,
    clippy::else_if_without_else,
    clippy::empty_drop,
    clippy::expect_used,
    clippy::field_reassign_with_default,
    clippy::format_push_string,
    clippy::if_then_some_else_none,
    clippy::indexing_slicing,
    clippy::integer_division,
    clippy::len_zero,
    clippy::let_and_return,
    clippy::let_underscore_must_use,
    clippy::manual_let_else,
    clippy::manual_range_contains,
    clippy::map_err_ignore,
    clippy::missing_const_for_fn,
    clippy::modulo_arithmetic,
    clippy::needless_range_loop,
    clippy::new_without_default,
    clippy::non_ascii_literal,
    clippy::nonminimal_bool,
    clippy::octal_escapes,
    clippy::overly_complex_bool_expr,
    clippy::redundant_clone,
    clippy::shadow_reuse,
    clippy::shadow_unrelated,
    clippy::similar_names,
    clippy::single_component_path_imports,
    clippy::single_match_else,
    clippy::str_to_string,
    clippy::string_slice,
    clippy::tests_outside_test_module,
    clippy::unnecessary_cast,
    clippy::unnecessary_get_then_check,
    clippy::unnecessary_unwrap,
    clippy::unseparated_literal_suffix,
    clippy::unwrap_or_default,
    clippy::unwrap_used,
    clippy::use_debug,
    clippy::useless_vec,
    clippy::wildcard_enum_match_arm,
    dead_code,
    deprecated,
    unreachable_pub,
    unused_assignments,
    unused_comparisons,
    unused_imports,
    unused_variables
)]

use data::parquet_persistence::{MarketDataEvent, ParquetMarketDataReader};
use std::mem::size_of;
use std::path::Path;
use std::time::Instant;
use trading_engine::types::metrics::MarketDataEventType;

// Test data directory
const REAL_DATA_PATH: &str = "test_data/real/parquet";
const BTC_FILE: &str = "BTC-USD_30day_2024-09.parquet";
const ETH_FILE: &str = "ETH-USD_30day_2024-09.parquet";

// Expected data characteristics (from VALIDATION_SUMMARY.md)
const BTC_EXPECTED_ROWS: usize = 41_550;
const ETH_EXPECTED_ROWS: usize = 42_220;
const BTC_EXPECTED_VENUE: &str = "yahoo_finance";
const ETH_EXPECTED_VENUE: &str = "yahoo_finance";

// September 2024 timestamp range (Unix epoch nanoseconds)
const SEPT_2024_START_NS: u64 = 1725148800000000000; // 2024-09-01 00:00:00
const SEPT_2024_END_NS: u64 = 1727740740000000000; // 2024-09-30 23:59:00

// BTC price range for September 2024 (from validation summary)
const BTC_MIN_PRICE: f64 = 50_000.0;
const BTC_MAX_PRICE: f64 = 70_000.0;

// ETH price range for September 2024 (from validation summary)
const ETH_MIN_PRICE: f64 = 2_000.0;
const ETH_MAX_PRICE: f64 = 3_000.0;

/// Helper to get absolute path to test data
fn get_test_data_path(filename: &str) -> String {
    // Try from workspace root first
    let workspace_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .unwrap()
        .join(REAL_DATA_PATH)
        .join(filename);

    if workspace_path.exists() {
        workspace_path.to_string_lossy().to_string()
    } else {
        // Fallback to relative path
        Path::new(REAL_DATA_PATH)
            .join(filename)
            .to_string_lossy()
            .to_string()
    }
}

/// Helper to check if test files exist
fn test_files_exist() -> bool {
    let btc_path = get_test_data_path(BTC_FILE);
    let eth_path = get_test_data_path(ETH_FILE);
    Path::new(&btc_path).exists() && Path::new(&eth_path).exists()
}

// ============================================================================
// TEST 1-2: Basic File Loading
// ============================================================================

#[tokio::test]
#[ignore = "requires real Parquet files"]
async fn test_01_load_btc_parquet() {
    if !test_files_exist() {
        eprintln!("SKIPPED: Test files not found at {}", REAL_DATA_PATH);
        return;
    }

    let base_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .unwrap()
        .join(REAL_DATA_PATH)
        .to_string_lossy()
        .to_string();

    let reader = ParquetMarketDataReader::new(base_path);
    let events = reader.read_file(BTC_FILE).await;

    // Smoke-level assertion: the reader is expected to succeed on the fixture.
    // Row-count / symbol assertions live in `test_03_btc_schema_validation`.
    assert!(events.is_ok(), "Failed to read BTC Parquet file");
}

#[tokio::test]
#[ignore = "requires real Parquet files"]
async fn test_02_load_eth_parquet() {
    if !test_files_exist() {
        eprintln!("SKIPPED: Test files not found at {}", REAL_DATA_PATH);
        return;
    }

    let base_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .unwrap()
        .join(REAL_DATA_PATH)
        .to_string_lossy()
        .to_string();

    let reader = ParquetMarketDataReader::new(base_path);
    let events = reader.read_file(ETH_FILE).await;

    // Smoke-level assertion; row-count / symbol checks live in
    // `test_04_eth_schema_validation`.
    assert!(events.is_ok(), "Failed to read ETH Parquet file");
}

// ============================================================================
// TEST 3-4: Schema Validation
// ============================================================================

#[tokio::test]
#[ignore = "requires real Parquet files and reader implementation"]
async fn test_03_btc_schema_validation() {
    if !test_files_exist() {
        eprintln!("SKIPPED: Test files not found");
        return;
    }

    let base_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .unwrap()
        .join(REAL_DATA_PATH)
        .to_string_lossy()
        .to_string();

    let reader = ParquetMarketDataReader::new(base_path);
    let events = reader.read_file(BTC_FILE).await.unwrap();

    // Validate all required fields present
    for (i, event) in events.iter().take(10).enumerate() {
        assert!(event.timestamp_ns > 0, "Row {}: Invalid timestamp", i);
        assert!(!event.symbol.is_empty(), "Row {}: Empty symbol", i);
        assert!(!event.venue.is_empty(), "Row {}: Empty venue", i);
        assert_eq!(event.venue, BTC_EXPECTED_VENUE, "Row {}: Wrong venue", i);

        // Validate timestamp in September 2024 range
        assert!(
            event.timestamp_ns >= SEPT_2024_START_NS && event.timestamp_ns <= SEPT_2024_END_NS,
            "Row {}: Timestamp {} outside September 2024 range",
            i,
            event.timestamp_ns
        );

        // Validate sequence numbers incrementing
        assert_eq!(event.sequence, i as u64, "Row {}: Sequence mismatch", i);
    }
}

#[tokio::test]
#[ignore = "requires real Parquet files and reader implementation"]
async fn test_04_eth_schema_validation() {
    if !test_files_exist() {
        eprintln!("SKIPPED: Test files not found");
        return;
    }

    let base_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .unwrap()
        .join(REAL_DATA_PATH)
        .to_string_lossy()
        .to_string();

    let reader = ParquetMarketDataReader::new(base_path);
    let events = reader.read_file(ETH_FILE).await.unwrap();

    // Validate schema compliance
    for (i, event) in events.iter().take(10).enumerate() {
        assert!(event.timestamp_ns > 0, "Row {}: Invalid timestamp", i);
        assert!(!event.symbol.is_empty(), "Row {}: Empty symbol", i);
        assert_eq!(event.venue, ETH_EXPECTED_VENUE, "Row {}: Wrong venue", i);

        // Validate event type is Trade (converted from OHLCV)
        assert_eq!(
            event.event_type,
            MarketDataEventType::Trade,
            "Row {}: Expected Trade event type",
            i
        );
    }
}

// ============================================================================
// TEST 5: Chronological Ordering
// ============================================================================

#[tokio::test]
#[ignore = "requires real Parquet files and reader implementation"]
async fn test_05_chronological_ordering() {
    if !test_files_exist() {
        eprintln!("SKIPPED: Test files not found");
        return;
    }

    let base_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .unwrap()
        .join(REAL_DATA_PATH)
        .to_string_lossy()
        .to_string();

    let reader = ParquetMarketDataReader::new(base_path);

    // Test both BTC and ETH
    for (file, symbol) in &[(BTC_FILE, "BTC"), (ETH_FILE, "ETH")] {
        let events = reader.read_file(file).await.unwrap();
        assert!(!events.is_empty(), "{}: No events loaded", symbol);

        // Verify timestamps are in descending order (newest first based on validation summary)
        let mut prev_timestamp = u64::MAX;
        for (i, event) in events.iter().enumerate() {
            assert!(
                event.timestamp_ns <= prev_timestamp,
                "{} Row {}: Timestamps not in descending order: {} > {}",
                symbol,
                i,
                event.timestamp_ns,
                prev_timestamp
            );
            prev_timestamp = event.timestamp_ns;

            // Verify no timestamps outside September 2024
            assert!(
                event.timestamp_ns >= SEPT_2024_START_NS && event.timestamp_ns <= SEPT_2024_END_NS,
                "{} Row {}: Timestamp outside September 2024 range",
                symbol,
                i
            );
        }
    }
}

// ============================================================================
// TEST 6: Price Sanity Checks
// ============================================================================

#[tokio::test]
#[ignore = "requires real Parquet files and reader implementation"]
async fn test_06_price_sanity() {
    if !test_files_exist() {
        eprintln!("SKIPPED: Test files not found");
        return;
    }

    let base_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .unwrap()
        .join(REAL_DATA_PATH)
        .to_string_lossy()
        .to_string();

    let reader = ParquetMarketDataReader::new(base_path);

    // Test BTC prices (September 2024: $50K-$70K range expected)
    let btc_events = reader.read_file(BTC_FILE).await.unwrap();
    for event in btc_events.iter() {
        if let Some(price) = event.price {
            assert!(price > 0.0, "BTC: Zero or negative price: {}", price);
            assert!(
                price >= BTC_MIN_PRICE && price <= BTC_MAX_PRICE,
                "BTC: Price {} outside expected range ${}-${}",
                price,
                BTC_MIN_PRICE,
                BTC_MAX_PRICE
            );
            assert!(!price.is_nan(), "BTC: NaN price detected");
            assert!(!price.is_infinite(), "BTC: Infinite price detected");
        }
    }

    // Test ETH prices (September 2024: $2K-$3K range expected)
    let eth_events = reader.read_file(ETH_FILE).await.unwrap();
    for event in eth_events.iter() {
        if let Some(price) = event.price {
            assert!(price > 0.0, "ETH: Zero or negative price: {}", price);
            assert!(
                price >= ETH_MIN_PRICE && price <= ETH_MAX_PRICE,
                "ETH: Price {} outside expected range ${}-${}",
                price,
                ETH_MIN_PRICE,
                ETH_MAX_PRICE
            );
            assert!(!price.is_nan(), "ETH: NaN price detected");
            assert!(!price.is_infinite(), "ETH: Infinite price detected");
        }
    }
}

// ============================================================================
// TEST 7: Quantity Validation
// ============================================================================

#[tokio::test]
#[ignore = "requires real Parquet files and reader implementation"]
async fn test_07_quantity_validation() {
    if !test_files_exist() {
        eprintln!("SKIPPED: Test files not found");
        return;
    }

    let base_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .unwrap()
        .join(REAL_DATA_PATH)
        .to_string_lossy()
        .to_string();

    let reader = ParquetMarketDataReader::new(base_path);

    // Test both BTC and ETH
    for (file, symbol) in &[(BTC_FILE, "BTC"), (ETH_FILE, "ETH")] {
        let events = reader.read_file(file).await.unwrap();

        for event in events.iter() {
            if let Some(quantity) = event.quantity {
                // No negative quantities
                assert!(
                    quantity >= 0.0,
                    "{}: Negative quantity: {}",
                    symbol,
                    quantity
                );

                // Check for NaN or infinity
                assert!(!quantity.is_nan(), "{}: NaN quantity", symbol);
                assert!(!quantity.is_infinite(), "{}: Infinite quantity", symbol);

                // Reasonable volume ranges (0 is acceptable for some periods)
                // Volume can be 0 for certain periods, so we just check it's not absurdly large
                assert!(
                    quantity < 1_000_000_000.0,
                    "{}: Unreasonably large quantity: {}",
                    symbol,
                    quantity
                );
            }
        }
    }
}

// ============================================================================
// TEST 8: Performance Benchmark (Load Time <5s)
// ============================================================================

#[tokio::test]
#[ignore = "requires real Parquet files and reader implementation"]
async fn test_08_load_performance() {
    if !test_files_exist() {
        eprintln!("SKIPPED: Test files not found");
        return;
    }

    let base_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .unwrap()
        .join(REAL_DATA_PATH)
        .to_string_lossy()
        .to_string();

    let reader = ParquetMarketDataReader::new(base_path);

    let start = Instant::now();
    let btc = reader.read_file(BTC_FILE).await.unwrap();
    let eth = reader.read_file(ETH_FILE).await.unwrap();
    let elapsed = start.elapsed();

    println!(
        "Loaded {} BTC + {} ETH events in {:?}",
        btc.len(),
        eth.len(),
        elapsed
    );

    assert!(
        elapsed.as_secs() < 5,
        "Load time {} seconds exceeded 5 second target",
        elapsed.as_secs()
    );

    assert_eq!(btc.len(), BTC_EXPECTED_ROWS, "BTC row count mismatch");
    assert_eq!(eth.len(), ETH_EXPECTED_ROWS, "ETH row count mismatch");

    // Calculate throughput
    let total_events = btc.len() + eth.len();
    let events_per_sec = total_events as f64 / elapsed.as_secs_f64();
    println!("Throughput: {:.0} events/second", events_per_sec);

    // Should be able to load at least 10K events/sec
    assert!(
        events_per_sec > 10_000.0,
        "Throughput {:.0} events/sec below 10K target",
        events_per_sec
    );
}

// ============================================================================
// TEST 9: Backtesting Integration (Placeholder)
// ============================================================================

#[tokio::test]
#[ignore = "requires backtesting service and reader implementation"]
async fn test_09_backtesting_integration() {
    if !test_files_exist() {
        eprintln!("SKIPPED: Test files not found");
        return;
    }

    // This test currently only verifies that the Parquet fixture can be
    // opened by the reader. End-to-end backtest wiring (config construction,
    // 1-day replay, PnL validation) is exercised by the backtesting crate's
    // own integration tests; repeating it here would duplicate that coverage.
    let base_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .unwrap()
        .join(REAL_DATA_PATH)
        .to_string_lossy()
        .to_string();

    let reader = ParquetMarketDataReader::new(base_path);
    let btc = reader.read_file(BTC_FILE).await;
    assert!(btc.is_ok(), "BTC file must be readable for backtesting");
}

// ============================================================================
// TEST 10: Feature Extraction (Placeholder)
// ============================================================================

#[tokio::test]
#[ignore = "requires feature processor and reader implementation"]
async fn test_10_feature_extraction() {
    if !test_files_exist() {
        eprintln!("SKIPPED: Test files not found");
        return;
    }

    // Feature-extraction correctness (32-dim OHLCV + 27 technical indicators)
    // is covered by the ml_features crate's own tests. This test only
    // verifies that the Parquet fixture the feature processor consumes is
    // readable from the data crate side.
    let base_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .unwrap()
        .join(REAL_DATA_PATH)
        .to_string_lossy()
        .to_string();

    let reader = ParquetMarketDataReader::new(base_path);
    let btc = reader.read_file(BTC_FILE).await;
    assert!(
        btc.is_ok(),
        "BTC file must be readable for feature extraction"
    );
}

// ============================================================================
// TEST 11: Memory Usage (<500MB)
// ============================================================================

#[tokio::test]
#[ignore = "requires reader implementation"]
async fn test_11_memory_usage() {
    if !test_files_exist() {
        eprintln!("SKIPPED: Test files not found");
        return;
    }

    let base_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .unwrap()
        .join(REAL_DATA_PATH)
        .to_string_lossy()
        .to_string();

    let reader = ParquetMarketDataReader::new(base_path);

    // Load both files
    let btc = reader.read_file(BTC_FILE).await.unwrap();
    let eth = reader.read_file(ETH_FILE).await.unwrap();

    // Calculate approximate memory usage
    // Each MarketDataEvent is roughly 128 bytes
    let btc_memory_bytes = btc.len() * size_of::<MarketDataEvent>();
    let eth_memory_bytes = eth.len() * size_of::<MarketDataEvent>();
    let total_memory_mb = (btc_memory_bytes + eth_memory_bytes) as f64 / (1024.0 * 1024.0);

    println!(
        "Memory usage: BTC {:.2} MB + ETH {:.2} MB = {:.2} MB total",
        btc_memory_bytes as f64 / (1024.0 * 1024.0),
        eth_memory_bytes as f64 / (1024.0 * 1024.0),
        total_memory_mb
    );

    assert!(
        total_memory_mb < 500.0,
        "Memory usage {:.2} MB exceeds 500 MB target",
        total_memory_mb
    );
}

// ============================================================================
// TEST 12: Error Handling (Invalid File)
// ============================================================================

#[tokio::test]
async fn test_12_invalid_file_handling() {
    let base_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .unwrap()
        .join(REAL_DATA_PATH)
        .to_string_lossy()
        .to_string();

    let reader = ParquetMarketDataReader::new(base_path);

    // A non-existent file surfaces the `File::open` error through anyhow's
    // context chain, so the reader returns `Err` rather than an empty Vec.
    let result = reader.read_file("nonexistent.parquet").await;
    assert!(
        result.is_err(),
        "Non-existent Parquet file should yield Err from the reader"
    );
}

// ============================================================================
// TEST 13: Simultaneous Load (BTC + ETH)
// ============================================================================

#[tokio::test]
#[ignore = "requires reader implementation"]
async fn test_13_simultaneous_load() {
    if !test_files_exist() {
        eprintln!("SKIPPED: Test files not found");
        return;
    }

    let base_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .unwrap()
        .join(REAL_DATA_PATH)
        .to_string_lossy()
        .to_string();

    let reader = ParquetMarketDataReader::new(base_path.clone());
    let reader2 = ParquetMarketDataReader::new(base_path);

    // Load both files concurrently
    let (btc_result, eth_result) =
        tokio::join!(reader.read_file(BTC_FILE), reader2.read_file(ETH_FILE));

    let btc = btc_result.unwrap();
    let eth = eth_result.unwrap();

    assert_eq!(btc.len(), BTC_EXPECTED_ROWS, "BTC: Wrong number of events");
    assert_eq!(eth.len(), ETH_EXPECTED_ROWS, "ETH: Wrong number of events");

    // Verify data integrity after concurrent load
    assert!(btc[0].symbol.contains("BTC"), "BTC: Symbol mismatch");
    assert!(eth[0].symbol.contains("ETH"), "ETH: Symbol mismatch");
}

// ============================================================================
// TEST 14: Reader File Listing
// ============================================================================

#[tokio::test]
async fn test_14_reader_file_listing() {
    if !test_files_exist() {
        eprintln!("SKIPPED: Test files not found");
        return;
    }

    let base_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .unwrap()
        .join(REAL_DATA_PATH)
        .to_string_lossy()
        .to_string();

    let reader = ParquetMarketDataReader::new(base_path);

    let files = reader.list_available_files().await.unwrap();

    // Should find both BTC and ETH files
    assert!(files.len() >= 2, "Expected at least 2 parquet files");

    assert!(
        files.contains(&BTC_FILE.to_string()),
        "BTC file not found in listing: {:?}",
        files
    );
    assert!(
        files.contains(&ETH_FILE.to_string()),
        "ETH file not found in listing: {:?}",
        files
    );

    // Files should be sorted
    let mut sorted = files.clone();
    sorted.sort();
    assert_eq!(files, sorted, "Files should be sorted alphabetically");
}

// ============================================================================
// TEST 15: Data Integrity (Sequence Numbers)
// ============================================================================

#[tokio::test]
#[ignore = "requires reader implementation"]
async fn test_15_sequence_integrity() {
    if !test_files_exist() {
        eprintln!("SKIPPED: Test files not found");
        return;
    }

    let base_path = Path::new(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .unwrap()
        .join(REAL_DATA_PATH)
        .to_string_lossy()
        .to_string();

    let reader = ParquetMarketDataReader::new(base_path);

    // Test BTC sequence numbers
    let btc = reader.read_file(BTC_FILE).await.unwrap();
    assert_eq!(btc.len(), BTC_EXPECTED_ROWS);

    // First event should have sequence 0
    assert_eq!(btc[0].sequence, 0, "First sequence should be 0");

    // Last event should have sequence N-1
    assert_eq!(
        btc[btc.len() - 1].sequence,
        (BTC_EXPECTED_ROWS - 1) as u64,
        "Last sequence should be {}",
        BTC_EXPECTED_ROWS - 1
    );

    // All sequences should be unique and ascending
    let mut sequences: Vec<u64> = btc.iter().map(|e| e.sequence).collect();
    sequences.sort();
    sequences.dedup();
    assert_eq!(
        sequences.len(),
        BTC_EXPECTED_ROWS,
        "All sequences should be unique"
    );
}