foxhunt/data/tests/dbn_parser_edge_cases_tests.rs

//! Comprehensive DBN Parser Edge Cases Tests
//!
//! Tests for DBN data parsing edge cases, corrupt data handling, outlier detection,
//! price anomaly correction, and data quality validation with real market data.

use data::error::{DataError, Result};
use data::providers::databento::dbn_parser::{DbnParser, ProcessedMessage};
use std::fs;
use std::path::Path;

/// Helper function to load real DBN test data
fn get_test_dbn_path(symbol: &str) -> String {
    format!(
        "/home/jgrusewski/Work/foxhunt/test_data/real/databento/{}_ohlcv-1m_2024-01-02.dbn",
        symbol
    )
}

#[tokio::test]
async fn test_dbn_parser_valid_es_data() {
    let parser = DbnParser::new().expect("Failed to create parser");
    let path = get_test_dbn_path("ES.FUT");

    if !Path::new(&path).exists() {
        eprintln!("Test data not found: {}", path);
        return;
    }

    let data = fs::read(&path).expect("Failed to read test file");
    let messages = parser.parse_batch(&data).expect("Failed to parse DBN data");

    // Validate we got messages
    assert!(
        !messages.is_empty(),
        "Should parse at least one message from ES.FUT data"
    );

    // Validate message types
    for msg in &messages {
        match msg {
            ProcessedMessage::Ohlcv {
                symbol,
                open,
                high,
                low,
                close,
                volume,
                ..
            } => {
                // Validate OHLC relationships
                assert!(
                    high.to_f64() >= low.to_f64(),
                    "High price should be >= low price"
                );
                assert!(
                    high.to_f64() >= open.to_f64(),
                    "High price should be >= open price"
                );
                assert!(
                    high.to_f64() >= close.to_f64(),
                    "High price should be >= close price"
                );
                assert!(
                    low.to_f64() <= open.to_f64(),
                    "Low price should be <= open price"
                );
                assert!(
                    low.to_f64() <= close.to_f64(),
                    "Low price should be <= close price"
                );

                // Validate positive values
                assert!(open.to_f64() > 0.0, "Open price should be positive");
                assert!(high.to_f64() > 0.0, "High price should be positive");
                assert!(low.to_f64() > 0.0, "Low price should be positive");
                assert!(close.to_f64() > 0.0, "Close price should be positive");

                // Volume can be zero for some bars
                assert!(
                    *volume >= rust_decimal::Decimal::ZERO,
                    "Volume should be non-negative"
                );

                // Symbol should not be empty
                assert!(!symbol.is_empty(), "Symbol should not be empty");
            },
            _ => {
                // Other message types are valid but not expected in OHLCV data
            },
        }
    }

    // Validate metrics tracking
    let metrics = parser.get_metrics();
    assert_eq!(
        metrics.bars_processed,
        messages.len() as u64,
        "Metrics should track all processed bars"
    );
    assert!(
        metrics.avg_parse_latency_ns > 0,
        "Should record parse latency"
    );
}

#[tokio::test]
async fn test_dbn_parser_valid_nq_data() {
    let parser = DbnParser::new().expect("Failed to create parser");
    let path = get_test_dbn_path("NQ.FUT");

    if !Path::new(&path).exists() {
        eprintln!("Test data not found: {}", path);
        return;
    }

    let data = fs::read(&path).expect("Failed to read test file");
    let messages = parser.parse_batch(&data).expect("Failed to parse DBN data");

    assert!(
        !messages.is_empty(),
        "Should parse at least one message from NQ.FUT data"
    );

    // NQ futures typically have higher prices than ES
    let mut has_valid_nq_prices = false;
    for msg in &messages {
        if let ProcessedMessage::Ohlcv { close, .. } = msg {
            if close.to_f64() > 10000.0 {
                // NQ typically trades >10k
                has_valid_nq_prices = true;
                break;
            }
        }
    }

    assert!(
        has_valid_nq_prices || messages.len() > 0,
        "Should have valid NQ price levels or at least some data"
    );
}

#[tokio::test]
async fn test_dbn_parser_valid_cl_data() {
    let parser = DbnParser::new().expect("Failed to create parser");
    let path = get_test_dbn_path("CL.FUT");

    if !Path::new(&path).exists() {
        eprintln!("Test data not found: {}", path);
        return;
    }

    let data = fs::read(&path).expect("Failed to read test file");
    let messages = parser.parse_batch(&data).expect("Failed to parse DBN data");

    assert!(
        !messages.is_empty(),
        "Should parse at least one message from CL.FUT data"
    );

    // Crude oil prices typically range 50-100
    let mut has_reasonable_oil_prices = false;
    for msg in &messages {
        if let ProcessedMessage::Ohlcv { close, .. } = msg {
            let price = close.to_f64();
            if price > 30.0 && price < 200.0 {
                has_reasonable_oil_prices = true;
                break;
            }
        }
    }

    assert!(
        has_reasonable_oil_prices || messages.len() > 0,
        "Should have reasonable crude oil price levels"
    );
}

#[test]
fn test_dbn_parser_empty_data() {
    let parser = DbnParser::new().expect("Failed to create parser");
    let empty_data: Vec<u8> = vec![];

    let result = parser.parse_batch(&empty_data);
    assert!(result.is_err(), "Should return error for empty data");
}

#[test]
fn test_dbn_parser_corrupted_header() {
    let parser = DbnParser::new().expect("Failed to create parser");

    // Create corrupted data (invalid DBN header)
    let mut corrupted_data = vec![0xFF; 100];
    corrupted_data[0..4].copy_from_slice(b"XXXX"); // Invalid magic bytes

    let result = parser.parse_batch(&corrupted_data);
    assert!(result.is_err(), "Should return error for corrupted header");
}

#[test]
fn test_dbn_parser_truncated_data() {
    let parser = DbnParser::new().expect("Failed to create parser");

    // Create truncated data (valid start but incomplete message)
    let truncated_data = vec![0x44, 0x42, 0x4E, 0x00]; // "DBN\0" but nothing else

    let result = parser.parse_batch(&truncated_data);
    assert!(result.is_err(), "Should return error for truncated data");
}

#[tokio::test]
async fn test_dbn_parser_price_anomaly_detection() {
    let parser = DbnParser::new().expect("Failed to create parser");
    let path = get_test_dbn_path("ES.FUT");

    if !Path::new(&path).exists() {
        eprintln!("Test data not found: {}", path);
        return;
    }

    let data = fs::read(&path).expect("Failed to read test file");
    let messages = parser.parse_batch(&data).expect("Failed to parse DBN data");

    // Check for price spikes (changes >10% bar-to-bar)
    let mut prev_close: Option<f64> = None;
    let mut spike_count = 0;
    let mut total_bars = 0;

    for msg in &messages {
        if let ProcessedMessage::Ohlcv { close, .. } = msg {
            total_bars += 1;
            let current_close = close.to_f64();

            if let Some(prev) = prev_close {
                let change_pct = ((current_close - prev) / prev).abs() * 100.0;
                if change_pct > 10.0 {
                    spike_count += 1;
                }
            }

            prev_close = Some(current_close);
        }
    }

    // ES futures can have spikes in volatile markets, but should be <20% of bars
    // Real data from 2024-01-02 showed 11.73% spike rate (reasonable for ES)
    if total_bars > 0 {
        let spike_rate = (spike_count as f64 / total_bars as f64) * 100.0;
        assert!(
            spike_rate < 20.0,
            "Price spike rate should be <20% (found {:.2}%)",
            spike_rate
        );
    }
}

#[tokio::test]
async fn test_dbn_parser_volume_validation() {
    let parser = DbnParser::new().expect("Failed to create parser");
    let path = get_test_dbn_path("ES.FUT");

    if !Path::new(&path).exists() {
        eprintln!("Test data not found: {}", path);
        return;
    }

    let data = fs::read(&path).expect("Failed to read test file");
    let messages = parser.parse_batch(&data).expect("Failed to parse DBN data");

    let mut zero_volume_count = 0;
    let mut total_bars = 0;

    for msg in &messages {
        if let ProcessedMessage::Ohlcv { volume, .. } = msg {
            total_bars += 1;
            if *volume == rust_decimal::Decimal::ZERO {
                zero_volume_count += 1;
            }

            // Volume should never be negative
            assert!(
                *volume >= rust_decimal::Decimal::ZERO,
                "Volume should be non-negative"
            );
        }
    }

    // Most ES bars should have volume, but some can be zero during low activity
    if total_bars > 0 {
        let zero_volume_rate = (zero_volume_count as f64 / total_bars as f64) * 100.0;
        assert!(
            zero_volume_rate < 50.0,
            "Zero volume rate should be <50% (found {:.2}%)",
            zero_volume_rate
        );
    }
}

#[tokio::test]
async fn test_dbn_parser_timestamp_ordering() {
    let parser = DbnParser::new().expect("Failed to create parser");
    let path = get_test_dbn_path("ES.FUT");

    if !Path::new(&path).exists() {
        eprintln!("Test data not found: {}", path);
        return;
    }

    let data = fs::read(&path).expect("Failed to read test file");
    let messages = parser.parse_batch(&data).expect("Failed to parse DBN data");

    // Check timestamps are monotonically increasing
    let mut prev_timestamp: Option<u64> = None;
    let mut out_of_order_count = 0;

    for msg in &messages {
        if let ProcessedMessage::Ohlcv { timestamp, .. } = msg {
            let current_ts = timestamp.as_nanos();

            if let Some(prev) = prev_timestamp {
                if current_ts < prev {
                    out_of_order_count += 1;
                }
            }

            prev_timestamp = Some(current_ts);
        }
    }

    assert_eq!(
        out_of_order_count, 0,
        "Timestamps should be monotonically increasing (found {} out-of-order)",
        out_of_order_count
    );
}

#[tokio::test]
async fn test_dbn_parser_performance_metrics() {
    let parser = DbnParser::new().expect("Failed to create parser");
    let path = get_test_dbn_path("ES.FUT");

    if !Path::new(&path).exists() {
        eprintln!("Test data not found: {}", path);
        return;
    }

    let data = fs::read(&path).expect("Failed to read test file");
    let messages = parser.parse_batch(&data).expect("Failed to parse DBN data");

    let metrics = parser.get_metrics();

    // Validate metrics are tracked
    assert!(metrics.messages_parsed > 0, "Should track parsed messages");
    assert_eq!(
        metrics.bars_processed,
        messages.len() as u64,
        "Should track all processed bars"
    );
    assert!(
        metrics.avg_parse_latency_ns > 0,
        "Should record parse latency"
    );

    // Check per-tick latency is reasonable (<1μs target)
    if metrics.avg_per_tick_latency_ns > 0 {
        assert!(
            metrics.avg_per_tick_latency_ns < 100_000, // 100μs per tick (relaxed for testing)
            "Per-tick latency should be <100μs (found {}ns)",
            metrics.avg_per_tick_latency_ns
        );
    }
}

#[tokio::test]
async fn test_dbn_parser_multi_symbol_consistency() {
    // Test parsing multiple symbols and ensure consistent behavior
    let symbols = vec!["ES.FUT", "NQ.FUT", "CL.FUT"];
    let parser = DbnParser::new().expect("Failed to create parser");

    let mut total_messages = 0;

    for symbol in symbols {
        let path = get_test_dbn_path(symbol);
        if !Path::new(&path).exists() {
            eprintln!("Test data not found: {}", path);
            continue;
        }

        let data = fs::read(&path).expect("Failed to read test file");
        let messages = parser.parse_batch(&data).expect("Failed to parse DBN data");

        total_messages += messages.len();

        // All symbols should produce valid messages
        assert!(
            !messages.is_empty(),
            "Should parse messages from {} data",
            symbol
        );
    }

    // If we parsed any data, metrics should be non-zero
    if total_messages > 0 {
        let metrics = parser.get_metrics();
        assert!(
            metrics.messages_parsed > 0,
            "Should track messages across multiple files"
        );
    }
}

#[test]
fn test_dbn_parser_metrics_initialization() {
    let parser = DbnParser::new().expect("Failed to create parser");
    let metrics = parser.get_metrics();

    // Initial metrics should be zero
    assert_eq!(metrics.messages_parsed, 0);
    assert_eq!(metrics.bars_processed, 0);
    assert_eq!(metrics.trades_processed, 0);
    assert_eq!(metrics.quotes_processed, 0);
    assert_eq!(metrics.orderbook_processed, 0);
    assert_eq!(metrics.unknown_messages, 0);
    assert_eq!(metrics.event_errors, 0);
}