foxhunt/crates/ml/tests/mbp10_parsing_test.rs

#![allow(
    clippy::assertions_on_constants,
    clippy::assertions_on_result_states,
    clippy::clone_on_copy,
    clippy::decimal_literal_representation,
    clippy::doc_markdown,
    clippy::empty_line_after_doc_comments,
    clippy::field_reassign_with_default,
    clippy::get_unwrap,
    clippy::identity_op,
    clippy::inconsistent_digit_grouping,
    clippy::indexing_slicing,
    clippy::integer_division,
    clippy::len_zero,
    clippy::let_underscore_must_use,
    clippy::manual_div_ceil,
    clippy::manual_let_else,
    clippy::manual_range_contains,
    clippy::modulo_arithmetic,
    clippy::needless_range_loop,
    clippy::non_ascii_literal,
    clippy::redundant_clone,
    clippy::shadow_reuse,
    clippy::shadow_same,
    clippy::shadow_unrelated,
    clippy::single_match_else,
    clippy::str_to_string,
    clippy::string_slice,
    clippy::tests_outside_test_module,
    clippy::too_many_lines,
    clippy::unnecessary_wraps,
    clippy::unseparated_literal_suffix,
    clippy::use_debug,
    clippy::useless_vec,
    clippy::wildcard_enum_match_arm,
    clippy::else_if_without_else,
    clippy::expect_used,
    clippy::missing_const_for_fn,
    clippy::similar_names,
    clippy::type_complexity,
    clippy::collapsible_else_if,
    clippy::doc_lazy_continuation,
    clippy::items_after_test_module,
    clippy::map_clone,
    clippy::multiple_unsafe_ops_per_block,
    clippy::unwrap_or_default,
    clippy::assign_op_pattern,
    clippy::needless_borrow,
    clippy::println_empty_string,
    clippy::unnecessary_cast,
    clippy::used_underscore_binding,
    clippy::create_dir,
    clippy::implicit_saturating_sub,
    clippy::exit,
    clippy::expect_fun_call,
    clippy::too_many_arguments,
    clippy::unnecessary_map_or,
    clippy::unwrap_used,
    dead_code,
    unused_imports,
    unused_variables,
    clippy::cloned_ref_to_slice_refs,
    clippy::neg_multiply,
    clippy::while_let_loop,
    clippy::bool_assert_comparison,
    clippy::excessive_precision,
    clippy::trivially_copy_pass_by_ref,
    clippy::op_ref,
    clippy::redundant_closure,
    clippy::unnecessary_lazy_evaluations,
    clippy::if_then_some_else_none,
    clippy::unnecessary_to_owned,
    clippy::single_component_path_imports,
)]
//! MBP-10 Parsing Tests
//!
//! Tests for parsing MBP-10 DBN files and extracting order book snapshots for OFI calculation.

use data::providers::databento::dbn_parser::DbnParser;
use std::path::Path;
use tracing::{info, warn};

#[tokio::test]
async fn test_parse_mbp10_single_day() {
    // Test parsing a single day of MBP-10 data
    let parser = DbnParser::new().expect("Failed to create parser");
    let path = Path::new("../test_data/mbp10-fixtures/glbx-mdp3-20240102.mbp-10.dbn.zst");

    if !path.exists() {
        warn!(path = ?path, "Test file not found, MBP-10 test data missing");
        panic!("Test data not available");
    }

    let snapshots = parser
        .parse_mbp10_file(path)
        .await
        .expect("Failed to parse MBP-10 file");

    info!(snapshot_count = snapshots.len(), path = %path.display(), "Parsed snapshots");

    // Validate snapshot count (expect millions of updates)
    assert!(
        snapshots.len() > 10_000,
        "Expected >10K snapshots, got {}",
        snapshots.len()
    );

    // Validate first snapshot structure
    let first = &snapshots[0];
    assert_eq!(
        first.levels.len(),
        10,
        "Expected 10 price levels, got {}",
        first.levels.len()
    );
    info!("First snapshot has 10 levels");

    // Validate data sanity (prices should be positive, spread should be positive)
    let (bid, ask) = first.get_best_bid_ask();
    assert!(bid > 0.0, "Bid price should be positive: {}", bid);
    assert!(ask > 0.0, "Ask price should be positive: {}", ask);
    assert!(ask > bid, "Ask ({}) should be > bid ({})", ask, bid);

    let spread = first.spread();
    info!(bid, ask, spread, "Best bid/ask/spread");
    assert!(spread > 0.0, "Spread should be positive: {}", spread);

    info!("Prices and spreads validated");

    // Validate timestamps are monotonic (mostly)
    let mut monotonic_violations = 0;
    for i in 1..snapshots.len().min(10000) {
        if snapshots[i].timestamp < snapshots[i - 1].timestamp {
            monotonic_violations += 1;
        }
    }

    let violation_pct = (monotonic_violations as f64 / 10000.0) * 100.0;
    info!(
        monotonic_violations,
        total_checked = 10000,
        violation_pct,
        "Timestamp monotonic violations"
    );

    // Allow up to 5% violations (reordering can happen in market data)
    assert!(
        violation_pct < 5.0,
        "Too many timestamp violations: {:.2}%",
        violation_pct
    );

    info!("Timestamps mostly monotonic");

    // Validate sizes are positive
    let invalid_sizes = snapshots
        .iter()
        .take(10000)
        .filter(|s| {
            let (bid_vol, ask_vol) = (s.total_bid_volume(), s.total_ask_volume());
            bid_vol == 0 && ask_vol == 0
        })
        .count();

    let invalid_pct = (invalid_sizes as f64 / 10000.0) * 100.0;
    info!(invalid_sizes, total_checked = 10000, invalid_pct, "Zero volume snapshots");

    // Allow up to 10% zero volume (market can be quiet)
    assert!(invalid_pct < 10.0, "Too many zero-volume snapshots: {:.2}%", invalid_pct);

    info!("Volumes validated");
}

#[tokio::test]
async fn test_parse_mbp10_multiple_days() {
    // Test parsing multiple days and aggregating results
    let parser = DbnParser::new().expect("Failed to create parser");

    let files = vec![
        "../test_data/mbp10-fixtures/glbx-mdp3-20240102.mbp-10.dbn.zst",
        "../test_data/mbp10-fixtures/glbx-mdp3-20240103.mbp-10.dbn.zst",
        "../test_data/mbp10-fixtures/glbx-mdp3-20240104.mbp-10.dbn.zst",
    ];

    let mut total_snapshots = 0;
    let mut daily_counts = Vec::new();

    for file in &files {
        let path = Path::new(file);
        if !path.exists() {
            warn!(path = ?path, "Skipping missing file");
            continue;
        }

        let snapshots = parser
            .parse_mbp10_file(path)
            .await
            .expect(&format!("Failed to parse {:?}", path));

        let count = snapshots.len();
        total_snapshots += count;
        daily_counts.push(count);

        info!(path = %path.display(), snapshot_count = count, "Parsed daily file");
    }

    info!(day_count = daily_counts.len(), total_snapshots, "Total snapshots across days");

    // Validate we got substantial data
    assert!(
        total_snapshots > 300_000,
        "Expected >30K total snapshots, got {}",
        total_snapshots
    );

    info!("Multi-day parsing successful");
}

#[tokio::test]
async fn test_mbp10_parsing_performance() {
    // Benchmark parsing performance
    use std::time::Instant;

    let parser = DbnParser::new().expect("Failed to create parser");
    let path = Path::new("../test_data/mbp10-fixtures/glbx-mdp3-20240102.mbp-10.dbn.zst");

    if !path.exists() {
        warn!("Test file not found, skipping performance test");
        return;
    }

    let start = Instant::now();
    let snapshots = parser
        .parse_mbp10_file(path)
        .await
        .expect("Failed to parse MBP-10 file");
    let duration = start.elapsed();

    let count = snapshots.len();
    let throughput = count as f64 / duration.as_secs_f64();
    let latency_us = duration.as_micros() as f64 / count as f64;

    info!(
        snapshot_count = count,
        ?duration,
        throughput,
        latency_us,
        "Performance metrics"
    );

    // Target: >100K snapshots/sec (10μs per snapshot)
    assert!(
        throughput > 100_000.0,
        "Parsing too slow: {:.0} snapshots/sec (target: 100K+)",
        throughput
    );

    info!("Performance target met");
}

#[tokio::test]
async fn test_mbp10_data_quality() {
    // Deep dive into data quality metrics
    let parser = DbnParser::new().expect("Failed to create parser");
    let path = Path::new("../test_data/mbp10-fixtures/glbx-mdp3-20240102.mbp-10.dbn.zst");

    if !path.exists() {
        warn!("Test file not found, skipping data quality test");
        return;
    }

    let snapshots = parser
        .parse_mbp10_file(path)
        .await
        .expect("Failed to parse MBP-10 file");

    info!("Data Quality Analysis (first 10,000 snapshots)");

    // Analyze spreads
    let spreads: Vec<f64> = snapshots.iter().take(10000).map(|s| s.spread()).collect();

    let avg_spread = spreads.iter().sum::<f64>() / spreads.len() as f64;
    let min_spread = spreads.iter().copied().fold(f64::INFINITY, f64::min);
    let max_spread = spreads.iter().copied().fold(f64::NEG_INFINITY, f64::max);

    info!(min_spread, max_spread, avg_spread, "Spread stats");

    // Spreads should be small and positive
    assert!(min_spread >= 0.0, "Negative spread detected: {}", min_spread);
    assert!(max_spread < 100.0, "Unrealistic spread: {}", max_spread);

    // Analyze volumes
    let bid_vols: Vec<u64> = snapshots.iter().take(10000).map(|s| s.total_bid_volume()).collect();
    let ask_vols: Vec<u64> = snapshots.iter().take(10000).map(|s| s.total_ask_volume()).collect();

    let avg_bid_vol = bid_vols.iter().sum::<u64>() as f64 / bid_vols.len() as f64;
    let avg_ask_vol = ask_vols.iter().sum::<u64>() as f64 / ask_vols.len() as f64;

    info!(avg_bid_vol, avg_ask_vol, "Volume stats");

    // Volumes should be positive
    assert!(avg_bid_vol > 0.0, "Zero average bid volume");
    assert!(avg_ask_vol > 0.0, "Zero average ask volume");

    // Analyze depth
    let depths: Vec<usize> = snapshots.iter().take(10000).map(|s| s.depth()).collect();
    let avg_depth = depths.iter().sum::<usize>() as f64 / depths.len() as f64;

    info!(avg_depth, "Order book depth (avg levels)");

    // Most snapshots should have some depth
    assert!(avg_depth > 1.0, "Insufficient order book depth: {:.2}", avg_depth);

    info!("Data quality validated");
}

#[tokio::test]
async fn test_mbp10_all_files_summary() {
    // Summary test that reports on all available files
    let parser = DbnParser::new().expect("Failed to create parser");

    let files = vec![
        "../test_data/mbp10-fixtures/glbx-mdp3-20240102.mbp-10.dbn.zst",
        "../test_data/mbp10-fixtures/glbx-mdp3-20240103.mbp-10.dbn.zst",
        "../test_data/mbp10-fixtures/glbx-mdp3-20240104.mbp-10.dbn.zst",
        "../test_data/mbp10-fixtures/glbx-mdp3-20240105.mbp-10.dbn.zst",
        "../test_data/mbp10-fixtures/glbx-mdp3-20240107.mbp-10.dbn.zst",
        "../test_data/mbp10-fixtures/glbx-mdp3-20240108.mbp-10.dbn.zst",
        "../test_data/mbp10-fixtures/glbx-mdp3-20240109.mbp-10.dbn.zst",
    ];

    info!("MBP-10 Data Summary Report");

    let mut total_snapshots = 0;
    let mut total_size_bytes = 0u64;

    for file in &files {
        let path = Path::new(file);
        if !path.exists() {
            warn!(path = %path.display(), "File missing");
            continue;
        }

        // Get file size
        let metadata = std::fs::metadata(path).expect("Failed to get file metadata");
        let size_mb = metadata.len() as f64 / (1024.0 * 1024.0);
        total_size_bytes += metadata.len();

        // Parse file
        let snapshots = match parser.parse_mbp10_file(path).await {
            Ok(s) => s,
            Err(e) => {
                warn!(path = %path.display(), error = %e, "Failed to parse file");
                continue;
            },
        };

        let count = snapshots.len();
        total_snapshots += count;

        info!(
            file = path.file_name().unwrap().to_string_lossy().as_ref(),
            snapshot_count = count,
            size_mb,
            "File summary"
        );
    }

    let total_gb = total_size_bytes as f64 / (1024.0 * 1024.0 * 1024.0);
    info!(total_snapshots, total_gb, "Total summary");

    info!("Summary report complete");
}