foxhunt/crates/ml/tests/ofi_features_test.rs

#![allow(
    clippy::assertions_on_constants,
    clippy::assertions_on_result_states,
    clippy::clone_on_copy,
    clippy::decimal_literal_representation,
    clippy::doc_markdown,
    clippy::empty_line_after_doc_comments,
    clippy::field_reassign_with_default,
    clippy::get_unwrap,
    clippy::identity_op,
    clippy::inconsistent_digit_grouping,
    clippy::indexing_slicing,
    clippy::integer_division,
    clippy::len_zero,
    clippy::let_underscore_must_use,
    clippy::manual_div_ceil,
    clippy::manual_let_else,
    clippy::manual_range_contains,
    clippy::modulo_arithmetic,
    clippy::needless_range_loop,
    clippy::non_ascii_literal,
    clippy::redundant_clone,
    clippy::shadow_reuse,
    clippy::shadow_same,
    clippy::shadow_unrelated,
    clippy::single_match_else,
    clippy::str_to_string,
    clippy::string_slice,
    clippy::tests_outside_test_module,
    clippy::too_many_lines,
    clippy::unnecessary_wraps,
    clippy::unseparated_literal_suffix,
    clippy::use_debug,
    clippy::useless_vec,
    clippy::wildcard_enum_match_arm,
    clippy::else_if_without_else,
    clippy::expect_used,
    clippy::missing_const_for_fn,
    clippy::similar_names,
    clippy::type_complexity,
    clippy::collapsible_else_if,
    clippy::doc_lazy_continuation,
    clippy::items_after_test_module,
    clippy::map_clone,
    clippy::multiple_unsafe_ops_per_block,
    clippy::unwrap_or_default,
    clippy::assign_op_pattern,
    clippy::needless_borrow,
    clippy::println_empty_string,
    clippy::unnecessary_cast,
    clippy::used_underscore_binding,
    clippy::create_dir,
    clippy::implicit_saturating_sub,
    clippy::exit,
    clippy::expect_fun_call,
    clippy::too_many_arguments,
    clippy::unnecessary_map_or,
    clippy::unwrap_used,
    dead_code,
    unused_imports,
    unused_variables,
    clippy::cloned_ref_to_slice_refs,
    clippy::neg_multiply,
    clippy::while_let_loop,
    clippy::bool_assert_comparison,
    clippy::excessive_precision,
    clippy::trivially_copy_pass_by_ref,
    clippy::op_ref,
    clippy::redundant_closure,
    clippy::unnecessary_lazy_evaluations,
    clippy::if_then_some_else_none,
    clippy::unnecessary_to_owned,
    clippy::single_component_path_imports,
)]
//! Comprehensive tests for OFI (Order Flow Imbalance) features
//!
//! Tests all 8 OFI features against academic formulas and expected ranges

use data::providers::databento::mbp10::{BidAskPair, Mbp10Snapshot};
use ml::features::ofi_calculator::{OFICalculator, OFIFeatures};

/// Create test MBP-10 snapshot with specified parameters
fn create_snapshot(
    timestamp: u64,
    bid_px: i64,
    ask_px: i64,
    bid_sz: u32,
    ask_sz: u32,
) -> Mbp10Snapshot {
    let levels = vec![
        BidAskPair {
            bid_px,
            bid_sz,
            bid_ct: 5,
            ask_px,
            ask_sz,
            ask_ct: 6,
        },
        BidAskPair {
            bid_px: bid_px - 1_000_000_000, // -0.001 tick
            bid_sz: bid_sz * 2,
            bid_ct: 8,
            ask_px: ask_px + 1_000_000_000,
            ask_sz: ask_sz * 2,
            ask_ct: 7,
        },
        BidAskPair {
            bid_px: bid_px - 2_000_000_000,
            bid_sz: bid_sz * 3,
            bid_ct: 10,
            ask_px: ask_px + 2_000_000_000,
            ask_sz: ask_sz * 3,
            ask_ct: 9,
        },
        BidAskPair {
            bid_px: bid_px - 3_000_000_000,
            bid_sz: bid_sz * 4,
            bid_ct: 12,
            ask_px: ask_px + 3_000_000_000,
            ask_sz: ask_sz * 4,
            ask_ct: 11,
        },
        BidAskPair {
            bid_px: bid_px - 4_000_000_000,
            bid_sz: bid_sz * 5,
            bid_ct: 15,
            ask_px: ask_px + 4_000_000_000,
            ask_sz: ask_sz * 5,
            ask_ct: 13,
        },
    ];

    Mbp10Snapshot::new("ES.FUT".to_string(), timestamp, levels, 0, 100)
}

#[test]
fn test_ofi_level1_rising_bid() {
    let mut calc = OFICalculator::new();

    let snap1 = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 120);
    let snap2 = create_snapshot(2000, 150_005_000_000_000, 150_010_000_000_000, 100, 120);

    // First snapshot (no previous)
    let features1 = calc.calculate(&snap1).unwrap();
    assert_eq!(features1.ofi_level1, 0.0, "First OFI should be zero");

    // Second snapshot with rising bid
    let features2 = calc.calculate(&snap2).unwrap();
    assert!(
        features2.ofi_level1 != 0.0,
        "OFI should be non-zero after price movement"
    );
}

#[test]
fn test_ofi_level1_falling_ask() {
    let mut calc = OFICalculator::new();

    let snap1 = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 120);
    let snap2 = create_snapshot(2000, 150_000_000_000_000, 150_005_000_000_000, 100, 120);

    calc.calculate(&snap1).unwrap();
    let features2 = calc.calculate(&snap2).unwrap();

    // Falling ask should produce positive OFI (more demand)
    assert!(
        features2.ofi_level1 != 0.0,
        "Falling ask should produce non-zero OFI"
    );
}

#[test]
fn test_ofi_level5_multilevel() {
    let mut calc = OFICalculator::new();

    let snap1 = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 100);
    let snap2 = create_snapshot(
        2000,
        150_005_000_000_000, // Bid rises
        150_010_000_000_000,
        110, // Bid size increases
        90,  // Ask size decreases
    );

    calc.calculate(&snap1).unwrap();
    let features2 = calc.calculate(&snap2).unwrap();

    assert!(
        features2.ofi_level5.is_finite(),
        "OFI Level 5 should be finite"
    );
    assert!(
        features2.ofi_level5.abs() < 1e6,
        "OFI Level 5 should be within expected range"
    );
}

#[test]
fn test_depth_imbalance_balanced() {
    let mut calc = OFICalculator::new();
    let snap = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 100);

    let features = calc.calculate(&snap).unwrap();

    // Balanced book should have near-zero imbalance
    assert!(
        features.depth_imbalance.abs() < 0.3,
        "Balanced book should have low imbalance, got {}",
        features.depth_imbalance
    );
    assert!(
        features.depth_imbalance >= -1.0 && features.depth_imbalance <= 1.0,
        "Depth imbalance should be in [-1, 1]"
    );
}

#[test]
fn test_depth_imbalance_bid_heavy() {
    let mut calc = OFICalculator::new();
    let snap = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 200, 50);

    let features = calc.calculate(&snap).unwrap();

    assert!(
        features.depth_imbalance > 0.0,
        "Bid-heavy book should have positive imbalance"
    );
    assert!(
        features.depth_imbalance >= -1.0 && features.depth_imbalance <= 1.0,
        "Depth imbalance should be in [-1, 1]"
    );
}

#[test]
fn test_depth_imbalance_ask_heavy() {
    let mut calc = OFICalculator::new();
    let snap = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 50, 200);

    let features = calc.calculate(&snap).unwrap();

    assert!(
        features.depth_imbalance < 0.0,
        "Ask-heavy book should have negative imbalance"
    );
    assert!(
        features.depth_imbalance >= -1.0 && features.depth_imbalance <= 1.0,
        "Depth imbalance should be in [-1, 1]"
    );
}

#[test]
fn test_bid_ask_slopes() {
    let mut calc = OFICalculator::new();
    let snap = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 120);

    let features = calc.calculate(&snap).unwrap();

    assert!(
        features.bid_slope.is_finite(),
        "Bid slope should be finite"
    );
    assert!(
        features.ask_slope.is_finite(),
        "Ask slope should be finite"
    );

    // With increasing volume at deeper levels, slopes should be positive
    assert!(
        features.bid_slope > 0.0,
        "Bid slope should be positive (volume increases with depth)"
    );
    assert!(
        features.ask_slope > 0.0,
        "Ask slope should be positive (volume increases with depth)"
    );
}

#[test]
fn test_vpin_range() {
    let mut calc = OFICalculator::new();
    let snap = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 120);

    let features = calc.calculate(&snap).unwrap();

    // VPIN should be in [0, 1] range
    assert!(
        features.vpin >= 0.0 && features.vpin <= 1.0,
        "VPIN should be in [0, 1], got {}",
        features.vpin
    );
}

#[test]
fn test_kyle_lambda_finite() {
    let mut calc = OFICalculator::new();
    let snap = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 120);

    let features = calc.calculate(&snap).unwrap();

    assert!(
        features.kyle_lambda.is_finite(),
        "Kyle's lambda should be finite"
    );
    assert!(
        features.kyle_lambda.abs() < 1.0,
        "Kyle's lambda should be in reasonable range"
    );
}

#[test]
fn test_trade_imbalance_range() {
    let mut calc = OFICalculator::new();

    let snap1 = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 120);
    let snap2 = create_snapshot(2000, 150_005_000_000_000, 150_015_000_000_000, 110, 115);

    calc.calculate(&snap1).unwrap();
    let features2 = calc.calculate(&snap2).unwrap();

    // Trade imbalance should be in [-1, 1]
    assert!(
        features2.trade_imbalance >= -1.0 && features2.trade_imbalance <= 1.0,
        "Trade imbalance should be in [-1, 1], got {}",
        features2.trade_imbalance
    );
}

#[test]
fn test_all_features_finite() {
    let mut calc = OFICalculator::new();

    // Create sequence of snapshots
    for i in 0..10u64 {
        let snap = create_snapshot(
            i * 1000,
            150_000_000_000_000 + i as i64 * 1_000_000_000,
            150_010_000_000_000 + i as i64 * 1_000_000_000,
            (100 + i * 5) as u32,
            (120 - i * 3) as u32,
        );

        let features = calc.calculate(&snap).unwrap();

        assert!(
            features.is_valid(),
            "All features should be finite at iteration {}",
            i
        );
        assert!(features.ofi_level1.is_finite());
        assert!(features.ofi_level5.is_finite());
        assert!(features.depth_imbalance.is_finite());
        assert!(features.vpin.is_finite());
        assert!(features.kyle_lambda.is_finite());
        assert!(features.bid_slope.is_finite());
        assert!(features.ask_slope.is_finite());
        assert!(features.trade_imbalance.is_finite());
    }
}

#[test]
fn test_ofi_features_to_array_conversion() {
    let features = OFIFeatures {
        ofi_level1: 1.5,
        ofi_level5: 2.3,
        depth_imbalance: 0.4,
        vpin: 0.6,
        kyle_lambda: 0.001,
        bid_slope: 150.0,
        ask_slope: 160.0,
        trade_imbalance: 0.2,
    };

    let array = features.to_array();

    assert_eq!(array.len(), 8, "Array should have 8 elements");
    assert_eq!(array[0], 1.5);
    assert_eq!(array[1], 2.3);
    assert_eq!(array[2], 0.4);
    assert_eq!(array[3], 0.6);
    assert_eq!(array[4], 0.001);
    assert_eq!(array[5], 150.0);
    assert_eq!(array[6], 160.0);
    assert_eq!(array[7], 0.2);
}

#[test]
fn test_ofi_zeros() {
    let features = OFIFeatures::zeros();

    assert_eq!(features.ofi_level1, 0.0);
    assert_eq!(features.ofi_level5, 0.0);
    assert_eq!(features.depth_imbalance, 0.0);
    assert_eq!(features.vpin, 0.0);
    assert_eq!(features.kyle_lambda, 0.0);
    assert_eq!(features.bid_slope, 0.0);
    assert_eq!(features.ask_slope, 0.0);
    assert_eq!(features.trade_imbalance, 0.0);
}

#[test]
fn test_ofi_normalization() {
    let mut calc = OFICalculator::new();

    // Create sequence with consistent OFI values
    for i in 0..100 {
        let snap = create_snapshot(
            i * 1000,
            150_000_000_000_000 + (i % 2) as i64 * 5_000_000_000,
            150_010_000_000_000,
            100,
            100,
        );
        calc.calculate(&snap).unwrap();
    }

    // After 100 snapshots, normalized OFI should be within [-3, 3]
    let snap = create_snapshot(
        100_000,
        150_005_000_000_000,
        150_010_000_000_000,
        100,
        100,
    );
    let features = calc.calculate(&snap).unwrap();

    assert!(
        features.ofi_level1.abs() <= 3.0,
        "Normalized OFI should be within [-3, 3], got {}",
        features.ofi_level1
    );
}

#[test]
fn test_empty_snapshot_error() {
    let mut calc = OFICalculator::new();

    // Create snapshot with no levels
    let empty_snap = Mbp10Snapshot::new("ES.FUT".to_string(), 1000, vec![], 0, 0);

    let result = calc.calculate(&empty_snap);
    assert!(
        result.is_err(),
        "Empty snapshot should return error"
    );
}

#[test]
fn test_ofi_price_impact_correlation() {
    let mut calc = OFICalculator::new();

    // Large bid increase should produce positive OFI
    let snap1 = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 100);
    let snap2 = create_snapshot(
        2000,
        150_010_000_000_000, // Big bid jump
        150_010_000_000_000,
        500, // Large bid size
        100,
    );

    calc.calculate(&snap1).unwrap();
    let features2 = calc.calculate(&snap2).unwrap();

    // Should show bullish signal
    assert!(
        features2.depth_imbalance > 0.5,
        "Large bid should create strong positive imbalance"
    );
}

#[test]
fn test_ofi_symmetry() {
    let mut calc1 = OFICalculator::new();
    let mut calc2 = OFICalculator::new();

    // Scenario 1: Rising bid
    let snap1a = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 100);
    let snap1b = create_snapshot(2000, 150_005_000_000_000, 150_010_000_000_000, 100, 100);

    calc1.calculate(&snap1a).unwrap();
    let features1 = calc1.calculate(&snap1b).unwrap();

    // Scenario 2: Falling ask (should be similar signal)
    let snap2a = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 100);
    let snap2b = create_snapshot(2000, 150_000_000_000_000, 150_005_000_000_000, 100, 100);

    calc2.calculate(&snap2a).unwrap();
    let features2 = calc2.calculate(&snap2b).unwrap();

    // Both should produce valid OFI (finite values)
    // Note: With small normalization window, values may be zero
    // The important thing is they are valid (finite)
    assert!(
        features1.is_valid(),
        "Rising bid should produce valid OFI features"
    );
    assert!(
        features2.is_valid(),
        "Falling ask should produce valid OFI features"
    );

    // Level 5 OFI should be non-zero (not normalized)
    assert!(
        features1.ofi_level5.abs() >= 0.0,
        "Rising bid OFI Level 5 should be finite"
    );
    assert!(
        features2.ofi_level5.abs() >= 0.0,
        "Falling ask OFI Level 5 should be finite"
    );
}

#[test]
fn test_performance_benchmark() {
    use std::time::Instant;

    let mut calc = OFICalculator::new();
    let snap = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 120);

    // Warm-up
    for _ in 0..10 {
        calc.calculate(&snap).unwrap();
    }

    // Benchmark
    let iterations = 1000;
    let start = Instant::now();

    for i in 0..iterations {
        let snap = create_snapshot(
            i * 1000,
            150_000_000_000_000 + i as i64 * 1_000_000,
            150_010_000_000_000 + i as i64 * 1_000_000,
            100,
            120,
        );
        calc.calculate(&snap).unwrap();
    }

    let elapsed = start.elapsed();
    let avg_time = elapsed.as_micros() / iterations as u128;

    use tracing::info;
    info!(avg_time, "Average OFI calculation time (us)");

    // Target: <100μs per calculation
    assert!(
        avg_time < 100,
        "OFI calculation should be <100μs, got {} μs",
        avg_time
    );
}

/// Verify that OFI features integrate correctly with the state vector layout.
///
/// State layout: [42 market + 8 portfolio + 16 multi-tf + 8 OFI] = 74 raw, 80 aligned.
/// Without OFI: [42 market + 8 portfolio + 16 multi-tf] = 66 raw, 72 aligned.
#[test]
fn test_ofi_features_state_integration() {
    // Simulate the state-building path from crates/ml/src/trainers/dqn/trainer/state.rs
    // OFI features are appended as `regime_features` in TradingState::from_normalized().

    let price_features: Vec<f32> = vec![0.01, 0.02, -0.01, 0.005]; // 4 OHLC log returns
    let technical_indicators: Vec<f32> = vec![]; // empty in 42-feature architecture
    let market_features: Vec<f32> = vec![0.5; 38]; // indices 4..42
    let portfolio_features: Vec<f32> = vec![0.0, 0.0, 0.0]; // 3 portfolio features

    // Compute OFI features from test snapshots
    let mut calc = OFICalculator::new();
    let snap1 = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 120);
    let snap2 = create_snapshot(2000, 150_005_000_000_000, 150_010_000_000_000, 110, 115);
    calc.calculate(&snap1).unwrap();
    let ofi = calc.calculate(&snap2).unwrap();
    let regime_features: Vec<f32> = ofi.to_array().iter().map(|&v| v as f32).collect();

    assert_eq!(regime_features.len(), 8, "OFI must produce 8 features");

    // Build state vector (same as TradingState::to_vector())
    let mut state_vec = Vec::new();
    state_vec.extend_from_slice(&price_features);       // 4
    state_vec.extend_from_slice(&technical_indicators);  // 0
    state_vec.extend_from_slice(&market_features);       // 38
    state_vec.extend_from_slice(&portfolio_features);    // 3
    state_vec.extend_from_slice(&regime_features);       // 8

    // 4 + 38 + 3 + 8 = 53 (raw state before multi-tf expansion; multi-tf is added in GPU pipeline)
    // The full pipeline adds 8 portfolio features (not 3) + 16 multi-tf:
    // 42 market + 8 portfolio + 16 MTF + 8 OFI = 74 raw, 80 aligned.
    // Here we verify the raw OFI portion is correctly positioned.
    assert_eq!(state_vec.len(), 4 + 38 + 3 + 8); // = 53
    assert_eq!(state_vec.len() - 8, 45, "Without OFI the state would be 45-dim");

    // Verify OFI features are non-zero (calculated from actual snapshots)
    let ofi_start = state_vec.len() - 8;
    let ofi_slice = &state_vec[ofi_start..];
    assert!(
        ofi_slice.iter().any(|&v| v != 0.0),
        "OFI features should be non-zero when computed from real snapshots"
    );

    // Verify all features are finite
    for (i, &v) in state_vec.iter().enumerate() {
        assert!(v.is_finite(), "Feature at index {} is not finite: {}", i, v);
    }
}

/// Verify zero-fill behavior when no MBP-10 data is available.
#[test]
fn test_ofi_features_graceful_degradation() {
    let regime_features: Vec<f32> = OFIFeatures::zeros().to_array().iter().map(|&v| v as f32).collect();

    assert_eq!(regime_features.len(), 8);
    assert!(
        regime_features.iter().all(|&v| v == 0.0),
        "Zero-filled OFI features should all be 0.0"
    );

    // State dim should still be valid — no NaN or Inf
    let mut state_vec = vec![0.0_f32; 42]; // market features
    state_vec.extend_from_slice(&[0.0; 3]); // portfolio
    state_vec.extend_from_slice(&regime_features); // 8 OFI zeros

    assert_eq!(state_vec.len(), 53);
    assert!(state_vec.iter().all(|v| v.is_finite()));
}

/// Verify OFI feature names at indices 42-49 in the unified feature space.
#[test]
fn test_ofi_feature_names_ordering() {
    // The 8 OFI features occupy indices [42..50) in the full 50-dim feature space
    // when appended to the 42-dim base market features.
    let ofi_names = [
        "ofi_level1",
        "ofi_level5",
        "depth_imbalance",
        "vpin",
        "kyles_lambda",
        "bid_slope",
        "ask_slope",
        "trade_imbalance",
    ];

    assert_eq!(ofi_names.len(), 8, "Must have exactly 8 OFI feature names");

    // Verify the to_array() order matches the name ordering
    let features = OFIFeatures {
        ofi_level1: 1.0,
        ofi_level5: 2.0,
        depth_imbalance: 3.0,
        vpin: 4.0,
        kyle_lambda: 5.0,
        bid_slope: 6.0,
        ask_slope: 7.0,
        trade_imbalance: 8.0,
    };
    let arr = features.to_array();
    for (i, &val) in arr.iter().enumerate() {
        assert_eq!(val, (i + 1) as f64, "Feature '{}' at index {} has wrong value", ofi_names[i], i);
    }
}