foxhunt/crates/ml-features/tests/imbalance_bars_parallel_bit_equiv_test.rs

//! Bit-equivalence test for `imbalance_bars_parallel` vs.
//! `imbalance_bars_sequential` (SP20 Bottleneck B, 2026-05-10).
//!
//! Asserts that the two-pass parallel implementation produces the SAME
//! imbalance bars as the single-threaded reference. Bars are constructed
//! from finite f64 reductions (sums, comparisons over identical trade
//! slices), so we expect EXACT equality — no tolerance.
//!
//! Test strategy
//! =============
//!   1. Generate synthetic `Mbp10Trade` ticks with a deterministic price
//!      walk + variable volumes. The walk drives both buy/sell direction
//!      transitions AND occasional unchanged-price ticks (which exercise
//!      the `last_direction` branch).
//!   2. Run `imbalance_bars_sequential` to capture the golden bars.
//!   3. Run `imbalance_bars_parallel` (always Pass 1 sequential to find
//!      emission boundaries; Pass 2 parallel rayon over segments). Vary
//!      `min_bars_per_task` to exercise both the sequential-fallback and
//!      par_iter cutoff paths.
//!   4. Assert bar lists are identical: same length, every field bitwise
//!      equal (timestamp, OHLCV all f64-bitwise + DateTime equal).
//!
//! Why synthetic data: the test must be hermetic — no dependence on
//! `test_data/futures-baseline-mbp10`. The walk produces dense (low-T)
//! and sparse (high-T) emission regimes both, exercising the cutoff path.
//!
//! Why two-pass (NOT time-bucket sharding): `ImbalanceBarSampler.cum_imb`
//! is path-dependent with no bounded-lookback window. Time-bucket sharding
//! with warmup overlap (the OFI pattern) does NOT yield bit-identical
//! emission indices — a phase offset persists indefinitely between shard-
//! local and sequential walks. The two-pass decomposition runs Pass 1
//! sequentially to capture the EXACT emission boundary set, then
//! parallelises only the per-segment OHLCV reduction (which is trivially
//! independent across segments).
//!
//! If this test fails: the two-pass implementation has diverged from
//! sequential semantics. Audit Pass 1's `update()` mirroring (direction
//! tie-break, zero-volume early-return) and Pass 2's segment-OHLCV
//! reduction. See the doc comment on `imbalance_bars_parallel` in
//! `mbp10_loader.rs`.

use chrono::{DateTime, TimeZone, Utc};
use ml_features::{
    imbalance_bars_parallel, imbalance_bars_sequential, Mbp10Trade, OHLCVBar,
    DEFAULT_IMBALANCE_BAR_MIN_BARS_PER_TASK,
};

/// Generate `n` synthetic trade ticks with a deterministic price/volume walk.
///
/// The walk produces:
/// - Buy ticks (price up) on `i % 5 ∈ {0, 1}`
/// - Sell ticks (price down) on `i % 5 ∈ {2, 3}`
/// - Unchanged-price ticks on `i % 5 == 4` (exercises `last_direction`)
/// - Volume varies by index parity (volumes 1..30) so imbalance accumulates
///   asymmetrically and emissions land at varied trade indices.
///
/// At ES.FUT-style threshold=20, this generates a bar emission roughly every
/// few hundred trades — well below the 10_000-trade warmup, so every shard's
/// warmup window covers MANY emissions, making the bit-equivalence guarantee
/// rock-solid.
fn make_trades(n: usize, base_price: f64) -> Vec<Mbp10Trade> {
    let start: DateTime<Utc> = Utc.with_ymd_and_hms(2025, 1, 1, 9, 30, 0).unwrap();
    let mut trades = Vec::with_capacity(n);
    let mut price = base_price;
    let mut last_change_dir: f64 = 0.0;
    for i in 0..n {
        // Price walk: deterministic but exercises all three direction branches.
        let r = i % 5;
        let dir = match r {
            0 | 1 => 1.0,
            2 | 3 => -1.0,
            _ => 0.0, // unchanged
        };
        if dir != 0.0 {
            price += dir * 0.25;
            last_change_dir = dir;
        }
        // Volume: vary 1..30 by index, occasionally inject larger blocks.
        let volume = if i % 47 == 0 {
            (((i % 17) + 5) * 3) as f64 // larger blocks at sparse positions
        } else {
            ((i % 29) + 1) as f64
        };
        let timestamp = start + chrono::Duration::milliseconds(i as i64 * 10);
        trades.push(Mbp10Trade {
            price,
            volume,
            timestamp,
            is_buy: last_change_dir >= 0.0, // unused by sampler; consistent for completeness
            instrument_id: 1,
        });
    }
    trades
}

/// Compare two `OHLCVBar` slices field-by-field with bitwise f64 equality
/// for floating-point fields. Returns the first mismatching index + a
/// human-readable summary, or `Ok(())` if identical.
fn assert_bars_bitwise_equal(golden: &[OHLCVBar], candidate: &[OHLCVBar], label: &str) {
    assert_eq!(
        golden.len(),
        candidate.len(),
        "{}: bar count mismatch (golden={}, candidate={})",
        label,
        golden.len(),
        candidate.len()
    );
    for (i, (g, c)) in golden.iter().zip(candidate.iter()).enumerate() {
        assert_eq!(g.timestamp, c.timestamp, "{}: bar[{}] timestamp mismatch", label, i);
        assert_eq!(
            g.open.to_bits(),
            c.open.to_bits(),
            "{}: bar[{}] open: golden={:?}, candidate={:?}",
            label,
            i,
            g.open,
            c.open
        );
        assert_eq!(
            g.high.to_bits(),
            c.high.to_bits(),
            "{}: bar[{}] high: golden={:?}, candidate={:?}",
            label,
            i,
            g.high,
            c.high
        );
        assert_eq!(
            g.low.to_bits(),
            c.low.to_bits(),
            "{}: bar[{}] low: golden={:?}, candidate={:?}",
            label,
            i,
            g.low,
            c.low
        );
        assert_eq!(
            g.close.to_bits(),
            c.close.to_bits(),
            "{}: bar[{}] close: golden={:?}, candidate={:?}",
            label,
            i,
            g.close,
            c.close
        );
        assert_eq!(
            g.volume.to_bits(),
            c.volume.to_bits(),
            "{}: bar[{}] volume: golden={:?}, candidate={:?}",
            label,
            i,
            g.volume,
            c.volume
        );
    }
}

#[test]
fn imbalance_bars_parallel_matches_sequential_default_cutoff() {
    // ~50k trades × threshold=20 → many hundreds of emissions. Default
    // `min_bars_per_task` triggers the parallel rayon path in Pass 2.
    let trades = make_trades(50_000, 4500.0);
    let threshold = 20.0_f64;

    let golden = imbalance_bars_sequential(&trades, threshold);
    assert!(
        !golden.is_empty(),
        "synthetic walk should produce at least one bar; produced 0"
    );

    let parallel = imbalance_bars_parallel(
        &trades,
        threshold,
        rayon::current_num_threads(),
        DEFAULT_IMBALANCE_BAR_MIN_BARS_PER_TASK,
    );

    assert_bars_bitwise_equal(&golden, &parallel, "default-cutoff");
    println!(
        "default-cutoff: {} bars, bit-identical to sequential",
        golden.len()
    );
}

#[test]
fn imbalance_bars_parallel_matches_sequential_force_par_iter() {
    // min_bars_per_task = 1 forces the par_iter path even for tiny segment
    // counts — exercises rayon execution explicitly.
    let trades = make_trades(50_000, 4500.0);
    let threshold = 20.0_f64;

    let golden = imbalance_bars_sequential(&trades, threshold);
    let parallel = imbalance_bars_parallel(&trades, threshold, 8, 1);

    assert_bars_bitwise_equal(&golden, &parallel, "force-par-iter");
    println!(
        "force-par-iter: {} bars, bit-identical to sequential",
        golden.len()
    );
}

#[test]
fn imbalance_bars_parallel_force_sequential_pass2() {
    // min_bars_per_task = usize::MAX forces the sequential Pass 2 fallback
    // even when many segments exist.
    let trades = make_trades(5_000, 4500.0);
    let threshold = 20.0_f64;

    let golden = imbalance_bars_sequential(&trades, threshold);
    let parallel = imbalance_bars_parallel(&trades, threshold, 1, usize::MAX);

    assert_bars_bitwise_equal(&golden, &parallel, "force-sequential-pass2");
}

#[test]
fn imbalance_bars_parallel_empty_input() {
    let golden = imbalance_bars_sequential(&[], 20.0);
    let parallel = imbalance_bars_parallel(
        &[],
        20.0,
        8,
        DEFAULT_IMBALANCE_BAR_MIN_BARS_PER_TASK,
    );
    assert!(golden.is_empty());
    assert!(parallel.is_empty());
}

#[test]
fn imbalance_bars_parallel_small_input() {
    // 100 trades with low threshold to force emissions. Pass 2 segment
    // count will be small (a few segments), which tests the cutoff path.
    let trades = make_trades(100, 4500.0);
    let threshold = 5.0_f64;

    let golden = imbalance_bars_sequential(&trades, threshold);
    let parallel = imbalance_bars_parallel(
        &trades,
        threshold,
        8,
        DEFAULT_IMBALANCE_BAR_MIN_BARS_PER_TASK,
    );

    assert_bars_bitwise_equal(&golden, &parallel, "small-input");
}

#[test]
fn imbalance_bars_parallel_high_threshold_few_emissions() {
    // High threshold → very sparse emissions across 50k trades. Pass 1
    // captures the exact emission set, so Pass 2 is bit-identical no
    // matter how few segments. THIS WAS THE FAILING CASE for the earlier
    // time-bucket-sharded prototype (phase-offset divergence between
    // shard-local and sequential cum_imb). The two-pass approach
    // resolves it definitively.
    let trades = make_trades(50_000, 4500.0);
    let threshold = 500.0_f64;

    let golden = imbalance_bars_sequential(&trades, threshold);
    let parallel_default = imbalance_bars_parallel(
        &trades,
        threshold,
        4,
        DEFAULT_IMBALANCE_BAR_MIN_BARS_PER_TASK,
    );
    let parallel_force_par = imbalance_bars_parallel(&trades, threshold, 8, 1);

    assert_bars_bitwise_equal(&golden, &parallel_default, "high-threshold default");
    assert_bars_bitwise_equal(&golden, &parallel_force_par, "high-threshold force-par");
    println!(
        "high-threshold (T=500): {} bars over 50k trades, default + force-par bit-identical",
        golden.len()
    );
}