OFI features at bar t were computed over [close(bar_t), close(bar_{t+1}))
— bar t+1's formation period — so 31 of 32 OFI dims at the policy's
input for bar t carried next-bar microstructure data. Per audit
`docs/lookahead-bias-audit-2026-04-28.md` §3, this is a DEFINITE leak
contaminating both training inputs and validation backtest.
Fix: shift the window backward to (close(bar_{t-1}), close(bar_t)] so
OFI at bar t uses only data accumulated during bar t's own formation.
Mirrors the correct `log_bar_duration` convention at
precompute_features.rs:567-575 (audit's smoking-gun citation).
Per feedback_no_partial_refactor, both write sites migrated in lockstep:
- crates/ml/examples/precompute_features.rs:441-475 (precompute path)
- crates/ml/src/trainers/dqn/data_loading.rs:396-448 (DBN fallback)
FXCACHE_VERSION bumped 6 → 7. PVC auto-regen via schema-hash check on
next deploy. Local regen:
./target/release/examples/precompute_features \
--data-dir test_data/futures-baseline \
--mbp10-data-dir test_data/futures-baseline-mbp10 \
--trades-data-dir test_data/futures-baseline-trades \
--output-dir test_data/feature-cache \
--symbol ES.FUT --data-source mbp10 --yes
Regression test added: tests/ofi_features_test.rs::
test_ofi_window_alignment_uses_formation_interval validates the new
partition_point predicates against a synthetic 5-bar dataset, including
an explicit anti-regression assertion that bar t+1 snapshots are NEVER
picked up for bar t.
dqn-wire-up-audit.md updated to reflect new contract for
`trainers/dqn/data_loading.rs`. Audit report
`docs/lookahead-bias-audit-2026-04-28.md` checked in alongside the fix.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
748 lines
24 KiB
Rust
748 lines
24 KiB
Rust
#![allow(
|
|
clippy::assertions_on_constants,
|
|
clippy::assertions_on_result_states,
|
|
clippy::clone_on_copy,
|
|
clippy::decimal_literal_representation,
|
|
clippy::doc_markdown,
|
|
clippy::empty_line_after_doc_comments,
|
|
clippy::field_reassign_with_default,
|
|
clippy::get_unwrap,
|
|
clippy::identity_op,
|
|
clippy::inconsistent_digit_grouping,
|
|
clippy::indexing_slicing,
|
|
clippy::integer_division,
|
|
clippy::len_zero,
|
|
clippy::let_underscore_must_use,
|
|
clippy::manual_div_ceil,
|
|
clippy::manual_let_else,
|
|
clippy::manual_range_contains,
|
|
clippy::modulo_arithmetic,
|
|
clippy::needless_range_loop,
|
|
clippy::non_ascii_literal,
|
|
clippy::redundant_clone,
|
|
clippy::shadow_reuse,
|
|
clippy::shadow_same,
|
|
clippy::shadow_unrelated,
|
|
clippy::single_match_else,
|
|
clippy::str_to_string,
|
|
clippy::string_slice,
|
|
clippy::tests_outside_test_module,
|
|
clippy::too_many_lines,
|
|
clippy::unnecessary_wraps,
|
|
clippy::unseparated_literal_suffix,
|
|
clippy::use_debug,
|
|
clippy::useless_vec,
|
|
clippy::wildcard_enum_match_arm,
|
|
clippy::else_if_without_else,
|
|
clippy::expect_used,
|
|
clippy::missing_const_for_fn,
|
|
clippy::similar_names,
|
|
clippy::type_complexity,
|
|
clippy::collapsible_else_if,
|
|
clippy::doc_lazy_continuation,
|
|
clippy::items_after_test_module,
|
|
clippy::map_clone,
|
|
clippy::multiple_unsafe_ops_per_block,
|
|
clippy::unwrap_or_default,
|
|
clippy::assign_op_pattern,
|
|
clippy::needless_borrow,
|
|
clippy::println_empty_string,
|
|
clippy::unnecessary_cast,
|
|
clippy::used_underscore_binding,
|
|
clippy::create_dir,
|
|
clippy::implicit_saturating_sub,
|
|
clippy::exit,
|
|
clippy::expect_fun_call,
|
|
clippy::too_many_arguments,
|
|
clippy::unnecessary_map_or,
|
|
clippy::unwrap_used,
|
|
dead_code,
|
|
unused_imports,
|
|
unused_variables,
|
|
clippy::cloned_ref_to_slice_refs,
|
|
clippy::neg_multiply,
|
|
clippy::while_let_loop,
|
|
clippy::bool_assert_comparison,
|
|
clippy::excessive_precision,
|
|
clippy::trivially_copy_pass_by_ref,
|
|
clippy::op_ref,
|
|
clippy::redundant_closure,
|
|
clippy::unnecessary_lazy_evaluations,
|
|
clippy::if_then_some_else_none,
|
|
clippy::unnecessary_to_owned,
|
|
clippy::single_component_path_imports,
|
|
)]
|
|
//! Comprehensive tests for OFI (Order Flow Imbalance) features
|
|
//!
|
|
//! Tests all 8 OFI features against academic formulas and expected ranges
|
|
|
|
use data::providers::databento::mbp10::{BidAskPair, Mbp10Snapshot};
|
|
use ml::features::ofi_calculator::{OFICalculator, OFIFeatures};
|
|
|
|
/// Create test MBP-10 snapshot with specified parameters
|
|
fn create_snapshot(
|
|
timestamp: u64,
|
|
bid_px: i64,
|
|
ask_px: i64,
|
|
bid_sz: u32,
|
|
ask_sz: u32,
|
|
) -> Mbp10Snapshot {
|
|
let levels = vec![
|
|
BidAskPair {
|
|
bid_px,
|
|
bid_sz,
|
|
bid_ct: 5,
|
|
ask_px,
|
|
ask_sz,
|
|
ask_ct: 6,
|
|
},
|
|
BidAskPair {
|
|
bid_px: bid_px - 1_000_000_000, // -0.001 tick
|
|
bid_sz: bid_sz * 2,
|
|
bid_ct: 8,
|
|
ask_px: ask_px + 1_000_000_000,
|
|
ask_sz: ask_sz * 2,
|
|
ask_ct: 7,
|
|
},
|
|
BidAskPair {
|
|
bid_px: bid_px - 2_000_000_000,
|
|
bid_sz: bid_sz * 3,
|
|
bid_ct: 10,
|
|
ask_px: ask_px + 2_000_000_000,
|
|
ask_sz: ask_sz * 3,
|
|
ask_ct: 9,
|
|
},
|
|
BidAskPair {
|
|
bid_px: bid_px - 3_000_000_000,
|
|
bid_sz: bid_sz * 4,
|
|
bid_ct: 12,
|
|
ask_px: ask_px + 3_000_000_000,
|
|
ask_sz: ask_sz * 4,
|
|
ask_ct: 11,
|
|
},
|
|
BidAskPair {
|
|
bid_px: bid_px - 4_000_000_000,
|
|
bid_sz: bid_sz * 5,
|
|
bid_ct: 15,
|
|
ask_px: ask_px + 4_000_000_000,
|
|
ask_sz: ask_sz * 5,
|
|
ask_ct: 13,
|
|
},
|
|
];
|
|
|
|
Mbp10Snapshot::new("ES.FUT".to_string(), timestamp, levels, 0, 100)
|
|
}
|
|
|
|
#[test]
|
|
fn test_ofi_level1_rising_bid() {
|
|
let mut calc = OFICalculator::new();
|
|
|
|
let snap1 = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 120);
|
|
let snap2 = create_snapshot(2000, 150_005_000_000_000, 150_010_000_000_000, 100, 120);
|
|
|
|
// First snapshot (no previous)
|
|
let features1 = calc.calculate(&snap1).unwrap();
|
|
assert_eq!(features1.ofi_level1, 0.0, "First OFI should be zero");
|
|
|
|
// Second snapshot with rising bid
|
|
let features2 = calc.calculate(&snap2).unwrap();
|
|
assert!(
|
|
features2.ofi_level1 != 0.0,
|
|
"OFI should be non-zero after price movement"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_ofi_level1_falling_ask() {
|
|
let mut calc = OFICalculator::new();
|
|
|
|
let snap1 = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 120);
|
|
let snap2 = create_snapshot(2000, 150_000_000_000_000, 150_005_000_000_000, 100, 120);
|
|
|
|
calc.calculate(&snap1).unwrap();
|
|
let features2 = calc.calculate(&snap2).unwrap();
|
|
|
|
// Falling ask should produce positive OFI (more demand)
|
|
assert!(
|
|
features2.ofi_level1 != 0.0,
|
|
"Falling ask should produce non-zero OFI"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_ofi_level5_multilevel() {
|
|
let mut calc = OFICalculator::new();
|
|
|
|
let snap1 = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 100);
|
|
let snap2 = create_snapshot(
|
|
2000,
|
|
150_005_000_000_000, // Bid rises
|
|
150_010_000_000_000,
|
|
110, // Bid size increases
|
|
90, // Ask size decreases
|
|
);
|
|
|
|
calc.calculate(&snap1).unwrap();
|
|
let features2 = calc.calculate(&snap2).unwrap();
|
|
|
|
assert!(
|
|
features2.ofi_level5.is_finite(),
|
|
"OFI Level 5 should be finite"
|
|
);
|
|
assert!(
|
|
features2.ofi_level5.abs() < 1e6,
|
|
"OFI Level 5 should be within expected range"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_depth_imbalance_balanced() {
|
|
let mut calc = OFICalculator::new();
|
|
let snap = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 100);
|
|
|
|
let features = calc.calculate(&snap).unwrap();
|
|
|
|
// Balanced book should have near-zero imbalance
|
|
assert!(
|
|
features.depth_imbalance.abs() < 0.3,
|
|
"Balanced book should have low imbalance, got {}",
|
|
features.depth_imbalance
|
|
);
|
|
assert!(
|
|
features.depth_imbalance >= -1.0 && features.depth_imbalance <= 1.0,
|
|
"Depth imbalance should be in [-1, 1]"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_depth_imbalance_bid_heavy() {
|
|
let mut calc = OFICalculator::new();
|
|
let snap = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 200, 50);
|
|
|
|
let features = calc.calculate(&snap).unwrap();
|
|
|
|
assert!(
|
|
features.depth_imbalance > 0.0,
|
|
"Bid-heavy book should have positive imbalance"
|
|
);
|
|
assert!(
|
|
features.depth_imbalance >= -1.0 && features.depth_imbalance <= 1.0,
|
|
"Depth imbalance should be in [-1, 1]"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_depth_imbalance_ask_heavy() {
|
|
let mut calc = OFICalculator::new();
|
|
let snap = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 50, 200);
|
|
|
|
let features = calc.calculate(&snap).unwrap();
|
|
|
|
assert!(
|
|
features.depth_imbalance < 0.0,
|
|
"Ask-heavy book should have negative imbalance"
|
|
);
|
|
assert!(
|
|
features.depth_imbalance >= -1.0 && features.depth_imbalance <= 1.0,
|
|
"Depth imbalance should be in [-1, 1]"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_bid_ask_slopes() {
|
|
let mut calc = OFICalculator::new();
|
|
let snap = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 120);
|
|
|
|
let features = calc.calculate(&snap).unwrap();
|
|
|
|
assert!(
|
|
features.bid_slope.is_finite(),
|
|
"Bid slope should be finite"
|
|
);
|
|
assert!(
|
|
features.ask_slope.is_finite(),
|
|
"Ask slope should be finite"
|
|
);
|
|
|
|
// With increasing volume at deeper levels, slopes should be positive
|
|
assert!(
|
|
features.bid_slope > 0.0,
|
|
"Bid slope should be positive (volume increases with depth)"
|
|
);
|
|
assert!(
|
|
features.ask_slope > 0.0,
|
|
"Ask slope should be positive (volume increases with depth)"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_vpin_range() {
|
|
let mut calc = OFICalculator::new();
|
|
let snap = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 120);
|
|
|
|
let features = calc.calculate(&snap).unwrap();
|
|
|
|
// VPIN should be in [0, 1] range
|
|
assert!(
|
|
features.vpin >= 0.0 && features.vpin <= 1.0,
|
|
"VPIN should be in [0, 1], got {}",
|
|
features.vpin
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_kyle_lambda_finite() {
|
|
let mut calc = OFICalculator::new();
|
|
let snap = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 120);
|
|
|
|
let features = calc.calculate(&snap).unwrap();
|
|
|
|
assert!(
|
|
features.kyle_lambda.is_finite(),
|
|
"Kyle's lambda should be finite"
|
|
);
|
|
assert!(
|
|
features.kyle_lambda.abs() < 1.0,
|
|
"Kyle's lambda should be in reasonable range"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_trade_imbalance_range() {
|
|
let mut calc = OFICalculator::new();
|
|
|
|
let snap1 = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 120);
|
|
let snap2 = create_snapshot(2000, 150_005_000_000_000, 150_015_000_000_000, 110, 115);
|
|
|
|
calc.calculate(&snap1).unwrap();
|
|
let features2 = calc.calculate(&snap2).unwrap();
|
|
|
|
// Trade imbalance should be in [-1, 1]
|
|
assert!(
|
|
features2.trade_imbalance >= -1.0 && features2.trade_imbalance <= 1.0,
|
|
"Trade imbalance should be in [-1, 1], got {}",
|
|
features2.trade_imbalance
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_all_features_finite() {
|
|
let mut calc = OFICalculator::new();
|
|
|
|
// Create sequence of snapshots
|
|
for i in 0..10u64 {
|
|
let snap = create_snapshot(
|
|
i * 1000,
|
|
150_000_000_000_000 + i as i64 * 1_000_000_000,
|
|
150_010_000_000_000 + i as i64 * 1_000_000_000,
|
|
(100 + i * 5) as u32,
|
|
(120 - i * 3) as u32,
|
|
);
|
|
|
|
let features = calc.calculate(&snap).unwrap();
|
|
|
|
assert!(
|
|
features.is_valid(),
|
|
"All features should be finite at iteration {}",
|
|
i
|
|
);
|
|
assert!(features.ofi_level1.is_finite());
|
|
assert!(features.ofi_level5.is_finite());
|
|
assert!(features.depth_imbalance.is_finite());
|
|
assert!(features.vpin.is_finite());
|
|
assert!(features.kyle_lambda.is_finite());
|
|
assert!(features.bid_slope.is_finite());
|
|
assert!(features.ask_slope.is_finite());
|
|
assert!(features.trade_imbalance.is_finite());
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn test_ofi_features_to_array_conversion() {
|
|
let features = OFIFeatures {
|
|
ofi_level1: 1.5,
|
|
ofi_level5: 2.3,
|
|
depth_imbalance: 0.4,
|
|
vpin: 0.6,
|
|
kyle_lambda: 0.001,
|
|
bid_slope: 150.0,
|
|
ask_slope: 160.0,
|
|
trade_imbalance: 0.2,
|
|
};
|
|
|
|
let array = features.to_array();
|
|
|
|
assert_eq!(array.len(), 8, "Array should have 8 elements");
|
|
assert_eq!(array[0], 1.5);
|
|
assert_eq!(array[1], 2.3);
|
|
assert_eq!(array[2], 0.4);
|
|
assert_eq!(array[3], 0.6);
|
|
assert_eq!(array[4], 0.001);
|
|
assert_eq!(array[5], 150.0);
|
|
assert_eq!(array[6], 160.0);
|
|
assert_eq!(array[7], 0.2);
|
|
}
|
|
|
|
#[test]
|
|
fn test_ofi_zeros() {
|
|
let features = OFIFeatures::zeros();
|
|
|
|
assert_eq!(features.ofi_level1, 0.0);
|
|
assert_eq!(features.ofi_level5, 0.0);
|
|
assert_eq!(features.depth_imbalance, 0.0);
|
|
assert_eq!(features.vpin, 0.0);
|
|
assert_eq!(features.kyle_lambda, 0.0);
|
|
assert_eq!(features.bid_slope, 0.0);
|
|
assert_eq!(features.ask_slope, 0.0);
|
|
assert_eq!(features.trade_imbalance, 0.0);
|
|
}
|
|
|
|
#[test]
|
|
fn test_ofi_normalization() {
|
|
let mut calc = OFICalculator::new();
|
|
|
|
// Create sequence with consistent OFI values
|
|
for i in 0..100 {
|
|
let snap = create_snapshot(
|
|
i * 1000,
|
|
150_000_000_000_000 + (i % 2) as i64 * 5_000_000_000,
|
|
150_010_000_000_000,
|
|
100,
|
|
100,
|
|
);
|
|
calc.calculate(&snap).unwrap();
|
|
}
|
|
|
|
// After 100 snapshots, normalized OFI should be within [-3, 3]
|
|
let snap = create_snapshot(
|
|
100_000,
|
|
150_005_000_000_000,
|
|
150_010_000_000_000,
|
|
100,
|
|
100,
|
|
);
|
|
let features = calc.calculate(&snap).unwrap();
|
|
|
|
assert!(
|
|
features.ofi_level1.abs() <= 3.0,
|
|
"Normalized OFI should be within [-3, 3], got {}",
|
|
features.ofi_level1
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_empty_snapshot_error() {
|
|
let mut calc = OFICalculator::new();
|
|
|
|
// Create snapshot with no levels
|
|
let empty_snap = Mbp10Snapshot::new("ES.FUT".to_string(), 1000, vec![], 0, 0);
|
|
|
|
let result = calc.calculate(&empty_snap);
|
|
assert!(
|
|
result.is_err(),
|
|
"Empty snapshot should return error"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_ofi_price_impact_correlation() {
|
|
let mut calc = OFICalculator::new();
|
|
|
|
// Large bid increase should produce positive OFI
|
|
let snap1 = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 100);
|
|
let snap2 = create_snapshot(
|
|
2000,
|
|
150_010_000_000_000, // Big bid jump
|
|
150_010_000_000_000,
|
|
500, // Large bid size
|
|
100,
|
|
);
|
|
|
|
calc.calculate(&snap1).unwrap();
|
|
let features2 = calc.calculate(&snap2).unwrap();
|
|
|
|
// Should show bullish signal
|
|
assert!(
|
|
features2.depth_imbalance > 0.5,
|
|
"Large bid should create strong positive imbalance"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_ofi_symmetry() {
|
|
let mut calc1 = OFICalculator::new();
|
|
let mut calc2 = OFICalculator::new();
|
|
|
|
// Scenario 1: Rising bid
|
|
let snap1a = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 100);
|
|
let snap1b = create_snapshot(2000, 150_005_000_000_000, 150_010_000_000_000, 100, 100);
|
|
|
|
calc1.calculate(&snap1a).unwrap();
|
|
let features1 = calc1.calculate(&snap1b).unwrap();
|
|
|
|
// Scenario 2: Falling ask (should be similar signal)
|
|
let snap2a = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 100);
|
|
let snap2b = create_snapshot(2000, 150_000_000_000_000, 150_005_000_000_000, 100, 100);
|
|
|
|
calc2.calculate(&snap2a).unwrap();
|
|
let features2 = calc2.calculate(&snap2b).unwrap();
|
|
|
|
// Both should produce valid OFI (finite values)
|
|
// Note: With small normalization window, values may be zero
|
|
// The important thing is they are valid (finite)
|
|
assert!(
|
|
features1.is_valid(),
|
|
"Rising bid should produce valid OFI features"
|
|
);
|
|
assert!(
|
|
features2.is_valid(),
|
|
"Falling ask should produce valid OFI features"
|
|
);
|
|
|
|
// Level 5 OFI should be non-zero (not normalized)
|
|
assert!(
|
|
features1.ofi_level5.abs() >= 0.0,
|
|
"Rising bid OFI Level 5 should be finite"
|
|
);
|
|
assert!(
|
|
features2.ofi_level5.abs() >= 0.0,
|
|
"Falling ask OFI Level 5 should be finite"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn test_performance_benchmark() {
|
|
use std::time::Instant;
|
|
|
|
let mut calc = OFICalculator::new();
|
|
let snap = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 120);
|
|
|
|
// Warm-up
|
|
for _ in 0..10 {
|
|
calc.calculate(&snap).unwrap();
|
|
}
|
|
|
|
// Benchmark
|
|
let iterations = 1000;
|
|
let start = Instant::now();
|
|
|
|
for i in 0..iterations {
|
|
let snap = create_snapshot(
|
|
i * 1000,
|
|
150_000_000_000_000 + i as i64 * 1_000_000,
|
|
150_010_000_000_000 + i as i64 * 1_000_000,
|
|
100,
|
|
120,
|
|
);
|
|
calc.calculate(&snap).unwrap();
|
|
}
|
|
|
|
let elapsed = start.elapsed();
|
|
let avg_time = elapsed.as_micros() / iterations as u128;
|
|
|
|
use tracing::info;
|
|
info!(avg_time, "Average OFI calculation time (us)");
|
|
|
|
// Target: <100μs per calculation
|
|
assert!(
|
|
avg_time < 100,
|
|
"OFI calculation should be <100μs, got {} μs",
|
|
avg_time
|
|
);
|
|
}
|
|
|
|
/// Verify that OFI features integrate correctly with the state vector layout.
|
|
///
|
|
/// State layout: [42 market + 8 portfolio + 16 multi-tf + 8 OFI] = 74 raw, 80 aligned.
|
|
/// Without OFI: [42 market + 8 portfolio + 16 multi-tf] = 66 raw, 72 aligned.
|
|
#[test]
|
|
fn test_ofi_features_state_integration() {
|
|
// Simulate the state-building path from crates/ml/src/trainers/dqn/trainer/state.rs
|
|
// OFI features are appended as `regime_features` in TradingState::from_normalized().
|
|
|
|
let price_features: Vec<f32> = vec![0.01, 0.02, -0.01, 0.005]; // 4 OHLC log returns
|
|
let technical_indicators: Vec<f32> = vec![]; // empty in 42-feature architecture
|
|
let market_features: Vec<f32> = vec![0.5; 38]; // indices 4..42
|
|
let portfolio_features: Vec<f32> = vec![0.0, 0.0, 0.0]; // 3 portfolio features
|
|
|
|
// Compute OFI features from test snapshots
|
|
let mut calc = OFICalculator::new();
|
|
let snap1 = create_snapshot(1000, 150_000_000_000_000, 150_010_000_000_000, 100, 120);
|
|
let snap2 = create_snapshot(2000, 150_005_000_000_000, 150_010_000_000_000, 110, 115);
|
|
calc.calculate(&snap1).unwrap();
|
|
let ofi = calc.calculate(&snap2).unwrap();
|
|
let regime_features: Vec<f32> = ofi.to_array().iter().map(|&v| v as f32).collect();
|
|
|
|
assert_eq!(regime_features.len(), 8, "OFI must produce 8 features");
|
|
|
|
// Build state vector (same as TradingState::to_vector())
|
|
let mut state_vec = Vec::new();
|
|
state_vec.extend_from_slice(&price_features); // 4
|
|
state_vec.extend_from_slice(&technical_indicators); // 0
|
|
state_vec.extend_from_slice(&market_features); // 38
|
|
state_vec.extend_from_slice(&portfolio_features); // 3
|
|
state_vec.extend_from_slice(®ime_features); // 8
|
|
|
|
// 4 + 38 + 3 + 8 = 53 (raw state before multi-tf expansion; multi-tf is added in GPU pipeline)
|
|
// The full pipeline adds 8 portfolio features (not 3) + 16 multi-tf:
|
|
// 42 market + 8 portfolio + 16 MTF + 8 OFI = 74 raw, 80 aligned.
|
|
// Here we verify the raw OFI portion is correctly positioned.
|
|
assert_eq!(state_vec.len(), 4 + 38 + 3 + 8); // = 53
|
|
assert_eq!(state_vec.len() - 8, 45, "Without OFI the state would be 45-dim");
|
|
|
|
// Verify OFI features are non-zero (calculated from actual snapshots)
|
|
let ofi_start = state_vec.len() - 8;
|
|
let ofi_slice = &state_vec[ofi_start..];
|
|
assert!(
|
|
ofi_slice.iter().any(|&v| v != 0.0),
|
|
"OFI features should be non-zero when computed from real snapshots"
|
|
);
|
|
|
|
// Verify all features are finite
|
|
for (i, &v) in state_vec.iter().enumerate() {
|
|
assert!(v.is_finite(), "Feature at index {} is not finite: {}", i, v);
|
|
}
|
|
}
|
|
|
|
/// Verify zero-fill behavior when no MBP-10 data is available.
|
|
#[test]
|
|
fn test_ofi_features_graceful_degradation() {
|
|
let regime_features: Vec<f32> = OFIFeatures::zeros().to_array().iter().map(|&v| v as f32).collect();
|
|
|
|
assert_eq!(regime_features.len(), 8);
|
|
assert!(
|
|
regime_features.iter().all(|&v| v == 0.0),
|
|
"Zero-filled OFI features should all be 0.0"
|
|
);
|
|
|
|
// State dim should still be valid — no NaN or Inf
|
|
let mut state_vec = vec![0.0_f32; 42]; // market features
|
|
state_vec.extend_from_slice(&[0.0; 3]); // portfolio
|
|
state_vec.extend_from_slice(®ime_features); // 8 OFI zeros
|
|
|
|
assert_eq!(state_vec.len(), 53);
|
|
assert!(state_vec.iter().all(|v| v.is_finite()));
|
|
}
|
|
|
|
/// Verify OFI feature names at indices 42-49 in the unified feature space.
|
|
#[test]
|
|
fn test_ofi_feature_names_ordering() {
|
|
// The 8 OFI features occupy indices [42..50) in the full 50-dim feature space
|
|
// when appended to the 42-dim base market features.
|
|
let ofi_names = [
|
|
"ofi_level1",
|
|
"ofi_level5",
|
|
"depth_imbalance",
|
|
"vpin",
|
|
"kyles_lambda",
|
|
"bid_slope",
|
|
"ask_slope",
|
|
"trade_imbalance",
|
|
];
|
|
|
|
assert_eq!(ofi_names.len(), 8, "Must have exactly 8 OFI feature names");
|
|
|
|
// Verify the to_array() order matches the name ordering
|
|
let features = OFIFeatures {
|
|
ofi_level1: 1.0,
|
|
ofi_level5: 2.0,
|
|
depth_imbalance: 3.0,
|
|
vpin: 4.0,
|
|
kyle_lambda: 5.0,
|
|
bid_slope: 6.0,
|
|
ask_slope: 7.0,
|
|
trade_imbalance: 8.0,
|
|
};
|
|
let arr = features.to_array();
|
|
for (i, &val) in arr.iter().enumerate() {
|
|
assert_eq!(val, (i + 1) as f64, "Feature '{}' at index {} has wrong value", ofi_names[i], i);
|
|
}
|
|
}
|
|
|
|
/// OFI window alignment regression — `precompute_features.rs:441-475` and
|
|
/// `data_loading.rs:396-448` build the per-bar OFI snapshot window via
|
|
/// `partition_point` predicates. The post-fix contract (audit
|
|
/// `docs/lookahead-bias-audit-2026-04-28.md` §3) is:
|
|
///
|
|
/// window(bar_t) = `(close(bar_{t-1}), close(bar_t)]`
|
|
///
|
|
/// i.e. the formation interval — strictly historical at the policy's decision
|
|
/// time. The pre-fix contract was `[close(bar_t), close(bar_{t+1}))`, which
|
|
/// leaked bar t+1's microstructure into 31/32 OFI dims. This test exercises the
|
|
/// new partition_point predicates against a synthetic 5-bar dataset with
|
|
/// strictly monotonic snapshot timestamps and asserts the slice indexes pick up
|
|
/// only snapshots in the legal interval.
|
|
#[test]
|
|
fn test_ofi_window_alignment_uses_formation_interval() {
|
|
// Bar close-times (= bar.timestamp for volume bars; see trades_loader.rs:124-176).
|
|
let bar_close_ts: [u64; 5] = [1_000, 2_000, 3_000, 4_000, 5_000];
|
|
|
|
// Synthetic snapshots strictly inside each bar's formation interval.
|
|
// Bar t formation interval = (bar_close_ts[t-1], bar_close_ts[t]].
|
|
// Bar 0 has no prior bar (interval empty) — no snapshot.
|
|
let snapshot_ts: Vec<u64> = vec![
|
|
// Bar 1 formation: (1000, 2000]
|
|
1_500,
|
|
// Bar 2 formation: (2000, 3000]
|
|
2_400, 2_700,
|
|
// Bar 3 formation: (3000, 4000]
|
|
3_500,
|
|
// Bar 4 formation: (4000, 5000]
|
|
4_100, 4_900, 5_000, // 5_000 is bar-4 close itself — included (closed end)
|
|
];
|
|
|
|
// Predicates mirror the new precompute_features.rs / data_loading.rs code:
|
|
// snap_start = partition_point(|t| t <= bar_start_ts)
|
|
// snap_end = partition_point(|t| t <= bar_ts)
|
|
// window = snapshots[snap_start..snap_end]
|
|
for t in 1..bar_close_ts.len() {
|
|
let bar_ts = bar_close_ts[t];
|
|
let bar_start_ts = bar_close_ts[t - 1];
|
|
|
|
let snap_start = snapshot_ts.partition_point(|&ts| ts <= bar_start_ts);
|
|
let snap_end = snapshot_ts.partition_point(|&ts| ts <= bar_ts);
|
|
|
|
// Every snapshot in the window MUST lie in (bar_start_ts, bar_ts].
|
|
for &ts in &snapshot_ts[snap_start..snap_end] {
|
|
assert!(
|
|
ts > bar_start_ts && ts <= bar_ts,
|
|
"bar t={t}: snapshot ts={ts} outside formation interval ({bar_start_ts}, {bar_ts}]"
|
|
);
|
|
}
|
|
|
|
// Conversely, every snapshot in the legal interval must be present.
|
|
for &ts in &snapshot_ts {
|
|
let in_window = ts > bar_start_ts && ts <= bar_ts;
|
|
let picked_up = snapshot_ts[snap_start..snap_end].contains(&ts);
|
|
assert_eq!(
|
|
in_window, picked_up,
|
|
"bar t={t}: snapshot ts={ts} in_window={in_window} but picked_up={picked_up}"
|
|
);
|
|
}
|
|
}
|
|
|
|
// Bar 0 (no prior bar) — bar_start_ts == bar_ts means an empty window
|
|
// (consistent with the first-bar handling in precompute_features.rs).
|
|
let bar_ts = bar_close_ts[0];
|
|
let bar_start_ts = bar_ts;
|
|
let snap_start = snapshot_ts.partition_point(|&ts| ts <= bar_start_ts);
|
|
let snap_end = snapshot_ts.partition_point(|&ts| ts <= bar_ts);
|
|
assert_eq!(
|
|
snap_start, snap_end,
|
|
"bar 0 must have an empty OFI window (no prior bar to bound the formation interval)"
|
|
);
|
|
|
|
// Anti-regression: confirm a snapshot at bar t+1's formation period is NEVER
|
|
// picked up for bar t (this was the leak being fixed).
|
|
let bar_t = 2_usize;
|
|
let bar_ts = bar_close_ts[bar_t]; // 3000
|
|
let bar_start_ts = bar_close_ts[bar_t - 1]; // 2000
|
|
let snap_start = snapshot_ts.partition_point(|&ts| ts <= bar_start_ts);
|
|
let snap_end = snapshot_ts.partition_point(|&ts| ts <= bar_ts);
|
|
let leaky_snap_ts = 3_500_u64; // belongs to bar 3's formation interval
|
|
assert!(
|
|
!snapshot_ts[snap_start..snap_end].contains(&leaky_snap_ts),
|
|
"bar t=2 must NOT pick up snapshot ts=3500 (that belongs to bar 3's formation interval)"
|
|
);
|
|
}
|