gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(data): MBP-10 decoder corrupted the inside quote (level 0)

Browse Source 
Both parse_mbp10_file and parse_mbp10_streaming wrote the single MBP-10
update event's (price,size) into levels[0] via update_level(0,...) and
then copied the authoritative book from mbp10.levels[1..] — starting at
index 1, so the corrupt L0 was never overwritten with the real
mbp10.levels[0]. Result on real cluster data (ES.FUT 2025-Q1 front-month,
2M records): 14.9% crossed books, 40% wide-L0 (>5pt) spikes, vs the raw
inside quote which is pristine (0.016% crossed, 0% wide, 0.25pt median).
Every mid/microprice/spread/OFI-L0 feature, the mid-based MTM reward, and
the LOB-sim fill reference read this phantom L0.

Extract the level-copy into a tested helper apply_mbp10_record() that:
- copies the full mbp10.levels[0..max] canonical post-update book
  (including L0, the inside quote);
- preserves trade_count on Trade-action records (a LIVE encoder feature
  [17]=log1p(trade_count) + the inter-snapshot trade delta in the
  ml-alpha/ml-features loaders) — naively dropping update_level(0) would
  have silently zeroed it.

Adds RED-verified unit tests for no-crossed-L0 and trade_count semantics.
cargo test -p data --lib: 377 passed.

Sidecars (.predecoded.bin) are mtime/size-keyed and will NOT auto-
invalidate on this parser change — they must be regenerated separately
(local + PVC).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-06-05 14:48:05 +02:00
parent 31f858b5ea
commit 6df3284d0d
1 changed files with 145 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

211
crates/data/src/providers/databento/dbn_parser.rs
View File

@@ -699,38 +699,19 @@ impl DbnParser {
if let RecordRefEnum::Mbp10(mbp10) = record_enum {
update_count += 1;
// Phase E.1 fix (2026-05-15): MBP-10 messages carry
// the FULL post-update top-10 book in
// `mbp10.levels: [BidAskPair; 10]` — not just the
// single update event. We copy the full top-10
// here so downstream consumers (OFI, microprice,
// FillModel L2/L3) see real data. Mirror of the
// parse_mbp10_streaming fix in the same file.
let is_bid = mbp10.side == b'B' as i8;
// MBP-10 messages carry the FULL post-update top-10 book
// in `mbp10.levels`; `apply_mbp10_record` overlays all 10
// levels (incl. L0, the inside quote) and bumps
// trade_count on Trade actions. See its doc comment.
let action = OrderBookAction::from(mbp10.action as u8);
current_snapshot.update_level(
0, // Level index
apply_mbp10_record(
&mut current_snapshot,
&mbp10.levels,
action,
mbp10.price,
mbp10.size,
1, // Order count (not available in MBP-10 single update)
is_bid,
mbp10.hd.ts_event,
mbp10.sequence,
);
let max_lvl = mbp10.levels.len().min(current_snapshot.levels.len());
for lvl in 1..max_lvl {
current_snapshot.levels[lvl].bid_px = mbp10.levels[lvl].bid_px;
current_snapshot.levels[lvl].bid_sz = mbp10.levels[lvl].bid_sz;
current_snapshot.levels[lvl].bid_ct = mbp10.levels[lvl].bid_ct;
current_snapshot.levels[lvl].ask_px = mbp10.levels[lvl].ask_px;
current_snapshot.levels[lvl].ask_sz = mbp10.levels[lvl].ask_sz;
current_snapshot.levels[lvl].ask_ct = mbp10.levels[lvl].ask_ct;
}
current_snapshot.timestamp = mbp10.hd.ts_event;
current_snapshot.sequence = mbp10.sequence;
// Create snapshot periodically to reduce memory
if update_count % SNAPSHOT_INTERVAL == 0 {
snapshots.push(current_snapshot.clone());
@@ -870,48 +851,23 @@ impl DbnParser {
kept += 1;
update_count += 1;
let is_bid = mbp10.side == b'B' as i8;
// The dbn `Mbp10Msg` carries the full 10-level
// post-update book in `levels: [BidAskPair; 10]`
// (index 0 = inside quote). `apply_mbp10_record` overlays
// ALL levels — including L0 — using the existing 1e9
// fixed-point scale convention, and bumps trade_count on
// Trade actions. Writing the single update event into L0
// (the previous behavior) corrupted the inside quote:
// ~6.4% crossed books, ~10% wide-L0 spikes.
let action = OrderBookAction::from(mbp10.action as u8);
current_snapshot.update_level(
0,
apply_mbp10_record(
&mut current_snapshot,
&mbp10.levels,
action,
mbp10.price,
mbp10.size,
1,
is_bid,
mbp10.hd.ts_event,
mbp10.sequence,
);
// Phase E.1 fix (2026-05-15): copy the full top-10
// post-update book snapshot from `mbp10.levels[1..]`
// into `current_snapshot.levels[1..]`. Without this,
// levels[1..10] stayed at default-empty, so downstream
// consumers (OFI calculator's L2-L5 reads, microprice
// at `snapshot.levels[1]`, FillModel L2/L3 distributions)
// received zeros for everything below L1. The dbn
// crate's `Mbp10Msg` carries the full 10-level
// post-update book in `levels: [BidAskPair; 10]`;
// previously only the single update event's
// (price, size) was captured (into level 0 via
// `update_level`).
//
// Field-by-field copy preserves the existing scale
// convention (raw 1e9 fixed-point i64; readers apply
// 1e-9 via `BidAskPair::price_to_f64` or local
// `raw_price_to_f32` workarounds).
let max_lvl = mbp10.levels.len().min(current_snapshot.levels.len());
for lvl in 1..max_lvl {
current_snapshot.levels[lvl].bid_px = mbp10.levels[lvl].bid_px;
current_snapshot.levels[lvl].bid_sz = mbp10.levels[lvl].bid_sz;
current_snapshot.levels[lvl].bid_ct = mbp10.levels[lvl].bid_ct;
current_snapshot.levels[lvl].ask_px = mbp10.levels[lvl].ask_px;
current_snapshot.levels[lvl].ask_sz = mbp10.levels[lvl].ask_sz;
current_snapshot.levels[lvl].ask_ct = mbp10.levels[lvl].ask_ct;
}
current_snapshot.timestamp = mbp10.hd.ts_event;
current_snapshot.sequence = mbp10.sequence;
if update_count % snapshot_interval == 0 {
callback(&current_snapshot);
snapshot_count += 1;
@@ -1289,6 +1245,43 @@ impl DbnParserMetrics {
}
}
/// Overlay an MBP-10 record's authoritative post-update book onto `snapshot`.
///
/// MBP-10 messages carry the full top-10 book in `levels` (index 0 = inside
/// quote). We copy **all** levels including L0. A prior version wrote only the
/// single update event's price into L0 via `update_level(0, …)` and then copied
/// `levels[1..]`, which left L0 holding the lone event price (typically one side
/// only) instead of the real inside quote → crossed books and wide-L0 spikes.
///
/// `trade_count` is bumped on Trade-action records, preserving the downstream
/// signal (encoder feature `[17] = log1p(trade_count)` and the inter-snapshot
/// trade delta in the ml-alpha / ml-features loaders). A trade does not alter
/// the book structure itself, so only the counter moves.
fn apply_mbp10_record(
snapshot: &mut Mbp10Snapshot,
levels: &[dbn::BidAskPair],
action: OrderBookAction,
ts_event: u64,
sequence: u32,
) {
let max_lvl = levels.len().min(snapshot.levels.len());
for lvl in 0..max_lvl {
snapshot.levels[lvl].bid_px = levels[lvl].bid_px;
snapshot.levels[lvl].bid_sz = levels[lvl].bid_sz;
snapshot.levels[lvl].bid_ct = levels[lvl].bid_ct;
snapshot.levels[lvl].ask_px = levels[lvl].ask_px;
snapshot.levels[lvl].ask_sz = levels[lvl].ask_sz;
snapshot.levels[lvl].ask_ct = levels[lvl].ask_ct;
}
if action == OrderBookAction::Trade {
snapshot.trade_count += 1;
}
snapshot.timestamp = ts_event;
snapshot.sequence = sequence;
}
/// Snapshot of DBN parser metrics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DbnParserMetricsSnapshot {
@@ -1348,6 +1341,92 @@ mod tests {
assert_eq!(parser.get_symbol(999), "UNKNOWN_999");
}
/// Regression: the MBP-10 decoder must populate L0 from the authoritative
/// `mbp10.levels[0]` (the inside quote), NOT from the lone update event.
/// Writing the event into L0 produced crossed/half-empty inside quotes
/// (root cause of ~6.4% crossed books + ~10% wide-L0 spikes).
#[test]
fn apply_mbp10_record_fills_inside_quote_without_crossing() {
use dbn::BidAskPair as DbnBidAskPair;
// Authoritative post-update book: L0 = normal uncrossed inside quote.
let mut levels: [DbnBidAskPair; 10] = std::array::from_fn(|_| DbnBidAskPair {
bid_px: 0,
ask_px: 0,
bid_sz: 0,
ask_sz: 0,
bid_ct: 0,
ask_ct: 0,
});
levels[0] = DbnBidAskPair {
bid_px: 5_000_000_000_000, // 5000.00 (×1e9)
ask_px: 5_000_250_000_000, // 5000.25 (×1e9)
bid_sz: 10,
ask_sz: 12,
bid_ct: 1,
ask_ct: 1,
};
levels[1] = DbnBidAskPair {
bid_px: 4_999_750_000_000,
ask_px: 5_000_500_000_000,
bid_sz: 7,
ask_sz: 9,
bid_ct: 1,
ask_ct: 1,
};
let mut snap = Mbp10Snapshot::empty("ES.FUT".to_string());
// A bid-side ADD event priced ABOVE the real ask — the classic
// corruptor. Pre-fix code wrote this into L0 and copied only levels[1..].
apply_mbp10_record(
&mut snap,
&levels,
OrderBookAction::Add,
1_700_000_000_000_000_000,
42,
);
assert_eq!(
snap.levels[0].bid_px, 5_000_000_000_000,
"L0 bid must be the real inside bid, not the event price"
);
assert_eq!(
snap.levels[0].ask_px, 5_000_250_000_000,
"L0 ask must be the real inside ask (event never set the ask side)"
);
assert!(
snap.levels[0].bid_px < snap.levels[0].ask_px,
"inside quote must not be crossed"
);
assert_eq!(snap.levels[1].ask_px, 5_000_500_000_000, "L1 still copied");
assert_eq!(snap.timestamp, 1_700_000_000_000_000_000);
assert_eq!(snap.sequence, 42);
}
/// `trade_count` is a live model feature; only Trade-action records bump it.
#[test]
fn apply_mbp10_record_counts_only_trade_actions() {
use dbn::BidAskPair as DbnBidAskPair;
let levels: [DbnBidAskPair; 10] = std::array::from_fn(|_| DbnBidAskPair {
bid_px: 5_000_000_000_000,
ask_px: 5_000_250_000_000,
bid_sz: 1,
ask_sz: 1,
bid_ct: 1,
ask_ct: 1,
});
let mut snap = Mbp10Snapshot::empty("ES.FUT".to_string());
apply_mbp10_record(&mut snap, &levels, OrderBookAction::Add, 1, 1);
assert_eq!(snap.trade_count, 0, "non-trade action must not bump count");
apply_mbp10_record(&mut snap, &levels, OrderBookAction::Trade, 2, 2);
apply_mbp10_record(&mut snap, &levels, OrderBookAction::Trade, 3, 3);
assert_eq!(snap.trade_count, 2, "Trade actions must increment trade_count");
}
#[test]
fn test_price_scaling() {
let parser = DbnParser::new().unwrap();