From 6df3284d0d005cba5d2db03a9af36eaa4a926820 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 5 Jun 2026 14:48:05 +0200 Subject: [PATCH] fix(data): MBP-10 decoder corrupted the inside quote (level 0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both parse_mbp10_file and parse_mbp10_streaming wrote the single MBP-10 update event's (price,size) into levels[0] via update_level(0,...) and then copied the authoritative book from mbp10.levels[1..] — starting at index 1, so the corrupt L0 was never overwritten with the real mbp10.levels[0]. Result on real cluster data (ES.FUT 2025-Q1 front-month, 2M records): 14.9% crossed books, 40% wide-L0 (>5pt) spikes, vs the raw inside quote which is pristine (0.016% crossed, 0% wide, 0.25pt median). Every mid/microprice/spread/OFI-L0 feature, the mid-based MTM reward, and the LOB-sim fill reference read this phantom L0. Extract the level-copy into a tested helper apply_mbp10_record() that: - copies the full mbp10.levels[0..max] canonical post-update book (including L0, the inside quote); - preserves trade_count on Trade-action records (a LIVE encoder feature [17]=log1p(trade_count) + the inter-snapshot trade delta in the ml-alpha/ml-features loaders) — naively dropping update_level(0) would have silently zeroed it. Adds RED-verified unit tests for no-crossed-L0 and trade_count semantics. cargo test -p data --lib: 377 passed. Sidecars (.predecoded.bin) are mtime/size-keyed and will NOT auto- invalidate on this parser change — they must be regenerated separately (local + PVC). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/providers/databento/dbn_parser.rs | 211 ++++++++++++------ 1 file changed, 145 insertions(+), 66 deletions(-) diff --git a/crates/data/src/providers/databento/dbn_parser.rs b/crates/data/src/providers/databento/dbn_parser.rs index ef74005d1..c88303901 100644 --- a/crates/data/src/providers/databento/dbn_parser.rs +++ b/crates/data/src/providers/databento/dbn_parser.rs @@ -699,38 +699,19 @@ impl DbnParser { if let RecordRefEnum::Mbp10(mbp10) = record_enum { update_count += 1; - // Phase E.1 fix (2026-05-15): MBP-10 messages carry - // the FULL post-update top-10 book in - // `mbp10.levels: [BidAskPair; 10]` — not just the - // single update event. We copy the full top-10 - // here so downstream consumers (OFI, microprice, - // FillModel L2/L3) see real data. Mirror of the - // parse_mbp10_streaming fix in the same file. - let is_bid = mbp10.side == b'B' as i8; + // MBP-10 messages carry the FULL post-update top-10 book + // in `mbp10.levels`; `apply_mbp10_record` overlays all 10 + // levels (incl. L0, the inside quote) and bumps + // trade_count on Trade actions. See its doc comment. let action = OrderBookAction::from(mbp10.action as u8); - - current_snapshot.update_level( - 0, // Level index + apply_mbp10_record( + &mut current_snapshot, + &mbp10.levels, action, - mbp10.price, - mbp10.size, - 1, // Order count (not available in MBP-10 single update) - is_bid, + mbp10.hd.ts_event, + mbp10.sequence, ); - let max_lvl = mbp10.levels.len().min(current_snapshot.levels.len()); - for lvl in 1..max_lvl { - current_snapshot.levels[lvl].bid_px = mbp10.levels[lvl].bid_px; - current_snapshot.levels[lvl].bid_sz = mbp10.levels[lvl].bid_sz; - current_snapshot.levels[lvl].bid_ct = mbp10.levels[lvl].bid_ct; - current_snapshot.levels[lvl].ask_px = mbp10.levels[lvl].ask_px; - current_snapshot.levels[lvl].ask_sz = mbp10.levels[lvl].ask_sz; - current_snapshot.levels[lvl].ask_ct = mbp10.levels[lvl].ask_ct; - } - - current_snapshot.timestamp = mbp10.hd.ts_event; - current_snapshot.sequence = mbp10.sequence; - // Create snapshot periodically to reduce memory if update_count % SNAPSHOT_INTERVAL == 0 { snapshots.push(current_snapshot.clone()); @@ -870,48 +851,23 @@ impl DbnParser { kept += 1; update_count += 1; - let is_bid = mbp10.side == b'B' as i8; + // The dbn `Mbp10Msg` carries the full 10-level + // post-update book in `levels: [BidAskPair; 10]` + // (index 0 = inside quote). `apply_mbp10_record` overlays + // ALL levels — including L0 — using the existing 1e9 + // fixed-point scale convention, and bumps trade_count on + // Trade actions. Writing the single update event into L0 + // (the previous behavior) corrupted the inside quote: + // ~6.4% crossed books, ~10% wide-L0 spikes. let action = OrderBookAction::from(mbp10.action as u8); - - current_snapshot.update_level( - 0, + apply_mbp10_record( + &mut current_snapshot, + &mbp10.levels, action, - mbp10.price, - mbp10.size, - 1, - is_bid, + mbp10.hd.ts_event, + mbp10.sequence, ); - // Phase E.1 fix (2026-05-15): copy the full top-10 - // post-update book snapshot from `mbp10.levels[1..]` - // into `current_snapshot.levels[1..]`. Without this, - // levels[1..10] stayed at default-empty, so downstream - // consumers (OFI calculator's L2-L5 reads, microprice - // at `snapshot.levels[1]`, FillModel L2/L3 distributions) - // received zeros for everything below L1. The dbn - // crate's `Mbp10Msg` carries the full 10-level - // post-update book in `levels: [BidAskPair; 10]`; - // previously only the single update event's - // (price, size) was captured (into level 0 via - // `update_level`). - // - // Field-by-field copy preserves the existing scale - // convention (raw 1e9 fixed-point i64; readers apply - // 1e-9 via `BidAskPair::price_to_f64` or local - // `raw_price_to_f32` workarounds). - let max_lvl = mbp10.levels.len().min(current_snapshot.levels.len()); - for lvl in 1..max_lvl { - current_snapshot.levels[lvl].bid_px = mbp10.levels[lvl].bid_px; - current_snapshot.levels[lvl].bid_sz = mbp10.levels[lvl].bid_sz; - current_snapshot.levels[lvl].bid_ct = mbp10.levels[lvl].bid_ct; - current_snapshot.levels[lvl].ask_px = mbp10.levels[lvl].ask_px; - current_snapshot.levels[lvl].ask_sz = mbp10.levels[lvl].ask_sz; - current_snapshot.levels[lvl].ask_ct = mbp10.levels[lvl].ask_ct; - } - - current_snapshot.timestamp = mbp10.hd.ts_event; - current_snapshot.sequence = mbp10.sequence; - if update_count % snapshot_interval == 0 { callback(¤t_snapshot); snapshot_count += 1; @@ -1289,6 +1245,43 @@ impl DbnParserMetrics { } } +/// Overlay an MBP-10 record's authoritative post-update book onto `snapshot`. +/// +/// MBP-10 messages carry the full top-10 book in `levels` (index 0 = inside +/// quote). We copy **all** levels including L0. A prior version wrote only the +/// single update event's price into L0 via `update_level(0, …)` and then copied +/// `levels[1..]`, which left L0 holding the lone event price (typically one side +/// only) instead of the real inside quote → crossed books and wide-L0 spikes. +/// +/// `trade_count` is bumped on Trade-action records, preserving the downstream +/// signal (encoder feature `[17] = log1p(trade_count)` and the inter-snapshot +/// trade delta in the ml-alpha / ml-features loaders). A trade does not alter +/// the book structure itself, so only the counter moves. +fn apply_mbp10_record( + snapshot: &mut Mbp10Snapshot, + levels: &[dbn::BidAskPair], + action: OrderBookAction, + ts_event: u64, + sequence: u32, +) { + let max_lvl = levels.len().min(snapshot.levels.len()); + for lvl in 0..max_lvl { + snapshot.levels[lvl].bid_px = levels[lvl].bid_px; + snapshot.levels[lvl].bid_sz = levels[lvl].bid_sz; + snapshot.levels[lvl].bid_ct = levels[lvl].bid_ct; + snapshot.levels[lvl].ask_px = levels[lvl].ask_px; + snapshot.levels[lvl].ask_sz = levels[lvl].ask_sz; + snapshot.levels[lvl].ask_ct = levels[lvl].ask_ct; + } + + if action == OrderBookAction::Trade { + snapshot.trade_count += 1; + } + + snapshot.timestamp = ts_event; + snapshot.sequence = sequence; +} + /// Snapshot of DBN parser metrics #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DbnParserMetricsSnapshot { @@ -1348,6 +1341,92 @@ mod tests { assert_eq!(parser.get_symbol(999), "UNKNOWN_999"); } + /// Regression: the MBP-10 decoder must populate L0 from the authoritative + /// `mbp10.levels[0]` (the inside quote), NOT from the lone update event. + /// Writing the event into L0 produced crossed/half-empty inside quotes + /// (root cause of ~6.4% crossed books + ~10% wide-L0 spikes). + #[test] + fn apply_mbp10_record_fills_inside_quote_without_crossing() { + use dbn::BidAskPair as DbnBidAskPair; + + // Authoritative post-update book: L0 = normal uncrossed inside quote. + let mut levels: [DbnBidAskPair; 10] = std::array::from_fn(|_| DbnBidAskPair { + bid_px: 0, + ask_px: 0, + bid_sz: 0, + ask_sz: 0, + bid_ct: 0, + ask_ct: 0, + }); + levels[0] = DbnBidAskPair { + bid_px: 5_000_000_000_000, // 5000.00 (×1e9) + ask_px: 5_000_250_000_000, // 5000.25 (×1e9) + bid_sz: 10, + ask_sz: 12, + bid_ct: 1, + ask_ct: 1, + }; + levels[1] = DbnBidAskPair { + bid_px: 4_999_750_000_000, + ask_px: 5_000_500_000_000, + bid_sz: 7, + ask_sz: 9, + bid_ct: 1, + ask_ct: 1, + }; + + let mut snap = Mbp10Snapshot::empty("ES.FUT".to_string()); + + // A bid-side ADD event priced ABOVE the real ask — the classic + // corruptor. Pre-fix code wrote this into L0 and copied only levels[1..]. + apply_mbp10_record( + &mut snap, + &levels, + OrderBookAction::Add, + 1_700_000_000_000_000_000, + 42, + ); + + assert_eq!( + snap.levels[0].bid_px, 5_000_000_000_000, + "L0 bid must be the real inside bid, not the event price" + ); + assert_eq!( + snap.levels[0].ask_px, 5_000_250_000_000, + "L0 ask must be the real inside ask (event never set the ask side)" + ); + assert!( + snap.levels[0].bid_px < snap.levels[0].ask_px, + "inside quote must not be crossed" + ); + assert_eq!(snap.levels[1].ask_px, 5_000_500_000_000, "L1 still copied"); + assert_eq!(snap.timestamp, 1_700_000_000_000_000_000); + assert_eq!(snap.sequence, 42); + } + + /// `trade_count` is a live model feature; only Trade-action records bump it. + #[test] + fn apply_mbp10_record_counts_only_trade_actions() { + use dbn::BidAskPair as DbnBidAskPair; + + let levels: [DbnBidAskPair; 10] = std::array::from_fn(|_| DbnBidAskPair { + bid_px: 5_000_000_000_000, + ask_px: 5_000_250_000_000, + bid_sz: 1, + ask_sz: 1, + bid_ct: 1, + ask_ct: 1, + }); + let mut snap = Mbp10Snapshot::empty("ES.FUT".to_string()); + + apply_mbp10_record(&mut snap, &levels, OrderBookAction::Add, 1, 1); + assert_eq!(snap.trade_count, 0, "non-trade action must not bump count"); + + apply_mbp10_record(&mut snap, &levels, OrderBookAction::Trade, 2, 2); + apply_mbp10_record(&mut snap, &levels, OrderBookAction::Trade, 3, 3); + assert_eq!(snap.trade_count, 2, "Trade actions must increment trade_count"); + } + #[test] fn test_price_scaling() { let parser = DbnParser::new().unwrap();