//! Production-Ready DBN (Databento Binary) Format Parser //! //! High-performance, zero-copy parser for Databento's binary format with SIMD optimizations //! for ultra-low latency HFT market data processing. Targets <1μs processing per tick. //! //! ## Performance Features //! //! - **Zero-Copy Deserialization**: Direct memory mapping with minimal allocations //! - **SIMD Optimizations**: Vectorized processing for batch operations //! - **Lock-Free Processing**: Atomic operations for concurrent access //! - **Hardware Timestamps**: RDTSC-based timing for latency measurement //! - **Memory Prefetching**: Cache-optimized data access patterns //! //! ## DBN Format Support //! //! - **Trade Messages**: Fast trade tick processing with price/size/conditions //! - **Quote Messages**: L1 BBO with bid/ask spreads //! - **Order Book**: L2/L3 depth with incremental updates //! - **Statistics**: OHLCV bars and session statistics //! - **Status Messages**: Market status and trading halts use crate::error::{DataError, Result}; use crate::providers::databento::mbp10::{Mbp10Snapshot, OrderBookAction}; use common::{OrderSide, Price}; use dbn::decode::{DbnDecoder, DbnMetadata, DecodeRecordRef}; use dbn::RecordRefEnum; use num_traits::{FromPrimitive, ToPrimitive}; use rust_decimal::Decimal; use serde::{Deserialize, Serialize}; use std::io::Cursor; use std::path::Path; use std::sync::{ atomic::{AtomicU64, Ordering}, Arc, }; use tracing::{debug, error, info, instrument, warn}; use trading_engine::{ events::event_types::{EventLevel, SystemEventType, TradingEvent}, events::EventProcessor, lockfree::{ring_buffer::LockFreeRingBuffer, HftMessage}, simd::{SafeSimdDispatcher, SimdMarketDataOps}, timing::HardwareTimestamp, }; /// DBN message header - optimized for zero-copy parsing #[repr(C, packed)] #[derive(Debug, Clone, Copy)] pub struct DbnMessageHeader { /// Message length in bytes pub length: u16, /// Record type identifier pub rtype: u8, /// Publisher ID pub publisher_id: u8, /// Instrument ID pub instrument_id: u32, /// Timestamp (nanoseconds since Unix epoch) pub ts_event: u64, } /// DBN trade message - zero-copy optimized #[repr(C, packed)] #[derive(Debug, Clone, Copy)] pub struct DbnTradeMessage { /// Message header with timestamp and symbol pub header: DbnMessageHeader, /// Trade price (scaled integer) pub price: i64, /// Trade size pub size: u32, /// Trade action (A=Add, C=Cancel, M=Modify, T=Trade, F=Fill) pub action: u8, /// Trade side (A=Ask, B=Bid, N=None) pub side: u8, /// Trade flags pub flags: u16, /// Depth level pub depth: u8, /// Sequence number pub sequence: u32, /// Reserved padding pub padding: u8, } /// DBN quote message - L1 BBO data #[repr(C, packed)] #[derive(Debug, Clone, Copy)] pub struct DbnQuoteMessage { /// Message header with timestamp and symbol pub header: DbnMessageHeader, /// Bid price (scaled integer) pub bid_px: i64, /// Ask price (scaled integer) pub ask_px: i64, /// Bid size pub bid_sz: u32, /// Ask size pub ask_sz: u32, /// Bid count pub bid_ct: u8, /// Ask count pub ask_ct: u8, /// Flags pub flags: u16, /// Sequence number pub sequence: u32, /// Reserved padding pub padding: [u8; 2], } /// DBN order book message - L2/L3 depth #[repr(C, packed)] #[derive(Debug, Clone, Copy)] pub struct DbnOrderBookMessage { /// Message header with timestamp and symbol pub header: DbnMessageHeader, /// Order ID pub order_id: u64, /// Price (scaled integer) pub price: i64, /// Size pub size: u32, /// Flags pub flags: u16, /// Channel ID pub channel_id: u8, /// Order count pub order_count: u8, /// Action (A=Add, C=Cancel, M=Modify, T=Trade, F=Fill) pub action: u8, /// Side (A=Ask, B=Bid) pub side: u8, /// Sequence number pub sequence: u32, /// Reserved padding pub padding: [u8; 2], } /// DBN OHLCV bar message #[repr(C, packed)] #[derive(Debug, Clone, Copy)] pub struct DbnOhlcvMessage { /// Message header with timestamp and symbol pub header: DbnMessageHeader, /// Open price (scaled integer) pub open: i64, /// High price (scaled integer) pub high: i64, /// Low price (scaled integer) pub low: i64, /// Close price (scaled integer) pub close: i64, /// Volume pub volume: u64, } /// DBN message types #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DbnMessageType { /// Trade message (tick data) Trade = 0x54, // 'T' /// Quote message (BBO) Quote = 0x51, // 'Q' /// Order book message (depth) OrderBook = 0x4F, // 'O' /// OHLCV bar message Ohlcv = 0x42, // 'B' (Bar) /// Status message Status = 0x53, // 'S' /// Error message Error = 0x45, // 'E' } impl From for DbnMessageType { fn from(value: u8) -> Self { match value { 0x54 => Self::Trade, 0x51 => Self::Quote, 0x4F => Self::OrderBook, 0x42 => Self::Ohlcv, 0x53 => Self::Status, 0x45 => Self::Error, _ => Self::Error, // Default to error for unknown types } } } /// High-performance DBN parser with SIMD optimizations pub struct DbnParser { /// SIMD dispatcher for optimal performance simd_dispatcher: SafeSimdDispatcher, /// Market data operations simd_ops: Option, /// Performance metrics metrics: Arc, /// Symbol mapping for instrument IDs symbol_map: Arc>>, /// Price scaling factors per instrument price_scales: Arc>>, /// Event processor for integration event_processor: Option>, /// Lock-free message buffer for high-frequency processing message_buffer: Arc>, } impl DbnParser { /// Create new high-performance DBN parser pub fn new() -> Result { let simd_dispatcher = SafeSimdDispatcher::new(); let simd_ops = simd_dispatcher.create_market_data_ops().ok(); if simd_ops.is_none() { warn!("AVX2 not available - falling back to scalar processing"); } else { debug!("DBN parser initialized with AVX2 SIMD optimizations"); } let message_buffer = Arc::new(LockFreeRingBuffer::new(0x8000).map_err(|e| { DataError::Initialization(format!("Failed to create message buffer: {}", e)) })?); Ok(Self { simd_dispatcher, simd_ops, metrics: Arc::new(DbnParserMetrics::new()), symbol_map: Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())), price_scales: Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())), event_processor: None, message_buffer, }) } /// Set event processor for integration with core event system pub fn set_event_processor(&mut self, processor: Arc) { self.event_processor = Some(processor); } /// Update symbol mapping for instrument IDs pub fn update_symbol_map(&self, mapping: std::collections::HashMap) { let mut symbol_map = self .symbol_map .write() .unwrap_or_else(|e| e.into_inner()); symbol_map.extend(mapping); debug!("Updated symbol map with {} instruments", symbol_map.len()); } /// Update price scaling factors pub fn update_price_scales(&self, scales: std::collections::HashMap) { let mut price_scales = self .price_scales .write() .unwrap_or_else(|e| e.into_inner()); price_scales.extend(scales); debug!( "Updated price scales for {} instruments", price_scales.len() ); } /// Parse DBN binary data using official dbn crate decoder /// /// Replaces custom binary parsing with production-ready decoder that correctly /// handles DataBento's binary format. Preserves HFT features (SIMD, metrics, timestamps). #[instrument(skip(self, data), level = "trace")] pub fn parse_batch(&self, data: &[u8]) -> Result> { let start_time = HardwareTimestamp::now(); let mut messages = Vec::new(); // Pre-allocate for common batch sizes messages.reserve(1000); // Create official DBN decoder let cursor = Cursor::new(data); let mut decoder = DbnDecoder::new(cursor) .map_err(|e| DataError::InvalidFormat(format!("DBN decode error: {}", e)))?; // Read metadata for symbol mapping let metadata = decoder.metadata(); let symbol = metadata .symbols .first() .map(|s| s.to_string()) .unwrap_or_else(|| "UNKNOWN".to_string()); debug!( "DBN metadata: dataset={:?}, schema={:?}, symbol={}", metadata.dataset, metadata.schema, symbol ); // Decode all records let mut record_count = 0; loop { match decoder.decode_record_ref() { Ok(Some(record)) => { record_count += 1; self.metrics.increment_messages_parsed(); // Convert RecordRef to RecordRefEnum for pattern matching let record_enum = record.as_enum().map_err(|e| { DataError::InvalidFormat(format!("Failed to convert record to enum: {}", e)) })?; // Parse based on record type match self.parse_dbn_record(record_enum, &symbol)? { Some(msg) => messages.push(msg), None => { // Unknown message type - skip self.metrics.increment_unknown_messages(); }, } }, Ok(None) => { // End of stream break; }, Err(e) => { return Err(DataError::InvalidFormat(format!( "Failed to decode record {}: {}", record_count, e ))); }, } } // SIMD batch processing for trade and quote messages if messages.len() >= 4 && self.simd_ops.is_some() { self.simd_batch_process(&mut messages)?; } let parse_time = HardwareTimestamp::now(); let latency_ns = parse_time.latency_ns(&start_time); self.metrics.record_parse_latency(latency_ns); // Check if we met the <1μs per tick target if messages.len() > 0 { let per_tick_latency = latency_ns / messages.len() as u64; if per_tick_latency > 1000 { warn!( "Parse latency {}ns/tick exceeds 1μs target", per_tick_latency ); } self.metrics.record_per_tick_latency(per_tick_latency); } debug!("Parsed {} messages from DBN data", messages.len()); Ok(messages) } /// Parse official dbn RecordRefEnum into ProcessedMessage /// /// Handles all DBN record types: OHLCV, Trade, MBP-1 (quotes), MBP-10 (order book). /// Preserves existing ProcessedMessage format for compatibility with trading_engine. fn parse_dbn_record( &self, record: RecordRefEnum<'_>, symbol: &str, ) -> Result> { match record { RecordRefEnum::Ohlcv(ohlcv) => { // OHLCV bars - primary data type for backtesting let timestamp = HardwareTimestamp::from_nanos(ohlcv.hd.ts_event); // Prices are i64 scaled by 1e-9 per DBN specification let open_f64 = ohlcv.open as f64 * 1e-9; let high_f64 = ohlcv.high as f64 * 1e-9; let low_f64 = ohlcv.low as f64 * 1e-9; let close_f64 = ohlcv.close as f64 * 1e-9; // Use absolute values for Price type (futures data can have negative values) let open = Price::from_f64(open_f64.abs())?; let high = Price::from_f64(high_f64.abs())?; let low = Price::from_f64(low_f64.abs())?; let close = Price::from_f64(close_f64.abs())?; let volume = Decimal::from(ohlcv.volume); self.metrics.increment_bars_processed(); Ok(Some(ProcessedMessage::Ohlcv { symbol: symbol.to_string(), timestamp, open, high, low, close, volume, })) }, RecordRefEnum::Trade(trade) => { // Trade ticks let timestamp = HardwareTimestamp::from_nanos(trade.hd.ts_event); // Prices are i64 scaled by 1e-9 per DBN specification let price_f64 = trade.price as f64 * 1e-9; let price = Price::from_f64(price_f64.abs())?; let size = Decimal::from(trade.size); // Determine side from trade action/flags (c_char is i8) let side = if trade.side == b'B' as i8 { OrderSide::Buy } else if trade.side == b'A' as i8 { OrderSide::Sell } else { OrderSide::Buy // Default }; self.metrics.increment_trades_processed(); Ok(Some(ProcessedMessage::Trade { symbol: symbol.to_string(), timestamp, price, size, side, trade_id: None, conditions: vec![], })) }, RecordRefEnum::Mbp1(mbp) => { // MBP-1 (Market By Price Level 1) - BBO quotes let timestamp = HardwareTimestamp::from_nanos(mbp.hd.ts_event); // Mbp1Msg has price/size/side, not separate bid/ask fields let price_f64 = mbp.price as f64 * 1e-9; let price = Price::from_f64(price_f64.abs())?; let size = Decimal::from(mbp.size); // Determine bid/ask from side field (c_char is i8) let (bid, ask, bid_size, ask_size) = if mbp.side == b'B' as i8 { // Bid side (Some(price), None, Some(size), None) } else if mbp.side == b'A' as i8 { // Ask side (None, Some(price), None, Some(size)) } else { // Unknown side - treat as bid (Some(price), None, Some(size), None) }; self.metrics.increment_quotes_processed(); Ok(Some(ProcessedMessage::Quote { symbol: symbol.to_string(), timestamp, bid, ask, bid_size, ask_size, exchange: Some("UNKNOWN".to_string()), })) }, RecordRefEnum::Mbp10(mbp10) => { // MBP-10 (Market By Price Level 2) - order book update event // Note: Mbp10Msg is a single update event (like Mbp1), not a full 10-level snapshot let timestamp = HardwareTimestamp::from_nanos(mbp10.hd.ts_event); let price_f64 = mbp10.price as f64 * 1e-9; let price = Price::from_f64(price_f64.abs())?; let size = Decimal::from(mbp10.size); // Determine bid/ask from side field (c_char is i8) let (bid, ask, bid_size, ask_size) = if mbp10.side == b'B' as i8 { // Bid side (Some(price), None, Some(size), None) } else if mbp10.side == b'A' as i8 { // Ask side (None, Some(price), None, Some(size)) } else { // Unknown side - treat as bid (Some(price), None, Some(size), None) }; self.metrics.increment_orderbook_processed(); Ok(Some(ProcessedMessage::Quote { symbol: symbol.to_string(), timestamp, bid, ask, bid_size, ask_size, exchange: Some("UNKNOWN".to_string()), })) }, _ => { // Skip other message types (Status, Error, etc.) Ok(None) }, } } /// SIMD batch processing for performance optimization fn simd_batch_process(&self, messages: &mut [ProcessedMessage]) -> Result<()> { use num_traits::ToPrimitive; if let Some(ref simd_ops) = self.simd_ops { // Group messages by type for SIMD processing let mut trade_prices = Vec::new(); let mut trade_volumes = Vec::new(); for msg in messages { if let ProcessedMessage::Trade { price, size, .. } = msg { trade_prices.push(price.to_f64()); // Handle Option from rust_decimal::Decimal::to_f64() if let Some(volume_f64) = size.to_f64() { trade_volumes.push(volume_f64); } else { warn!("Failed to convert trade volume to f64, skipping"); continue; } } } // Ensure both vectors have the same length after filtering let min_len = trade_prices.len().min(trade_volumes.len()); trade_prices.truncate(min_len); trade_volumes.truncate(min_len); // Calculate VWAP using SIMD if we have enough trades if trade_prices.len() >= 4 { let vwap = unsafe { simd_ops.calculate_vwap(&trade_prices, &trade_volumes) }; // SAFETY: Unsafe operation validated - invariants maintained by surrounding code debug!("Batch VWAP calculated: {:.4}", vwap); self.metrics.record_vwap(vwap); } } Ok(()) } /// Get symbol name for instrument ID fn get_symbol(&self, instrument_id: u32) -> String { self.symbol_map .read() .unwrap_or_else(|e| e.into_inner()) .get(&instrument_id) .cloned() .unwrap_or_else(|| format!("UNKNOWN_{}", instrument_id)) } /// Scale integer price to decimal using instrument-specific scaling fn scale_price(&self, price: i64, instrument_id: u32) -> Result { let scale = self .price_scales .read() .unwrap_or_else(|e| e.into_inner()) .get(&instrument_id) .copied() .unwrap_or(4); // Default to 4 decimal places let decimal_price = Decimal::from(price); let scale_factor = Decimal::from(10_i64.pow(scale as u32)); let scaled_decimal = decimal_price / scale_factor; let result_f64 = scaled_decimal.to_f64().ok_or_else(|| { DataError::InvalidFormat("Failed to convert decimal to f64".to_string()) })?; Price::from_f64(result_f64) .map_err(|e| DataError::InvalidFormat(format!("Failed to convert price: {}", e))) } /// Send processed messages to event system pub async fn send_to_event_system(&self, messages: Vec) -> Result<()> { if let Some(ref processor) = self.event_processor { for msg in messages { let trading_event = self.convert_to_trading_event(msg)?; // Capture event with sub-microsecond latency if let Err(e) = processor.capture_event(trading_event).await { error!("Failed to capture trading event: {}", e); self.metrics.increment_event_errors(); } } } Ok(()) } /// Convert processed message to trading event fn convert_to_trading_event(&self, msg: ProcessedMessage) -> Result { match msg { ProcessedMessage::Trade { symbol, timestamp, price, size, trade_id, .. } => Ok(TradingEvent::OrderExecuted { trade_id: trade_id.unwrap_or_default(), symbol, quantity: Decimal::from(size), price: Decimal::from_f64(price.to_f64()).unwrap_or(Decimal::ZERO), timestamp, sequence_number: None, metadata: None, }), ProcessedMessage::Quote { symbol, timestamp, .. } => Ok(TradingEvent::SystemEvent { event_type: SystemEventType::MarketDataFeed, message: format!("Quote update for {}", symbol), level: EventLevel::Info, timestamp, sequence_number: None, metadata: None, }), ProcessedMessage::OrderBook { symbol, timestamp, .. } => Ok(TradingEvent::SystemEvent { event_type: SystemEventType::MarketDataFeed, message: format!("OrderBook update for {}", symbol), level: EventLevel::Info, timestamp, sequence_number: None, metadata: None, }), _ => Err(DataError::Conversion( "Unsupported message type for trading event".to_string(), )), } } /// Get performance metrics pub fn get_metrics(&self) -> DbnParserMetricsSnapshot { self.metrics.get_snapshot() } /// Parse MBP-10 file and aggregate into order book snapshots /// /// # Arguments /// * `path` - Path to DBN file containing MBP-10 data /// /// # Returns /// Vector of aggregated 10-level order book snapshots /// /// # Example /// ```no_run /// use data::providers::databento::dbn_parser::DbnParser; /// /// # async fn example() -> anyhow::Result<()> { /// let parser = DbnParser::new()?; /// let snapshots = parser.parse_mbp10_file("test_data/ES.FUT.mbp10.dbn").await?; /// println!("Loaded {} snapshots", snapshots.len()); /// # Ok(()) /// # } /// ``` pub async fn parse_mbp10_file>(&self, path: P) -> Result> { use std::fs::File; use std::io::BufReader; let path = path.as_ref(); info!("📖 Parsing MBP-10 file: {:?}", path); // Open file and create DBN decoder (auto-detect .dbn.zst vs .dbn) let file = File::open(path)?; let is_zstd = path.to_string_lossy().ends_with(".dbn.zst"); let reader: Box = if is_zstd { Box::new(zstd::Decoder::new(BufReader::new(file)).map_err(|e| { DataError::InvalidFormat(format!("Failed to create zstd decoder: {}", e)) })?) } else { Box::new(BufReader::new(file)) }; let mut decoder = DbnDecoder::new(reader).map_err(|e| { DataError::InvalidFormat(format!("Failed to create DBN decoder: {}", e)) })?; // Read metadata let metadata = decoder.metadata(); let symbol = metadata .symbols .first() .map(|s| s.to_string()) .unwrap_or_else(|| "UNKNOWN".to_string()); info!(" Symbol: {}, Schema: {:?}", symbol, metadata.schema); // Track snapshot aggregation // MBP-10 messages are incremental updates, so we need to aggregate them into full snapshots let mut current_snapshot = Mbp10Snapshot::empty(symbol.clone()); let mut snapshots = Vec::new(); let mut update_count = 0; const SNAPSHOT_INTERVAL: usize = 100; // Create snapshot every 100 updates // Decode all MBP-10 records loop { match decoder.decode_record_ref() { Ok(Some(record)) => { let record_enum = record.as_enum().map_err(|e| { DataError::InvalidFormat(format!("Failed to convert record: {}", e)) })?; if let RecordRefEnum::Mbp10(mbp10) = record_enum { update_count += 1; // Phase E.1 fix (2026-05-15): MBP-10 messages carry // the FULL post-update top-10 book in // `mbp10.levels: [BidAskPair; 10]` — not just the // single update event. We copy the full top-10 // here so downstream consumers (OFI, microprice, // FillModel L2/L3) see real data. Mirror of the // parse_mbp10_streaming fix in the same file. let is_bid = mbp10.side == b'B' as i8; let action = OrderBookAction::from(mbp10.action as u8); current_snapshot.update_level( 0, // Level index action, mbp10.price, mbp10.size, 1, // Order count (not available in MBP-10 single update) is_bid, ); let max_lvl = mbp10.levels.len().min(current_snapshot.levels.len()); for lvl in 1..max_lvl { current_snapshot.levels[lvl].bid_px = mbp10.levels[lvl].bid_px; current_snapshot.levels[lvl].bid_sz = mbp10.levels[lvl].bid_sz; current_snapshot.levels[lvl].bid_ct = mbp10.levels[lvl].bid_ct; current_snapshot.levels[lvl].ask_px = mbp10.levels[lvl].ask_px; current_snapshot.levels[lvl].ask_sz = mbp10.levels[lvl].ask_sz; current_snapshot.levels[lvl].ask_ct = mbp10.levels[lvl].ask_ct; } current_snapshot.timestamp = mbp10.hd.ts_event; current_snapshot.sequence = mbp10.sequence; // Create snapshot periodically to reduce memory if update_count % SNAPSHOT_INTERVAL == 0 { snapshots.push(current_snapshot.clone()); if snapshots.len() % 1000 == 0 { info!(" Progress: {} snapshots aggregated", snapshots.len()); } } self.metrics.increment_orderbook_processed(); } }, Ok(None) => { // End of stream break; }, Err(e) => { return Err(DataError::InvalidFormat(format!( "Failed to decode MBP-10 record: {}", e ))); }, } } // Add final snapshot if there are pending updates if update_count % SNAPSHOT_INTERVAL != 0 { snapshots.push(current_snapshot); } info!( "✅ Parsed {} MBP-10 snapshots from {} updates", snapshots.len(), update_count ); Ok(snapshots) } /// Stream MBP-10 records through a callback without materializing all snapshots. /// /// This is significantly faster than `parse_mbp10_file` for large files because it /// avoids allocating and cloning millions of snapshots. The callback receives each /// aggregated snapshot by reference at the configured interval. /// /// # Arguments /// /// * `path` - Path to the .dbn or .dbn.zst file /// * `snapshot_interval` - Create a snapshot every N raw updates (e.g. 100) /// * `filter` - Instrument selection strategy. See [`InstrumentFilter`] for variants. /// ES.FUT files from databento are parent-symbol expansions and contain ~14 distinct /// instrument_ids per file (one per contract month). Without filtering, K-window /// labels span contract boundaries and produce $5000+ ΔP artifacts. /// * `callback` - Called with each aggregated snapshot (by reference, not cloned) /// /// # Returns /// /// Total number of snapshots processed. pub fn parse_mbp10_streaming( &self, path: P, snapshot_interval: usize, filter: InstrumentFilter, mut callback: F, ) -> Result where P: AsRef, F: FnMut(&Mbp10Snapshot), { let path = path.as_ref(); // FrontMonth is two-pass: pass 1 detects the dominant id and validates // it resolves to an ES contract via SymbolMapping records; pass 2 // streams emission. Resolve to Id(winning_id) so the hot loop below // stays branch-free. let resolved_filter = match filter { InstrumentFilter::FrontMonth => { let winning_id = self.detect_front_month_id(path)?; InstrumentFilter::Id(winning_id) } other => other, }; let id_filter: Option = match resolved_filter { InstrumentFilter::All => None, InstrumentFilter::Id(id) => Some(id), InstrumentFilter::FrontMonth => unreachable!("resolved above"), }; use std::fs::File; use std::io::BufReader; let file = File::open(path)?; let is_zstd = path.to_string_lossy().ends_with(".dbn.zst"); let reader: Box = if is_zstd { Box::new(zstd::Decoder::new(BufReader::new(file)).map_err(|e| { DataError::InvalidFormat(format!("Failed to create zstd decoder: {}", e)) })?) } else { Box::new(BufReader::new(file)) }; let mut decoder = DbnDecoder::new(reader).map_err(|e| { DataError::InvalidFormat(format!("Failed to create DBN decoder: {}", e)) })?; let symbol = decoder .metadata() .symbols .first() .map(|s| s.to_string()) .unwrap_or_else(|| "UNKNOWN".to_string()); let mut current_snapshot = Mbp10Snapshot::empty(symbol); let mut update_count: usize = 0; let mut snapshot_count: usize = 0; // Diagnostic counters: how many records matched vs. were skipped by // the instrument_id filter. Logged at end-of-parse so multi-instrument // DBN files are visible in training-job stdout. let mut kept: u64 = 0; let mut skipped: u64 = 0; loop { match decoder.decode_record_ref() { Ok(Some(record)) => { let record_enum = record.as_enum().map_err(|e| { DataError::InvalidFormat(format!("Failed to convert record: {}", e)) })?; if let RecordRefEnum::Mbp10(mbp10) = record_enum { if let Some(filter_id) = id_filter { if mbp10.hd.instrument_id != filter_id { skipped += 1; continue; } } kept += 1; update_count += 1; let is_bid = mbp10.side == b'B' as i8; let action = OrderBookAction::from(mbp10.action as u8); current_snapshot.update_level( 0, action, mbp10.price, mbp10.size, 1, is_bid, ); // Phase E.1 fix (2026-05-15): copy the full top-10 // post-update book snapshot from `mbp10.levels[1..]` // into `current_snapshot.levels[1..]`. Without this, // levels[1..10] stayed at default-empty, so downstream // consumers (OFI calculator's L2-L5 reads, microprice // at `snapshot.levels[1]`, FillModel L2/L3 distributions) // received zeros for everything below L1. The dbn // crate's `Mbp10Msg` carries the full 10-level // post-update book in `levels: [BidAskPair; 10]`; // previously only the single update event's // (price, size) was captured (into level 0 via // `update_level`). // // Field-by-field copy preserves the existing scale // convention (raw 1e9 fixed-point i64; readers apply // 1e-9 via `BidAskPair::price_to_f64` or local // `raw_price_to_f32` workarounds). let max_lvl = mbp10.levels.len().min(current_snapshot.levels.len()); for lvl in 1..max_lvl { current_snapshot.levels[lvl].bid_px = mbp10.levels[lvl].bid_px; current_snapshot.levels[lvl].bid_sz = mbp10.levels[lvl].bid_sz; current_snapshot.levels[lvl].bid_ct = mbp10.levels[lvl].bid_ct; current_snapshot.levels[lvl].ask_px = mbp10.levels[lvl].ask_px; current_snapshot.levels[lvl].ask_sz = mbp10.levels[lvl].ask_sz; current_snapshot.levels[lvl].ask_ct = mbp10.levels[lvl].ask_ct; } current_snapshot.timestamp = mbp10.hd.ts_event; current_snapshot.sequence = mbp10.sequence; if update_count % snapshot_interval == 0 { callback(¤t_snapshot); snapshot_count += 1; } } } Ok(None) => break, Err(e) => { return Err(DataError::InvalidFormat(format!( "Failed to decode MBP-10 record: {}", e ))); } } } // Final partial snapshot if update_count % snapshot_interval != 0 { callback(¤t_snapshot); snapshot_count += 1; } if let Some(filter_id) = id_filter { tracing::info!( kept, skipped, filter = filter_id, path = %path.display(), "instrument filter applied" ); } Ok(snapshot_count) } /// First pass for [`InstrumentFilter::FrontMonth`]: stream-decodes the DBN /// file once to identify the most-frequent `instrument_id` (the front-month /// contract has the bulk of trading volume) and validates that its /// resolved symbol matches an ES futures contract pattern using /// SymbolMapping records. /// /// Returns the winning instrument_id on success. Errors when: /// - The file contains no MBP-10 records. /// - The dominant id cannot be resolved to a symbol via SymbolMapping /// records (suggests a malformed or non-databento DBN file). /// - The resolved symbol does not match the ES contract pattern /// (suggests a calendar spread or non-ES instrument is dominating — /// loud failure so the caller surfaces the data anomaly). fn detect_front_month_id(&self, path: &Path) -> Result { use std::collections::HashMap; use std::fs::File; use std::io::BufReader; let file = File::open(path)?; let is_zstd = path.to_string_lossy().ends_with(".dbn.zst"); let reader: Box = if is_zstd { Box::new(zstd::Decoder::new(BufReader::new(file)).map_err(|e| { DataError::InvalidFormat(format!("Failed to create zstd decoder: {}", e)) })?) } else { Box::new(BufReader::new(file)) }; let mut decoder = DbnDecoder::new(reader).map_err(|e| { DataError::InvalidFormat(format!("Failed to create DBN decoder: {}", e)) })?; let mut counts: HashMap = HashMap::new(); let mut id_to_symbol: HashMap = HashMap::new(); loop { match decoder.decode_record_ref() { Ok(Some(record)) => { let record_enum = record.as_enum().map_err(|e| { DataError::InvalidFormat(format!("Failed to convert record: {}", e)) })?; match record_enum { RecordRefEnum::Mbp10(mbp10) => { *counts.entry(mbp10.hd.instrument_id).or_insert(0) += 1; } RecordRefEnum::SymbolMapping(sym) => { if let Ok(out) = sym.stype_out_symbol() { id_to_symbol .insert(sym.hd.instrument_id, out.to_string()); } } _ => {} } } Ok(None) => break, Err(e) => { return Err(DataError::InvalidFormat(format!( "front-month detect: decode failed: {}", e ))); } } } let (winner_id, winner_count) = counts .iter() .max_by_key(|(_, c)| *c) .map(|(id, c)| (*id, *c)) .ok_or_else(|| { DataError::InvalidFormat(format!( "front-month detect: no MBP-10 records in {}", path.display() )) })?; let total: u64 = counts.values().sum(); // Soft-validate via SymbolMapping records when present. Databento // historical bulk files often emit mappings only in the metadata // header (date-range form) and skip in-stream SymbolMappingMsg // entirely; in that case we trust the dominant-id (volume leader = // front-month) and proceed. When the in-stream symbol IS present, // we check it against the ES contract regex and warn (not fail) on // mismatch so calendar-spread anomalies surface in logs without // blocking training. let winner_symbol = id_to_symbol.get(&winner_id).cloned(); let es_re = regex::Regex::new(r"^ES[FGHJKMNQUVXZ]\d{1,2}$") .expect("static ES futures regex compiles"); let symbol_status: &str = match &winner_symbol { Some(sym) if es_re.is_match(sym) => "es-contract", Some(_) => "symbol-mismatch", None => "no-streaming-mapping", }; if symbol_status == "symbol-mismatch" { warn!( instrument_id = winner_id, symbol = %winner_symbol.as_deref().unwrap_or(""), path = %path.display(), "front-month dominant id resolves to a non-ES symbol via SymbolMapping; \ proceeding anyway (calendar spread or unexpected contract?)" ); } info!( instrument_id = winner_id, symbol = %winner_symbol.as_deref().unwrap_or("(metadata-only)"), symbol_status, count = winner_count, total = total, distinct_ids = counts.len(), path = %path.display(), "front-month detected" ); Ok(winner_id) } } /// Strategy for selecting which `instrument_id` records to keep when streaming /// MBP-10 from a DBN file. ES.FUT parent-expanded files contain ~14 distinct /// contract months in one stream; without filtering, downstream K-window /// labels span contract boundaries and produce ΔP artifacts at the rolls. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum InstrumentFilter { /// Emit all MBP-10 records regardless of `instrument_id`. Preserves the /// pre-refactor behavior for callers that need multi-instrument streams /// (e.g., DQN training that doesn't compute multi-step labels). All, /// Emit only records where `record.hd.instrument_id == id`. Fails silently /// for files where the configured id never appears (caller must validate). Id(u32), /// Two-pass: detect the most-frequent `instrument_id` in the file (volume /// leader = front-month for ES.FUT), validate it resolves to a valid ES /// contract via SymbolMapping records, then emit only its records. /// Self-tuning across quarterly contract rolls. FrontMonth, } /// Processed message types from DBN parsing #[derive(Debug, Clone)] pub enum ProcessedMessage { /// Trade tick message Trade { /// Trading symbol symbol: String, /// Event timestamp timestamp: HardwareTimestamp, /// Trade price price: Price, /// Trade size size: Decimal, /// Trade side (buy/sell) side: OrderSide, /// Optional trade identifier trade_id: Option, /// Trade conditions conditions: Vec, }, /// Quote (BBO) message Quote { /// Trading symbol symbol: String, /// Event timestamp timestamp: HardwareTimestamp, /// Bid price bid: Option, /// Ask price ask: Option, /// Bid size bid_size: Option, /// Ask size ask_size: Option, /// Exchange identifier exchange: Option, }, /// Order book depth update OrderBook { /// Trading symbol symbol: String, /// Event timestamp timestamp: HardwareTimestamp, /// Price level price: Price, /// Size at level size: Decimal, /// Side (bid/ask) side: OrderSide, /// Action (add/modify/cancel) action: OrderBookAction, /// Depth level level: usize, /// Optional order ID order_id: Option, }, /// OHLCV bar message Ohlcv { /// Trading symbol symbol: String, /// Bar timestamp timestamp: HardwareTimestamp, /// Open price open: Price, /// High price high: Price, /// Low price low: Price, /// Close price close: Price, /// Volume volume: Decimal, }, /// Status message Status { /// Status timestamp timestamp: HardwareTimestamp, /// Status message message: String, }, } // OrderBookAction is now imported from mbp10 module (line 23) // Removed duplicate definition to avoid conflict /// Performance metrics for DBN parser #[derive(Debug)] pub struct DbnParserMetrics { messages_parsed: AtomicU64, trades_processed: AtomicU64, quotes_processed: AtomicU64, orderbook_processed: AtomicU64, bars_processed: AtomicU64, unknown_messages: AtomicU64, event_errors: AtomicU64, parse_latency_sum_ns: AtomicU64, parse_latency_count: AtomicU64, per_tick_latency_sum_ns: AtomicU64, per_tick_latency_count: AtomicU64, vwap_sum: AtomicU64, // Store as u64 (scaled by 10000) vwap_count: AtomicU64, } impl DbnParserMetrics { pub fn new() -> Self { Self { messages_parsed: AtomicU64::new(0), trades_processed: AtomicU64::new(0), quotes_processed: AtomicU64::new(0), orderbook_processed: AtomicU64::new(0), bars_processed: AtomicU64::new(0), unknown_messages: AtomicU64::new(0), event_errors: AtomicU64::new(0), parse_latency_sum_ns: AtomicU64::new(0), parse_latency_count: AtomicU64::new(0), per_tick_latency_sum_ns: AtomicU64::new(0), per_tick_latency_count: AtomicU64::new(0), vwap_sum: AtomicU64::new(0), vwap_count: AtomicU64::new(0), } } pub fn increment_messages_parsed(&self) { self.messages_parsed.fetch_add(1, Ordering::Relaxed); } pub fn increment_trades_processed(&self) { self.trades_processed.fetch_add(1, Ordering::Relaxed); } pub fn increment_quotes_processed(&self) { self.quotes_processed.fetch_add(1, Ordering::Relaxed); } pub fn increment_orderbook_processed(&self) { self.orderbook_processed.fetch_add(1, Ordering::Relaxed); } pub fn increment_bars_processed(&self) { self.bars_processed.fetch_add(1, Ordering::Relaxed); } pub fn increment_unknown_messages(&self) { self.unknown_messages.fetch_add(1, Ordering::Relaxed); } pub fn increment_event_errors(&self) { self.event_errors.fetch_add(1, Ordering::Relaxed); } pub fn record_parse_latency(&self, latency_ns: u64) { self.parse_latency_sum_ns .fetch_add(latency_ns, Ordering::Relaxed); self.parse_latency_count.fetch_add(1, Ordering::Relaxed); } pub fn record_per_tick_latency(&self, latency_ns: u64) { self.per_tick_latency_sum_ns .fetch_add(latency_ns, Ordering::Relaxed); self.per_tick_latency_count.fetch_add(1, Ordering::Relaxed); } pub fn record_vwap(&self, vwap: f64) { let scaled_vwap = (vwap * 10000.0) as u64; self.vwap_sum.fetch_add(scaled_vwap, Ordering::Relaxed); self.vwap_count.fetch_add(1, Ordering::Relaxed); } pub fn get_snapshot(&self) -> DbnParserMetricsSnapshot { let parse_count = self.parse_latency_count.load(Ordering::Relaxed); let avg_parse_latency_ns = if parse_count > 0 { self.parse_latency_sum_ns.load(Ordering::Relaxed) / parse_count } else { 0 }; let tick_count = self.per_tick_latency_count.load(Ordering::Relaxed); let avg_per_tick_latency_ns = if tick_count > 0 { self.per_tick_latency_sum_ns.load(Ordering::Relaxed) / tick_count } else { 0 }; let vwap_count = self.vwap_count.load(Ordering::Relaxed); let avg_vwap = if vwap_count > 0 { (self.vwap_sum.load(Ordering::Relaxed) / vwap_count) as f64 / 10000.0 } else { 0.0 }; DbnParserMetricsSnapshot { messages_parsed: self.messages_parsed.load(Ordering::Relaxed), trades_processed: self.trades_processed.load(Ordering::Relaxed), quotes_processed: self.quotes_processed.load(Ordering::Relaxed), orderbook_processed: self.orderbook_processed.load(Ordering::Relaxed), bars_processed: self.bars_processed.load(Ordering::Relaxed), unknown_messages: self.unknown_messages.load(Ordering::Relaxed), event_errors: self.event_errors.load(Ordering::Relaxed), avg_parse_latency_ns, avg_per_tick_latency_ns, avg_vwap, } } } /// Snapshot of DBN parser metrics #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DbnParserMetricsSnapshot { pub messages_parsed: u64, pub trades_processed: u64, pub quotes_processed: u64, pub orderbook_processed: u64, pub bars_processed: u64, pub unknown_messages: u64, pub event_errors: u64, pub avg_parse_latency_ns: u64, pub avg_per_tick_latency_ns: u64, pub avg_vwap: f64, } #[cfg(test)] mod tests { use super::*; #[test] fn test_dbn_message_sizes() { // Verify packed struct sizes for zero-copy parsing // DbnMessageHeader: 16 bytes (2+1+1+4+8) assert_eq!(size_of::(), 16); // DbnTradeMessage: 38 bytes (header:16 + price:8 + size:4 + action:1 + side:1 + flags:2 + depth:1 + sequence:4 + padding:1) assert_eq!(size_of::(), 38); // DbnQuoteMessage: 50 bytes (header:16 + bid_px:8 + ask_px:8 + bid_sz:4 + ask_sz:4 + bid_ct:1 + ask_ct:1 + flags:2 + sequence:4 + padding:2) assert_eq!(size_of::(), 50); // DbnOrderBookMessage: 48 bytes assert_eq!(size_of::(), 48); // DbnOhlcvMessage: 56 bytes assert_eq!(size_of::(), 56); } #[test] fn test_dbn_parser_creation() { let parser = DbnParser::new(); assert!(parser.is_ok()); let parser = parser.unwrap(); let metrics = parser.get_metrics(); assert_eq!(metrics.messages_parsed, 0); } #[tokio::test] async fn test_symbol_mapping() { let parser = DbnParser::new().unwrap(); let mut mapping = std::collections::HashMap::new(); mapping.insert(1, "AAPL".to_string()); mapping.insert(2, "MSFT".to_string()); parser.update_symbol_map(mapping); assert_eq!(parser.get_symbol(1), "AAPL"); assert_eq!(parser.get_symbol(2), "MSFT"); assert_eq!(parser.get_symbol(999), "UNKNOWN_999"); } #[test] fn test_price_scaling() { let parser = DbnParser::new().unwrap(); let mut scales = std::collections::HashMap::new(); scales.insert(1, 4); // 4 decimal places scales.insert(2, 2); // 2 decimal places parser.update_price_scales(scales); let price1 = parser.scale_price(123450, 1).unwrap(); // 123450 / 10^4 = 12.3450 let price2 = parser.scale_price(12345, 2).unwrap(); // 12345 / 10^2 = 123.45 // Price::from_f64() stores values with 8 decimal places (multiplies by 100_000_000) assert_eq!(price1, Price::from_f64(12.3450).unwrap()); assert_eq!(price2, Price::from_f64(123.45).unwrap()); } }