// Wave 16S-P3: DBN Raw Price Inspector // Investigates 60.7% data corruption by examining raw i64 values from DBN files // Hypothesis: 1e-9 scaling is incorrect, prices might use different scaling factor use dbn::decode::dbn::Decoder; use dbn::decode::{DbnMetadata, DecodeRecordRef}; use std::fs::File; use std::io::BufReader; fn main() -> Result<(), Box> { let files = vec![ "test_data/ES_FUT_180d.dbn", "test_data/6E_FUT_180d.dbn", "test_data/ZN_FUT_180d.dbn", ]; for file_path in files { println!("\n{}", "=".repeat(80)); println!("📂 File: {}", file_path); println!("{}", "=".repeat(80)); let file = File::open(file_path)?; let reader = BufReader::new(file); let mut decoder = Decoder::new(reader)?; let metadata = decoder.metadata(); println!("Metadata:"); println!(" Dataset: {:?}", metadata.dataset); println!(" Schema: {:?}", metadata.schema); println!(" Symbols: {:?}", metadata.symbols); println!(" SType In: {:?}", metadata.stype_in); println!(" SType Out: {:?}", metadata.stype_out); let mut count = 0; let mut prices_1e9 = Vec::new(); let mut prices_1e_minus_9 = Vec::new(); let mut prices_raw = Vec::new(); loop { match decoder.decode_record_ref() { Ok(Some(record)) => { count += 1; let record_enum = record.as_enum()?; if let dbn::RecordRefEnum::Ohlcv(ohlcv) = record_enum { let close_raw = ohlcv.close; let close_1e9 = close_raw as f64 * 1e-9; let close_1e_minus_9 = close_raw as f64 / 1e9; // Alternative interpretation if count <= 5 { println!("\nRecord #{}", count); println!(" Raw i64: {:20} (0x{:016X})", close_raw, close_raw as u64); println!(" × 1e-9: {:20.10} (current implementation)", close_1e9); println!(" ÷ 1e9: {:20.10} (alternative)", close_1e_minus_9); println!(" As cents: {:20.2} (close_raw / 100.0)", close_raw as f64 / 100.0); println!(" As ticks: {:20.2} (close_raw / 4.0)", close_raw as f64 / 4.0); } prices_raw.push(close_raw); prices_1e9.push(close_1e9); prices_1e_minus_9.push(close_1e_minus_9); if count >= 100 { break; } } } Ok(None) => break, Err(e) => return Err(e.into()), } } if !prices_1e9.is_empty() { println!("\n📊 Statistics (first 100 OHLCV bars):"); println!(" Raw i64 range: {} to {}", prices_raw.iter().min().unwrap(), prices_raw.iter().max().unwrap()); println!(" × 1e-9 range: {:.6} to {:.6}", prices_1e9.iter().cloned().fold(f64::INFINITY, f64::min), prices_1e9.iter().cloned().fold(f64::NEG_INFINITY, f64::max)); println!(" ÷ 1e9 range: {:.2} to {:.2}", prices_1e_minus_9.iter().cloned().fold(f64::INFINITY, f64::min), prices_1e_minus_9.iter().cloned().fold(f64::NEG_INFINITY, f64::max)); // Check for negative raw values let negative_count = prices_raw.iter().filter(|&&x| x < 0).count(); if negative_count > 0 { println!("\n ⚠️ WARNING: {} negative raw i64 values found! ({:.1}%)", negative_count, (negative_count as f64 / prices_raw.len() as f64) * 100.0); } } } Ok(()) }