- Archive: 85 agent .txt files → docs/archive/agents/legacy_txt/ - Scripts: Move 110 shell scripts → scripts/ (keep deploy.sh in root) - Models: Move 18 .safetensors → ml/models/checkpoints/training_artifacts/ - Delete: 34 directories (~33GB freed) - target/, coverage_*, test artifacts - Build: Clean 14 build artifacts (.rlib, .o, .pid, binaries) - Tests: Move 14 .rs files → tests/standalone/ - SQL: Move 5 files → sql/ (keep init-db*.sql for Docker) - Wave 153: Archive to docs/archive/historical/wave153/ - Docs: Archive 9 markdown files to wave_d/reports/ and historical/ Total impact: ~34GB freed (both waves), root directory cleaned from 583 to ~40 essential files Directory count reduced from 65 to 31 (52% reduction) All historical data preserved in organized archive structure
74 lines
2.2 KiB
Rust
74 lines
2.2 KiB
Rust
use data::providers::databento::dbn_parser::{DbnParser, ProcessedMessage};
|
|
use std::collections::HashMap;
|
|
|
|
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
|
// Create parser
|
|
let parser = DbnParser::new()?;
|
|
|
|
// Configure symbol map
|
|
let mut symbol_map = HashMap::new();
|
|
symbol_map.insert(0, "6E.FUT".to_string());
|
|
symbol_map.insert(1, "6E.FUT".to_string());
|
|
parser.update_symbol_map(symbol_map);
|
|
|
|
// Configure price scales
|
|
let mut price_scales = HashMap::new();
|
|
price_scales.insert(0, 4);
|
|
price_scales.insert(1, 4);
|
|
parser.update_price_scales(price_scales);
|
|
|
|
// Read and parse file
|
|
let file_path = "test_data/real/databento/ml_training_small/6E.FUT_ohlcv-1m_2024-01-02.dbn";
|
|
let dbn_bytes = std::fs::read(file_path)?;
|
|
|
|
println!("File size: {} bytes", dbn_bytes.len());
|
|
|
|
let messages = parser.parse_batch(&dbn_bytes)?;
|
|
|
|
println!("Total messages: {}", messages.len());
|
|
|
|
// Count message types
|
|
let mut ohlcv_count = 0;
|
|
let mut trade_count = 0;
|
|
let mut quote_count = 0;
|
|
let mut other_count = 0;
|
|
|
|
for (i, msg) in messages.iter().enumerate() {
|
|
match msg {
|
|
ProcessedMessage::Ohlcv { open, high, low, close, volume, .. } => {
|
|
ohlcv_count += 1;
|
|
if i < 5 {
|
|
println!("OHLCV #{}: open={}, high={}, low={}, close={}, volume={}",
|
|
i+1, open, high, low, close, volume);
|
|
}
|
|
}
|
|
ProcessedMessage::Trade { .. } => {
|
|
trade_count += 1;
|
|
if i < 5 {
|
|
println!("Trade #{}", i+1);
|
|
}
|
|
}
|
|
ProcessedMessage::Quote { .. } => {
|
|
quote_count += 1;
|
|
if i < 5 {
|
|
println!("Quote #{}", i+1);
|
|
}
|
|
}
|
|
_ => {
|
|
other_count += 1;
|
|
if i < 5 {
|
|
println!("Other message type #{}", i+1);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
println!("\nMessage type breakdown:");
|
|
println!(" OHLCV: {}", ohlcv_count);
|
|
println!(" Trade: {}", trade_count);
|
|
println!(" Quote: {}", quote_count);
|
|
println!(" Other: {}", other_count);
|
|
|
|
Ok(())
|
|
}
|