e8a68ee39f
- Created data/examples/download_ml_training_data.rs using reqwest + Databento HTTP API - Downloaded 90 days × 4 symbols (ES.FUT, NQ.FUT, ZN.FUT, 6E.FUT) - Files saved to test_data/real/databento/ml_training/ - Total: 360 files, 15 MB compressed DBN format - Used existing Rust pattern from download_nq_fut.rs - API key loaded from .env file - 100% success rate (360/360 files) - Ready for ML training benchmarks Next: Create simplified training benchmark for RTX 3050 Ti GPU measurements
83 lines
3.5 KiB
Rust
83 lines
3.5 KiB
Rust
//! Convert ES.FUT DBN file to Parquet format
|
|
//!
|
|
//! This example demonstrates converting a real Databento DBN file containing
|
|
//! ES.FUT (E-mini S&P 500 Futures) OHLCV data to Parquet format for backtesting.
|
|
//!
|
|
//! Input: test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn (96 KB, ~390 bars)
|
|
//! Output: test_data/real/parquet/ES.FUT_ohlcv-1m_2024-01-02.parquet
|
|
//!
|
|
//! Schema: 11 columns (timestamp_ns, symbol, venue, event_type, price, quantity,
|
|
//! sequence, latency_ns, open, high, low)
|
|
//!
|
|
//! Example usage:
|
|
//! ```bash
|
|
//! cargo run --example convert_es_fut_to_parquet
|
|
//! ```
|
|
|
|
use anyhow::Result;
|
|
use data::providers::databento::DbnToParquetConverter;
|
|
use data::parquet_persistence::ParquetConfig;
|
|
|
|
#[tokio::main]
|
|
async fn main() -> Result<()> {
|
|
// Initialize tracing for logging
|
|
tracing_subscriber::fmt()
|
|
.with_env_filter(
|
|
tracing_subscriber::EnvFilter::from_default_env()
|
|
.add_directive(tracing::Level::INFO.into()),
|
|
)
|
|
.init();
|
|
|
|
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
|
|
println!(" DBN to Parquet Converter - ES.FUT OHLCV Example");
|
|
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
|
|
println!();
|
|
|
|
// Configure Parquet output
|
|
let config = ParquetConfig {
|
|
base_path: "test_data/real/parquet".to_string(),
|
|
batch_size: 10000,
|
|
..Default::default()
|
|
};
|
|
|
|
println!("📁 Input: test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn");
|
|
println!("📂 Output: test_data/real/parquet/");
|
|
println!();
|
|
|
|
// Create converter
|
|
let mut converter = DbnToParquetConverter::new(config).await?;
|
|
|
|
// Convert ES.FUT file
|
|
println!("⚙️ Converting DBN to Parquet...");
|
|
let report = converter
|
|
.convert_file("test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn")
|
|
.await?;
|
|
|
|
println!();
|
|
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
|
|
println!(" Conversion Results");
|
|
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
|
|
println!();
|
|
println!("✅ Events processed: {}", report.events_processed);
|
|
println!("⏭️ Events skipped: {}", report.events_skipped);
|
|
println!("❌ Events failed: {}", report.events_failed);
|
|
println!("📈 Success rate: {:.2}%", report.success_rate());
|
|
println!("⏱️ Duration: {:?}", report.duration);
|
|
println!("🚀 Throughput: {} events/sec", report.throughput_events_per_sec);
|
|
println!();
|
|
|
|
if report.is_success() {
|
|
println!("✅ SUCCESS! All events converted without errors.");
|
|
println!();
|
|
println!("📦 Output file ready for backtesting:");
|
|
println!(" test_data/real/parquet/ES.FUT_ohlcv-1m_2024-01-02.parquet");
|
|
} else {
|
|
println!("⚠️ WARNING: Some events failed to convert ({} failures)", report.events_failed);
|
|
}
|
|
|
|
println!();
|
|
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
|
|
|
|
Ok(())
|
|
}
|