foxhunt/data/examples/convert_es_fut_to_parquet.rs

//! Convert ES.FUT DBN file to Parquet format
//!
//! This example demonstrates converting a real Databento DBN file containing
//! ES.FUT (E-mini S&P 500 Futures) OHLCV data to Parquet format for backtesting.
//!
//! Input: test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn (96 KB, ~390 bars)
//! Output: test_data/real/parquet/ES.FUT_ohlcv-1m_2024-01-02.parquet
//!
//! Schema: 11 columns (timestamp_ns, symbol, venue, event_type, price, quantity,
//!         sequence, latency_ns, open, high, low)
//!
//! Example usage:
//! ```bash
//! cargo run --example convert_es_fut_to_parquet
//! ```

use anyhow::Result;
use data::parquet_persistence::ParquetConfig;
use data::providers::databento::DbnToParquetConverter;

#[tokio::main]
async fn main() -> Result<()> {
    // Initialize tracing for logging
    tracing_subscriber::fmt()
        .with_env_filter(
            tracing_subscriber::EnvFilter::from_default_env()
                .add_directive(tracing::Level::INFO.into()),
        )
        .init();

    println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
    println!("  DBN to Parquet Converter - ES.FUT OHLCV Example");
    println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
    println!();

    // Configure Parquet output
    let config = ParquetConfig {
        base_path: "test_data/real/parquet".to_string(),
        batch_size: 10000,
        ..Default::default()
    };

    println!("📁 Input:  test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn");
    println!("📂 Output: test_data/real/parquet/");
    println!();

    // Create converter
    let mut converter = DbnToParquetConverter::new(config).await?;

    // Convert ES.FUT file
    println!("⚙️  Converting DBN to Parquet...");
    let report = converter
        .convert_file("test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn")
        .await?;

    println!();
    println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
    println!("  Conversion Results");
    println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
    println!();
    println!("✅ Events processed: {}", report.events_processed);
    println!("⏭️  Events skipped:   {}", report.events_skipped);
    println!("❌ Events failed:    {}", report.events_failed);
    println!("📈 Success rate:     {:.2}%", report.success_rate());
    println!("⏱️  Duration:         {:?}", report.duration);
    println!(
        "🚀 Throughput:       {} events/sec",
        report.throughput_events_per_sec
    );
    println!();

    if report.is_success() {
        println!("✅ SUCCESS! All events converted without errors.");
        println!();
        println!("📦 Output file ready for backtesting:");
        println!("   test_data/real/parquet/ES.FUT_ohlcv-1m_2024-01-02.parquet");
    } else {
        println!(
            "⚠️  WARNING: Some events failed to convert ({} failures)",
            report.events_failed
        );
    }

    println!();
    println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");

    Ok(())
}