foxhunt/data/examples/convert_nq_fut_to_parquet.rs

//! Convert NQ.FUT DBN file to Parquet format
//!
//! This example converts the Databento DBN file containing
//! NQ.FUT (E-mini NASDAQ 100 Futures) OHLCV data to Parquet format for ML training.
//!
//! Input: test_data/NQ_FUT_180d.dbn (4.10 MB, ~262,442 bars)
//! Output: test_data/NQ_FUT_180d.parquet
//!
//! Example usage:
//! ```bash
//! cargo run -p data --example convert_nq_fut_to_parquet --release
//! ```

use anyhow::Result;
use data::parquet_persistence::ParquetConfig;
use data::providers::databento::DbnToParquetConverter;

#[tokio::main]
async fn main() -> Result<()> {
    // Initialize tracing for logging
    tracing_subscriber::fmt()
        .with_env_filter(
            tracing_subscriber::EnvFilter::from_default_env()
                .add_directive(tracing::Level::INFO.into()),
        )
        .init();

    println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
    println!("  DBN to Parquet Converter - NQ.FUT 180d OHLCV");
    println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
    println!();

    // Configure Parquet output
    let config = ParquetConfig {
        base_path: "test_data".to_string(),
        batch_size: 10000,
        compression: parquet::basic::Compression::SNAPPY,
        ..Default::default()
    };

    println!("📁 Input:  test_data/NQ_FUT_180d_uncompressed.dbn");
    println!("📂 Output: test_data/NQ_FUT_180d.parquet");
    println!();

    // Create converter
    let mut converter = DbnToParquetConverter::new(config).await?;

    // Convert NQ.FUT file
    println!("⚙️  Converting DBN to Parquet...");
    let report = converter
        .convert_file("test_data/NQ_FUT_180d_uncompressed.dbn")
        .await?;

    println!();
    println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
    println!("  Conversion Results");
    println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
    println!();
    println!("✅ Events processed: {}", report.events_processed);
    println!("⏭️  Events skipped:   {}", report.events_skipped);
    println!("❌ Events failed:    {}", report.events_failed);
    println!("📈 Success rate:     {:.2}%", report.success_rate());
    println!("⏱️  Duration:         {:?}", report.duration);
    println!(
        "🚀 Throughput:       {} events/sec",
        report.throughput_events_per_sec
    );
    println!();

    if report.is_success() {
        println!("✅ SUCCESS! All events converted without errors.");
        println!();
        println!("📦 Output file ready for ML training:");
        println!("   test_data/NQ_FUT_180d.parquet");
    } else {
        println!(
            "⚠️  WARNING: Some events failed to convert ({} failures)",
            report.events_failed
        );
    }

    println!();
    println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");

    Ok(())
}