foxhunt/data/examples/convert_dbn_to_parquet.rs

//! CLI Tool: DBN to Parquet Converter
//!
//! Command-line tool for converting Databento binary format (DBN) files to Parquet format.
//!
//! ## Usage
//!
//! ```bash
//! # Convert a single DBN file
//! cargo run --example convert_dbn_to_parquet -- \
//!     --input test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn \
//!     --output market_data/converted
//!
//! # With custom configuration
//! cargo run --example convert_dbn_to_parquet -- \
//!     --input data.dbn \
//!     --output ./parquet \
//!     --batch-size 5000 \
//!     --compression snappy
//! ```

use anyhow::{Context, Result};
use clap::Parser;
use data::parquet_persistence::ParquetConfig;
use data::providers::databento::{ConversionReport, DbnToParquetConverter};
use std::path::PathBuf;
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};

#[derive(Parser, Debug)]
#[clap(name = "dbn-to-parquet")]
#[clap(about = "Convert Databento DBN files to Parquet format", long_about = None)]
struct Args {
    /// Input DBN file path
    #[clap(short, long)]
    input: PathBuf,

    /// Output directory for Parquet files
    #[clap(short, long, default_value = "./market_data")]
    output: PathBuf,

    /// Batch size for processing (events per batch)
    #[clap(short, long, default_value_t = 10000)]
    batch_size: usize,

    /// Compression algorithm (snappy, gzip, lz4, zstd, none)
    #[clap(short, long, default_value = "snappy")]
    compression: String,

    /// Enable verbose logging
    #[clap(short, long)]
    verbose: bool,
}

#[tokio::main]
async fn main() -> Result<()> {
    let args = Args::parse();

    // Initialize tracing
    let log_level = if args.verbose {
        tracing::Level::DEBUG
    } else {
        tracing::Level::INFO
    };

    tracing_subscriber::registry()
        .with(
            tracing_subscriber::fmt::layer()
                .with_target(false)
                .with_level(true),
        )
        .with(
            tracing_subscriber::filter::LevelFilter::from_level(log_level),
        )
        .init();

    // Validate input file exists
    if !args.input.exists() {
        anyhow::bail!("Input file does not exist: {:?}", args.input);
    }

    // Create output directory if needed
    if !args.output.exists() {
        std::fs::create_dir_all(&args.output)
            .with_context(|| format!("Failed to create output directory: {:?}", args.output))?;
    }

    // Parse compression algorithm
    let compression = match args.compression.to_lowercase().as_str() {
        "snappy" => parquet::basic::Compression::SNAPPY,
        "gzip" => parquet::basic::Compression::GZIP(parquet::basic::GzipLevel::default()),
        "lz4" => parquet::basic::Compression::LZ4,
        "zstd" => parquet::basic::Compression::ZSTD(parquet::basic::ZstdLevel::default()),
        "none" => parquet::basic::Compression::UNCOMPRESSED,
        other => anyhow::bail!("Unknown compression algorithm: {}", other),
    };

    // Configure Parquet writer
    let config = ParquetConfig {
        base_path: args.output.to_string_lossy().to_string(),
        batch_size: args.batch_size,
        compression,
        ..Default::default()
    };

    // Create converter
    tracing::info!("Initializing DBN to Parquet converter...");
    let mut converter = DbnToParquetConverter::new(config).await?;

    // Convert file
    tracing::info!("Converting {:?}...", args.input);
    let report = converter.convert_file(&args.input).await?;

    // Print results
    print_report(&report);

    if !report.is_success() {
        anyhow::bail!("Conversion completed with {} errors", report.events_failed);
    }

    tracing::info!("✓ Conversion successful!");
    Ok(())
}

fn print_report(report: &ConversionReport) {
    println!("\n{}", "=".repeat(60));
    println!("Conversion Report");
    println!("{}", "=".repeat(60));
    println!("Events processed:    {}", report.events_processed);
    println!("Events skipped:      {}", report.events_skipped);
    println!("Events failed:       {}", report.events_failed);
    println!("Duration:            {:?}", report.duration);
    println!("Throughput:          {} events/sec", report.throughput_events_per_sec);
    println!("Success rate:        {:.2}%", report.success_rate());
    println!("{}", "=".repeat(60));
}