//! CLI Tool: DBN to Parquet Converter //! //! Command-line tool for converting Databento binary format (DBN) files to Parquet format. //! //! ## Usage //! //! ```bash //! # Convert a single DBN file //! cargo run --example convert_dbn_to_parquet -- \ //! --input test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn \ //! --output market_data/converted //! //! # With custom configuration //! cargo run --example convert_dbn_to_parquet -- \ //! --input data.dbn \ //! --output ./parquet \ //! --batch-size 5000 \ //! --compression snappy //! ``` use anyhow::{Context, Result}; use clap::Parser; use data::parquet_persistence::ParquetConfig; use data::providers::databento::{ConversionReport, DbnToParquetConverter}; use std::path::PathBuf; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; #[derive(Parser, Debug)] #[clap(name = "dbn-to-parquet")] #[clap(about = "Convert Databento DBN files to Parquet format", long_about = None)] struct Args { /// Input DBN file path #[clap(short, long)] input: PathBuf, /// Output directory for Parquet files #[clap(short, long, default_value = "./market_data")] output: PathBuf, /// Batch size for processing (events per batch) #[clap(short, long, default_value_t = 10000)] batch_size: usize, /// Compression algorithm (snappy, gzip, lz4, zstd, none) #[clap(short, long, default_value = "snappy")] compression: String, /// Enable verbose logging #[clap(short, long)] verbose: bool, } #[tokio::main] async fn main() -> Result<()> { let args = Args::parse(); // Initialize tracing let log_level = if args.verbose { tracing::Level::DEBUG } else { tracing::Level::INFO }; tracing_subscriber::registry() .with( tracing_subscriber::fmt::layer() .with_target(false) .with_level(true), ) .with( tracing_subscriber::filter::LevelFilter::from_level(log_level), ) .init(); // Validate input file exists if !args.input.exists() { anyhow::bail!("Input file does not exist: {:?}", args.input); } // Create output directory if needed if !args.output.exists() { std::fs::create_dir_all(&args.output) .with_context(|| format!("Failed to create output directory: {:?}", args.output))?; } // Parse compression algorithm let compression = match args.compression.to_lowercase().as_str() { "snappy" => parquet::basic::Compression::SNAPPY, "gzip" => parquet::basic::Compression::GZIP(parquet::basic::GzipLevel::default()), "lz4" => parquet::basic::Compression::LZ4, "zstd" => parquet::basic::Compression::ZSTD(parquet::basic::ZstdLevel::default()), "none" => parquet::basic::Compression::UNCOMPRESSED, other => anyhow::bail!("Unknown compression algorithm: {}", other), }; // Configure Parquet writer let config = ParquetConfig { base_path: args.output.to_string_lossy().to_string(), batch_size: args.batch_size, compression, ..Default::default() }; // Create converter tracing::info!("Initializing DBN to Parquet converter..."); let mut converter = DbnToParquetConverter::new(config).await?; // Convert file tracing::info!("Converting {:?}...", args.input); let report = converter.convert_file(&args.input).await?; // Print results print_report(&report); if !report.is_success() { anyhow::bail!("Conversion completed with {} errors", report.events_failed); } tracing::info!("✓ Conversion successful!"); Ok(()) } fn print_report(report: &ConversionReport) { println!("\n{}", "=".repeat(60)); println!("Conversion Report"); println!("{}", "=".repeat(60)); println!("Events processed: {}", report.events_processed); println!("Events skipped: {}", report.events_skipped); println!("Events failed: {}", report.events_failed); println!("Duration: {:?}", report.duration); println!("Throughput: {} events/sec", report.throughput_events_per_sec); println!("Success rate: {:.2}%", report.success_rate()); println!("{}", "=".repeat(60)); }