foxhunt/ml/examples/convert_6e_parquet_simple.rs

//! Simple 6E.FUT DBN → Parquet converter using dbn crate directly
//!
//! Agent W12-09: Convert 6E_FUT_180d.dbn to Parquet format

use anyhow::{Context, Result};
use arrow::array::{Array, Float64Array, StringArray, TimestampNanosecondArray, UInt64Array};
use arrow::datatypes::{DataType, Field, Schema, TimeUnit};
use arrow::record_batch::RecordBatch;
use dbn::decode::{DbnDecoder, DbnMetadata, DecodeRecordRef};
use dbn::RecordRefEnum;
use parquet::arrow::ArrowWriter;
use parquet::file::properties::WriterProperties;
use std::fs::File;
use std::sync::Arc;

#[tokio::main]
async fn main() -> Result<()> {
    println!("=== 6E.FUT DBN → Parquet Conversion (dbn crate) ===\n");

    let input_path = "test_data/6E_FUT_180d_decompressed.dbn";
    let output_path = "test_data/6E_FUT_180d.parquet";

    println!("Input:  {}", input_path);
    println!("Output: {}", output_path);

    // Open and decode DBN file
    let file = File::open(input_path).with_context(|| format!("Failed to open {}", input_path))?;

    let mut decoder = DbnDecoder::new(file)?;
    let _metadata = decoder.metadata().clone();

    // Collect OHLCV bars
    let mut timestamps: Vec<i64> = Vec::new();
    let mut symbols = Vec::new();
    let mut opens = Vec::new();
    let mut highs = Vec::new();
    let mut lows = Vec::new();
    let mut closes = Vec::new();
    let mut volumes = Vec::new();

    let mut bar_count = 0;
    let start_time = std::time::Instant::now();

    while let Some(record) = decoder.decode_record_ref()? {
        if let Ok(record_enum) = record.as_enum() {
            if let RecordRefEnum::Ohlcv(ohlcv) = record_enum {
                bar_count += 1;

                timestamps.push(ohlcv.hd.ts_event as i64);
                symbols.push(format!("6E.FUT")); // Simplified
                opens.push(ohlcv.open as f64 / 1e9); // Convert fixed-point to float
                highs.push(ohlcv.high as f64 / 1e9);
                lows.push(ohlcv.low as f64 / 1e9);
                closes.push(ohlcv.close as f64 / 1e9);
                volumes.push(ohlcv.volume);
            }
        }
    }

    let duration = start_time.elapsed();

    println!("\n=== Parsing Results ===");
    println!("Bars parsed: {}", bar_count);
    println!("Duration: {:?}", duration);
    println!("Bars/sec: {:.0}", bar_count as f64 / duration.as_secs_f64());

    // Create Arrow schema
    let schema = Schema::new(vec![
        Field::new(
            "timestamp_ns",
            DataType::Timestamp(TimeUnit::Nanosecond, None),
            false,
        ),
        Field::new("symbol", DataType::Utf8, false),
        Field::new("open", DataType::Float64, false),
        Field::new("high", DataType::Float64, false),
        Field::new("low", DataType::Float64, false),
        Field::new("close", DataType::Float64, false),
        Field::new("volume", DataType::UInt64, false),
    ]);

    // Create Arrow arrays
    let timestamp_array = TimestampNanosecondArray::from(timestamps);
    let symbol_array = StringArray::from(symbols);
    let open_array = Float64Array::from(opens);
    let high_array = Float64Array::from(highs);
    let low_array = Float64Array::from(lows);
    let close_array = Float64Array::from(closes);
    let volume_array = UInt64Array::from(volumes);

    // Create record batch
    let batch = RecordBatch::try_new(
        Arc::new(schema.clone()),
        vec![
            Arc::new(timestamp_array) as Arc<dyn Array>,
            Arc::new(symbol_array) as Arc<dyn Array>,
            Arc::new(open_array) as Arc<dyn Array>,
            Arc::new(high_array) as Arc<dyn Array>,
            Arc::new(low_array) as Arc<dyn Array>,
            Arc::new(close_array) as Arc<dyn Array>,
            Arc::new(volume_array) as Arc<dyn Array>,
        ],
    )?;

    // Write Parquet file
    let output_file = File::create(output_path)?;
    let props = WriterProperties::builder()
        .set_compression(parquet::basic::Compression::SNAPPY)
        .build();

    let mut writer = ArrowWriter::try_new(output_file, Arc::new(schema), Some(props))?;
    writer.write(&batch)?;
    writer.close()?;

    println!("\n=== Parquet Output ===");
    let metadata = std::fs::metadata(output_path)?;
    let file_size_mb = metadata.len() as f64 / (1024.0 * 1024.0);
    println!("File: {}", output_path);
    println!("Size: {:.2} MB", file_size_mb);
    println!("Rows: {}", bar_count);

    // Calculate compression ratio
    let input_metadata = std::fs::metadata(input_path)?;
    let input_size_mb = input_metadata.len() as f64 / (1024.0 * 1024.0);
    let compression_ratio = (1.0 - (file_size_mb / input_size_mb)) * 100.0;
    println!("Input size: {:.2} MB", input_size_mb);
    println!("Compression: {:.1}%", compression_ratio);

    println!("\n✅ Conversion complete!");

    Ok(())
}