foxhunt/data/examples/download_ml_training_data.rs

//! Download 90 days of real market data from Databento for ML training
//!
//! Downloads OHLCV-1m data for multiple futures symbols for ML model training.
//!
//! Symbols downloaded:
//! - ES.FUT (E-mini S&P 500) - Stock index
//! - NQ.FUT (E-mini NASDAQ) - Tech index
//! - ZN.FUT (10-Year Treasury) - Fixed income
//! - 6E.FUT (Euro FX) - Currency
//!
//! Usage:
//!     source .env && cargo run -p data --example download_ml_training_data

use chrono::{Datelike, NaiveDate};
use std::env;
use std::time::Duration;

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("================================================================================");
    println!("ML Training Data Download - Databento (Rust)");
    println!("================================================================================\n");

    // Load API key from environment
    let api_key = env::var("DATABENTO_API_KEY")
        .map_err(|_| "DATABENTO_API_KEY environment variable not set")?;

    println!(
        "✅ API Key found: {}...{}",
        &api_key[0..10],
        &api_key[api_key.len() - 10..]
    );
    println!();

    // Configuration
    let symbols = vec!["ES.FUT", "NQ.FUT", "ZN.FUT", "6E.FUT"];
    let dataset = "GLBX.MDP3";
    let schema = "ohlcv-1m";
    let start_date = NaiveDate::from_ymd_opt(2024, 1, 2).unwrap();
    let num_days = 90;

    // Generate trading dates (excluding weekends)
    let mut dates = Vec::new();
    let mut current = start_date;
    while dates.len() < num_days {
        if current.weekday().num_days_from_monday() < 5 {
            dates.push(current);
        }
        current = current.succ_opt().ok_or("Date overflow")?;
    }

    // Estimate cost ($0.12 per symbol per day)
    let estimated_cost = dates.len() as f64 * symbols.len() as f64 * 0.12;

    println!("📊 Download Configuration:");
    println!("  Start date: {}", start_date);
    println!("  Trading days: {}", dates.len());
    println!("  Symbols: {} ({})", symbols.len(), symbols.join(", "));
    println!("  Schema: {}", schema);
    println!("  Dataset: {}", dataset);
    println!("  Output: test_data/real/databento/ml_training/");
    println!();
    println!("📦 Total Downloads: {} files", dates.len() * symbols.len());
    println!("💰 Estimated Cost: ${:.2}", estimated_cost);
    println!();

    println!(
        "⚠️  This will download data and incur costs (~${:.2})",
        estimated_cost
    );
    println!("Press Ctrl+C to cancel, or press Enter to continue...");
    let mut input = String::new();
    std::io::stdin().read_line(&mut input)?;
    println!();

    // Create output directory
    std::fs::create_dir_all("test_data/real/databento/ml_training")?;
    println!("📁 Created output directory: test_data/real/databento/ml_training");
    println!();

    // Create HTTP client
    let client = reqwest::Client::builder()
        .timeout(Duration::from_secs(60))
        .build()?;

    println!("✅ HTTP client initialized");
    println!();

    // Track statistics
    let mut successful = 0;
    let mut failed = 0;
    let mut skipped = 0;
    let mut total_bytes = 0u64;
    let total_files = dates.len() * symbols.len();

    // Download all combinations
    let mut current_file = 0;

    for symbol in &symbols {
        println!("{:-<80}", "");
        println!("📥 Downloading: {}", symbol);
        println!("{:-<80}", "");
        println!();

        for date in &dates {
            current_file += 1;
            let progress = (current_file as f64 / total_files as f64) * 100.0;
            let date_str = date.format("%Y-%m-%d").to_string();

            print!(
                "[{}/{} - {:.1}%] {} @ {}... ",
                current_file, total_files, progress, symbol, date_str
            );
            std::io::Write::flush(&mut std::io::stdout())?;

            // Check if file already exists
            let output_path = format!(
                "test_data/real/databento/ml_training/{}_ohlcv-1m_{}.dbn",
                symbol.replace("/", "_"),
                date_str
            );

            if std::path::Path::new(&output_path).exists() {
                let size = std::fs::metadata(&output_path)?.len();
                successful += 1;
                total_bytes += size;
                println!("✅ {} KB (exists)", size / 1024);
                skipped += 1;
                continue;
            }

            // Build request URL
            let url = format!(
                "https://hist.databento.com/v0/timeseries.get_range?dataset={}&symbols={}&schema={}&start={}T00:00:00Z&end={}T23:59:59Z&encoding=dbn&stype_in=parent",
                dataset, symbol, schema, date_str, date_str
            );

            // Make request
            match client.get(&url).basic_auth(&api_key, Some("")).send().await {
                Ok(response) if response.status().is_success() => match response.bytes().await {
                    Ok(body) => {
                        let size = body.len() as u64;
                        std::fs::write(&output_path, &body)?;
                        successful += 1;
                        total_bytes += size;
                        println!("✅ {} KB", size / 1024);
                    },
                    Err(e) => {
                        failed += 1;
                        println!("❌ Error: {}", e);
                    },
                },
                Ok(response) => {
                    let status = response.status();
                    if status.as_u16() == 404 {
                        failed += 1;
                        println!("⚠️  No data (holiday/no trading)");
                    } else {
                        let error_text = response.text().await.unwrap_or_default();
                        failed += 1;
                        println!("❌ Error {}: {}", status, error_text);
                    }
                },
                Err(e) => {
                    failed += 1;
                    println!("❌ Network error: {}", e);
                },
            }

            // Small delay to avoid rate limits
            tokio::time::sleep(Duration::from_millis(100)).await;
        }
        println!();
    }

    // Summary
    println!();
    println!("================================================================================");
    println!("📊 DOWNLOAD SUMMARY");
    println!("================================================================================");
    println!();
    println!("✅ Successful: {}/{}", successful, total_files);
    println!("⏭️  Skipped:    {}/{}", skipped, total_files);
    println!("❌ Failed:     {}/{}", failed, total_files);
    println!();
    println!("💾 Total Size: {:.1} MB", total_bytes as f64 / 1_048_576.0);
    println!("💰 Estimated Cost: ${:.2}", estimated_cost);
    println!();

    let success_rate = (successful as f64 / total_files as f64) * 100.0;

    println!("📋 NEXT STEPS:");
    println!("1. Run ML readiness validation with new data:");
    println!("   cargo test -p ml --test ml_readiness_validation_tests");
    println!();
    println!("2. Run training time benchmarks:");
    println!("   cargo run -p ml --example benchmark_training_time --release");
    println!();

    if success_rate >= 80.0 {
        println!(
            "✅ SUCCESS: Downloaded {:.1}% of requested data!",
            success_rate
        );
        println!("   Ready for ML training benchmarks on RTX 3050 Ti");
    } else if success_rate >= 50.0 {
        println!(
            "⚠️  PARTIAL SUCCESS: Downloaded {:.1}% of data",
            success_rate
        );
        println!(
            "   May be sufficient for benchmarking, but consider re-downloading missing files"
        );
    } else {
        println!("❌ ERROR: Only downloaded {:.1}% of data", success_rate);
        println!("   Check errors above and retry");
    }

    Ok(())
}