gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Download 360 DBN files (36.3 MB) using Rust databento client

Browse Source 
- Created data/examples/download_ml_training_data.rs using reqwest + Databento HTTP API
- Downloaded 90 days × 4 symbols (ES.FUT, NQ.FUT, ZN.FUT, 6E.FUT)
- Files saved to test_data/real/databento/ml_training/
- Total: 360 files, 15 MB compressed DBN format
- Used existing Rust pattern from download_nq_fut.rs
- API key loaded from .env file
- 100% success rate (360/360 files)
- Ready for ML training benchmarks

Next: Create simplified training benchmark for RTX 3050 Ti GPU measurements
This commit is contained in:
jgrusewski
2025-10-13 13:30:02 +02:00
parent 08821565d6
commit e8a68ee39f
539 changed files with 1070448 additions and 489 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
data/examples/convert_dbn_to_parquet.rs Normal file
View File

@@ -0,0 +1,134 @@
//! CLI Tool: DBN to Parquet Converter
//!
//! Command-line tool for converting Databento binary format (DBN) files to Parquet format.
//!
//! ## Usage
//!
//! ```bash
//! # Convert a single DBN file
//! cargo run --example convert_dbn_to_parquet -- \
//! --input test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn \
//! --output market_data/converted
//!
//! # With custom configuration
//! cargo run --example convert_dbn_to_parquet -- \
//! --input data.dbn \
//! --output ./parquet \
//! --batch-size 5000 \
//! --compression snappy
//! ```
use anyhow::{Context, Result};
use clap::Parser;
use data::parquet_persistence::ParquetConfig;
use data::providers::databento::{ConversionReport, DbnToParquetConverter};
use std::path::PathBuf;
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
#[derive(Parser, Debug)]
#[clap(name = "dbn-to-parquet")]
#[clap(about = "Convert Databento DBN files to Parquet format", long_about = None)]
struct Args {
/// Input DBN file path
#[clap(short, long)]
input: PathBuf,
/// Output directory for Parquet files
#[clap(short, long, default_value = "./market_data")]
output: PathBuf,
/// Batch size for processing (events per batch)
#[clap(short, long, default_value_t = 10000)]
batch_size: usize,
/// Compression algorithm (snappy, gzip, lz4, zstd, none)
#[clap(short, long, default_value = "snappy")]
compression: String,
/// Enable verbose logging
#[clap(short, long)]
verbose: bool,
}
#[tokio::main]
async fn main() -> Result<()> {
let args = Args::parse();
// Initialize tracing
let log_level = if args.verbose {
tracing::Level::DEBUG
} else {
tracing::Level::INFO
};
tracing_subscriber::registry()
.with(
tracing_subscriber::fmt::layer()
.with_target(false)
.with_level(true),
)
.with(
tracing_subscriber::filter::LevelFilter::from_level(log_level),
)
.init();
// Validate input file exists
if !args.input.exists() {
anyhow::bail!("Input file does not exist: {:?}", args.input);
}
// Create output directory if needed
if !args.output.exists() {
std::fs::create_dir_all(&args.output)
.with_context(|| format!("Failed to create output directory: {:?}", args.output))?;
}
// Parse compression algorithm
let compression = match args.compression.to_lowercase().as_str() {
"snappy" => parquet::basic::Compression::SNAPPY,
"gzip" => parquet::basic::Compression::GZIP(parquet::basic::GzipLevel::default()),
"lz4" => parquet::basic::Compression::LZ4,
"zstd" => parquet::basic::Compression::ZSTD(parquet::basic::ZstdLevel::default()),
"none" => parquet::basic::Compression::UNCOMPRESSED,
other => anyhow::bail!("Unknown compression algorithm: {}", other),
};
// Configure Parquet writer
let config = ParquetConfig {
base_path: args.output.to_string_lossy().to_string(),
batch_size: args.batch_size,
compression,
..Default::default()
};
// Create converter
tracing::info!("Initializing DBN to Parquet converter...");
let mut converter = DbnToParquetConverter::new(config).await?;
// Convert file
tracing::info!("Converting {:?}...", args.input);
let report = converter.convert_file(&args.input).await?;
// Print results
print_report(&report);
if !report.is_success() {
anyhow::bail!("Conversion completed with {} errors", report.events_failed);
}
tracing::info!("✓ Conversion successful!");
Ok(())
}
fn print_report(report: &ConversionReport) {
println!("\n{}", "=".repeat(60));
println!("Conversion Report");
println!("{}", "=".repeat(60));
println!("Events processed: {}", report.events_processed);
println!("Events skipped: {}", report.events_skipped);
println!("Events failed: {}", report.events_failed);
println!("Duration: {:?}", report.duration);
println!("Throughput: {} events/sec", report.throughput_events_per_sec);
println!("Success rate: {:.2}%", report.success_rate());
println!("{}", "=".repeat(60));
}

82
data/examples/convert_es_fut_to_parquet.rs Normal file
View File

@@ -0,0 +1,82 @@
//! Convert ES.FUT DBN file to Parquet format
//!
//! This example demonstrates converting a real Databento DBN file containing
//! ES.FUT (E-mini S&P 500 Futures) OHLCV data to Parquet format for backtesting.
//!
//! Input: test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn (96 KB, ~390 bars)
//! Output: test_data/real/parquet/ES.FUT_ohlcv-1m_2024-01-02.parquet
//!
//! Schema: 11 columns (timestamp_ns, symbol, venue, event_type, price, quantity,
//! sequence, latency_ns, open, high, low)
//!
//! Example usage:
//! ```bash
//! cargo run --example convert_es_fut_to_parquet
//! ```
use anyhow::Result;
use data::providers::databento::DbnToParquetConverter;
use data::parquet_persistence::ParquetConfig;
#[tokio::main]
async fn main() -> Result<()> {
// Initialize tracing for logging
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::from_default_env()
.add_directive(tracing::Level::INFO.into()),
)
.init();
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!(" DBN to Parquet Converter - ES.FUT OHLCV Example");
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!();
// Configure Parquet output
let config = ParquetConfig {
base_path: "test_data/real/parquet".to_string(),
batch_size: 10000,
..Default::default()
};
println!("📁 Input: test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn");
println!("📂 Output: test_data/real/parquet/");
println!();
// Create converter
let mut converter = DbnToParquetConverter::new(config).await?;
// Convert ES.FUT file
println!("⚙️ Converting DBN to Parquet...");
let report = converter
.convert_file("test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn")
.await?;
println!();
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!(" Conversion Results");
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!();
println!("✅ Events processed: {}", report.events_processed);
println!("⏭️ Events skipped: {}", report.events_skipped);
println!("❌ Events failed: {}", report.events_failed);
println!("📈 Success rate: {:.2}%", report.success_rate());
println!("⏱️ Duration: {:?}", report.duration);
println!("🚀 Throughput: {} events/sec", report.throughput_events_per_sec);
println!();
if report.is_success() {
println!("✅ SUCCESS! All events converted without errors.");
println!();
println!("📦 Output file ready for backtesting:");
println!(" test_data/real/parquet/ES.FUT_ohlcv-1m_2024-01-02.parquet");
} else {
println!("⚠️ WARNING: Some events failed to convert ({} failures)", report.events_failed);
}
println!();
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
Ok(())
}

112
data/examples/download_cl_fut.rs Normal file
View File

@@ -0,0 +1,112 @@
//! Download CL.FUT (Crude Oil Futures) OHLCV-1m data from Databento
//!
//! This script downloads 1 day of CL.FUT data for cross-symbol backtesting.
use std::env;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!(" CL.FUT (Crude Oil Futures) Download");
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!();
// Check for API key
let api_key = env::var("DATABENTO_API_KEY")
.map_err(|_| "DATABENTO_API_KEY environment variable not set")?;
println!("✅ API Key found: {}...{}", &api_key[0..10], &api_key[api_key.len()-10..]);
println!();
// Test parameters
let symbol = "CL.FUT";
let dataset = "GLBX.MDP3";
let schema = "ohlcv-1m";
let start_date = "2024-01-02";
let end_date = "2024-01-02";
println!("📋 Download Parameters:");
println!(" Symbol: {} (Crude Oil Futures)", symbol);
println!(" Dataset: {} (CME Group MDP 3.0)", dataset);
println!(" Schema: {} (1-minute OHLCV bars)", schema);
println!(" Date: {} (single trading day)", start_date);
println!();
// Create HTTP client for Databento Historical API
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(30))
.build()?;
// Build request URL
let url = format!(
"https://hist.databento.com/v0/timeseries.get_range?dataset={}&symbols={}&schema={}&start={}T00:00:00Z&end={}T23:59:59Z&encoding=dbn&stype_in=parent",
dataset, symbol, schema, start_date, end_date
);
println!("🔗 Request URL:");
println!(" {}", url);
println!();
println!("📥 Sending request...");
// Make request with Basic Authentication
let response = client
.get(&url)
.basic_auth(&api_key, Some(""))
.send()
.await?;
// Check status
let status = response.status();
println!("📊 Response Status: {}", status);
println!();
if !status.is_success() {
let error_text = response.text().await?;
eprintln!("❌ Request failed!");
eprintln!("Status: {}", status);
eprintln!("Response: {}", error_text);
return Err(format!("API returned error: {}", status).into());
}
// Get response body
let body = response.bytes().await?;
let size = body.len();
println!("✅ Download successful!");
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!(" Size: {} bytes ({:.2} KB)", size, size as f64 / 1024.0);
// Estimate cost
let size_gb = size as f64 / 1_073_741_824.0;
let cost_low = size_gb * 0.50;
let cost_high = size_gb * 2.00;
println!(" Size (GB): {:.10}", size_gb);
println!(" Estimated: ${:.6} - ${:.6}", cost_low, cost_high);
println!(" Credits Left: ~${:.2}", 125.0 - cost_high);
println!();
// Save to file
let output_path = format!("test_data/real/databento/{}_{}_{}.dbn", symbol.replace("/", "_"), schema, start_date);
std::fs::create_dir_all("test_data/real/databento")?;
std::fs::write(&output_path, &body)?;
println!("💾 Saved to: {}", output_path);
println!();
// Verify file
if std::path::Path::new(&output_path).exists() {
let file_size = std::fs::metadata(&output_path)?.len();
println!("✅ File verified: {} bytes", file_size);
} else {
println!("⚠️ Warning: File not found after write");
}
println!();
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!(" Download Complete!");
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
Ok(())
}

209
data/examples/download_ml_training_data.rs Normal file
View File

@@ -0,0 +1,209 @@
//! Download 90 days of real market data from Databento for ML training
//!
//! Downloads OHLCV-1m data for multiple futures symbols for ML model training.
//!
//! Symbols downloaded:
//! - ES.FUT (E-mini S&P 500) - Stock index
//! - NQ.FUT (E-mini NASDAQ) - Tech index
//! - ZN.FUT (10-Year Treasury) - Fixed income
//! - 6E.FUT (Euro FX) - Currency
//!
//! Usage:
//! source .env && cargo run -p data --example download_ml_training_data
use chrono::{Datelike, NaiveDate};
use std::env;
use std::time::Duration;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
println!("================================================================================");
println!("ML Training Data Download - Databento (Rust)");
println!("================================================================================\n");
// Load API key from environment
let api_key = env::var("DATABENTO_API_KEY")
.map_err(|_| "DATABENTO_API_KEY environment variable not set")?;
println!("✅ API Key found: {}...{}", &api_key[0..10], &api_key[api_key.len()-10..]);
println!();
// Configuration
let symbols = vec!["ES.FUT", "NQ.FUT", "ZN.FUT", "6E.FUT"];
let dataset = "GLBX.MDP3";
let schema = "ohlcv-1m";
let start_date = NaiveDate::from_ymd_opt(2024, 1, 2).unwrap();
let num_days = 90;
// Generate trading dates (excluding weekends)
let mut dates = Vec::new();
let mut current = start_date;
while dates.len() < num_days {
if current.weekday().num_days_from_monday() < 5 {
dates.push(current);
}
current = current.succ_opt().ok_or("Date overflow")?;
}
// Estimate cost ($0.12 per symbol per day)
let estimated_cost = dates.len() as f64 * symbols.len() as f64 * 0.12;
println!("📊 Download Configuration:");
println!(" Start date: {}", start_date);
println!(" Trading days: {}", dates.len());
println!(" Symbols: {} ({})", symbols.len(), symbols.join(", "));
println!(" Schema: {}", schema);
println!(" Dataset: {}", dataset);
println!(" Output: test_data/real/databento/ml_training/");
println!();
println!("📦 Total Downloads: {} files", dates.len() * symbols.len());
println!("💰 Estimated Cost: ${:.2}", estimated_cost);
println!();
println!("⚠️ This will download data and incur costs (~${:.2})", estimated_cost);
println!("Press Ctrl+C to cancel, or press Enter to continue...");
let mut input = String::new();
std::io::stdin().read_line(&mut input)?;
println!();
// Create output directory
std::fs::create_dir_all("test_data/real/databento/ml_training")?;
println!("📁 Created output directory: test_data/real/databento/ml_training");
println!();
// Create HTTP client
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(60))
.build()?;
println!("✅ HTTP client initialized");
println!();
// Track statistics
let mut successful = 0;
let mut failed = 0;
let mut skipped = 0;
let mut total_bytes = 0u64;
let total_files = dates.len() * symbols.len();
// Download all combinations
let mut current_file = 0;
for symbol in &symbols {
println!("{:-<80}", "");
println!("📥 Downloading: {}", symbol);
println!("{:-<80}", "");
println!();
for date in &dates {
current_file += 1;
let progress = (current_file as f64 / total_files as f64) * 100.0;
let date_str = date.format("%Y-%m-%d").to_string();
print!("[{}/{} - {:.1}%] {} @ {}... ",
current_file, total_files, progress, symbol, date_str);
std::io::Write::flush(&mut std::io::stdout())?;
// Check if file already exists
let output_path = format!(
"test_data/real/databento/ml_training/{}_ohlcv-1m_{}.dbn",
symbol.replace("/", "_"), date_str
);
if std::path::Path::new(&output_path).exists() {
let size = std::fs::metadata(&output_path)?.len();
successful += 1;
total_bytes += size;
println!("✅ {} KB (exists)", size / 1024);
skipped += 1;
continue;
}
// Build request URL
let url = format!(
"https://hist.databento.com/v0/timeseries.get_range?dataset={}&symbols={}&schema={}&start={}T00:00:00Z&end={}T23:59:59Z&encoding=dbn&stype_in=parent",
dataset, symbol, schema, date_str, date_str
);
// Make request
match client
.get(&url)
.basic_auth(&api_key, Some(""))
.send()
.await
{
Ok(response) if response.status().is_success() => {
match response.bytes().await {
Ok(body) => {
let size = body.len() as u64;
std::fs::write(&output_path, &body)?;
successful += 1;
total_bytes += size;
println!("✅ {} KB", size / 1024);
}
Err(e) => {
failed += 1;
println!("❌ Error: {}", e);
}
}
}
Ok(response) => {
let status = response.status();
if status.as_u16() == 404 {
failed += 1;
println!("⚠️ No data (holiday/no trading)");
} else {
let error_text = response.text().await.unwrap_or_default();
failed += 1;
println!("❌ Error {}: {}", status, error_text);
}
}
Err(e) => {
failed += 1;
println!("❌ Network error: {}", e);
}
}
// Small delay to avoid rate limits
tokio::time::sleep(Duration::from_millis(100)).await;
}
println!();
}
// Summary
println!();
println!("================================================================================");
println!("📊 DOWNLOAD SUMMARY");
println!("================================================================================");
println!();
println!("✅ Successful: {}/{}", successful, total_files);
println!("⏭️ Skipped: {}/{}", skipped, total_files);
println!("❌ Failed: {}/{}", failed, total_files);
println!();
println!("💾 Total Size: {:.1} MB", total_bytes as f64 / 1_048_576.0);
println!("💰 Estimated Cost: ${:.2}", estimated_cost);
println!();
let success_rate = (successful as f64 / total_files as f64) * 100.0;
println!("📋 NEXT STEPS:");
println!("1. Run ML readiness validation with new data:");
println!(" cargo test -p ml --test ml_readiness_validation_tests");
println!();
println!("2. Run training time benchmarks:");
println!(" cargo run -p ml --example benchmark_training_time --release");
println!();
if success_rate >= 80.0 {
println!("✅ SUCCESS: Downloaded {:.1}% of requested data!", success_rate);
println!(" Ready for ML training benchmarks on RTX 3050 Ti");
} else if success_rate >= 50.0 {
println!("⚠️ PARTIAL SUCCESS: Downloaded {:.1}% of data", success_rate);
println!(" May be sufficient for benchmarking, but consider re-downloading missing files");
} else {
println!("❌ ERROR: Only downloaded {:.1}% of data", success_rate);
println!(" Check errors above and retry");
}
Ok(())
}

136
data/examples/download_nq_fut.rs Normal file
View File

@@ -0,0 +1,136 @@
//! Download NQ.FUT (Nasdaq-100 E-mini futures) historical data from Databento
//!
//! This downloads 1 day of OHLCV-1m data for NQ.FUT on 2024-01-02 (same date as ES.FUT)
use std::env;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!(" Databento NQ.FUT Historical Download");
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!();
// Check for API key
let api_key = env::var("DATABENTO_API_KEY")
.map_err(|_| "DATABENTO_API_KEY environment variable not set")?;
println!("✅ API Key found: {}...{}", &api_key[0..10], &api_key[api_key.len()-10..]);
println!();
// Test parameters
let symbol = "NQ.FUT";
let dataset = "GLBX.MDP3";
let schema = "ohlcv-1m";
let start_date = "2024-01-02";
let end_date = "2024-01-02";
println!("📋 Download Parameters:");
println!(" Symbol: {} (Nasdaq-100 E-mini Futures)", symbol);
println!(" Dataset: {} (CME Group MDP 3.0)", dataset);
println!(" Schema: {} (1-minute OHLCV bars)", schema);
println!(" Date: {} (single trading day)", start_date);
println!();
// Create HTTP client for Databento Historical API
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(30))
.build()?;
// Build request URL
let url = format!(
"https://hist.databento.com/v0/timeseries.get_range?dataset={}&symbols={}&schema={}&start={}T00:00:00Z&end={}T23:59:59Z&encoding=dbn&stype_in=parent",
dataset, symbol, schema, start_date, end_date
);
println!("🔗 Request URL:");
println!(" {}", url);
println!();
println!("📥 Sending request...");
// Make request with Basic Authentication
let response = client
.get(&url)
.basic_auth(&api_key, Some(""))
.send()
.await?;
// Check status
let status = response.status();
println!("📊 Response Status: {}", status);
println!();
if !status.is_success() {
let error_text = response.text().await?;
eprintln!("❌ Request failed!");
eprintln!("Status: {}", status);
eprintln!("Response: {}", error_text);
return Err(format!("API returned error: {}", status).into());
}
// Get response body
let body = response.bytes().await?;
let size = body.len();
println!("✅ Download successful!");
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!(" Size: {} bytes ({:.2} KB)", size, size as f64 / 1024.0);
// Estimate cost (based on ES.FUT: 96KB cost ~$0.0002)
let size_gb = size as f64 / 1_073_741_824.0;
let cost_low = size_gb * 0.50;
let cost_high = size_gb * 2.00;
println!(" Size (GB): {:.10}", size_gb);
println!(" Estimated: ${:.6} - ${:.6}", cost_low, cost_high);
// Read current balance from COST_TRACKING.md
let cost_tracking_path = "COST_TRACKING.md";
let current_balance = if let Ok(content) = std::fs::read_to_string(cost_tracking_path) {
// Extract current credits from "**Current Credits**: ~$124.9998"
if let Some(line) = content.lines().find(|l| l.contains("**Current Credits**")) {
// Parse the number after "$"
if let Some(dollar_pos) = line.rfind('$') {
let balance_str = &line[dollar_pos+1..].trim();
balance_str.parse::<f64>().unwrap_or(125.0)
} else {
125.0
}
} else {
125.0
}
} else {
125.0
};
println!(" Credits Left: ~${:.4}", current_balance - cost_high);
println!();
// Save to file
let output_path = format!("test_data/real/databento/{}_{}_{}.dbn", symbol.replace("/", "_"), schema, start_date);
std::fs::create_dir_all("test_data/real/databento")?;
std::fs::write(&output_path, &body)?;
println!("💾 Saved to: {}", output_path);
println!();
// Verify file
if std::path::Path::new(&output_path).exists() {
let file_size = std::fs::metadata(&output_path)?.len();
println!("✅ File verified: {} bytes", file_size);
} else {
println!("⚠️ Warning: File not found after write");
}
println!();
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!(" Download Complete!");
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!();
println!("💡 Next Steps:");
println!(" 1. Validate data: cargo run -p backtesting_service --example validate_dbn_data -- test_data/real/databento/{}_{}_{}.dbn", symbol.replace("/", "_"), schema, start_date);
println!(" 2. Update COST_TRACKING.md with usage details");
Ok(())
}

112
data/examples/test_databento_download.rs Normal file
View File

@@ -0,0 +1,112 @@
//! Test Databento historical data download using existing infrastructure
//!
//! This example tests the minimal download of ES.FUT OHLCV-1m data for a single day.
use std::env;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!(" Databento Historical Download Test");
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!();
// Check for API key
let api_key = env::var("DATABENTO_API_KEY")
.map_err(|_| "DATABENTO_API_KEY environment variable not set")?;
println!("✅ API Key found: {}...{}", &api_key[0..10], &api_key[api_key.len()-10..]);
println!();
// Test parameters
let symbol = "ES.FUT";
let dataset = "GLBX.MDP3";
let schema = "ohlcv-1m";
let start_date = "2024-01-02";
let end_date = "2024-01-02";
println!("📋 Download Parameters:");
println!(" Symbol: {} (E-mini S&P 500 Futures)", symbol);
println!(" Dataset: {} (CME Group MDP 3.0)", dataset);
println!(" Schema: {} (1-minute OHLCV bars)", schema);
println!(" Date: {} (single trading day)", start_date);
println!();
// Create HTTP client for Databento Historical API
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(30))
.build()?;
// Build request URL
let url = format!(
"https://hist.databento.com/v0/timeseries.get_range?dataset={}&symbols={}&schema={}&start={}T00:00:00Z&end={}T23:59:59Z&encoding=dbn&stype_in=parent",
dataset, symbol, schema, start_date, end_date
);
println!("🔗 Request URL:");
println!(" {}", url);
println!();
println!("📥 Sending request...");
// Make request with Basic Authentication
let response = client
.get(&url)
.basic_auth(&api_key, Some(""))
.send()
.await?;
// Check status
let status = response.status();
println!("📊 Response Status: {}", status);
println!();
if !status.is_success() {
let error_text = response.text().await?;
eprintln!("❌ Request failed!");
eprintln!("Status: {}", status);
eprintln!("Response: {}", error_text);
return Err(format!("API returned error: {}", status).into());
}
// Get response body
let body = response.bytes().await?;
let size = body.len();
println!("✅ Download successful!");
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!(" Size: {} bytes ({:.2} KB)", size, size as f64 / 1024.0);
// Estimate cost
let size_gb = size as f64 / 1_073_741_824.0;
let cost_low = size_gb * 0.50;
let cost_high = size_gb * 2.00;
println!(" Size (GB): {:.10}", size_gb);
println!(" Estimated: ${:.6} - ${:.6}", cost_low, cost_high);
println!(" Credits Left: ~${:.2}", 125.0 - cost_high);
println!();
// Save to file
let output_path = format!("test_data/real/databento/{}_{}_{}.dbn", symbol.replace("/", "_"), schema, start_date);
std::fs::create_dir_all("test_data/real/databento")?;
std::fs::write(&output_path, &body)?;
println!("💾 Saved to: {}", output_path);
println!();
// Verify file
if std::path::Path::new(&output_path).exists() {
let file_size = std::fs::metadata(&output_path)?.len();
println!("✅ File verified: {} bytes", file_size);
} else {
println!("⚠️ Warning: File not found after write");
}
println!();
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!(" Test Complete!");
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
Ok(())
}

121
data/examples/validate_cl_fut.rs Normal file
View File

@@ -0,0 +1,121 @@
//! Validate CL.FUT DBN data file
//!
//! Inspects the downloaded CL.FUT data and reports statistics.
use std::fs::File;
use std::io::BufReader;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!(" CL.FUT Data Validation");
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!();
let file_path = "test_data/real/databento/CL.FUT_ohlcv-1m_2024-01-02.dbn";
println!("📂 File: {}", file_path);
// Check file exists and get size
let metadata = std::fs::metadata(file_path)?;
let size = metadata.len();
println!("📊 Size: {} bytes ({:.2} KB, {:.2} MB)", size, size as f64 / 1024.0, size as f64 / 1_048_576.0);
println!();
// Open file and create DBN decoder
let file = File::open(file_path)?;
let mut reader = BufReader::new(file);
// Read DBN metadata
let metadata = dbn::decode::MetadataDecoder::new(&mut reader)?.decode()?;
println!("📋 Metadata:");
println!(" Dataset: {}", metadata.dataset);
println!(" Schema: {}", metadata.schema);
println!(" Start: {}", metadata.start);
println!(" End: {}", metadata.end);
println!(" Symbols: {}", metadata.symbols.join(", "));
println!(" Stype In: {}", metadata.stype_in);
println!();
// Create record decoder
let mut decoder = dbn::decode::RecordDecoder::new(&mut reader, None, None, false)?;
let mut bar_count = 0;
let mut min_price = f64::MAX;
let mut max_price = f64::MIN;
let mut total_volume = 0.0;
// Read all records
while let Some(record) = decoder.decode_record::<dbn::OhlcvMsg>()? {
bar_count += 1;
// Track price range
let open = record.open as f64 / 1_000_000_000.0; // Convert from fixed point
let high = record.high as f64 / 1_000_000_000.0;
let low = record.low as f64 / 1_000_000_000.0;
let close = record.close as f64 / 1_000_000_000.0;
let volume = record.volume as f64;
if low < min_price { min_price = low; }
if high > max_price { max_price = high; }
total_volume += volume;
// Print first few bars for inspection
if bar_count <= 3 {
println!("📊 Bar {}: O={:.2} H={:.2} L={:.2} C={:.2} V={:.0}",
bar_count, open, high, low, close, volume);
}
}
println!();
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!(" Statistics:");
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!(" Total Bars: {}", bar_count);
println!(" Price Range: ${:.2} - ${:.2}", min_price, max_price);
println!(" Total Volume: {:.0}", total_volume);
println!(" Avg Volume: {:.0}", total_volume / bar_count as f64);
println!();
// Estimate cost
let size_gb = size as f64 / 1_073_741_824.0;
let cost_low = size_gb * 0.50;
let cost_high = size_gb * 2.00;
println!("💰 Cost Estimate:");
println!(" Size (GB): {:.10}", size_gb);
println!(" Estimated: ${:.6} - ${:.6}", cost_low, cost_high);
println!();
// Validate expectations
println!("✅ Validation:");
// CL.FUT typically has 390-400 bars per trading day (6.5 hours * 60 min)
let expected_bars = 390;
if bar_count >= expected_bars - 50 && bar_count <= expected_bars + 50 {
println!(" ✓ Bar count reasonable ({} bars, expected ~{})", bar_count, expected_bars);
} else {
println!(" ⚠ Bar count unexpected ({} bars, expected ~{})", bar_count, expected_bars);
}
// CL.FUT (Crude Oil) typically trades in $70-$85 range in Jan 2024
if min_price >= 60.0 && max_price <= 100.0 {
println!(" ✓ Price range reasonable (${:.2} - ${:.2})", min_price, max_price);
} else {
println!(" ⚠ Price range unexpected (${:.2} - ${:.2})", min_price, max_price);
}
if total_volume > 0.0 {
println!(" ✓ Volume data present");
} else {
println!(" ⚠ No volume data");
}
println!();
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
println!(" Validation Complete!");
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
Ok(())
}