Download 360 DBN files (36.3 MB) using Rust databento client

- Created data/examples/download_ml_training_data.rs using reqwest + Databento HTTP API
- Downloaded 90 days × 4 symbols (ES.FUT, NQ.FUT, ZN.FUT, 6E.FUT)
- Files saved to test_data/real/databento/ml_training/
- Total: 360 files, 15 MB compressed DBN format
- Used existing Rust pattern from download_nq_fut.rs
- API key loaded from .env file
- 100% success rate (360/360 files)
- Ready for ML training benchmarks

Next: Create simplified training benchmark for RTX 3050 Ti GPU measurements
This commit is contained in:
jgrusewski
2025-10-13 13:30:02 +02:00
parent 08821565d6
commit e8a68ee39f
539 changed files with 1070448 additions and 489 deletions

View File

@@ -0,0 +1,154 @@
//! DBN Backtesting Integration Example
//!
//! This example demonstrates using DBN data with the backtesting service's
//! MarketDataRepository interface.
//!
//! ## Usage
//!
//! ```bash
//! cargo run --example dbn_backtesting_integration
//! ```
use backtesting_service::{
dbn_repository::DbnMarketDataRepository,
repositories::MarketDataRepository,
};
use chrono::{TimeZone, Utc};
use std::collections::HashMap;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
println!("=== DBN Backtesting Integration Example ===\n");
// 1. Setup repository with DBN data
let mut file_mapping = HashMap::new();
file_mapping.insert(
"ES.FUT".to_string(),
"test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn".to_string(),
);
println!("Creating MarketDataRepository with DBN backend...");
let repo = DbnMarketDataRepository::new(file_mapping).await?;
println!("Available symbols: {:?}", repo.available_symbols());
// 2. Define backtest time range
let start_time = Utc
.with_ymd_and_hms(2024, 1, 2, 14, 30, 0)
.unwrap()
.timestamp_nanos_opt()
.unwrap();
let end_time = Utc
.with_ymd_and_hms(2024, 1, 2, 16, 0, 0)
.unwrap()
.timestamp_nanos_opt()
.unwrap();
println!(
"\nBacktest window: {} to {}",
Utc.timestamp_nanos(start_time).format("%Y-%m-%d %H:%M:%S"),
Utc.timestamp_nanos(end_time).format("%Y-%m-%d %H:%M:%S")
);
// 3. Load historical data via repository interface
let symbols = vec!["ES.FUT".to_string()];
println!("\nLoading historical data for {:?}...", symbols);
let data = repo.load_historical_data(&symbols, start_time, end_time).await?;
println!("✅ Loaded {} bars via repository interface\n", data.len());
// 4. Check data availability
println!("=== Data Availability Check ===");
let availability = repo
.check_data_availability(&symbols, start_time, end_time)
.await?;
for (symbol, available) in availability.iter() {
println!(
"{}: {}",
symbol,
if *available { "✅ Available" } else { "❌ Not available" }
);
}
// 5. Simulate simple backtest logic
println!("\n=== Simulating Simple Backtest ===");
let mut position = 0i32;
let mut pnl = 0.0;
let mut trades = 0;
for (i, bar) in data.iter().enumerate() {
let close_f64 = bar.close.to_string().parse::<f64>().unwrap();
// Simple strategy: Buy when price drops, sell when price rises
if i > 0 {
let prev_close = data[i - 1].close.to_string().parse::<f64>().unwrap();
let price_change = close_f64 - prev_close;
if position == 0 && price_change < -1.0 {
// Buy signal
position = 1;
pnl -= close_f64; // Entry cost
trades += 1;
println!(" [{}] BUY @ {:.2}", bar.timestamp.format("%H:%M"), close_f64);
} else if position == 1 && price_change > 1.0 {
// Sell signal
position = 0;
pnl += close_f64; // Exit proceeds
trades += 1;
println!(" [{}] SELL @ {:.2}", bar.timestamp.format("%H:%M"), close_f64);
}
}
}
// Close any open position
if position != 0 {
let last_close = data.last().unwrap().close.to_string().parse::<f64>().unwrap();
pnl += last_close * position as f64;
trades += 1;
println!(
" [{}] CLOSE @ {:.2}",
data.last().unwrap().timestamp.format("%H:%M"),
last_close
);
}
println!("\n=== Backtest Results ===");
println!("Total trades: {}", trades);
println!("Final PnL: ${:.2}", pnl);
// 6. Advanced repository features
println!("\n=== Advanced Repository Features ===");
// Load with volume filter
let min_volume = rust_decimal::Decimal::from(50);
let high_volume_bars = repo
.load_with_volume_filter(&symbols, min_volume, start_time, end_time)
.await?;
println!(
"High-volume bars (volume >= {}): {}",
min_volume,
high_volume_bars.len()
);
// Get date range
let (first_ts, last_ts) = repo.get_date_range("ES.FUT").await?;
println!("Data range: {} to {}", first_ts, last_ts);
// Generate summary statistics
let stats = repo.generate_summary_stats(&data);
println!("\nSummary Statistics:");
println!(" Count: {}", stats.get("count").unwrap());
println!(" Mean close: ${:.2}", stats.get("mean_close").unwrap());
println!(" Std close: ${:.2}", stats.get("std_close").unwrap());
println!(" Min close: ${:.2}", stats.get("min_close").unwrap());
println!(" Max close: ${:.2}", stats.get("max_close").unwrap());
println!("\n✅ Example completed successfully!");
Ok(())
}

View File

@@ -0,0 +1,112 @@
//! Basic DBN Loading Example
//!
//! This example demonstrates the simplest way to load OHLCV bars from a DBN file.
//!
//! ## Usage
//!
//! ```bash
//! cargo run --example dbn_basic_loading
//! ```
use backtesting_service::dbn_data_source::DbnDataSource;
use std::collections::HashMap;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
println!("=== DBN Basic Loading Example ===\n");
// 1. Create file mapping (symbol -> file path)
let mut file_mapping = HashMap::new();
file_mapping.insert(
"ES.FUT".to_string(),
"test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn".to_string(),
);
println!("Creating data source with 1 symbol...");
// 2. Create data source
let data_source = DbnDataSource::new(file_mapping).await?;
println!("Available symbols: {:?}", data_source.available_symbols());
// 3. Load OHLCV bars
println!("\nLoading OHLCV bars for ES.FUT...");
let bars = data_source.load_ohlcv_bars("ES.FUT").await?;
println!("✅ Loaded {} bars from DBN file\n", bars.len());
// 4. Display first and last bars
println!("First bar:");
let first = &bars[0];
println!(
" {} @ {} (open={}, high={}, low={}, close={}, volume={})",
first.symbol,
first.timestamp,
first.open,
first.high,
first.low,
first.close,
first.volume
);
println!("\nLast bar:");
let last = &bars[bars.len() - 1];
println!(
" {} @ {} (open={}, high={}, low={}, close={}, volume={})",
last.symbol,
last.timestamp,
last.open,
last.high,
last.low,
last.close,
last.volume
);
// 5. Validate data quality
println!("\n=== Data Quality Checks ===");
// Check OHLCV relationships
let mut valid_count = 0;
for bar in &bars {
if bar.high >= bar.low
&& bar.high >= bar.open
&& bar.high >= bar.close
&& bar.low <= bar.open
&& bar.low <= bar.close
{
valid_count += 1;
}
}
println!("OHLCV validation: {}/{} bars passed", valid_count, bars.len());
// Check timestamp ordering
let mut ordered = true;
for i in 1..bars.len() {
if bars[i].timestamp < bars[i - 1].timestamp {
ordered = false;
break;
}
}
println!("Timestamp ordering: {}", if ordered { "✅ Sorted" } else { "❌ Not sorted" });
// Check price ranges (ES.FUT typical: $3,000-$6,000)
let mut realistic_prices = 0;
for bar in &bars {
let close_f64 = bar.close.to_string().parse::<f64>().unwrap();
if close_f64 > 3000.0 && close_f64 < 6000.0 {
realistic_prices += 1;
}
}
println!(
"Realistic prices: {}/{} bars in range",
realistic_prices,
bars.len()
);
println!("\n✅ Example completed successfully!");
Ok(())
}

View File

@@ -0,0 +1,116 @@
//! Multi-Day DBN Loading Example
//!
//! This example shows how to load data from multiple DBN files (multi-day backtests).
//!
//! ## Usage
//!
//! ```bash
//! cargo run --example dbn_multi_day_loading
//! ```
use backtesting_service::dbn_data_source::DbnDataSource;
use std::collections::HashMap;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
println!("=== DBN Multi-Day Loading Example ===\n");
// 1. Create file mapping with multiple files per symbol
let mut file_mapping = HashMap::new();
file_mapping.insert(
"ESH4".to_string(),
vec![
"test_data/real/databento/ESH4_ohlcv-1m_2024-01-03.dbn".to_string(),
"test_data/real/databento/ESH4_ohlcv-1m_2024-01-04.dbn".to_string(),
"test_data/real/databento/ESH4_ohlcv-1m_2024-01-05.dbn".to_string(),
],
);
println!("Creating data source with 3 files for ESH4...");
// 2. Create data source (multi-file mode)
let data_source = DbnDataSource::new_multi_file(file_mapping).await?;
println!("Files configured: {}", data_source.get_file_count("ESH4"));
// 3. Load all files (merged and sorted)
println!("\nLoading all days for ESH4...");
let start = std::time::Instant::now();
let bars = data_source.load_ohlcv_bars_all("ESH4").await?;
let duration = start.elapsed();
println!(
"✅ Loaded {} bars from {} files in {:?} ({:.2}ms)",
bars.len(),
3,
duration,
duration.as_secs_f64() * 1000.0
);
// 4. Analyze data by day
println!("\n=== Data by Day ===");
let mut day_stats: HashMap<String, Vec<_>> = HashMap::new();
for bar in &bars {
let day = bar.timestamp.format("%Y-%m-%d").to_string();
day_stats.entry(day).or_default().push(bar);
}
for (day, bars_for_day) in day_stats.iter() {
let first = bars_for_day.first().unwrap();
let last = bars_for_day.last().unwrap();
println!(
"{}: {} bars | {} to {} | Range: {} to {}",
day,
bars_for_day.len(),
first.timestamp.format("%H:%M"),
last.timestamp.format("%H:%M"),
first.close,
last.close
);
}
// 5. Verify chronological ordering across all days
println!("\n=== Cross-Day Validation ===");
let mut ordered = true;
for i in 1..bars.len() {
if bars[i].timestamp < bars[i - 1].timestamp {
println!(
"❌ Ordering issue at index {}: {} < {}",
i,
bars[i].timestamp,
bars[i - 1].timestamp
);
ordered = false;
break;
}
}
if ordered {
println!("✅ All bars properly ordered across {} days", day_stats.len());
}
// 6. Calculate daily returns
println!("\n=== Daily Returns ===");
for (day, bars_for_day) in day_stats.iter() {
if bars_for_day.len() > 1 {
let first_close = bars_for_day.first().unwrap().close;
let last_close = bars_for_day.last().unwrap().close;
let return_pct =
((last_close - first_close) / first_close * rust_decimal::Decimal::from(100))
.to_string()
.parse::<f64>()
.unwrap();
println!("{}: {:.2}% return", day, return_pct);
}
}
println!("\n✅ Example completed successfully!");
Ok(())
}

View File

@@ -0,0 +1,198 @@
//! DBN Statistical Analysis Example
//!
//! This example demonstrates advanced statistical analysis and data transformation
//! features of the DBN repository.
//!
//! ## Usage
//!
//! ```bash
//! cargo run --example dbn_statistical_analysis
//! ```
use backtesting_service::dbn_repository::DbnMarketDataRepository;
use std::collections::HashMap;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
println!("=== DBN Statistical Analysis Example ===\n");
// 1. Setup repository
let mut file_mapping = HashMap::new();
file_mapping.insert(
"ES.FUT".to_string(),
"test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn".to_string(),
);
let repo = DbnMarketDataRepository::new(file_mapping).await?;
// 2. Load all data
println!("Loading data...");
let symbols = vec!["ES.FUT".to_string()];
let start_time = 1704153600_000_000_000i64; // 2024-01-02 00:00:00
let end_time = 1704240000_000_000_000i64; // 2024-01-03 00:00:00
let bars = repo.load_historical_data(&symbols, start_time, end_time).await?;
println!("✅ Loaded {} bars\n", bars.len());
// 3. Summary Statistics
println!("=== Summary Statistics ===");
let stats = repo.generate_summary_stats(&bars);
println!("Count: {}", stats.get("count").unwrap());
println!("Mean Close: ${:.2}", stats.get("mean_close").unwrap());
println!("Std Close: ${:.2}", stats.get("std_close").unwrap());
println!("Min Close: ${:.2}", stats.get("min_close").unwrap());
println!("Max Close: ${:.2}", stats.get("max_close").unwrap());
println!("Mean Volume: {:.0}", stats.get("mean_volume").unwrap());
println!("Total Volume: {:.0}", stats.get("total_volume").unwrap());
// 4. Rolling Window Statistics
println!("\n=== Rolling 20-Bar Window Statistics ===");
let window_size = 20;
let rolling_stats = repo.calculate_rolling_stats(&bars, window_size);
println!("Window size: {} bars", window_size);
println!("Windows calculated: {}", rolling_stats.len());
// Display last 5 windows
println!("\nLast 5 windows:");
for (i, (mean, std, min, max)) in rolling_stats.iter().rev().take(5).enumerate() {
println!(
" Window {}: mean=${:.2}, std=${:.2}, range=[${:.2}, ${:.2}]",
rolling_stats.len() - i,
mean,
std,
min,
max
);
}
// 5. Bar Resampling
println!("\n=== Bar Resampling ===");
// Resample to different timeframes
for target_minutes in [5, 15, 60] {
let resampled = repo.resample_bars(&bars, target_minutes)?;
let reduction = (1.0 - (resampled.len() as f64 / bars.len() as f64)) * 100.0;
println!(
"{:>2}-minute bars: {:>3} bars ({:.1}% reduction)",
target_minutes,
resampled.len(),
reduction
);
// Show first resampled bar
if let Some(first) = resampled.first() {
println!(
" First: {} @ {} | O={} H={} L={} C={} V={}",
first.symbol,
first.timestamp.format("%H:%M"),
first.open,
first.high,
first.low,
first.close,
first.volume
);
}
}
// 6. Regime Detection (Simple Heuristic)
println!("\n=== Regime Detection ===");
let regime_types = ["trending", "ranging", "volatile", "stable"];
for regime_type in regime_types.iter() {
match repo.load_regime_samples(regime_type, 5, &symbols).await {
Ok(samples) => {
println!("{:<10} regime: {} sample bars found", regime_type, samples.len());
if !samples.is_empty() {
let sample = &samples[0];
let range = sample.high - sample.low;
let avg_price = (sample.high + sample.low) / rust_decimal::Decimal::from(2);
let range_pct = (range / avg_price * rust_decimal::Decimal::from(10000))
.to_string()
.parse::<f64>()
.unwrap()
/ 100.0;
println!(" Example: {} @ {} (range: {:.2}%)", sample.symbol, sample.timestamp, range_pct);
}
}
Err(e) => {
println!("{:<10} regime: Error - {}", regime_type, e);
}
}
}
// 7. Price Return Analysis
println!("\n=== Price Return Analysis ===");
let mut returns: Vec<f64> = Vec::new();
for i in 1..bars.len() {
let prev_close = bars[i - 1].close.to_string().parse::<f64>().unwrap();
let curr_close = bars[i].close.to_string().parse::<f64>().unwrap();
let ret = (curr_close - prev_close) / prev_close;
returns.push(ret);
}
let mean_return = returns.iter().sum::<f64>() / returns.len() as f64;
let variance = returns
.iter()
.map(|r| (r - mean_return).powi(2))
.sum::<f64>()
/ returns.len() as f64;
let std_dev = variance.sqrt();
println!("Mean return: {:.6} ({:.4}%)", mean_return, mean_return * 100.0);
println!("Std dev: {:.6} ({:.4}%)", std_dev, std_dev * 100.0);
println!(
"Min return: {:.6} ({:.4}%)",
returns.iter().cloned().fold(f64::INFINITY, f64::min),
returns.iter().cloned().fold(f64::INFINITY, f64::min) * 100.0
);
println!(
"Max return: {:.6} ({:.4}%)",
returns.iter().cloned().fold(f64::NEG_INFINITY, f64::max),
returns.iter().cloned().fold(f64::NEG_INFINITY, f64::max) * 100.0
);
// Sharpe ratio (annualized, assuming 252 trading days)
let sharpe = (mean_return / std_dev) * (252.0 * 390.0_f64).sqrt(); // 390 bars per day
println!("Sharpe ratio: {:.2}", sharpe);
// 8. Volume Analysis
println!("\n=== Volume Analysis ===");
let volumes: Vec<f64> = bars
.iter()
.map(|b| b.volume.to_string().parse().unwrap())
.collect();
let mean_volume = volumes.iter().sum::<f64>() / volumes.len() as f64;
let max_volume = volumes.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
let min_volume = volumes.iter().cloned().fold(f64::INFINITY, f64::min);
println!("Mean volume: {:.0}", mean_volume);
println!("Max volume: {:.0}", max_volume);
println!("Min volume: {:.0}", min_volume);
// Find high-volume bars
let high_volume_threshold = mean_volume * 2.0;
let high_volume_bars: Vec<_> = bars
.iter()
.filter(|b| b.volume.to_string().parse::<f64>().unwrap() > high_volume_threshold)
.collect();
println!("\nHigh-volume bars (>2x mean): {}", high_volume_bars.len());
for bar in high_volume_bars.iter().take(3) {
println!(
" {} @ {} | Volume: {}",
bar.symbol, bar.timestamp, bar.volume
);
}
println!("\n✅ Example completed successfully!");
Ok(())
}