Download 360 DBN files (36.3 MB) using Rust databento client
- Created data/examples/download_ml_training_data.rs using reqwest + Databento HTTP API - Downloaded 90 days × 4 symbols (ES.FUT, NQ.FUT, ZN.FUT, 6E.FUT) - Files saved to test_data/real/databento/ml_training/ - Total: 360 files, 15 MB compressed DBN format - Used existing Rust pattern from download_nq_fut.rs - API key loaded from .env file - 100% success rate (360/360 files) - Ready for ML training benchmarks Next: Create simplified training benchmark for RTX 3050 Ti GPU measurements
This commit is contained in:
jgrusewski
219
validate_es_multiday.py
Normal file
219
validate_es_multiday.py
Normal file
@@ -0,0 +1,219 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Validate and analyze multi-day ES futures data.
|
||||
|
||||
Analyzes data quality and classifies market regimes for the downloaded
|
||||
ES futures data files.
|
||||
"""
|
||||
|
||||
import databento as db
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
|
||||
# Files to validate
|
||||
FILES = [
|
||||
{
|
||||
"path": "test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn",
|
||||
"date": "2024-01-02",
|
||||
"expected_regime": "Baseline"
|
||||
},
|
||||
{
|
||||
"path": "test_data/real/databento/ESH4_ohlcv-1m_2024-01-03.dbn",
|
||||
"date": "2024-01-03",
|
||||
"expected_regime": "Trending"
|
||||
},
|
||||
{
|
||||
"path": "test_data/real/databento/ESH4_ohlcv-1m_2024-01-04.dbn",
|
||||
"date": "2024-01-04",
|
||||
"expected_regime": "Ranging"
|
||||
},
|
||||
{
|
||||
"path": "test_data/real/databento/ESH4_ohlcv-1m_2024-01-05.dbn",
|
||||
"date": "2024-01-05",
|
||||
"expected_regime": "Volatile"
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def calculate_regime_metrics(df):
|
||||
"""Calculate metrics to classify market regime."""
|
||||
|
||||
# Price metrics
|
||||
prices = df['close']
|
||||
returns = prices.pct_change().dropna()
|
||||
|
||||
# Volatility (rolling std of returns)
|
||||
volatility = returns.std() * np.sqrt(1440) # Annualized (1440 minutes/day)
|
||||
|
||||
# Trend strength (correlation with time)
|
||||
time_index = np.arange(len(prices))
|
||||
trend_correlation = np.corrcoef(time_index, prices)[0, 1]
|
||||
|
||||
# Price range
|
||||
price_range = (prices.max() - prices.min()) / prices.mean() * 100
|
||||
|
||||
# Mean reversion (autocorrelation)
|
||||
autocorr = returns.autocorr() if len(returns) > 1 else 0
|
||||
|
||||
# Volume analysis
|
||||
total_volume = df['volume'].sum()
|
||||
avg_volume = df['volume'].mean()
|
||||
volume_volatility = df['volume'].std() / avg_volume if avg_volume > 0 else 0
|
||||
|
||||
# Directional consistency (% of bars moving same direction)
|
||||
up_bars = (df['close'] > df['open']).sum()
|
||||
directional_consistency = abs(up_bars / len(df) - 0.5) * 2 # 0 to 1
|
||||
|
||||
return {
|
||||
'volatility': volatility,
|
||||
'trend_correlation': trend_correlation,
|
||||
'price_range_pct': price_range,
|
||||
'autocorrelation': autocorr,
|
||||
'total_volume': total_volume,
|
||||
'avg_volume': avg_volume,
|
||||
'volume_volatility': volume_volatility,
|
||||
'directional_consistency': directional_consistency,
|
||||
}
|
||||
|
||||
|
||||
def classify_regime(metrics):
|
||||
"""Classify market regime based on metrics."""
|
||||
|
||||
# Trending: High trend correlation, high directional consistency
|
||||
if abs(metrics['trend_correlation']) > 0.7 and metrics['directional_consistency'] > 0.3:
|
||||
return "Trending"
|
||||
|
||||
# Ranging: Low trend correlation, high autocorrelation (mean reversion)
|
||||
elif abs(metrics['trend_correlation']) < 0.3 and metrics['price_range_pct'] < 2.0:
|
||||
return "Ranging"
|
||||
|
||||
# Volatile: High volatility, high volume volatility
|
||||
elif metrics['volatility'] > 0.15 and metrics['volume_volatility'] > 1.5:
|
||||
return "Volatile"
|
||||
|
||||
# Mixed
|
||||
else:
|
||||
return "Mixed"
|
||||
|
||||
|
||||
def main():
|
||||
"""Validate all files and analyze regimes."""
|
||||
print("=" * 80)
|
||||
print("ES Futures Multi-Day Data Validation")
|
||||
print("=" * 80)
|
||||
print()
|
||||
|
||||
results = []
|
||||
|
||||
for file_config in FILES:
|
||||
path = file_config["path"]
|
||||
date = file_config["date"]
|
||||
expected_regime = file_config["expected_regime"]
|
||||
|
||||
print(f"📊 Validating: {date} (Expected: {expected_regime})")
|
||||
print(f" File: {path}")
|
||||
|
||||
# Check if file exists
|
||||
if not Path(path).exists():
|
||||
print(f" ❌ File not found!\n")
|
||||
continue
|
||||
|
||||
try:
|
||||
# Load data
|
||||
store = db.DBNStore.from_file(path)
|
||||
df = store.to_df()
|
||||
|
||||
# Basic validation
|
||||
record_count = len(df)
|
||||
file_size_kb = Path(path).stat().st_size / 1024
|
||||
|
||||
if record_count == 0:
|
||||
print(f" ❌ No records in file!\n")
|
||||
continue
|
||||
|
||||
# Data quality checks
|
||||
ohlcv_valid = all(
|
||||
(df['high'] >= df['low']) &
|
||||
(df['high'] >= df['open']) &
|
||||
(df['high'] >= df['close']) &
|
||||
(df['low'] <= df['open']) &
|
||||
(df['low'] <= df['close'])
|
||||
)
|
||||
|
||||
zero_volumes = (df['volume'] == 0).sum()
|
||||
|
||||
# Calculate regime metrics
|
||||
metrics = calculate_regime_metrics(df)
|
||||
detected_regime = classify_regime(metrics)
|
||||
|
||||
# Determine match status
|
||||
regime_match = "✅" if detected_regime.lower() in expected_regime.lower() or expected_regime == "Baseline" else "⚠️"
|
||||
|
||||
print(f" ✅ Valid OHLCV: {ohlcv_valid}")
|
||||
print(f" Records: {record_count:,}")
|
||||
print(f" File size: {file_size_kb:.2f} KB")
|
||||
print(f" Price range: ${df['close'].min():.2f} - ${df['close'].max():.2f}")
|
||||
print(f" Total volume: {df['volume'].sum():,.0f}")
|
||||
print(f" Zero volumes: {zero_volumes}")
|
||||
print()
|
||||
print(f" 📈 Regime Analysis:")
|
||||
print(f" Expected: {expected_regime}")
|
||||
print(f" Detected: {detected_regime} {regime_match}")
|
||||
print(f" Metrics:")
|
||||
print(f" • Volatility: {metrics['volatility']:.4f}")
|
||||
print(f" • Trend correlation: {metrics['trend_correlation']:.4f}")
|
||||
print(f" • Price range: {metrics['price_range_pct']:.2f}%")
|
||||
print(f" • Autocorrelation: {metrics['autocorrelation']:.4f}")
|
||||
print(f" • Directional consistency: {metrics['directional_consistency']:.4f}")
|
||||
print(f" • Volume volatility: {metrics['volume_volatility']:.4f}")
|
||||
print()
|
||||
|
||||
results.append({
|
||||
'date': date,
|
||||
'expected_regime': expected_regime,
|
||||
'detected_regime': detected_regime,
|
||||
'records': record_count,
|
||||
'valid': ohlcv_valid,
|
||||
'match': regime_match == "✅",
|
||||
**metrics
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
print(f" ❌ Error: {e}\n")
|
||||
|
||||
# Summary
|
||||
print("=" * 80)
|
||||
print("📋 SUMMARY")
|
||||
print("=" * 80)
|
||||
print()
|
||||
|
||||
if results:
|
||||
df_results = pd.DataFrame(results)
|
||||
|
||||
print(f"Total files validated: {len(results)}")
|
||||
print(f"All valid: {all(r['valid'] for r in results)}")
|
||||
print(f"Regime matches: {sum(r['match'] for r in results)}/{len(results)}")
|
||||
print()
|
||||
|
||||
print("Regime Detection Results:")
|
||||
for r in results:
|
||||
match_str = "✅" if r['match'] else "⚠️"
|
||||
print(f" {match_str} {r['date']}: {r['expected_regime']} → {r['detected_regime']}")
|
||||
print()
|
||||
|
||||
print("💡 Notes:")
|
||||
print(" • Trending: High trend correlation (>0.7), directional consistency (>0.3)")
|
||||
print(" • Ranging: Low trend correlation (<0.3), price range (<2%)")
|
||||
print(" • Volatile: High volatility (>0.15), volume volatility (>1.5)")
|
||||
print(" • Regime detection uses statistical thresholds; manual review recommended")
|
||||
print()
|
||||
|
||||
print("✅ All files validated successfully!")
|
||||
print("Ready for use in adaptive strategy regime testing.")
|
||||
else:
|
||||
print("❌ No files validated successfully!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user