e8a68ee39f
- Created data/examples/download_ml_training_data.rs using reqwest + Databento HTTP API - Downloaded 90 days × 4 symbols (ES.FUT, NQ.FUT, ZN.FUT, 6E.FUT) - Files saved to test_data/real/databento/ml_training/ - Total: 360 files, 15 MB compressed DBN format - Used existing Rust pattern from download_nq_fut.rs - API key loaded from .env file - 100% success rate (360/360 files) - Ready for ML training benchmarks Next: Create simplified training benchmark for RTX 3050 Ti GPU measurements
220 lines
7.2 KiB
Python
220 lines
7.2 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Validate and analyze multi-day ES futures data.
|
|
|
|
Analyzes data quality and classifies market regimes for the downloaded
|
|
ES futures data files.
|
|
"""
|
|
|
|
import databento as db
|
|
import pandas as pd
|
|
import numpy as np
|
|
from pathlib import Path
|
|
|
|
# Files to validate
|
|
FILES = [
|
|
{
|
|
"path": "test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn",
|
|
"date": "2024-01-02",
|
|
"expected_regime": "Baseline"
|
|
},
|
|
{
|
|
"path": "test_data/real/databento/ESH4_ohlcv-1m_2024-01-03.dbn",
|
|
"date": "2024-01-03",
|
|
"expected_regime": "Trending"
|
|
},
|
|
{
|
|
"path": "test_data/real/databento/ESH4_ohlcv-1m_2024-01-04.dbn",
|
|
"date": "2024-01-04",
|
|
"expected_regime": "Ranging"
|
|
},
|
|
{
|
|
"path": "test_data/real/databento/ESH4_ohlcv-1m_2024-01-05.dbn",
|
|
"date": "2024-01-05",
|
|
"expected_regime": "Volatile"
|
|
},
|
|
]
|
|
|
|
|
|
def calculate_regime_metrics(df):
|
|
"""Calculate metrics to classify market regime."""
|
|
|
|
# Price metrics
|
|
prices = df['close']
|
|
returns = prices.pct_change().dropna()
|
|
|
|
# Volatility (rolling std of returns)
|
|
volatility = returns.std() * np.sqrt(1440) # Annualized (1440 minutes/day)
|
|
|
|
# Trend strength (correlation with time)
|
|
time_index = np.arange(len(prices))
|
|
trend_correlation = np.corrcoef(time_index, prices)[0, 1]
|
|
|
|
# Price range
|
|
price_range = (prices.max() - prices.min()) / prices.mean() * 100
|
|
|
|
# Mean reversion (autocorrelation)
|
|
autocorr = returns.autocorr() if len(returns) > 1 else 0
|
|
|
|
# Volume analysis
|
|
total_volume = df['volume'].sum()
|
|
avg_volume = df['volume'].mean()
|
|
volume_volatility = df['volume'].std() / avg_volume if avg_volume > 0 else 0
|
|
|
|
# Directional consistency (% of bars moving same direction)
|
|
up_bars = (df['close'] > df['open']).sum()
|
|
directional_consistency = abs(up_bars / len(df) - 0.5) * 2 # 0 to 1
|
|
|
|
return {
|
|
'volatility': volatility,
|
|
'trend_correlation': trend_correlation,
|
|
'price_range_pct': price_range,
|
|
'autocorrelation': autocorr,
|
|
'total_volume': total_volume,
|
|
'avg_volume': avg_volume,
|
|
'volume_volatility': volume_volatility,
|
|
'directional_consistency': directional_consistency,
|
|
}
|
|
|
|
|
|
def classify_regime(metrics):
|
|
"""Classify market regime based on metrics."""
|
|
|
|
# Trending: High trend correlation, high directional consistency
|
|
if abs(metrics['trend_correlation']) > 0.7 and metrics['directional_consistency'] > 0.3:
|
|
return "Trending"
|
|
|
|
# Ranging: Low trend correlation, high autocorrelation (mean reversion)
|
|
elif abs(metrics['trend_correlation']) < 0.3 and metrics['price_range_pct'] < 2.0:
|
|
return "Ranging"
|
|
|
|
# Volatile: High volatility, high volume volatility
|
|
elif metrics['volatility'] > 0.15 and metrics['volume_volatility'] > 1.5:
|
|
return "Volatile"
|
|
|
|
# Mixed
|
|
else:
|
|
return "Mixed"
|
|
|
|
|
|
def main():
|
|
"""Validate all files and analyze regimes."""
|
|
print("=" * 80)
|
|
print("ES Futures Multi-Day Data Validation")
|
|
print("=" * 80)
|
|
print()
|
|
|
|
results = []
|
|
|
|
for file_config in FILES:
|
|
path = file_config["path"]
|
|
date = file_config["date"]
|
|
expected_regime = file_config["expected_regime"]
|
|
|
|
print(f"📊 Validating: {date} (Expected: {expected_regime})")
|
|
print(f" File: {path}")
|
|
|
|
# Check if file exists
|
|
if not Path(path).exists():
|
|
print(f" ❌ File not found!\n")
|
|
continue
|
|
|
|
try:
|
|
# Load data
|
|
store = db.DBNStore.from_file(path)
|
|
df = store.to_df()
|
|
|
|
# Basic validation
|
|
record_count = len(df)
|
|
file_size_kb = Path(path).stat().st_size / 1024
|
|
|
|
if record_count == 0:
|
|
print(f" ❌ No records in file!\n")
|
|
continue
|
|
|
|
# Data quality checks
|
|
ohlcv_valid = all(
|
|
(df['high'] >= df['low']) &
|
|
(df['high'] >= df['open']) &
|
|
(df['high'] >= df['close']) &
|
|
(df['low'] <= df['open']) &
|
|
(df['low'] <= df['close'])
|
|
)
|
|
|
|
zero_volumes = (df['volume'] == 0).sum()
|
|
|
|
# Calculate regime metrics
|
|
metrics = calculate_regime_metrics(df)
|
|
detected_regime = classify_regime(metrics)
|
|
|
|
# Determine match status
|
|
regime_match = "✅" if detected_regime.lower() in expected_regime.lower() or expected_regime == "Baseline" else "⚠️"
|
|
|
|
print(f" ✅ Valid OHLCV: {ohlcv_valid}")
|
|
print(f" Records: {record_count:,}")
|
|
print(f" File size: {file_size_kb:.2f} KB")
|
|
print(f" Price range: ${df['close'].min():.2f} - ${df['close'].max():.2f}")
|
|
print(f" Total volume: {df['volume'].sum():,.0f}")
|
|
print(f" Zero volumes: {zero_volumes}")
|
|
print()
|
|
print(f" 📈 Regime Analysis:")
|
|
print(f" Expected: {expected_regime}")
|
|
print(f" Detected: {detected_regime} {regime_match}")
|
|
print(f" Metrics:")
|
|
print(f" • Volatility: {metrics['volatility']:.4f}")
|
|
print(f" • Trend correlation: {metrics['trend_correlation']:.4f}")
|
|
print(f" • Price range: {metrics['price_range_pct']:.2f}%")
|
|
print(f" • Autocorrelation: {metrics['autocorrelation']:.4f}")
|
|
print(f" • Directional consistency: {metrics['directional_consistency']:.4f}")
|
|
print(f" • Volume volatility: {metrics['volume_volatility']:.4f}")
|
|
print()
|
|
|
|
results.append({
|
|
'date': date,
|
|
'expected_regime': expected_regime,
|
|
'detected_regime': detected_regime,
|
|
'records': record_count,
|
|
'valid': ohlcv_valid,
|
|
'match': regime_match == "✅",
|
|
**metrics
|
|
})
|
|
|
|
except Exception as e:
|
|
print(f" ❌ Error: {e}\n")
|
|
|
|
# Summary
|
|
print("=" * 80)
|
|
print("📋 SUMMARY")
|
|
print("=" * 80)
|
|
print()
|
|
|
|
if results:
|
|
df_results = pd.DataFrame(results)
|
|
|
|
print(f"Total files validated: {len(results)}")
|
|
print(f"All valid: {all(r['valid'] for r in results)}")
|
|
print(f"Regime matches: {sum(r['match'] for r in results)}/{len(results)}")
|
|
print()
|
|
|
|
print("Regime Detection Results:")
|
|
for r in results:
|
|
match_str = "✅" if r['match'] else "⚠️"
|
|
print(f" {match_str} {r['date']}: {r['expected_regime']} → {r['detected_regime']}")
|
|
print()
|
|
|
|
print("💡 Notes:")
|
|
print(" • Trending: High trend correlation (>0.7), directional consistency (>0.3)")
|
|
print(" • Ranging: Low trend correlation (<0.3), price range (<2%)")
|
|
print(" • Volatile: High volatility (>0.15), volume volatility (>1.5)")
|
|
print(" • Regime detection uses statistical thresholds; manual review recommended")
|
|
print()
|
|
|
|
print("✅ All files validated successfully!")
|
|
print("Ready for use in adaptive strategy regime testing.")
|
|
else:
|
|
print("❌ No files validated successfully!")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|