chore: Major codebase cleanup - remove deprecated files and organize structure
- Docker: Delete 23 deprecated Dockerfiles, fix CI/CD to use Dockerfile.foxhunt-build - Config: Remove 36 .env files, keep 4 essential, delete config/environments/ - Docs: Archive 614 Wave D files to docs/archive/wave_d/, 95% reduction in root - Scripts: Delete 56 deprecated scripts, keep 58 production-critical (49% reduction) - Python: Organize 37 scripts into scripts/python/ subdirectories, delete ml/python/ - Build: Remove 1GB artifacts, delete old venvs, clean Python cache from git - Migrations: Delete deprecated directory (4,432 lines), remove duplicate database/migrations/ - Infrastructure: Delete deployment/ (61 files), docs/scripts/ (8 files) Total impact: ~2,500 files cleaned, 750MB+ space freed, zero production impact All deleted scripts backed up to archives. runpod/ and tests/runpod/ preserved. data_acquisition_service retained per user request.
This commit is contained in:
jgrusewski
166
scripts/python/data/download_es_databento.py
Normal file
166
scripts/python/data/download_es_databento.py
Normal file
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Download ES.FUT data from Databento for different market regimes.
|
||||
|
||||
This script downloads multiple days of E-mini S&P 500 futures data
|
||||
to enable regime testing (trending, ranging, volatile).
|
||||
|
||||
Usage:
|
||||
python3 download_es_databento.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
import databento as db
|
||||
|
||||
# Configuration
|
||||
API_KEY = os.getenv("DATABENTO_API_KEY", "db-95LEt9gtDRPJfc55NVUB5KL3A3uf6")
|
||||
OUTPUT_DIR = "test_data/real/databento"
|
||||
SYMBOL = "ES.FUT"
|
||||
SCHEMA = "ohlcv-1m"
|
||||
DATASET = "GLBX.MDP3"
|
||||
|
||||
# Target dates for different market regimes
|
||||
# Based on historical ES.FUT analysis (early 2024):
|
||||
# - 2024-01-03: Trending day (strong uptrend after Jan 2nd)
|
||||
# - 2024-01-04: Ranging day (consolidation, sideways movement)
|
||||
# - 2024-01-05: Volatile day (whipsaws, high volatility)
|
||||
DOWNLOAD_DATES = [
|
||||
"2024-01-03", # Trending regime
|
||||
"2024-01-04", # Ranging regime
|
||||
"2024-01-05", # Volatile regime
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
"""Download ES.FUT data for multiple days."""
|
||||
print("=" * 80)
|
||||
print("ES.FUT Multi-Day Databento Download")
|
||||
print("=" * 80)
|
||||
print()
|
||||
|
||||
# Check API key
|
||||
if not API_KEY:
|
||||
print("❌ ERROR: DATABENTO_API_KEY not found in environment!")
|
||||
print("Set it with: export DATABENTO_API_KEY='your-key-here'")
|
||||
sys.exit(1)
|
||||
|
||||
# Create output directory
|
||||
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
||||
print(f"📁 Output directory: {OUTPUT_DIR}")
|
||||
print(f"🎯 Symbol: {SYMBOL}")
|
||||
print(f"📊 Schema: {SCHEMA}")
|
||||
print(f"📦 Dataset: {DATASET}")
|
||||
print(f"📅 Dates: {', '.join(DOWNLOAD_DATES)}")
|
||||
print()
|
||||
|
||||
# Initialize Databento client
|
||||
try:
|
||||
client = db.Historical(API_KEY)
|
||||
print("✅ Databento client initialized")
|
||||
except Exception as e:
|
||||
print(f"❌ Failed to initialize Databento client: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
# Track results
|
||||
total_cost = 0.0
|
||||
successful_downloads = []
|
||||
failed_downloads = []
|
||||
|
||||
# Download each date
|
||||
for date_str in DOWNLOAD_DATES:
|
||||
print()
|
||||
print("-" * 80)
|
||||
print(f"📥 Downloading: {date_str}")
|
||||
print("-" * 80)
|
||||
|
||||
try:
|
||||
# Parse date (start at midnight UTC, end at 23:59:59 UTC)
|
||||
start_date = datetime.strptime(date_str, "%Y-%m-%d").replace(tzinfo=timezone.utc)
|
||||
end_date = start_date.replace(hour=23, minute=59, second=59)
|
||||
|
||||
# Build output filename
|
||||
output_file = os.path.join(OUTPUT_DIR, f"{SYMBOL}_{SCHEMA}_{date_str}.dbn")
|
||||
|
||||
# Download data
|
||||
print(f" Start: {start_date.isoformat()}")
|
||||
print(f" End: {end_date.isoformat()}")
|
||||
print(f" Output: {output_file}")
|
||||
print()
|
||||
|
||||
# Request data
|
||||
data = client.timeseries.get_range(
|
||||
dataset=DATASET,
|
||||
symbols=[SYMBOL],
|
||||
schema=SCHEMA,
|
||||
start=start_date.isoformat(),
|
||||
end=end_date.isoformat(),
|
||||
)
|
||||
|
||||
# Write to file
|
||||
data.to_file(output_file)
|
||||
|
||||
# Get file size
|
||||
file_size = os.path.getsize(output_file)
|
||||
file_size_kb = file_size / 1024
|
||||
|
||||
print(f"✅ Download complete!")
|
||||
print(f" File size: {file_size:,} bytes ({file_size_kb:.2f} KB)")
|
||||
|
||||
# Estimate cost (rough: ~$0.10 per day of 1-minute OHLCV data)
|
||||
estimated_cost = 0.10
|
||||
total_cost += estimated_cost
|
||||
|
||||
print(f" Estimated cost: ${estimated_cost:.2f}")
|
||||
|
||||
successful_downloads.append((date_str, output_file, file_size_kb))
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Download failed for {date_str}: {e}")
|
||||
failed_downloads.append((date_str, str(e)))
|
||||
|
||||
# Summary
|
||||
print()
|
||||
print("=" * 80)
|
||||
print("📊 DOWNLOAD SUMMARY")
|
||||
print("=" * 80)
|
||||
print()
|
||||
print(f"✅ Successful: {len(successful_downloads)}/{len(DOWNLOAD_DATES)}")
|
||||
print(f"❌ Failed: {len(failed_downloads)}/{len(DOWNLOAD_DATES)}")
|
||||
print(f"💰 Total estimated cost: ${total_cost:.2f}")
|
||||
print()
|
||||
|
||||
if successful_downloads:
|
||||
print("✅ Successfully downloaded files:")
|
||||
for date, path, size in successful_downloads:
|
||||
print(f" • {date}: {path} ({size:.2f} KB)")
|
||||
print()
|
||||
|
||||
if failed_downloads:
|
||||
print("❌ Failed downloads:")
|
||||
for date, error in failed_downloads:
|
||||
print(f" • {date}: {error}")
|
||||
print()
|
||||
|
||||
# Regime classification guidance
|
||||
print("📋 NEXT STEPS:")
|
||||
print("1. Validate each file with: cargo run --example validate_dbn_data")
|
||||
print("2. Analyze market regimes:")
|
||||
print(" • Trending: Strong directional moves, consistent momentum")
|
||||
print(" • Ranging: Sideways consolidation, mean-reverting")
|
||||
print(" • Volatile: High volatility, whipsaws, rapid reversals")
|
||||
print("3. Use data for regime detection testing in adaptive strategy")
|
||||
print()
|
||||
|
||||
if len(successful_downloads) >= 2:
|
||||
print("✅ SUCCESS: Downloaded sufficient data for regime testing!")
|
||||
elif len(successful_downloads) >= 1:
|
||||
print("⚠️ WARNING: Only 1 day downloaded. Consider downloading more.")
|
||||
else:
|
||||
print("❌ ERROR: No data downloaded successfully!")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user