foxhunt/scripts/python/data/download_6e_fut.py

#!/usr/bin/env python3
"""
Download 6E.FUT (Euro FX Futures) OHLCV-1m data from Databento.
"""
import os
import databento as db
from datetime import datetime
import sys

# Configuration
DATASET = "GLBX.MDP3"
SYMBOLS = ["6E.FUT"]
SCHEMA = "ohlcv-1m"
START_DATE = "2024-01-02"
END_DATE = "2024-01-31"
OUTPUT_DIR = "/home/jgrusewski/Work/foxhunt/test_data/real/databento"
OUTPUT_FILE = f"{OUTPUT_DIR}/6E.FUT_ohlcv-1m_2024-01-02_to_2024-01-31.dbn"

def main():
    # Check for API key
    api_key = os.environ.get("DATABENTO_API_KEY")
    if not api_key:
        print("ERROR: DATABENTO_API_KEY environment variable not set")
        print("Please set it with: export DATABENTO_API_KEY='your-key-here'")
        sys.exit(1)

    # Create client
    client = db.Historical(api_key)

    print(f"Dataset: {DATASET}")
    print(f"Symbols: {SYMBOLS}")
    print(f"Schema: {SCHEMA}")
    print(f"Date Range: {START_DATE} to {END_DATE}")
    print(f"Output: {OUTPUT_FILE}")
    print()

    # Step 1: Get cost estimate
    print("=" * 70)
    print("STEP 1: Cost Estimation")
    print("=" * 70)
    try:
        cost = client.metadata.get_cost(
            dataset=DATASET,
            symbols=SYMBOLS,
            schema=SCHEMA,
            start=START_DATE,
            end=END_DATE
        )
        print(f"Estimated Cost: ${cost:.4f} USD")
        print()

        if cost > 1.0:
            print(f"WARNING: Cost ${cost:.4f} exceeds $1.00 threshold")
            print("Consider trying specific contracts instead: 6EH24, 6EM24, 6EU24")
            response = input("Continue anyway? (yes/no): ")
            if response.lower() not in ["yes", "y"]:
                print("Download cancelled.")
                sys.exit(0)
    except Exception as e:
        print(f"WARNING: Could not estimate cost: {e}")
        print("Proceeding with download...")

    # Step 2: Download data
    print("=" * 70)
    print("STEP 2: Downloading Data")
    print("=" * 70)
    try:
        # Create output directory if it doesn't exist
        os.makedirs(OUTPUT_DIR, exist_ok=True)

        # Download data to DBN file
        client.timeseries.get_range(
            dataset=DATASET,
            symbols=SYMBOLS,
            schema=SCHEMA,
            start=START_DATE,
            end=END_DATE,
            path=OUTPUT_FILE
        )

        print(f"✅ Download complete: {OUTPUT_FILE}")

    except Exception as e:
        print(f"❌ Download failed: {e}")
        sys.exit(1)

    # Step 3: Verify data quality
    print()
    print("=" * 70)
    print("STEP 3: Data Verification")
    print("=" * 70)
    try:
        # Get file size
        file_size = os.path.getsize(OUTPUT_FILE)
        file_size_mb = file_size / (1024 * 1024)
        print(f"File Size: {file_size:,} bytes ({file_size_mb:.2f} MB)")

        # Read and analyze data
        store = db.DBNStore.from_file(OUTPUT_FILE)

        # Count records
        record_count = 0
        first_record = None
        last_record = None
        sample_prices = []

        for record in store:
            if record_count == 0:
                first_record = record
            last_record = record

            # Sample some prices (every 1000th record)
            if record_count % 1000 == 0 and hasattr(record, 'close'):
                sample_prices.append(float(record.close) / 1e9)  # Price is in fixed-point

            record_count += 1

        print(f"Total Records: {record_count:,} bars")

        if first_record:
            print(f"First Timestamp: {first_record.ts_event}")
        if last_record:
            print(f"Last Timestamp: {last_record.ts_event}")

        # Check price sanity for EUR/USD (typically 1.05-1.15)
        if sample_prices:
            min_price = min(sample_prices)
            max_price = max(sample_prices)
            avg_price = sum(sample_prices) / len(sample_prices)

            print(f"\nPrice Range (sampled {len(sample_prices)} bars):")
            print(f"  Min: {min_price:.5f}")
            print(f"  Max: {max_price:.5f}")
            print(f"  Avg: {avg_price:.5f}")

            # EUR/USD typically trades in 1.05-1.15 range
            if 1.00 < avg_price < 1.20:
                print("  ✅ Prices look reasonable for EUR/USD")
            else:
                print(f"  ⚠️  WARNING: Unusual price range for EUR/USD (expected 1.05-1.15)")

        print()
        print("=" * 70)
        print("SUMMARY")
        print("=" * 70)
        print(f"✅ Successfully downloaded {record_count:,} bars")
        print(f"✅ File size: {file_size_mb:.2f} MB")
        print(f"✅ Output: {OUTPUT_FILE}")

    except Exception as e:
        print(f"⚠️  Verification warning: {e}")
        print(f"File was downloaded but could not be fully verified")

if __name__ == "__main__":
    main()