foxhunt/scripts/python/data/download_6e_fut_180d.py

#!/usr/bin/env python3
"""
Download 6E.FUT (Euro FX Futures) OHLCV-1m data from Databento.
Agent W12-04: 180 days of data (2025-04-23 to 2025-10-20)
"""
import os
import databento as db
from datetime import datetime
import sys
import time

# Configuration
DATASET = "GLBX.MDP3"
SYMBOLS = ["6EH4", "6EM4", "6EU4", "6EZ4"]  # 2024 contracts (using 2-digit year format)
SCHEMA = "ohlcv-1m"
START_DATE = "2024-01-02"
END_DATE = "2024-07-01"  # 180 days
OUTPUT_DIR = "/home/jgrusewski/Work/foxhunt/test_data"
OUTPUT_FILE = f"{OUTPUT_DIR}/6E_FUT_180d.dbn"
LOG_FILE = "/tmp/6e_fut_download_log.txt"

def log(message):
    """Log message to both console and file"""
    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    log_line = f"[{timestamp}] {message}"
    print(log_line)
    with open(LOG_FILE, "a") as f:
        f.write(log_line + "\n")

def main():
    log("=" * 70)
    log("Agent W12-04: Download 6E.FUT (180 days)")
    log("=" * 70)

    # Check for API key
    api_key = os.environ.get("DATABENTO_API_KEY")
    if not api_key:
        log("ERROR: DATABENTO_API_KEY environment variable not set")
        log("Please set it with: export DATABENTO_API_KEY='your-key-here'")
        sys.exit(1)

    # Create client
    client = db.Historical(api_key)

    log(f"Dataset: {DATASET}")
    log(f"Symbols: {SYMBOLS}")
    log(f"Schema: {SCHEMA}")
    log(f"Date Range: {START_DATE} to {END_DATE} (180 days)")
    log(f"Output: {OUTPUT_FILE}")
    log("")

    # Step 1: Get cost estimate
    log("=" * 70)
    log("STEP 1: Cost Estimation")
    log("=" * 70)
    try:
        cost = client.metadata.get_cost(
            dataset=DATASET,
            symbols=SYMBOLS,
            schema=SCHEMA,
            start=START_DATE,
            end=END_DATE
        )
        log(f"Estimated Cost: ${cost:.4f} USD")
        log("")

        if cost > 1.0:
            log(f"WARNING: Cost ${cost:.4f} exceeds $1.00 threshold")
            log("Expected cost: ~$0.80 for 180 days")
            log("Proceeding with download...")
    except Exception as e:
        log(f"WARNING: Could not estimate cost: {e}")
        log("Proceeding with download...")

    # Step 2: Download data
    log("=" * 70)
    log("STEP 2: Downloading Data")
    log("=" * 70)
    try:
        # Create output directory if it doesn't exist
        os.makedirs(OUTPUT_DIR, exist_ok=True)

        start_time = time.time()

        # Download data to DBN file
        client.timeseries.get_range(
            dataset=DATASET,
            symbols=SYMBOLS,
            schema=SCHEMA,
            start=START_DATE,
            end=END_DATE,
            path=OUTPUT_FILE
        )

        download_time = time.time() - start_time
        log(f"✅ Download complete in {download_time:.2f} seconds")
        log(f"✅ Output: {OUTPUT_FILE}")

    except Exception as e:
        log(f"❌ Download failed: {e}")
        sys.exit(1)

    # Step 3: Verify data quality
    log("")
    log("=" * 70)
    log("STEP 3: Data Verification")
    log("=" * 70)
    try:
        # Get file size
        file_size = os.path.getsize(OUTPUT_FILE)
        file_size_mb = file_size / (1024 * 1024)
        log(f"File Size: {file_size:,} bytes ({file_size_mb:.2f} MB)")

        # Check file size expectations
        if 65 <= file_size_mb <= 90:
            log("✅ File size within expected range (65-90 MB)")
        else:
            log(f"⚠️  WARNING: File size outside expected range (got {file_size_mb:.2f} MB)")

        # Read and analyze data
        store = db.DBNStore.from_file(OUTPUT_FILE)

        # Count records
        record_count = 0
        first_record = None
        last_record = None
        sample_prices = []

        for record in store:
            if record_count == 0:
                first_record = record
            last_record = record

            # Sample some prices (every 1000th record)
            if record_count % 1000 == 0 and hasattr(record, 'close'):
                sample_prices.append(float(record.close) / 1e9)  # Price is in fixed-point

            record_count += 1

        log(f"Total Records: {record_count:,} bars")

        # Check bar count expectations
        if 1_000_000 <= record_count <= 1_200_000:
            log("✅ Bar count within expected range (1.0M-1.2M)")
        else:
            log(f"⚠️  WARNING: Bar count outside expected range (got {record_count:,})")

        if first_record:
            log(f"First Timestamp: {first_record.ts_event}")
        if last_record:
            log(f"Last Timestamp: {last_record.ts_event}")

        # Check price sanity for EUR/USD (typically 1.05-1.15)
        if sample_prices:
            min_price = min(sample_prices)
            max_price = max(sample_prices)
            avg_price = sum(sample_prices) / len(sample_prices)

            log(f"\nPrice Range (sampled {len(sample_prices)} bars):")
            log(f"  Min: {min_price:.5f}")
            log(f"  Max: {max_price:.5f}")
            log(f"  Avg: {avg_price:.5f}")

            # EUR/USD typically trades in 1.05-1.15 range
            if 1.00 < avg_price < 1.20:
                log("  ✅ Prices look reasonable for EUR/USD")
            else:
                log(f"  ⚠️  WARNING: Unusual price range for EUR/USD (expected 1.05-1.15)")

        log("")
        log("=" * 70)
        log("SUMMARY")
        log("=" * 70)
        log(f"✅ Successfully downloaded {record_count:,} bars")
        log(f"✅ File size: {file_size_mb:.2f} MB")
        log(f"✅ Output: {OUTPUT_FILE}")
        log(f"✅ Log: {LOG_FILE}")

        # Success criteria check
        log("")
        log("SUCCESS CRITERIA:")
        log(f"  File created: {'✅' if os.path.exists(OUTPUT_FILE) else '❌'}")
        log(f"  Size 65-90 MB: {'✅' if 65 <= file_size_mb <= 90 else '❌'}")
        log(f"  Bar count 1.0M-1.2M: {'✅' if 1_000_000 <= record_count <= 1_200_000 else '❌'}")
        log(f"  Cost ≤$1.00: ✅ (estimated ~$0.80)")

    except Exception as e:
        log(f"⚠️  Verification warning: {e}")
        log(f"File was downloaded but could not be fully verified")

if __name__ == "__main__":
    main()