foxhunt/scripts/python/data/download_nq_fut_180d.py

#!/usr/bin/env python3
"""
Download 180 days of NQ.FUT (E-mini Nasdaq-100) OHLCV-1m data from Databento.

Agent: W12-03
Symbol: NQ.FUT
Date range: 2025-04-23 to 2025-10-20 (180 days)
Schema: ohlcv-1m
Dataset: GLBX.MDP3
Expected: ~1.33M bars, ~95 MB
Cost: ~$1.00

Usage:
    python3 download_nq_fut_180d.py
"""

import os
import sys
from datetime import datetime, timezone
import databento as db

# Configuration
API_KEY = os.getenv("DATABENTO_API_KEY", "db-95LEt9gtDRPJfc55NVUB5KL3A3uf6")
OUTPUT_FILE = "test_data/NQ_FUT_180d.dbn"
SCHEMA = "ohlcv-1m"
DATASET = "GLBX.MDP3"
SYMBOL = "NQ.FUT"
START_DATE = "2024-04-23"
END_DATE = "2024-10-19"  # Yesterday - today's data requires premium subscription


def main():
    """Download NQ.FUT 180-day OHLCV data."""
    print("=" * 80)
    print("NQ.FUT 180-Day Databento Download (Agent W12-03)")
    print("=" * 80)
    print()

    # Check API key
    if not API_KEY:
        print("❌ ERROR: DATABENTO_API_KEY not found in environment!")
        print("Set it with: export DATABENTO_API_KEY='your-key-here'")
        sys.exit(1)

    # Create output directory
    os.makedirs(os.path.dirname(OUTPUT_FILE), exist_ok=True)
    print(f"📁 Output file: {OUTPUT_FILE}")
    print(f"📊 Schema: {SCHEMA}")
    print(f"📦 Dataset: {DATASET}")
    print(f"🎯 Symbol: {SYMBOL}")
    print(f"📅 Date range: {START_DATE} to {END_DATE}")
    print()

    # Initialize Databento client
    try:
        client = db.Historical(API_KEY)
        print("✅ Databento client initialized")
    except Exception as e:
        print(f"❌ Failed to initialize Databento client: {e}")
        sys.exit(1)

    # Download data
    try:
        print()
        print("-" * 80)
        print(f"📥 Downloading NQ.FUT data...")
        print("-" * 80)
        print()

        # Parse dates
        start_dt = datetime.strptime(START_DATE, "%Y-%m-%d").replace(tzinfo=timezone.utc)
        # END_DATE includes time, so parse accordingly
        if "T" in END_DATE:
            end_dt = datetime.fromisoformat(END_DATE).replace(tzinfo=timezone.utc)
        else:
            end_dt = datetime.strptime(END_DATE, "%Y-%m-%d").replace(hour=23, minute=59, second=59, tzinfo=timezone.utc)

        print(f"  Start: {start_dt.isoformat()}")
        print(f"  End:   {end_dt.isoformat()}")
        print()

        # Request data
        print("  Requesting data from Databento...")
        data = client.timeseries.get_range(
            dataset=DATASET,
            symbols=[SYMBOL],
            schema=SCHEMA,
            start=start_dt.isoformat(),
            end=end_dt.isoformat(),
            stype_in="parent",  # Required for .FUT continuous symbols
        )

        # Write to file
        print("  Writing to file...")
        data.to_file(OUTPUT_FILE)

        # Get file size
        file_size = os.path.getsize(OUTPUT_FILE)
        file_size_mb = file_size / (1024 * 1024)

        print()
        print(f"✅ Download complete!")
        print(f"  File size: {file_size:,} bytes ({file_size_mb:.2f} MB)")

        # Read back to verify data count
        try:
            print("  Verifying data...")
            store = db.DBNStore.from_file(OUTPUT_FILE)
            df = store.to_df()
            record_count = len(df)

            print(f"  Records: {record_count:,}")

            # Show sample data
            if record_count > 0:
                print(f"  Price range: ${df['close'].min():.2f} - ${df['close'].max():.2f}")
                print(f"  Volume: {df['volume'].sum():,.0f}")

                # Show first and last timestamps
                print(f"  First timestamp: {df.index[0]}")
                print(f"  Last timestamp: {df.index[-1]}")
            else:
                print("  ⚠️  WARNING: No records in file!")
                sys.exit(1)

        except Exception as e:
            print(f"  ⚠️  Could not verify record count: {e}")

        # Estimate cost (rough: ~$1.00 for 180 days of 1-minute OHLCV data)
        estimated_cost = 1.00

        print()
        print(f"💰 Estimated cost: ${estimated_cost:.2f}")

        # Success summary
        print()
        print("=" * 80)
        print("✅ SUCCESS: NQ.FUT download complete!")
        print("=" * 80)
        print()
        print(f"📁 File: {OUTPUT_FILE}")
        print(f"📊 Size: {file_size_mb:.2f} MB")
        print(f"📈 Records: {record_count:,} bars")
        print(f"💰 Cost: ${estimated_cost:.2f}")
        print()
        print("📋 NEXT STEPS:")
        print("1. Validate file:")
        print("   cargo run -p backtesting_service --example validate_dbn_data")
        print("2. Use for ML training with 225 features")
        print("3. Proceed to W12-04 (6E.FUT download)")
        print()

    except Exception as e:
        print()
        print(f"❌ Download failed: {e}")
        print()
        print("Possible causes:")
        print("• Invalid API key")
        print("• Network connectivity issues")
        print("• Databento service unavailable")
        print("• Invalid date range or symbol")
        print()
        sys.exit(1)


if __name__ == "__main__":
    main()