foxhunt/scripts/python/data/final_6e_download.py

#!/usr/bin/env python3
"""
Final attempt to download 6E (Euro FX) data from Databento with correct date handling.
"""
import os
import databento as db
import sys

def main():
    api_key = os.environ.get("DATABENTO_API_KEY")
    if not api_key:
        print("ERROR: DATABENTO_API_KEY environment variable not set")
        sys.exit(1)

    client = db.Historical(api_key)

    print("=" * 70)
    print("Attempting 6E (Euro FX Futures) Download")
    print("=" * 70)

    dataset = "GLBX.MDP3"
    schema = "ohlcv-1m"
    # Use date strings without time component
    start = "2024-01-02"
    end = "2024-02-01"  # Exclusive end date (so includes up to 2024-01-31)

    # Try multiple symbol variations
    symbol_attempts = [
        ("ES.FUT", "S&P 500 E-mini (test if CME data works at all)"),
        ("ES", "S&P 500 E-mini (root symbol)"),
        ("6E", "Euro FX (root symbol)"),
        ("6E.FUT", "Euro FX (with .FUT suffix)"),
        ("6EH4", "Euro FX March 2024 (2-digit year)"),
        ("6EH24", "Euro FX March 2024 (4-digit year)"),
    ]

    print(f"\nDataset: {dataset}")
    print(f"Schema: {schema}")
    print(f"Date Range: {start} to {end} (exclusive)")
    print()

    successful_downloads = []

    for symbol, description in symbol_attempts:
        print("=" * 70)
        print(f"Attempting: {symbol} ({description})")
        print("=" * 70)

        try:
            # Step 1: Check cost
            print("Checking cost... ", end="", flush=True)
            cost = client.metadata.get_cost(
                dataset=dataset,
                symbols=[symbol],
                schema=schema,
                start=start,
                end=end
            )
            print(f"${cost:.4f}")

            if cost == 0:
                print("  ⚠️  Cost is $0 - no data available for this symbol/period")
                continue

            if cost > 5.0:
                print(f"  ⚠️  Cost ${cost:.4f} exceeds $5.00 - skipping")
                continue

            # Step 2: Download
            print("Downloading data... ", end="", flush=True)
            output_file = f"/home/jgrusewski/Work/foxhunt/test_data/real/databento/{symbol}_ohlcv-1m_2024-01-02_to_2024-01-31.dbn"

            client.timeseries.get_range(
                dataset=dataset,
                symbols=[symbol],
                schema=schema,
                start=start,
                end=end,
                path=output_file
            )

            # Step 3: Verify
            file_size = os.path.getsize(output_file)
            if file_size < 500:  # Less than 500 bytes suggests empty file
                print(f"❌ File too small ({file_size} bytes) - likely no data")
                os.remove(output_file)
                continue

            # Count records
            store = db.DBNStore.from_file(output_file)
            record_count = sum(1 for _ in store)

            print(f"✅ SUCCESS!")
            print(f"   Records: {record_count:,}")
            print(f"   Size: {file_size / (1024*1024):.2f} MB")
            print(f"   File: {output_file}")

            successful_downloads.append((symbol, record_count, file_size, cost))

        except Exception as e:
            error_msg = str(e)
            if "symbology" in error_msg.lower():
                print(f"❌ Symbol not found/resolved")
            elif "no data" in error_msg.lower():
                print(f"❌ No data available")
            else:
                print(f"❌ Error: {error_msg[:80]}")

        print()

    # Final summary
    print("=" * 70)
    print("DOWNLOAD SUMMARY")
    print("=" * 70)

    if successful_downloads:
        print(f"\n✅ Successfully downloaded {len(successful_downloads)} file(s):\n")
        total_cost = 0
        for symbol, records, size, cost in successful_downloads:
            print(f"  {symbol:12s} - {records:,} records, {size/(1024*1024):.2f} MB, ${cost:.4f}")
            total_cost += cost

        print(f"\n💰 Total Cost: ${total_cost:.4f}")
    else:
        print("\n❌ No data could be downloaded.")
        print("\nPossible reasons:")
        print("  1. Databento subscription doesn't include CME/GLBX.MDP3 data")
        print("  2. Symbology format is incorrect for this dataset")
        print("  3. Data not available for the requested time period")
        print("\nRecommendations:")
        print("  • Check subscription at: https://databento.com/account")
        print("  • Try a different data vendor (Alpaca, Polygon.io, IB)")
        print("  • Use synthetic/mock data for development")

if __name__ == "__main__":
    main()