foxhunt/download_sequential_validation.py

#!/usr/bin/env python3
"""
Download sequential validation data for DQN evaluation.

Training data ends: 2025-10-19 23:59:00+00:00
Unseen data should start: 2025-10-20 00:00:00+00:00
Today: 2025-11-08

We can download ~19 days of sequential data (Oct 20 - Nov 8).
"""

import os
import sys
from datetime import datetime, timezone
import databento as db

# Configuration
API_KEY = os.getenv("DATABENTO_API_KEY", "db-95LEt9gtDRPJfc55NVUB5KL3A3uf6")
OUTPUT_DIR = "test_data"
SYMBOL = "ESZ5"  # December 2025 contract (frontmonth for Oct-Nov 2025)
SCHEMA = "ohlcv-1m"
DATASET = "GLBX.MDP3"

# Date range: Sequential after training (Oct 20 - Nov 7, 2025)
# Training ends: 2025-10-19 23:59:00
# Unseen starts: 2025-10-20 00:00:00
# End: 2025-11-07 23:59:59 (leave 1 day buffer)
START_DATE = "2025-10-20"
END_DATE = "2025-11-07"
OUTPUT_FILE_DBN = "ES_FUT_unseen_sequential.dbn"


def main():
    """Download sequential validation data."""
    print("=" * 80)
    print("ES.FUT Sequential Validation Data Download")
    print("=" * 80)
    print()

    # Check API key
    if not API_KEY or API_KEY == "your-key-here":
        print("❌ ERROR: DATABENTO_API_KEY not found in environment!")
        print("Set it with: export DATABENTO_API_KEY='your-key-here'")
        sys.exit(1)

    # Create output directory
    os.makedirs(OUTPUT_DIR, exist_ok=True)
    print(f"📁 Output directory: {OUTPUT_DIR}")
    print(f"🎯 Symbol: {SYMBOL}")
    print(f"📊 Schema: {SCHEMA}")
    print(f"📦 Dataset: {DATASET}")
    print(f"📅 Date range: {START_DATE} to {END_DATE}")
    print()
    print("📌 Context:")
    print("  Training data ends: 2025-10-19 23:59:00+00:00")
    print("  Unseen data starts: 2025-10-20 00:00:00+00:00")
    print("  Duration: ~19 days (sequential continuation)")
    print()

    # Initialize Databento client
    try:
        client = db.Historical(API_KEY)
        print("✅ Databento client initialized")
    except Exception as e:
        print(f"❌ Failed to initialize Databento client: {e}")
        sys.exit(1)

    print()
    print("-" * 80)
    print(f"📥 Downloading sequential validation data: {START_DATE} to {END_DATE}")
    print("-" * 80)

    try:
        # Parse dates
        start_dt = datetime.strptime(START_DATE, "%Y-%m-%d").replace(
            hour=0, minute=0, second=0, tzinfo=timezone.utc
        )
        end_dt = datetime.strptime(END_DATE, "%Y-%m-%d").replace(
            hour=23, minute=59, second=59, tzinfo=timezone.utc
        )

        # Build output path
        output_path = os.path.join(OUTPUT_DIR, OUTPUT_FILE_DBN)

        print(f"  Start: {start_dt.isoformat()}")
        print(f"  End:   {end_dt.isoformat()}")
        print(f"  Output: {output_path}")
        print()

        # Download data
        days_count = (end_dt - start_dt).days + 1
        print(f"⏳ Downloading... (this may take 1-3 minutes for {days_count} days)")
        data = client.timeseries.get_range(
            dataset=DATASET,
            symbols=[SYMBOL],
            schema=SCHEMA,
            start=start_dt.isoformat(),
            end=end_dt.isoformat(),
        )

        # Write to file
        print("💾 Writing to DBN file...")
        data.to_file(output_path)

        # Get file size
        file_size = os.path.getsize(output_path)
        file_size_mb = file_size / (1024 * 1024)

        print()
        print("✅ Download complete!")
        print(f"  File: {output_path}")
        print(f"  Size: {file_size:,} bytes ({file_size_mb:.2f} MB)")
        print()

        # Verify data with databento
        try:
            store = db.DBNStore.from_file(output_path)
            df = store.to_df()
            record_count = len(df)

            print("📊 Data Summary:")
            print(f"  Total bars: {record_count:,}")
            print(f"  Expected bars ({days_count} days × ~150 bars/day): ~{days_count * 150:,}")

            if record_count > 0:
                print(f"  Date range: {df.index[0]} to {df.index[-1]}")
                print(f"  Price range: ${df['close'].min():.2f} - ${df['close'].max():.2f}")
                print(f"  Total volume: {df['volume'].sum():,.0f}")

                # Market balance check
                bullish_bars = (df['close'] > df['open']).sum()
                bullish_pct = 100 * bullish_bars / record_count
                trend_pct = 100 * (df['close'].iloc[-1] - df['close'].iloc[0]) / df['close'].iloc[0]

                print(f"  Bullish bars: {bullish_pct:.1f}%")
                print(f"  Overall trend: {trend_pct:+.2f}%")

                # Assess balance
                if 40 <= bullish_pct <= 60:
                    print(f"  ✅ Market balance: GOOD (40-60% range)")
                elif 30 <= bullish_pct <= 70:
                    print(f"  ⚠️  Market balance: ACCEPTABLE (30-70% range)")
                else:
                    print(f"  ❌ Market balance: BIASED (outside 30-70% range)")
            else:
                print("  ⚠️  WARNING: No records in file!")

        except Exception as e:
            print(f"⚠️  Could not verify data with databento: {e}")
            print("   (File downloaded but verification failed)")

        # Estimate cost
        estimated_cost = 0.10 * days_count  # ~$0.10 per day
        print()
        print(f"💰 Estimated cost: ${estimated_cost:.2f}")
        print()

        print("=" * 80)
        print("📋 NEXT STEPS")
        print("=" * 80)
        print()
        print("The DBN file has been downloaded. It will be automatically")
        print("converted to parquet format and renamed.")
        print()
        print("To manually convert (if needed):")
        print(f"  cargo run -p data --example convert_dbn_to_parquet --release -- \\")
        print(f"    --input {output_path} \\")
        print(f"    --output test_data")
        print()
        print("✅ SUCCESS: Sequential validation data downloaded!")

    except Exception as e:
        print()
        print(f"❌ Download failed: {e}")
        print()
        print("Possible issues:")
        print("  • API key invalid or expired")
        print("  • Databento API rate limit exceeded")
        print("  • Network connectivity issues")
        print("  • Data not available for requested date range (future dates?)")
        sys.exit(1)


if __name__ == "__main__":
    main()