foxhunt/check_data_sequence.py

#!/usr/bin/env python3
"""
Verify chronological sequence of training and unseen validation data.
Uses databento to read parquet files.
"""

import sys
import os

def main():
    try:
        import databento as db
        from datetime import datetime

        print("=" * 60)
        print("TRAINING DATA (ES_FUT_180d.parquet)")
        print("=" * 60)

        # Read training data
        train_store = db.DBNStore.from_file('test_data/ES_FUT_180d.parquet')
        train_df = train_store.to_df()
        train_rows = len(train_df)

        print(f"Rows: {train_rows:,}")
        print(f"File size: 2.9M")
        print(f"Columns: {list(train_df.columns)}")

        train_start = train_df.index[0]
        train_end = train_df.index[-1]

        print(f"\nDate range:")
        print(f"  Start: {train_start}")
        print(f"  End:   {train_end}")

        # Read unseen data
        print("\n" + "=" * 60)
        print("UNSEEN VALIDATION DATA (ES_FUT_unseen.parquet)")
        print("=" * 60)

        unseen_store = db.DBNStore.from_file('test_data/ES_FUT_unseen.parquet')
        unseen_df = unseen_store.to_df()
        unseen_rows = len(unseen_df)

        print(f"Rows: {unseen_rows:,}")
        print(f"File size: 224K")
        print(f"Columns: {list(unseen_df.columns)}")

        unseen_start = unseen_df.index[0]
        unseen_end = unseen_df.index[-1]

        print(f"\nDate range:")
        print(f"  Start: {unseen_start}")
        print(f"  End:   {unseen_end}")

        # Check chronological sequence
        print("\n" + "=" * 60)
        print("CHRONOLOGICAL SEQUENCE CHECK")
        print("=" * 60)

        gap = unseen_start - train_end
        gap_seconds = gap.total_seconds()
        gap_days = gap_seconds / 86400

        print(f"Training ends:     {train_end}")
        print(f"Unseen starts:     {unseen_start}")
        print(f"Gap:               {gap} ({gap_seconds:,.0f} seconds = {gap_days:.1f} days)")

        if gap_seconds < 0:
            print(f"⚠️  OVERLAP: Unseen data starts BEFORE training ends!")
            status = "FAILED"
        elif gap_seconds == 0:
            print(f"✅ PERFECT: No gap, immediate continuation")
            status = "EXISTS"
        elif gap_seconds <= 86400:
            print(f"✅ ACCEPTABLE: Gap is less than 1 day")
            status = "EXISTS"
        else:
            print(f"⚠️  GAP: {gap_days:.1f} days between training and unseen data")
            status = "FAILED"

        # Calculate unseen duration
        unseen_duration = unseen_end - unseen_start
        days = unseen_duration.total_seconds() / 86400
        print(f"\nUnseen data duration: {days:.1f} days")

        short_data = False
        if days < 30:
            print(f"⚠️  WARNING: Only {days:.1f} days (recommended: 30-90 days)")
            short_data = True
        else:
            print(f"✅ GOOD: {days:.1f} days of validation data")

        # Data quality check
        print("\n" + "=" * 60)
        print("DATA QUALITY CHECK")
        print("=" * 60)

        null_counts = unseen_df.isnull().sum()
        total_nulls = null_counts.sum()
        print(f"Null values: {dict(null_counts[null_counts > 0])}")
        print(f"Total nulls: {total_nulls}")

        has_nulls = total_nulls > 0
        if has_nulls:
            print(f"⚠️  WARNING: {total_nulls} null values detected")
        else:
            print("✅ No null values")

        print(f"\nPrice stats (close):")
        print(f"  Min: ${unseen_df['close'].min():.2f}")
        print(f"  Max: ${unseen_df['close'].max():.2f}")
        print(f"  Mean: ${unseen_df['close'].mean():.2f}")

        print(f"\nVolume stats:")
        print(f"  Total: {unseen_df['volume'].sum():,.0f}")
        print(f"  Mean: {unseen_df['volume'].mean():,.0f}")

        # Final verdict
        print("\n" + "=" * 60)
        print("FINAL VERDICT")
        print("=" * 60)

        if status == "FAILED":
            print(f"❌ FAILED: Need to re-download - chronological gap/overlap issue")
            ready = "NO"
        elif has_nulls:
            print(f"❌ FAILED: Null values detected - need to re-download")
            ready = "NO"
        elif short_data:
            print(f"⚠️  WARNING: Data exists but only {days:.1f} days (recommend 30-90)")
            ready = "PARTIAL"
        else:
            print(f"✅ PASSED: Data is valid and ready for backtest")
            ready = "YES"

        print(f"\nStatus: {status}")
        print(f"Ready for backtest: {ready}")

        return 0 if ready in ["YES", "PARTIAL"] else 1

    except ImportError as e:
        print(f"ERROR: {e}")
        print("\nDatabento is not installed. Installing...")
        os.system("pip3 install --user databento --break-system-packages")
        print("\nPlease run this script again after installation.")
        return 2
    except FileNotFoundError as e:
        print(f"ERROR: File not found: {e}")
        print("\nThe unseen validation data doesn't exist.")
        print("Will need to download it from Databento.")
        return 3
    except Exception as e:
        print(f"ERROR: {e}")
        import traceback
        traceback.print_exc()
        return 4

if __name__ == "__main__":
    sys.exit(main())