foxhunt/check_data_simple.py

#!/usr/bin/env python3
"""
Verify chronological sequence of training and unseen validation data.
Uses pandas to read standard parquet files.
"""

import sys
import pandas as pd
from datetime import datetime

def main():
    try:
        print("=" * 60)
        print("TRAINING DATA (ES_FUT_180d.parquet)")
        print("=" * 60)

        # Read training data
        train_df = pd.read_parquet('test_data/ES_FUT_180d.parquet')
        train_rows = len(train_df)

        print(f"Rows: {train_rows:,}")
        print(f"File size: 2.9M")
        print(f"Columns: {list(train_df.columns)}")
        print(f"Index name: {train_df.index.name}")

        # Timestamp is in the index for DBN parquet files
        train_start = train_df.index.min()
        train_end = train_df.index.max()

        print(f"\nDate range (using index):")
        print(f"  Start: {train_start}")
        print(f"  End:   {train_end}")

        # Read unseen data
        print("\n" + "=" * 60)
        print("UNSEEN VALIDATION DATA (ES_FUT_unseen.parquet)")
        print("=" * 60)

        unseen_df = pd.read_parquet('test_data/ES_FUT_unseen.parquet')
        unseen_rows = len(unseen_df)

        print(f"Rows: {unseen_rows:,}")
        print(f"File size: 224K")
        print(f"Columns: {list(unseen_df.columns)}")
        print(f"Index name: {unseen_df.index.name}")

        unseen_start = unseen_df.index.min()
        unseen_end = unseen_df.index.max()

        print(f"\nDate range (using index):")
        print(f"  Start: {unseen_start}")
        print(f"  End:   {unseen_end}")

        # Check chronological sequence
        print("\n" + "=" * 60)
        print("CHRONOLOGICAL SEQUENCE CHECK")
        print("=" * 60)

        # Convert to pandas Timestamp if not already
        if isinstance(train_end, pd.Timestamp):
            gap = unseen_start - train_end
            gap_seconds = gap.total_seconds()
        else:
            # Nanoseconds timestamps
            gap_ns = int(unseen_start) - int(train_end)
            gap_seconds = gap_ns / 1e9
            gap = pd.Timedelta(seconds=gap_seconds)

        gap_days = gap_seconds / 86400

        print(f"Training ends:     {train_end}")
        print(f"Unseen starts:     {unseen_start}")
        print(f"Gap:               {gap} ({gap_seconds:,.0f} seconds = {gap_days:.1f} days)")

        if gap_seconds < 0:
            print(f"⚠️  OVERLAP: Unseen data starts BEFORE training ends!")
            status = "FAILED"
        elif gap_seconds == 0:
            print(f"✅ PERFECT: No gap, immediate continuation")
            status = "EXISTS"
        elif gap_seconds <= 86400:
            print(f"✅ ACCEPTABLE: Gap is less than 1 day")
            status = "EXISTS"
        else:
            print(f"⚠️  GAP: {gap_days:.1f} days between training and unseen data")
            status = "FAILED"

        # Calculate unseen duration
        if isinstance(unseen_end, pd.Timestamp):
            unseen_duration = unseen_end - unseen_start
            days = unseen_duration.total_seconds() / 86400
        else:
            duration_ns = int(unseen_end) - int(unseen_start)
            days = duration_ns / (1e9 * 86400)

        print(f"\nUnseen data duration: {days:.1f} days")

        short_data = False
        if days < 30:
            print(f"⚠️  WARNING: Only {days:.1f} days (recommended: 30-90 days)")
            short_data = True
        else:
            print(f"✅ GOOD: {days:.1f} days of validation data")

        # Data quality check
        print("\n" + "=" * 60)
        print("DATA QUALITY CHECK")
        print("=" * 60)

        null_counts = unseen_df.isnull().sum()
        total_nulls = null_counts.sum()

        if total_nulls > 0:
            print(f"Null values: {dict(null_counts[null_counts > 0])}")
        print(f"Total nulls: {total_nulls}")

        has_nulls = total_nulls > 0
        if has_nulls:
            print(f"⚠️  WARNING: {total_nulls} null values detected")
        else:
            print("✅ No null values")

        # Check for OHLCV columns
        if 'close' in unseen_df.columns:
            print(f"\nPrice stats (close):")
            print(f"  Min: ${unseen_df['close'].min():.2f}")
            print(f"  Max: ${unseen_df['close'].max():.2f}")
            print(f"  Mean: ${unseen_df['close'].mean():.2f}")

        if 'volume' in unseen_df.columns:
            print(f"\nVolume stats:")
            print(f"  Total: {unseen_df['volume'].sum():,.0f}")
            print(f"  Mean: {unseen_df['volume'].mean():,.0f}")

        # Final verdict
        print("\n" + "=" * 60)
        print("FINAL VERDICT")
        print("=" * 60)

        if status == "FAILED":
            print(f"❌ FAILED: Need to re-download - chronological gap/overlap issue")
            ready = "NO"
        elif has_nulls:
            print(f"⚠️  WARNING: Null values detected (may need cleaning)")
            ready = "PARTIAL"
        elif short_data:
            print(f"⚠️  WARNING: Data exists but only {days:.1f} days (recommend 30-90)")
            ready = "PARTIAL"
        else:
            print(f"✅ PASSED: Data is valid and ready for backtest")
            ready = "YES"

        print(f"\nStatus: {status}")
        print(f"Ready for backtest: {ready}")
        print(f"\nTraining data: {train_start} to {train_end}")
        print(f"Unseen data: {unseen_start} to {unseen_end}")
        print(f"File sizes: training=2.9M, unseen=224K")
        print(f"Row counts: training={train_rows:,}, unseen={unseen_rows:,}")

        return 0 if ready in ["YES", "PARTIAL"] else 1

    except FileNotFoundError as e:
        print(f"ERROR: File not found: {e}")
        print("\nThe unseen validation data doesn't exist.")
        print("Will need to download it from Databento.")
        return 3
    except Exception as e:
        print(f"ERROR: {e}")
        import traceback
        traceback.print_exc()
        return 4

if __name__ == "__main__":
    sys.exit(main())