foxhunt/scripts/python/data/validate_es_multiday.py

#!/usr/bin/env python3
"""
Validate and analyze multi-day ES futures data.

Analyzes data quality and classifies market regimes for the downloaded
ES futures data files.
"""

import databento as db
import pandas as pd
import numpy as np
from pathlib import Path

# Files to validate
FILES = [
    {
        "path": "test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn",
        "date": "2024-01-02",
        "expected_regime": "Baseline"
    },
    {
        "path": "test_data/real/databento/ESH4_ohlcv-1m_2024-01-03.dbn",
        "date": "2024-01-03",
        "expected_regime": "Trending"
    },
    {
        "path": "test_data/real/databento/ESH4_ohlcv-1m_2024-01-04.dbn",
        "date": "2024-01-04",
        "expected_regime": "Ranging"
    },
    {
        "path": "test_data/real/databento/ESH4_ohlcv-1m_2024-01-05.dbn",
        "date": "2024-01-05",
        "expected_regime": "Volatile"
    },
]


def calculate_regime_metrics(df):
    """Calculate metrics to classify market regime."""

    # Price metrics
    prices = df['close']
    returns = prices.pct_change().dropna()

    # Volatility (rolling std of returns)
    volatility = returns.std() * np.sqrt(1440)  # Annualized (1440 minutes/day)

    # Trend strength (correlation with time)
    time_index = np.arange(len(prices))
    trend_correlation = np.corrcoef(time_index, prices)[0, 1]

    # Price range
    price_range = (prices.max() - prices.min()) / prices.mean() * 100

    # Mean reversion (autocorrelation)
    autocorr = returns.autocorr() if len(returns) > 1 else 0

    # Volume analysis
    total_volume = df['volume'].sum()
    avg_volume = df['volume'].mean()
    volume_volatility = df['volume'].std() / avg_volume if avg_volume > 0 else 0

    # Directional consistency (% of bars moving same direction)
    up_bars = (df['close'] > df['open']).sum()
    directional_consistency = abs(up_bars / len(df) - 0.5) * 2  # 0 to 1

    return {
        'volatility': volatility,
        'trend_correlation': trend_correlation,
        'price_range_pct': price_range,
        'autocorrelation': autocorr,
        'total_volume': total_volume,
        'avg_volume': avg_volume,
        'volume_volatility': volume_volatility,
        'directional_consistency': directional_consistency,
    }


def classify_regime(metrics):
    """Classify market regime based on metrics."""

    # Trending: High trend correlation, high directional consistency
    if abs(metrics['trend_correlation']) > 0.7 and metrics['directional_consistency'] > 0.3:
        return "Trending"

    # Ranging: Low trend correlation, high autocorrelation (mean reversion)
    elif abs(metrics['trend_correlation']) < 0.3 and metrics['price_range_pct'] < 2.0:
        return "Ranging"

    # Volatile: High volatility, high volume volatility
    elif metrics['volatility'] > 0.15 and metrics['volume_volatility'] > 1.5:
        return "Volatile"

    # Mixed
    else:
        return "Mixed"


def main():
    """Validate all files and analyze regimes."""
    print("=" * 80)
    print("ES Futures Multi-Day Data Validation")
    print("=" * 80)
    print()

    results = []

    for file_config in FILES:
        path = file_config["path"]
        date = file_config["date"]
        expected_regime = file_config["expected_regime"]

        print(f"📊 Validating: {date} (Expected: {expected_regime})")
        print(f"   File: {path}")

        # Check if file exists
        if not Path(path).exists():
            print(f"   ❌ File not found!\n")
            continue

        try:
            # Load data
            store = db.DBNStore.from_file(path)
            df = store.to_df()

            # Basic validation
            record_count = len(df)
            file_size_kb = Path(path).stat().st_size / 1024

            if record_count == 0:
                print(f"   ❌ No records in file!\n")
                continue

            # Data quality checks
            ohlcv_valid = all(
                (df['high'] >= df['low']) &
                (df['high'] >= df['open']) &
                (df['high'] >= df['close']) &
                (df['low'] <= df['open']) &
                (df['low'] <= df['close'])
            )

            zero_volumes = (df['volume'] == 0).sum()

            # Calculate regime metrics
            metrics = calculate_regime_metrics(df)
            detected_regime = classify_regime(metrics)

            # Determine match status
            regime_match = "✅" if detected_regime.lower() in expected_regime.lower() or expected_regime == "Baseline" else "⚠️"

            print(f"   ✅ Valid OHLCV: {ohlcv_valid}")
            print(f"   Records: {record_count:,}")
            print(f"   File size: {file_size_kb:.2f} KB")
            print(f"   Price range: ${df['close'].min():.2f} - ${df['close'].max():.2f}")
            print(f"   Total volume: {df['volume'].sum():,.0f}")
            print(f"   Zero volumes: {zero_volumes}")
            print()
            print(f"   📈 Regime Analysis:")
            print(f"      Expected: {expected_regime}")
            print(f"      Detected: {detected_regime} {regime_match}")
            print(f"      Metrics:")
            print(f"        • Volatility: {metrics['volatility']:.4f}")
            print(f"        • Trend correlation: {metrics['trend_correlation']:.4f}")
            print(f"        • Price range: {metrics['price_range_pct']:.2f}%")
            print(f"        • Autocorrelation: {metrics['autocorrelation']:.4f}")
            print(f"        • Directional consistency: {metrics['directional_consistency']:.4f}")
            print(f"        • Volume volatility: {metrics['volume_volatility']:.4f}")
            print()

            results.append({
                'date': date,
                'expected_regime': expected_regime,
                'detected_regime': detected_regime,
                'records': record_count,
                'valid': ohlcv_valid,
                'match': regime_match == "✅",
                **metrics
            })

        except Exception as e:
            print(f"   ❌ Error: {e}\n")

    # Summary
    print("=" * 80)
    print("📋 SUMMARY")
    print("=" * 80)
    print()

    if results:
        df_results = pd.DataFrame(results)

        print(f"Total files validated: {len(results)}")
        print(f"All valid: {all(r['valid'] for r in results)}")
        print(f"Regime matches: {sum(r['match'] for r in results)}/{len(results)}")
        print()

        print("Regime Detection Results:")
        for r in results:
            match_str = "✅" if r['match'] else "⚠️"
            print(f"  {match_str} {r['date']}: {r['expected_regime']} → {r['detected_regime']}")
        print()

        print("💡 Notes:")
        print("  • Trending: High trend correlation (>0.7), directional consistency (>0.3)")
        print("  • Ranging: Low trend correlation (<0.3), price range (<2%)")
        print("  • Volatile: High volatility (>0.15), volume volatility (>1.5)")
        print("  • Regime detection uses statistical thresholds; manual review recommended")
        print()

        print("✅ All files validated successfully!")
        print("Ready for use in adaptive strategy regime testing.")
    else:
        print("❌ No files validated successfully!")


if __name__ == "__main__":
    main()