foxhunt/services/ml_training_service/validate_test_data.py

#!/usr/bin/env python3
"""
Validate test data for hyperparameter optimization.

Checks data integrity, size, and suitability for training.
"""

import argparse
import sys
from pathlib import Path

import pandas as pd


def validate_parquet_file(file_path: Path) -> dict:
    """Validate a Parquet file for ML training."""
    print(f"\nValidating: {file_path}")
    print("=" * 80)

    try:
        # Read Parquet file
        df = pd.read_parquet(file_path)

        # Basic statistics
        num_rows = len(df)
        num_cols = len(df.columns)
        memory_mb = df.memory_usage(deep=True).sum() / (1024 ** 2)

        print(f"✓ File loaded successfully")
        print(f"  - Rows: {num_rows:,}")
        print(f"  - Columns: {num_cols}")
        print(f"  - Memory: {memory_mb:.2f} MB")

        # Check for required columns (OHLCV)
        required_cols = ['timestamp', 'open', 'high', 'low', 'close', 'volume']
        missing_cols = [col for col in required_cols if col not in df.columns]

        if missing_cols:
            print(f"⚠️  Missing columns: {', '.join(missing_cols)}")
        else:
            print(f"✓ All required columns present: {', '.join(required_cols)}")

        # Time range
        if 'timestamp' in df.columns:
            time_range = df['timestamp'].max() - df['timestamp'].min()
            print(f"✓ Time range: {time_range}")

        # Check for nulls
        null_counts = df.isnull().sum()
        if null_counts.sum() > 0:
            print(f"⚠️  Null values found:")
            for col, count in null_counts[null_counts > 0].items():
                print(f"    - {col}: {count} nulls ({count/num_rows*100:.2f}%)")
        else:
            print(f"✓ No null values")

        # Training suitability assessment
        min_rows_required = 10000  # Minimum for meaningful training
        suitability = "EXCELLENT" if num_rows >= min_rows_required * 10 else \
                      "GOOD" if num_rows >= min_rows_required else \
                      "INSUFFICIENT"

        print(f"\nTraining Suitability: {suitability}")
        if num_rows < min_rows_required:
            print(f"  ⚠️  Warning: Less than {min_rows_required:,} rows may not be sufficient")

        return {
            "file_path": str(file_path),
            "valid": True,
            "num_rows": num_rows,
            "num_cols": num_cols,
            "memory_mb": memory_mb,
            "missing_columns": missing_cols,
            "null_values": null_counts.sum(),
            "suitability": suitability
        }

    except Exception as e:
        print(f"❌ Validation failed: {e}")
        return {
            "file_path": str(file_path),
            "valid": False,
            "error": str(e)
        }


def main():
    parser = argparse.ArgumentParser(
        description="Validate test data for hyperparameter optimization"
    )
    parser.add_argument(
        "file_path",
        type=str,
        help="Path to Parquet file to validate"
    )

    args = parser.parse_args()

    file_path = Path(args.file_path)
    if not file_path.exists():
        print(f"❌ File not found: {file_path}")
        sys.exit(1)

    result = validate_parquet_file(file_path)

    if result["valid"] and result.get("suitability") in ["EXCELLENT", "GOOD"]:
        print("\n✅ Data validation PASSED - Ready for hyperparameter optimization")
        sys.exit(0)
    else:
        print("\n⚠️  Data validation completed with warnings")
        sys.exit(0)


if __name__ == "__main__":
    main()