foxhunt/wave153_bakeoff_cryptodatadownload/analyze_quality.py

#!/usr/bin/env python3
"""
CryptoDataDownload Quality Analysis Script
Analyzes BTC/USD and ETH/USD 1-minute OHLCV data quality
"""

import pandas as pd
import json
from datetime import datetime, timedelta

def analyze_file(filepath, symbol):
    """Analyze a single CSV file for quality metrics"""

    # Read CSV (skip first row which is the website URL)
    df = pd.read_csv(filepath, skiprows=1)

    # Parse dates
    df['date'] = pd.to_datetime(df['date'])

    # Sort by date ascending
    df = df.sort_values('date')

    # Basic stats
    row_count = len(df)
    date_range = f"{df['date'].min()} to {df['date'].max()}"

    # Calculate expected rows (1 minute bars for entire year 2024)
    # 2024 is a leap year: 366 days * 24 hours * 60 minutes = 527,040 minutes
    expected_rows = 366 * 24 * 60
    completeness_pct = (row_count / expected_rows) * 100

    # OHLCV Validation
    ohlcv_violations = 0
    # Check: high >= low
    violations_high_low = (df['high'] < df['low']).sum()
    ohlcv_violations += violations_high_low

    # Check: no negative prices
    violations_negative = ((df['open'] < 0) | (df['high'] < 0) | (df['low'] < 0) | (df['close'] < 0)).sum()
    ohlcv_violations += violations_negative

    # Check: no negative volumes
    # Detect volume column name (BTC or ETH)
    volume_col = 'Volume BTC' if 'Volume BTC' in df.columns else 'Volume ETH'
    violations_volume = (df[volume_col] < 0).sum()
    ohlcv_violations += violations_volume

    # Outlier Detection (>10% price change in 1 minute)
    df['pct_change'] = df['close'].pct_change().abs() * 100
    outliers_count = (df['pct_change'] > 10).sum()

    # Volume Quality
    zero_volume_count = (df[volume_col] == 0).sum()
    zero_volume_pct = (zero_volume_count / row_count) * 100

    # Missing timestamps (gaps)
    df['time_diff'] = df['date'].diff()
    expected_diff = timedelta(minutes=1)
    gaps = (df['time_diff'] > expected_diff).sum()

    # Price statistics
    price_stats = {
        'min_price': float(df['low'].min()),
        'max_price': float(df['high'].max()),
        'avg_price': float(df['close'].mean()),
        'median_price': float(df['close'].median())
    }

    # Volume statistics
    volume_stats = {
        'total_volume': float(df[volume_col].sum()),
        'avg_volume': float(df[volume_col].mean()),
        'median_volume': float(df[volume_col].median()),
        'max_volume': float(df[volume_col].max()),
        'volume_column': volume_col
    }

    return {
        'download_url': f"https://www.cryptodatadownload.com/cdd/Bitstamp_{symbol}_2024_minute.csv",
        'exchange': 'Bitstamp',
        'date_tested': datetime.now().strftime('%Y-%m-%d'),
        'date_range': date_range,
        'row_count': int(row_count),
        'expected_rows': int(expected_rows),
        'completeness_pct': round(completeness_pct, 2),
        'ohlcv_violations': int(ohlcv_violations),
        'ohlcv_details': {
            'high_low_violations': int(violations_high_low),
            'negative_price_violations': int(violations_negative),
            'negative_volume_violations': int(violations_volume)
        },
        'outliers_count': int(outliers_count),
        'outliers_pct': round((outliers_count / row_count) * 100, 2),
        'zero_volume_count': int(zero_volume_count),
        'zero_volume_pct': round(zero_volume_pct, 2),
        'gaps_detected': int(gaps),
        'price_stats': price_stats,
        'volume_stats': volume_stats
    }

def calculate_quality_score(btc_data, eth_data):
    """Calculate overall quality score (0-10)"""

    def score_symbol(data):
        score = 10.0

        # Completeness (max -2 points)
        if data['completeness_pct'] < 95:
            score -= 2.0
        elif data['completeness_pct'] < 98:
            score -= 1.0
        elif data['completeness_pct'] < 99:
            score -= 0.5

        # OHLCV violations (max -3 points)
        if data['ohlcv_violations'] > 100:
            score -= 3.0
        elif data['ohlcv_violations'] > 10:
            score -= 1.5
        elif data['ohlcv_violations'] > 0:
            score -= 0.5

        # Outliers (max -2 points)
        if data['outliers_pct'] > 1.0:
            score -= 2.0
        elif data['outliers_pct'] > 0.5:
            score -= 1.0
        elif data['outliers_pct'] > 0.1:
            score -= 0.5

        # Zero volume (max -2 points)
        if data['zero_volume_pct'] > 10:
            score -= 2.0
        elif data['zero_volume_pct'] > 5:
            score -= 1.0
        elif data['zero_volume_pct'] > 1:
            score -= 0.5

        # Gaps (max -1 point)
        gap_pct = (data['gaps_detected'] / data['row_count']) * 100
        if gap_pct > 5:
            score -= 1.0
        elif gap_pct > 1:
            score -= 0.5

        return max(0, score)

    btc_score = score_symbol(btc_data)
    eth_score = score_symbol(eth_data)

    return round((btc_score + eth_score) / 2, 1)

def generate_recommendation(overall_score, btc_data, eth_data):
    """Generate recommendation based on quality score"""

    if overall_score >= 9.0:
        return "EXCELLENT: Highly recommended for production use. Data quality is exceptional with minimal issues."
    elif overall_score >= 8.0:
        return "VERY GOOD: Recommended for production use. Minor quality issues present but acceptable."
    elif overall_score >= 7.0:
        return "GOOD: Acceptable for production use with monitoring. Some data quality concerns."
    elif overall_score >= 6.0:
        return "FAIR: Use with caution. Moderate data quality issues require validation."
    else:
        return "POOR: Not recommended for production use. Significant data quality problems detected."

def main():
    print("Analyzing CryptoDataDownload data quality...")

    # Analyze BTC
    print("  Analyzing BTC/USD...")
    btc_data = analyze_file('Bitstamp_BTCUSD_2024_minute.csv', 'BTCUSD')

    # Analyze ETH
    print("  Analyzing ETH/USD...")
    eth_data = analyze_file('Bitstamp_ETHUSD_2024_minute.csv', 'ETHUSD')

    # Calculate overall score
    overall_score = calculate_quality_score(btc_data, eth_data)

    # Generate recommendation
    recommendation = generate_recommendation(overall_score, btc_data, eth_data)

    # Build final report
    report = {
        'source': 'CryptoDataDownload',
        'analysis_date': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
        'btc_usd': btc_data,
        'eth_usd': eth_data,
        'overall_quality_score': overall_score,
        'recommendation': recommendation,
        'notes': [
            'Data is in reverse chronological order (newest first)',
            'Full year 2024 coverage (366 days - leap year)',
            'Free access via direct CSV download',
            'No API key required',
            'Daily updates available'
        ]
    }

    # Save report
    with open('analysis_report.json', 'w') as f:
        json.dump(report, f, indent=2)

    print(f"\n✅ Analysis complete!")
    print(f"   Overall Quality Score: {overall_score}/10")
    print(f"   Recommendation: {recommendation}")
    print(f"\n📊 Report saved to: analysis_report.json")

    # Print summary
    print("\n" + "="*70)
    print("SUMMARY")
    print("="*70)
    print(f"\nBTC/USD:")
    print(f"  Rows: {btc_data['row_count']:,} / {btc_data['expected_rows']:,} ({btc_data['completeness_pct']}%)")
    print(f"  OHLCV Violations: {btc_data['ohlcv_violations']}")
    print(f"  Outliers: {btc_data['outliers_count']} ({btc_data['outliers_pct']}%)")
    print(f"  Zero Volume: {btc_data['zero_volume_pct']}%")
    print(f"  Gaps: {btc_data['gaps_detected']}")

    print(f"\nETH/USD:")
    print(f"  Rows: {eth_data['row_count']:,} / {eth_data['expected_rows']:,} ({eth_data['completeness_pct']}%)")
    print(f"  OHLCV Violations: {eth_data['ohlcv_violations']}")
    print(f"  Outliers: {eth_data['outliers_count']} ({eth_data['outliers_pct']}%)")
    print(f"  Zero Volume: {eth_data['zero_volume_pct']}%")
    print(f"  Gaps: {eth_data['gaps_detected']}")

if __name__ == '__main__':
    main()