foxhunt/scripts/validate_data_quality.sh

#!/bin/bash
# DBN Data Quality Validation Script for CI/CD
#
# This script runs comprehensive data quality validation and fails
# the build if quality standards are not met.
#
# Usage:
#   ./scripts/validate_data_quality.sh [options]
#
# Options:
#   --min-quality SCORE    Minimum quality score (default: 70)
#   --output DIR          Output directory for reports (default: ./validation_reports)
#   --symbol SYMBOL       Validate single symbol
#   --all                 Validate all symbols (default)
#   --verbose             Enable verbose output
#
# Exit codes:
#   0 - All validations passed
#   1 - Quality check failed
#   2 - Validation error (no data, missing files)

set -e

# Default configuration
MIN_QUALITY=70
OUTPUT_DIR="./validation_reports"
SYMBOL=""
VALIDATE_ALL="--all"
VERBOSE=""
DATA_DIR="test_data/real/databento"

# Parse arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        --min-quality)
            MIN_QUALITY="$2"
            shift 2
            ;;
        --output)
            OUTPUT_DIR="$2"
            shift 2
            ;;
        --symbol)
            SYMBOL="$2"
            VALIDATE_ALL=""
            shift 2
            ;;
        --all)
            VALIDATE_ALL="--all"
            SYMBOL=""
            shift
            ;;
        --verbose)
            VERBOSE="--verbose"
            shift
            ;;
        --data-dir)
            DATA_DIR="$2"
            shift 2
            ;;
        *)
            echo "Unknown option: $1"
            exit 2
            ;;
    esac
done

# Create output directory
mkdir -p "$OUTPUT_DIR"

# Timestamp for report files
TIMESTAMP=$(date +%Y%m%d_%H%M%S)

echo "════════════════════════════════════════════════════════"
echo "DBN Data Quality Validation"
echo "════════════════════════════════════════════════════════"
echo "Timestamp: $(date --rfc-3339=seconds)"
echo "Min Quality Score: $MIN_QUALITY"
echo "Output Directory: $OUTPUT_DIR"
echo "Data Directory: $DATA_DIR"

if [ -n "$SYMBOL" ]; then
    echo "Symbol: $SYMBOL"
else
    echo "Mode: All symbols"
fi

echo "════════════════════════════════════════════════════════"
echo

# Check if data directory exists
if [ ! -d "$DATA_DIR" ]; then
    echo "❌ ERROR: Data directory not found: $DATA_DIR"
    exit 2
fi

# Count DBN files
DBN_COUNT=$(find "$DATA_DIR" -name "*.dbn" | wc -l)
if [ "$DBN_COUNT" -eq 0 ]; then
    echo "❌ ERROR: No DBN files found in: $DATA_DIR"
    exit 2
fi

echo "Found $DBN_COUNT DBN files"
echo

# Build validation tool if not already built
echo "Building validation tool..."
cargo build --release -p backtesting-service --bin validate_dbn_data 2>&1 | grep -v "Compiling\|Finished" || true
echo

# Run validation
VALIDATION_CMD="cargo run --release -p backtesting-service --bin validate_dbn_data --"

if [ -n "$SYMBOL" ]; then
    VALIDATION_CMD="$VALIDATION_CMD --symbol $SYMBOL"
else
    VALIDATION_CMD="$VALIDATION_CMD $VALIDATE_ALL"
fi

VALIDATION_CMD="$VALIDATION_CMD --data-dir $DATA_DIR"
VALIDATION_CMD="$VALIDATION_CMD --min-quality-score $MIN_QUALITY"
VALIDATION_CMD="$VALIDATION_CMD --fail-on-poor-quality"
VALIDATION_CMD="$VALIDATION_CMD $VERBOSE"

# Generate reports in multiple formats
echo "Generating validation reports..."
echo

# Text report (stdout + file)
TEXT_REPORT="$OUTPUT_DIR/validation_report_${TIMESTAMP}.txt"
echo "📄 Text report: $TEXT_REPORT"
$VALIDATION_CMD --format text 2>&1 | tee "$TEXT_REPORT"
TEXT_EXIT_CODE=${PIPESTATUS[0]}

# JSON report (if text passed)
if [ $TEXT_EXIT_CODE -eq 0 ]; then
    JSON_REPORT="$OUTPUT_DIR/validation_report_${TIMESTAMP}.json"
    echo
    echo "📊 JSON report: $JSON_REPORT"
    $VALIDATION_CMD --format json --output "$JSON_REPORT" >/dev/null 2>&1 || true

    # HTML report
    HTML_REPORT="$OUTPUT_DIR/validation_report_${TIMESTAMP}.html"
    echo "📈 HTML report: $HTML_REPORT"
    $VALIDATION_CMD --format html --output "$HTML_REPORT" >/dev/null 2>&1 || true

    # Create latest symlinks
    ln -sf "$(basename "$TEXT_REPORT")" "$OUTPUT_DIR/latest.txt"
    ln -sf "$(basename "$JSON_REPORT")" "$OUTPUT_DIR/latest.json"
    ln -sf "$(basename "$HTML_REPORT")" "$OUTPUT_DIR/latest.html"

    echo
    echo "════════════════════════════════════════════════════════"
    echo "✅ VALIDATION PASSED"
    echo "════════════════════════════════════════════════════════"
    echo
    echo "Reports generated:"
    echo "  • Text: $TEXT_REPORT"
    echo "  • JSON: $JSON_REPORT"
    echo "  • HTML: $HTML_REPORT"
    echo
    echo "View HTML report:"
    echo "  xdg-open $HTML_REPORT"
    echo
    exit 0
else
    echo
    echo "════════════════════════════════════════════════════════"
    echo "❌ VALIDATION FAILED"
    echo "════════════════════════════════════════════════════════"
    echo
    echo "Quality score below minimum threshold ($MIN_QUALITY)"
    echo "See report for details: $TEXT_REPORT"
    echo
    exit 1
fi