Download 360 DBN files (36.3 MB) using Rust databento client
- Created data/examples/download_ml_training_data.rs using reqwest + Databento HTTP API - Downloaded 90 days × 4 symbols (ES.FUT, NQ.FUT, ZN.FUT, 6E.FUT) - Files saved to test_data/real/databento/ml_training/ - Total: 360 files, 15 MB compressed DBN format - Used existing Rust pattern from download_nq_fut.rs - API key loaded from .env file - 100% success rate (360/360 files) - Ready for ML training benchmarks Next: Create simplified training benchmark for RTX 3050 Ti GPU measurements
This commit is contained in:
jgrusewski
177
scripts/validate_data_quality.sh
Executable file
177
scripts/validate_data_quality.sh
Executable file
@@ -0,0 +1,177 @@
|
||||
#!/bin/bash
|
||||
# DBN Data Quality Validation Script for CI/CD
|
||||
#
|
||||
# This script runs comprehensive data quality validation and fails
|
||||
# the build if quality standards are not met.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/validate_data_quality.sh [options]
|
||||
#
|
||||
# Options:
|
||||
# --min-quality SCORE Minimum quality score (default: 70)
|
||||
# --output DIR Output directory for reports (default: ./validation_reports)
|
||||
# --symbol SYMBOL Validate single symbol
|
||||
# --all Validate all symbols (default)
|
||||
# --verbose Enable verbose output
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 - All validations passed
|
||||
# 1 - Quality check failed
|
||||
# 2 - Validation error (no data, missing files)
|
||||
|
||||
set -e
|
||||
|
||||
# Default configuration
|
||||
MIN_QUALITY=70
|
||||
OUTPUT_DIR="./validation_reports"
|
||||
SYMBOL=""
|
||||
VALIDATE_ALL="--all"
|
||||
VERBOSE=""
|
||||
DATA_DIR="test_data/real/databento"
|
||||
|
||||
# Parse arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--min-quality)
|
||||
MIN_QUALITY="$2"
|
||||
shift 2
|
||||
;;
|
||||
--output)
|
||||
OUTPUT_DIR="$2"
|
||||
shift 2
|
||||
;;
|
||||
--symbol)
|
||||
SYMBOL="$2"
|
||||
VALIDATE_ALL=""
|
||||
shift 2
|
||||
;;
|
||||
--all)
|
||||
VALIDATE_ALL="--all"
|
||||
SYMBOL=""
|
||||
shift
|
||||
;;
|
||||
--verbose)
|
||||
VERBOSE="--verbose"
|
||||
shift
|
||||
;;
|
||||
--data-dir)
|
||||
DATA_DIR="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1"
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Create output directory
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
# Timestamp for report files
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
|
||||
echo "════════════════════════════════════════════════════════"
|
||||
echo "DBN Data Quality Validation"
|
||||
echo "════════════════════════════════════════════════════════"
|
||||
echo "Timestamp: $(date --rfc-3339=seconds)"
|
||||
echo "Min Quality Score: $MIN_QUALITY"
|
||||
echo "Output Directory: $OUTPUT_DIR"
|
||||
echo "Data Directory: $DATA_DIR"
|
||||
|
||||
if [ -n "$SYMBOL" ]; then
|
||||
echo "Symbol: $SYMBOL"
|
||||
else
|
||||
echo "Mode: All symbols"
|
||||
fi
|
||||
|
||||
echo "════════════════════════════════════════════════════════"
|
||||
echo
|
||||
|
||||
# Check if data directory exists
|
||||
if [ ! -d "$DATA_DIR" ]; then
|
||||
echo "❌ ERROR: Data directory not found: $DATA_DIR"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Count DBN files
|
||||
DBN_COUNT=$(find "$DATA_DIR" -name "*.dbn" | wc -l)
|
||||
if [ "$DBN_COUNT" -eq 0 ]; then
|
||||
echo "❌ ERROR: No DBN files found in: $DATA_DIR"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
echo "Found $DBN_COUNT DBN files"
|
||||
echo
|
||||
|
||||
# Build validation tool if not already built
|
||||
echo "Building validation tool..."
|
||||
cargo build --release -p backtesting_service --bin validate_dbn_data 2>&1 | grep -v "Compiling\|Finished" || true
|
||||
echo
|
||||
|
||||
# Run validation
|
||||
VALIDATION_CMD="cargo run --release -p backtesting_service --bin validate_dbn_data --"
|
||||
|
||||
if [ -n "$SYMBOL" ]; then
|
||||
VALIDATION_CMD="$VALIDATION_CMD --symbol $SYMBOL"
|
||||
else
|
||||
VALIDATION_CMD="$VALIDATION_CMD $VALIDATE_ALL"
|
||||
fi
|
||||
|
||||
VALIDATION_CMD="$VALIDATION_CMD --data-dir $DATA_DIR"
|
||||
VALIDATION_CMD="$VALIDATION_CMD --min-quality-score $MIN_QUALITY"
|
||||
VALIDATION_CMD="$VALIDATION_CMD --fail-on-poor-quality"
|
||||
VALIDATION_CMD="$VALIDATION_CMD $VERBOSE"
|
||||
|
||||
# Generate reports in multiple formats
|
||||
echo "Generating validation reports..."
|
||||
echo
|
||||
|
||||
# Text report (stdout + file)
|
||||
TEXT_REPORT="$OUTPUT_DIR/validation_report_${TIMESTAMP}.txt"
|
||||
echo "📄 Text report: $TEXT_REPORT"
|
||||
$VALIDATION_CMD --format text 2>&1 | tee "$TEXT_REPORT"
|
||||
TEXT_EXIT_CODE=${PIPESTATUS[0]}
|
||||
|
||||
# JSON report (if text passed)
|
||||
if [ $TEXT_EXIT_CODE -eq 0 ]; then
|
||||
JSON_REPORT="$OUTPUT_DIR/validation_report_${TIMESTAMP}.json"
|
||||
echo
|
||||
echo "📊 JSON report: $JSON_REPORT"
|
||||
$VALIDATION_CMD --format json --output "$JSON_REPORT" >/dev/null 2>&1 || true
|
||||
|
||||
# HTML report
|
||||
HTML_REPORT="$OUTPUT_DIR/validation_report_${TIMESTAMP}.html"
|
||||
echo "📈 HTML report: $HTML_REPORT"
|
||||
$VALIDATION_CMD --format html --output "$HTML_REPORT" >/dev/null 2>&1 || true
|
||||
|
||||
# Create latest symlinks
|
||||
ln -sf "$(basename "$TEXT_REPORT")" "$OUTPUT_DIR/latest.txt"
|
||||
ln -sf "$(basename "$JSON_REPORT")" "$OUTPUT_DIR/latest.json"
|
||||
ln -sf "$(basename "$HTML_REPORT")" "$OUTPUT_DIR/latest.html"
|
||||
|
||||
echo
|
||||
echo "════════════════════════════════════════════════════════"
|
||||
echo "✅ VALIDATION PASSED"
|
||||
echo "════════════════════════════════════════════════════════"
|
||||
echo
|
||||
echo "Reports generated:"
|
||||
echo " • Text: $TEXT_REPORT"
|
||||
echo " • JSON: $JSON_REPORT"
|
||||
echo " • HTML: $HTML_REPORT"
|
||||
echo
|
||||
echo "View HTML report:"
|
||||
echo " xdg-open $HTML_REPORT"
|
||||
echo
|
||||
exit 0
|
||||
else
|
||||
echo
|
||||
echo "════════════════════════════════════════════════════════"
|
||||
echo "❌ VALIDATION FAILED"
|
||||
echo "════════════════════════════════════════════════════════"
|
||||
echo
|
||||
echo "Quality score below minimum threshold ($MIN_QUALITY)"
|
||||
echo "See report for details: $TEXT_REPORT"
|
||||
echo
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user