e8a68ee39f
- Created data/examples/download_ml_training_data.rs using reqwest + Databento HTTP API - Downloaded 90 days × 4 symbols (ES.FUT, NQ.FUT, ZN.FUT, 6E.FUT) - Files saved to test_data/real/databento/ml_training/ - Total: 360 files, 15 MB compressed DBN format - Used existing Rust pattern from download_nq_fut.rs - API key loaded from .env file - 100% success rate (360/360 files) - Ready for ML training benchmarks Next: Create simplified training benchmark for RTX 3050 Ti GPU measurements
75 lines
2.7 KiB
Bash
Executable File
75 lines
2.7 KiB
Bash
Executable File
#!/bin/bash
|
|
# Pre-commit hook for DBN data quality validation
|
|
#
|
|
# This hook runs when new DBN files are added to the repository.
|
|
# It validates data quality before allowing the commit.
|
|
#
|
|
# Installation:
|
|
# ln -sf ../../.githooks/pre-commit-data-validation .git/hooks/pre-commit
|
|
#
|
|
# Or configure git hooks path:
|
|
# git config core.hooksPath .githooks
|
|
|
|
# Check if any .dbn files are being added/modified
|
|
DBN_FILES=$(git diff --cached --name-only --diff-filter=AM | grep '\.dbn$' || true)
|
|
|
|
if [ -z "$DBN_FILES" ]; then
|
|
# No DBN files in this commit, skip validation
|
|
exit 0
|
|
fi
|
|
|
|
echo "════════════════════════════════════════════════════════"
|
|
echo "Pre-commit: DBN Data Quality Validation"
|
|
echo "════════════════════════════════════════════════════════"
|
|
echo
|
|
echo "Detected DBN file changes:"
|
|
for file in $DBN_FILES; do
|
|
echo " • $file"
|
|
done
|
|
echo
|
|
|
|
# Extract symbols from filenames
|
|
SYMBOLS=$(echo "$DBN_FILES" | xargs -n1 basename | cut -d'_' -f1 | sort -u)
|
|
|
|
# Run validation for each affected symbol
|
|
VALIDATION_FAILED=0
|
|
|
|
for SYMBOL in $SYMBOLS; do
|
|
echo "Validating $SYMBOL..."
|
|
|
|
if ! cargo run --release --bin validate_dbn_data -- \
|
|
--symbol "$SYMBOL" \
|
|
--min-quality-score 60 \
|
|
--fail-on-poor-quality \
|
|
--format text >/dev/null 2>&1; then
|
|
|
|
echo "❌ Validation failed for $SYMBOL"
|
|
echo
|
|
echo "Run full validation to see details:"
|
|
echo " cargo run --bin validate_dbn_data -- --symbol $SYMBOL"
|
|
echo
|
|
VALIDATION_FAILED=1
|
|
else
|
|
echo "✅ $SYMBOL passed validation"
|
|
echo
|
|
fi
|
|
done
|
|
|
|
if [ $VALIDATION_FAILED -eq 1 ]; then
|
|
echo "════════════════════════════════════════════════════════"
|
|
echo "❌ COMMIT BLOCKED: Data quality check failed"
|
|
echo "════════════════════════════════════════════════════════"
|
|
echo
|
|
echo "To skip this check (NOT RECOMMENDED):"
|
|
echo " git commit --no-verify"
|
|
echo
|
|
exit 1
|
|
fi
|
|
|
|
echo "════════════════════════════════════════════════════════"
|
|
echo "✅ All data quality checks passed"
|
|
echo "════════════════════════════════════════════════════════"
|
|
echo
|
|
|
|
exit 0
|