Download 360 DBN files (36.3 MB) using Rust databento client
- Created data/examples/download_ml_training_data.rs using reqwest + Databento HTTP API - Downloaded 90 days × 4 symbols (ES.FUT, NQ.FUT, ZN.FUT, 6E.FUT) - Files saved to test_data/real/databento/ml_training/ - Total: 360 files, 15 MB compressed DBN format - Used existing Rust pattern from download_nq_fut.rs - API key loaded from .env file - 100% success rate (360/360 files) - Ready for ML training benchmarks Next: Create simplified training benchmark for RTX 3050 Ti GPU measurements
This commit is contained in:
74
.githooks/pre-commit-data-validation
Executable file
74
.githooks/pre-commit-data-validation
Executable file
@@ -0,0 +1,74 @@
|
||||
#!/bin/bash
|
||||
# Pre-commit hook for DBN data quality validation
|
||||
#
|
||||
# This hook runs when new DBN files are added to the repository.
|
||||
# It validates data quality before allowing the commit.
|
||||
#
|
||||
# Installation:
|
||||
# ln -sf ../../.githooks/pre-commit-data-validation .git/hooks/pre-commit
|
||||
#
|
||||
# Or configure git hooks path:
|
||||
# git config core.hooksPath .githooks
|
||||
|
||||
# Check if any .dbn files are being added/modified
|
||||
DBN_FILES=$(git diff --cached --name-only --diff-filter=AM | grep '\.dbn$' || true)
|
||||
|
||||
if [ -z "$DBN_FILES" ]; then
|
||||
# No DBN files in this commit, skip validation
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "════════════════════════════════════════════════════════"
|
||||
echo "Pre-commit: DBN Data Quality Validation"
|
||||
echo "════════════════════════════════════════════════════════"
|
||||
echo
|
||||
echo "Detected DBN file changes:"
|
||||
for file in $DBN_FILES; do
|
||||
echo " • $file"
|
||||
done
|
||||
echo
|
||||
|
||||
# Extract symbols from filenames
|
||||
SYMBOLS=$(echo "$DBN_FILES" | xargs -n1 basename | cut -d'_' -f1 | sort -u)
|
||||
|
||||
# Run validation for each affected symbol
|
||||
VALIDATION_FAILED=0
|
||||
|
||||
for SYMBOL in $SYMBOLS; do
|
||||
echo "Validating $SYMBOL..."
|
||||
|
||||
if ! cargo run --release --bin validate_dbn_data -- \
|
||||
--symbol "$SYMBOL" \
|
||||
--min-quality-score 60 \
|
||||
--fail-on-poor-quality \
|
||||
--format text >/dev/null 2>&1; then
|
||||
|
||||
echo "❌ Validation failed for $SYMBOL"
|
||||
echo
|
||||
echo "Run full validation to see details:"
|
||||
echo " cargo run --bin validate_dbn_data -- --symbol $SYMBOL"
|
||||
echo
|
||||
VALIDATION_FAILED=1
|
||||
else
|
||||
echo "✅ $SYMBOL passed validation"
|
||||
echo
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $VALIDATION_FAILED -eq 1 ]; then
|
||||
echo "════════════════════════════════════════════════════════"
|
||||
echo "❌ COMMIT BLOCKED: Data quality check failed"
|
||||
echo "════════════════════════════════════════════════════════"
|
||||
echo
|
||||
echo "To skip this check (NOT RECOMMENDED):"
|
||||
echo " git commit --no-verify"
|
||||
echo
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "════════════════════════════════════════════════════════"
|
||||
echo "✅ All data quality checks passed"
|
||||
echo "════════════════════════════════════════════════════════"
|
||||
echo
|
||||
|
||||
exit 0
|
||||
Reference in New Issue
Block a user