Download 360 DBN files (36.3 MB) using Rust databento client
- Created data/examples/download_ml_training_data.rs using reqwest + Databento HTTP API - Downloaded 90 days × 4 symbols (ES.FUT, NQ.FUT, ZN.FUT, 6E.FUT) - Files saved to test_data/real/databento/ml_training/ - Total: 360 files, 15 MB compressed DBN format - Used existing Rust pattern from download_nq_fut.rs - API key loaded from .env file - 100% success rate (360/360 files) - Ready for ML training benchmarks Next: Create simplified training benchmark for RTX 3050 Ti GPU measurements
This commit is contained in:
jgrusewski
165
.github/workflows/data-quality-validation.yml
vendored
Normal file
165
.github/workflows/data-quality-validation.yml
vendored
Normal file
@@ -0,0 +1,165 @@
|
||||
name: DBN Data Quality Validation
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- 'test_data/**/*.dbn'
|
||||
- 'scripts/validate_data_quality.sh'
|
||||
- 'services/backtesting_service/src/bin/validate_dbn_data.rs'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'test_data/**/*.dbn'
|
||||
- 'scripts/validate_data_quality.sh'
|
||||
- 'services/backtesting_service/src/bin/validate_dbn_data.rs'
|
||||
schedule:
|
||||
# Run daily at 2 AM UTC to catch any data degradation
|
||||
- cron: '0 2 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
min_quality:
|
||||
description: 'Minimum quality score (0-100)'
|
||||
required: false
|
||||
default: '70'
|
||||
symbol:
|
||||
description: 'Specific symbol to validate (leave empty for all)'
|
||||
required: false
|
||||
default: ''
|
||||
|
||||
jobs:
|
||||
validate-data-quality:
|
||||
name: Validate DBN Data Quality
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Rust toolchain
|
||||
uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
toolchain: stable
|
||||
components: rustfmt, clippy
|
||||
|
||||
- name: Cache Cargo dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: |
|
||||
~/.cargo/bin/
|
||||
~/.cargo/registry/index/
|
||||
~/.cargo/registry/cache/
|
||||
~/.cargo/git/db/
|
||||
target/
|
||||
key: ${{ runner.os }}-cargo-validation-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-cargo-validation-
|
||||
${{ runner.os }}-cargo-
|
||||
|
||||
- name: Check for DBN files
|
||||
id: check_files
|
||||
run: |
|
||||
if [ -d "test_data/real/databento" ]; then
|
||||
COUNT=$(find test_data/real/databento -name "*.dbn" | wc -l)
|
||||
echo "count=$COUNT" >> $GITHUB_OUTPUT
|
||||
echo "Found $COUNT DBN files"
|
||||
else
|
||||
echo "count=0" >> $GITHUB_OUTPUT
|
||||
echo "No test_data directory found"
|
||||
fi
|
||||
|
||||
- name: Build validation tool
|
||||
if: steps.check_files.outputs.count > 0
|
||||
run: |
|
||||
cargo build --release --bin validate_dbn_data
|
||||
|
||||
- name: Run data quality validation
|
||||
if: steps.check_files.outputs.count > 0
|
||||
env:
|
||||
MIN_QUALITY: ${{ github.event.inputs.min_quality || '70' }}
|
||||
SYMBOL: ${{ github.event.inputs.symbol || '' }}
|
||||
run: |
|
||||
mkdir -p validation_reports
|
||||
|
||||
# Build validation command
|
||||
if [ -z "$SYMBOL" ]; then
|
||||
# Validate all symbols
|
||||
./scripts/validate_data_quality.sh \
|
||||
--min-quality "$MIN_QUALITY" \
|
||||
--output validation_reports \
|
||||
--all
|
||||
else
|
||||
# Validate specific symbol
|
||||
./scripts/validate_data_quality.sh \
|
||||
--min-quality "$MIN_QUALITY" \
|
||||
--output validation_reports \
|
||||
--symbol "$SYMBOL"
|
||||
fi
|
||||
|
||||
- name: Upload validation reports
|
||||
if: always() && steps.check_files.outputs.count > 0
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: validation-reports
|
||||
path: validation_reports/
|
||||
retention-days: 30
|
||||
|
||||
- name: Comment PR with validation results
|
||||
if: github.event_name == 'pull_request' && steps.check_files.outputs.count > 0
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{secrets.GITHUB_TOKEN}}
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const reportPath = 'validation_reports/latest.txt';
|
||||
|
||||
if (fs.existsSync(reportPath)) {
|
||||
const report = fs.readFileSync(reportPath, 'utf8');
|
||||
const summary = report.split('\n').slice(0, 30).join('\n');
|
||||
|
||||
await github.rest.issues.createComment({
|
||||
issue_number: context.issue.number,
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: `## 📊 DBN Data Quality Validation\n\n\`\`\`\n${summary}\n\`\`\`\n\n<details>\n<summary>View full report</summary>\n\n\`\`\`\n${report}\n\`\`\`\n\n</details>`
|
||||
});
|
||||
}
|
||||
|
||||
- name: Fail if validation failed
|
||||
if: failure() && steps.check_files.outputs.count > 0
|
||||
run: |
|
||||
echo "❌ Data quality validation failed!"
|
||||
echo "See validation reports artifact for details"
|
||||
exit 1
|
||||
|
||||
summary:
|
||||
name: Validation Summary
|
||||
runs-on: ubuntu-latest
|
||||
needs: validate-data-quality
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: Download validation reports
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: validation-reports
|
||||
path: validation_reports/
|
||||
continue-on-error: true
|
||||
|
||||
- name: Create job summary
|
||||
run: |
|
||||
echo "# DBN Data Quality Validation Summary" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
|
||||
if [ -f "validation_reports/latest.json" ]; then
|
||||
# Parse JSON report for summary
|
||||
echo "✅ Validation completed successfully" >> $GITHUB_STEP_SUMMARY
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "Download the full validation report from the artifacts." >> $GITHUB_STEP_SUMMARY
|
||||
else
|
||||
echo "ℹ️ No validation report generated (no DBN files or validation skipped)" >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
|
||||
if [ "${{ needs.validate-data-quality.result }}" == "failure" ]; then
|
||||
echo "" >> $GITHUB_STEP_SUMMARY
|
||||
echo "❌ **Validation failed - quality score below threshold**" >> $GITHUB_STEP_SUMMARY
|
||||
fi
|
||||
Reference in New Issue
Block a user