foxhunt/.github/workflows/data-quality-validation.yml

name: DBN Data Quality Validation

on:
  push:
    paths:
      - 'test_data/**/*.dbn'
      - 'scripts/validate_data_quality.sh'
      - 'services/backtesting_service/src/bin/validate_dbn_data.rs'
  pull_request:
    paths:
      - 'test_data/**/*.dbn'
      - 'scripts/validate_data_quality.sh'
      - 'services/backtesting_service/src/bin/validate_dbn_data.rs'
  schedule:
    # Run daily at 2 AM UTC to catch any data degradation
    - cron: '0 2 * * *'
  workflow_dispatch:
    inputs:
      min_quality:
        description: 'Minimum quality score (0-100)'
        required: false
        default: '70'
      symbol:
        description: 'Specific symbol to validate (leave empty for all)'
        required: false
        default: ''

jobs:
  validate-data-quality:
    name: Validate DBN Data Quality
    runs-on: ubuntu-latest
    timeout-minutes: 15

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Setup Rust toolchain
        uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          toolchain: stable
          components: rustfmt, clippy

      - name: Cache Cargo dependencies
        uses: actions/cache@v4
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
            target/
          key: ${{ runner.os }}-cargo-validation-${{ hashFiles('**/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-validation-
            ${{ runner.os }}-cargo-

      - name: Check for DBN files
        id: check_files
        run: |
          if [ -d "test_data/real/databento" ]; then
            COUNT=$(find test_data/real/databento -name "*.dbn" | wc -l)
            echo "count=$COUNT" >> $GITHUB_OUTPUT
            echo "Found $COUNT DBN files"
          else
            echo "count=0" >> $GITHUB_OUTPUT
            echo "No test_data directory found"
          fi

      - name: Build validation tool
        if: steps.check_files.outputs.count > 0
        run: |
          cargo build --release --bin validate_dbn_data

      - name: Run data quality validation
        if: steps.check_files.outputs.count > 0
        env:
          MIN_QUALITY: ${{ github.event.inputs.min_quality || '70' }}
          SYMBOL: ${{ github.event.inputs.symbol || '' }}
        run: |
          mkdir -p validation_reports

          # Build validation command
          if [ -z "$SYMBOL" ]; then
            # Validate all symbols
            ./scripts/validate_data_quality.sh \
              --min-quality "$MIN_QUALITY" \
              --output validation_reports \
              --all
          else
            # Validate specific symbol
            ./scripts/validate_data_quality.sh \
              --min-quality "$MIN_QUALITY" \
              --output validation_reports \
              --symbol "$SYMBOL"
          fi

      - name: Upload validation reports
        if: always() && steps.check_files.outputs.count > 0
        uses: actions/upload-artifact@v4
        with:
          name: validation-reports
          path: validation_reports/
          retention-days: 30

      - name: Comment PR with validation results
        if: github.event_name == 'pull_request' && steps.check_files.outputs.count > 0
        uses: actions/github-script@v7
        with:
          github-token: ${{secrets.GITHUB_TOKEN}}
          script: |
            const fs = require('fs');
            const reportPath = 'validation_reports/latest.txt';

            if (fs.existsSync(reportPath)) {
              const report = fs.readFileSync(reportPath, 'utf8');
              const summary = report.split('\n').slice(0, 30).join('\n');

              await github.rest.issues.createComment({
                issue_number: context.issue.number,
                owner: context.repo.owner,
                repo: context.repo.repo,
                body: `## 📊 DBN Data Quality Validation\n\n\`\`\`\n${summary}\n\`\`\`\n\n<details>\n<summary>View full report</summary>\n\n\`\`\`\n${report}\n\`\`\`\n\n</details>`
              });
            }

      - name: Fail if validation failed
        if: failure() && steps.check_files.outputs.count > 0
        run: |
          echo "❌ Data quality validation failed!"
          echo "See validation reports artifact for details"
          exit 1

  summary:
    name: Validation Summary
    runs-on: ubuntu-latest
    needs: validate-data-quality
    if: always()

    steps:
      - name: Download validation reports
        uses: actions/download-artifact@v4
        with:
          name: validation-reports
          path: validation_reports/
        continue-on-error: true

      - name: Create job summary
        run: |
          echo "# DBN Data Quality Validation Summary" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY

          if [ -f "validation_reports/latest.json" ]; then
            # Parse JSON report for summary
            echo "✅ Validation completed successfully" >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            echo "Download the full validation report from the artifacts." >> $GITHUB_STEP_SUMMARY
          else
            echo "ℹ️ No validation report generated (no DBN files or validation skipped)" >> $GITHUB_STEP_SUMMARY
          fi

          if [ "${{ needs.validate-data-quality.result }}" == "failure" ]; then
            echo "" >> $GITHUB_STEP_SUMMARY
            echo "❌ **Validation failed - quality score below threshold**" >> $GITHUB_STEP_SUMMARY
          fi