#!/bin/bash
# Verify current DBN dataset coverage

DATA_DIR="test_data/real/databento/ml_training"
SYMBOLS=("ES.FUT" "NQ.FUT" "ZN.FUT" "6E.FUT")

echo "========================================"
echo "DBN Dataset Coverage Verification"
echo "========================================"
echo ""

# Check total files
total_files=$(ls -1 ${DATA_DIR}/*.dbn 2>/dev/null | wc -l)
echo "Total files: ${total_files}"

# Check files per symbol
echo ""
echo "Files per symbol:"
for symbol in "${SYMBOLS[@]}"; do
  count=$(ls -1 ${DATA_DIR}/${symbol}_*.dbn 2>/dev/null | wc -l)
  echo "  ${symbol}: ${count} files"
done

# Check unique dates
echo ""
echo "Unique trading days:"
unique_dates=$(ls -1 ${DATA_DIR}/*.dbn | awk -F'_' '{print $3}' | awk -F'.' '{print $1}' | sort -u | wc -l)
echo "  Total: ${unique_dates} days"

# Check date range
echo ""
echo "Date range:"
first_date=$(ls -1 ${DATA_DIR}/*.dbn | awk -F'_' '{print $3}' | awk -F'.' '{print $1}' | sort -u | head -1)
last_date=$(ls -1 ${DATA_DIR}/*.dbn | awk -F'_' '{print $3}' | awk -F'.' '{print $1}' | sort -u | tail -1)
echo "  Start: ${first_date}"
echo "  End: ${last_date}"

# Calculate calendar days
if [[ ! -z "${first_date}" && ! -z "${last_date}" ]]; then
  start_epoch=$(date -d "${first_date}" +%s)
  end_epoch=$(date -d "${last_date}" +%s)
  calendar_days=$(( (end_epoch - start_epoch) / 86400 ))
  echo "  Calendar days: ${calendar_days}"
fi

# Check total size
echo ""
echo "Total size:"
du -sh ${DATA_DIR}/

# Sample 5 files
echo ""
echo "Sample file sizes:"
ls -lh ${DATA_DIR}/*.dbn | head -5 | awk '{print "  " $9 ": " $5}'

# List unique dates per symbol (to check if all symbols have same coverage)
echo ""
echo "Date coverage per symbol:"
for symbol in "${SYMBOLS[@]}"; do
  dates=$(ls -1 ${DATA_DIR}/${symbol}_*.dbn 2>/dev/null | awk -F'_' '{print $3}' | awk -F'.' '{print $1}' | sort -u | wc -l)
  echo "  ${symbol}: ${dates} unique dates"
done

# Check for gaps in date sequence
echo ""
echo "Checking for date gaps..."
ls -1 ${DATA_DIR}/*.dbn | awk -F'_' '{print $3}' | awk -F'.' '{print $1}' | sort -u > /tmp/dbn_dates.txt
gap_count=0
prev_date=""
while IFS= read -r date; do
  if [[ ! -z "${prev_date}" ]]; then
    prev_epoch=$(date -d "${prev_date}" +%s)
    curr_epoch=$(date -d "${date}" +%s)
    day_diff=$(( (curr_epoch - prev_epoch) / 86400 ))

    # Allow up to 3-day gaps (weekends)
    if [ ${day_diff} -gt 3 ]; then
      echo "  ⚠️  Gap detected: ${prev_date} → ${date} (${day_diff} days)"
      gap_count=$((gap_count + 1))
    fi
  fi
  prev_date="${date}"
done < /tmp/dbn_dates.txt

if [ ${gap_count} -eq 0 ]; then
  echo "  ✅ No significant gaps (>3 days) detected"
else
  echo "  ⚠️  Found ${gap_count} gaps in date sequence"
fi

# Recommendation
echo ""
echo "========================================"
echo "Recommendation:"
echo "========================================"
if [ ${unique_dates} -ge 90 ]; then
  echo "✅ Dataset appears to cover 90+ trading days."
  echo "   No download needed. Proceed to Phase 2 (data loader enhancement)."
elif [ ${unique_dates} -ge 30 ]; then
  echo "⚠️  Dataset covers ${unique_dates} trading days (need 90)."
  echo "   Download remaining $((90 - unique_dates)) days."
else
  echo "❌ Dataset incomplete (only ${unique_dates} days)."
  echo "   Download full 90-day dataset."
fi

echo ""
echo "Next steps:"
if [ ${unique_dates} -lt 90 ]; then
  echo "  1. Set DATABENTO_API_KEY in .env"
  echo "  2. Run dry run: cargo run -p ml --example download_training_data -- --dry-run"
  echo "  3. Download data: cargo run -p ml --example download_training_data --release"
else
  echo "  1. Implement 3-way split in DbnSequenceLoader"
  echo "  2. Update training examples with CLI flags"
  echo "  3. Execute training pipeline"
fi

# Cleanup
rm -f /tmp/dbn_dates.txt