8d89fe80ff
- Archive: 85 agent .txt files → docs/archive/agents/legacy_txt/ - Scripts: Move 110 shell scripts → scripts/ (keep deploy.sh in root) - Models: Move 18 .safetensors → ml/models/checkpoints/training_artifacts/ - Delete: 34 directories (~33GB freed) - target/, coverage_*, test artifacts - Build: Clean 14 build artifacts (.rlib, .o, .pid, binaries) - Tests: Move 14 .rs files → tests/standalone/ - SQL: Move 5 files → sql/ (keep init-db*.sql for Docker) - Wave 153: Archive to docs/archive/historical/wave153/ - Docs: Archive 9 markdown files to wave_d/reports/ and historical/ Total impact: ~34GB freed (both waves), root directory cleaned from 583 to ~40 essential files Directory count reduced from 65 to 31 (52% reduction) All historical data preserved in organized archive structure
121 lines
3.7 KiB
Bash
Executable File
121 lines
3.7 KiB
Bash
Executable File
#!/bin/bash
|
|
# Verify current DBN dataset coverage
|
|
|
|
DATA_DIR="test_data/real/databento/ml_training"
|
|
SYMBOLS=("ES.FUT" "NQ.FUT" "ZN.FUT" "6E.FUT")
|
|
|
|
echo "========================================"
|
|
echo "DBN Dataset Coverage Verification"
|
|
echo "========================================"
|
|
echo ""
|
|
|
|
# Check total files
|
|
total_files=$(ls -1 ${DATA_DIR}/*.dbn 2>/dev/null | wc -l)
|
|
echo "Total files: ${total_files}"
|
|
|
|
# Check files per symbol
|
|
echo ""
|
|
echo "Files per symbol:"
|
|
for symbol in "${SYMBOLS[@]}"; do
|
|
count=$(ls -1 ${DATA_DIR}/${symbol}_*.dbn 2>/dev/null | wc -l)
|
|
echo " ${symbol}: ${count} files"
|
|
done
|
|
|
|
# Check unique dates
|
|
echo ""
|
|
echo "Unique trading days:"
|
|
unique_dates=$(ls -1 ${DATA_DIR}/*.dbn | awk -F'_' '{print $3}' | awk -F'.' '{print $1}' | sort -u | wc -l)
|
|
echo " Total: ${unique_dates} days"
|
|
|
|
# Check date range
|
|
echo ""
|
|
echo "Date range:"
|
|
first_date=$(ls -1 ${DATA_DIR}/*.dbn | awk -F'_' '{print $3}' | awk -F'.' '{print $1}' | sort -u | head -1)
|
|
last_date=$(ls -1 ${DATA_DIR}/*.dbn | awk -F'_' '{print $3}' | awk -F'.' '{print $1}' | sort -u | tail -1)
|
|
echo " Start: ${first_date}"
|
|
echo " End: ${last_date}"
|
|
|
|
# Calculate calendar days
|
|
if [[ ! -z "${first_date}" && ! -z "${last_date}" ]]; then
|
|
start_epoch=$(date -d "${first_date}" +%s)
|
|
end_epoch=$(date -d "${last_date}" +%s)
|
|
calendar_days=$(( (end_epoch - start_epoch) / 86400 ))
|
|
echo " Calendar days: ${calendar_days}"
|
|
fi
|
|
|
|
# Check total size
|
|
echo ""
|
|
echo "Total size:"
|
|
du -sh ${DATA_DIR}/
|
|
|
|
# Sample 5 files
|
|
echo ""
|
|
echo "Sample file sizes:"
|
|
ls -lh ${DATA_DIR}/*.dbn | head -5 | awk '{print " " $9 ": " $5}'
|
|
|
|
# List unique dates per symbol (to check if all symbols have same coverage)
|
|
echo ""
|
|
echo "Date coverage per symbol:"
|
|
for symbol in "${SYMBOLS[@]}"; do
|
|
dates=$(ls -1 ${DATA_DIR}/${symbol}_*.dbn 2>/dev/null | awk -F'_' '{print $3}' | awk -F'.' '{print $1}' | sort -u | wc -l)
|
|
echo " ${symbol}: ${dates} unique dates"
|
|
done
|
|
|
|
# Check for gaps in date sequence
|
|
echo ""
|
|
echo "Checking for date gaps..."
|
|
ls -1 ${DATA_DIR}/*.dbn | awk -F'_' '{print $3}' | awk -F'.' '{print $1}' | sort -u > /tmp/dbn_dates.txt
|
|
gap_count=0
|
|
prev_date=""
|
|
while IFS= read -r date; do
|
|
if [[ ! -z "${prev_date}" ]]; then
|
|
prev_epoch=$(date -d "${prev_date}" +%s)
|
|
curr_epoch=$(date -d "${date}" +%s)
|
|
day_diff=$(( (curr_epoch - prev_epoch) / 86400 ))
|
|
|
|
# Allow up to 3-day gaps (weekends)
|
|
if [ ${day_diff} -gt 3 ]; then
|
|
echo " ⚠️ Gap detected: ${prev_date} → ${date} (${day_diff} days)"
|
|
gap_count=$((gap_count + 1))
|
|
fi
|
|
fi
|
|
prev_date="${date}"
|
|
done < /tmp/dbn_dates.txt
|
|
|
|
if [ ${gap_count} -eq 0 ]; then
|
|
echo " ✅ No significant gaps (>3 days) detected"
|
|
else
|
|
echo " ⚠️ Found ${gap_count} gaps in date sequence"
|
|
fi
|
|
|
|
# Recommendation
|
|
echo ""
|
|
echo "========================================"
|
|
echo "Recommendation:"
|
|
echo "========================================"
|
|
if [ ${unique_dates} -ge 90 ]; then
|
|
echo "✅ Dataset appears to cover 90+ trading days."
|
|
echo " No download needed. Proceed to Phase 2 (data loader enhancement)."
|
|
elif [ ${unique_dates} -ge 30 ]; then
|
|
echo "⚠️ Dataset covers ${unique_dates} trading days (need 90)."
|
|
echo " Download remaining $((90 - unique_dates)) days."
|
|
else
|
|
echo "❌ Dataset incomplete (only ${unique_dates} days)."
|
|
echo " Download full 90-day dataset."
|
|
fi
|
|
|
|
echo ""
|
|
echo "Next steps:"
|
|
if [ ${unique_dates} -lt 90 ]; then
|
|
echo " 1. Set DATABENTO_API_KEY in .env"
|
|
echo " 2. Run dry run: cargo run -p ml --example download_training_data -- --dry-run"
|
|
echo " 3. Download data: cargo run -p ml --example download_training_data --release"
|
|
else
|
|
echo " 1. Implement 3-way split in DbnSequenceLoader"
|
|
echo " 2. Update training examples with CLI flags"
|
|
echo " 3. Execute training pipeline"
|
|
fi
|
|
|
|
# Cleanup
|
|
rm -f /tmp/dbn_dates.txt
|