e8a68ee39f
- Created data/examples/download_ml_training_data.rs using reqwest + Databento HTTP API - Downloaded 90 days × 4 symbols (ES.FUT, NQ.FUT, ZN.FUT, 6E.FUT) - Files saved to test_data/real/databento/ml_training/ - Total: 360 files, 15 MB compressed DBN format - Used existing Rust pattern from download_nq_fut.rs - API key loaded from .env file - 100% success rate (360/360 files) - Ready for ML training benchmarks Next: Create simplified training benchmark for RTX 3050 Ti GPU measurements
184 lines
6.4 KiB
Bash
Executable File
184 lines
6.4 KiB
Bash
Executable File
#!/bin/bash
|
||
# Databento Minimal Test Download Script
|
||
#
|
||
# Purpose: Download smallest possible dataset to validate API and measure actual cost
|
||
# Budget Impact: ~$0.00002 (0.00002% of $125 budget)
|
||
# Safety: Explicit confirmation required before downloading
|
||
|
||
set -eo pipefail
|
||
|
||
# Colors for output
|
||
RED='\033[0;31m'
|
||
GREEN='\033[0;32m'
|
||
YELLOW='\033[1;33m'
|
||
BLUE='\033[0;34m'
|
||
NC='\033[0m' # No Color
|
||
|
||
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||
echo -e "${BLUE} Databento Minimal Test Download Script${NC}"
|
||
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||
echo
|
||
|
||
# Check for API key
|
||
if [ -z "${DATABENTO_API_KEY:-}" ]; then
|
||
echo -e "${RED}❌ ERROR: DATABENTO_API_KEY not set${NC}"
|
||
echo
|
||
echo "Please set your API key:"
|
||
echo " export DATABENTO_API_KEY=db-95LEt9gtDRPJfc55NVUB5KL3A3uf6"
|
||
echo
|
||
echo "Or load from .env:"
|
||
echo " source .env"
|
||
exit 1
|
||
fi
|
||
|
||
echo -e "${GREEN}✅ API Key found: ${DATABENTO_API_KEY:0:10}...${DATABENTO_API_KEY: -10}${NC}"
|
||
echo
|
||
|
||
# Download parameters
|
||
SYMBOL="ES.FUT"
|
||
DATASET="GLBX.MDP3"
|
||
SCHEMA="ohlcv-1m"
|
||
START_DATE="2024-01-02T00:00:00Z"
|
||
END_DATE="2024-01-02T23:59:59Z"
|
||
OUTPUT_DIR="test_data/real/databento"
|
||
OUTPUT_FILE="${OUTPUT_DIR}/${SYMBOL//\//_}_${SCHEMA}_2024-01-02.dbn"
|
||
|
||
# Create output directory
|
||
mkdir -p "$OUTPUT_DIR"
|
||
|
||
# Display download plan
|
||
echo -e "${YELLOW}📋 Download Plan:${NC}"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo " Symbol: $SYMBOL (E-mini S&P 500 Futures)"
|
||
echo " Dataset: $DATASET (CME Group MDP 3.0)"
|
||
echo " Schema: $SCHEMA (1-minute OHLCV bars)"
|
||
echo " Date: 2024-01-02 (single trading day)"
|
||
echo " Output: $OUTPUT_FILE"
|
||
echo
|
||
echo -e "${GREEN}💰 Cost Estimate:${NC}"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo " Expected Size: ~12 KB (390 bars × 32 bytes)"
|
||
echo " Cost per GB: $0.50 - $2.00"
|
||
echo " Estimated Cost: $0.00001 - $0.00002"
|
||
echo " Budget Impact: 0.00002% of $125"
|
||
echo " Credits After: ~$124.99998"
|
||
echo
|
||
|
||
# Confirmation prompt
|
||
echo -e "${YELLOW}⚠️ This will download data and consume credits.${NC}"
|
||
echo
|
||
read -p "Continue with download? (yes/no): " -r
|
||
echo
|
||
if [[ ! $REPLY =~ ^[Yy][Ee][Ss]$ ]]; then
|
||
echo -e "${YELLOW}Download cancelled by user.${NC}"
|
||
exit 0
|
||
fi
|
||
|
||
# Download using curl (HTTP API)
|
||
echo -e "${BLUE}📥 Downloading data...${NC}"
|
||
echo
|
||
|
||
TEMP_FILE="${OUTPUT_FILE}.tmp"
|
||
|
||
curl -X GET "https://hist.databento.com/v0/timeseries.get_range" \
|
||
-H "Authorization: Bearer ${DATABENTO_API_KEY}" \
|
||
-H "Content-Type: application/json" \
|
||
-d "{
|
||
\"dataset\": \"${DATASET}\",
|
||
\"symbols\": [\"${SYMBOL}\"],
|
||
\"schema\": \"${SCHEMA}\",
|
||
\"start\": \"${START_DATE}\",
|
||
\"end\": \"${END_DATE}\",
|
||
\"encoding\": \"dbn\",
|
||
\"stype_in\": \"parent\"
|
||
}" \
|
||
--output "$TEMP_FILE" \
|
||
--fail-with-body \
|
||
--progress-bar
|
||
|
||
# Check if download succeeded
|
||
if [ -f "$TEMP_FILE" ]; then
|
||
FILE_SIZE=$(stat -f%z "$TEMP_FILE" 2>/dev/null || stat -c%s "$TEMP_FILE" 2>/dev/null)
|
||
|
||
if [ "$FILE_SIZE" -gt 0 ]; then
|
||
mv "$TEMP_FILE" "$OUTPUT_FILE"
|
||
|
||
echo
|
||
echo -e "${GREEN}✅ Download successful!${NC}"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo " File: $OUTPUT_FILE"
|
||
echo " Size: $FILE_SIZE bytes ($(numfmt --to=iec-i --suffix=B $FILE_SIZE 2>/dev/null || echo "${FILE_SIZE} bytes"))"
|
||
|
||
# Calculate actual cost
|
||
SIZE_GB=$(echo "scale=10; $FILE_SIZE / 1073741824" | bc)
|
||
COST_LOW=$(echo "scale=6; $SIZE_GB * 0.50" | bc)
|
||
COST_HIGH=$(echo "scale=6; $SIZE_GB * 2.00" | bc)
|
||
|
||
echo " Size (GB): $SIZE_GB"
|
||
echo " Actual Cost: \$$COST_LOW - \$$COST_HIGH"
|
||
echo
|
||
|
||
# Update cost tracking
|
||
TRACKING_FILE="${OUTPUT_DIR}/COST_TRACKING.md"
|
||
if [ ! -f "$TRACKING_FILE" ]; then
|
||
cat > "$TRACKING_FILE" <<EOF
|
||
# Databento Download Cost Tracking
|
||
|
||
## Total Budget: \$125.00
|
||
## Starting Balance: \$125.00
|
||
|
||
---
|
||
|
||
EOF
|
||
fi
|
||
|
||
cat >> "$TRACKING_FILE" <<EOF
|
||
## Download $(date +"%Y-%m-%d %H:%M:%S")
|
||
- Symbol: $SYMBOL
|
||
- Schema: $SCHEMA
|
||
- Date Range: 2024-01-02
|
||
- File: $OUTPUT_FILE
|
||
- File Size: $FILE_SIZE bytes ($SIZE_GB GB)
|
||
- Estimated Cost: \$$COST_LOW - \$$COST_HIGH
|
||
- Credits Remaining: ~\$$(echo "125.00 - $COST_HIGH" | bc)
|
||
|
||
EOF
|
||
|
||
echo -e "${GREEN}📝 Cost tracking updated: $TRACKING_FILE${NC}"
|
||
echo
|
||
|
||
# Next steps
|
||
echo -e "${BLUE}🎯 Next Steps:${NC}"
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||
echo "1. Verify data format:"
|
||
echo " file $OUTPUT_FILE"
|
||
echo
|
||
echo "2. Parse DBN format (requires databento Rust crate):"
|
||
echo " cargo run --example parse_dbn -- $OUTPUT_FILE"
|
||
echo
|
||
echo "3. Convert to Parquet for backtesting:"
|
||
echo " cargo run --bin convert_dbn_to_parquet -- $OUTPUT_FILE"
|
||
echo
|
||
echo "4. Check actual cost in Databento portal:"
|
||
echo " https://databento.com/portal/billing"
|
||
echo
|
||
echo "5. If successful, scale up to:"
|
||
echo " - Multiple days (5-10 days)"
|
||
echo " - Additional symbols (NQ.FUT, SPY, QQQ)"
|
||
echo " - Richer schemas (trades, quotes)"
|
||
echo
|
||
|
||
else
|
||
echo -e "${RED}❌ Download failed: Empty response${NC}"
|
||
rm -f "$TEMP_FILE"
|
||
exit 1
|
||
fi
|
||
else
|
||
echo -e "${RED}❌ Download failed: No output file created${NC}"
|
||
exit 1
|
||
fi
|
||
|
||
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||
echo -e "${GREEN} Download Complete!${NC}"
|
||
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|