#!/bin/bash
# Databento Minimal Test Download Script
# 
# Purpose: Download smallest possible dataset to validate API and measure actual cost
# Budget Impact: ~$0.00002 (0.00002% of $125 budget)
# Safety: Explicit confirmation required before downloading

set -eo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e "${BLUE}   Databento Minimal Test Download Script${NC}"
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo

# Check for API key
if [ -z "${DATABENTO_API_KEY:-}" ]; then
    echo -e "${RED}❌ ERROR: DATABENTO_API_KEY not set${NC}"
    echo
    echo "Please set your API key:"
    echo "  export DATABENTO_API_KEY=db-95LEt9gtDRPJfc55NVUB5KL3A3uf6"
    echo
    echo "Or load from .env:"
    echo "  source .env"
    exit 1
fi

echo -e "${GREEN}✅ API Key found: ${DATABENTO_API_KEY:0:10}...${DATABENTO_API_KEY: -10}${NC}"
echo

# Download parameters
SYMBOL="ES.FUT"
DATASET="GLBX.MDP3"
SCHEMA="ohlcv-1m"
START_DATE="2024-01-02T00:00:00Z"
END_DATE="2024-01-02T23:59:59Z"
OUTPUT_DIR="test_data/real/databento"
OUTPUT_FILE="${OUTPUT_DIR}/${SYMBOL//\//_}_${SCHEMA}_2024-01-02.dbn"

# Create output directory
mkdir -p "$OUTPUT_DIR"

# Display download plan
echo -e "${YELLOW}📋 Download Plan:${NC}"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "  Symbol:      $SYMBOL (E-mini S&P 500 Futures)"
echo "  Dataset:     $DATASET (CME Group MDP 3.0)"
echo "  Schema:      $SCHEMA (1-minute OHLCV bars)"
echo "  Date:        2024-01-02 (single trading day)"
echo "  Output:      $OUTPUT_FILE"
echo
echo -e "${GREEN}💰 Cost Estimate:${NC}"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "  Expected Size:    ~12 KB (390 bars × 32 bytes)"
echo "  Cost per GB:      $0.50 - $2.00"
echo "  Estimated Cost:   $0.00001 - $0.00002"
echo "  Budget Impact:    0.00002% of $125"
echo "  Credits After:    ~$124.99998"
echo

# Confirmation prompt
echo -e "${YELLOW}⚠️  This will download data and consume credits.${NC}"
echo
read -p "Continue with download? (yes/no): " -r
echo
if [[ ! $REPLY =~ ^[Yy][Ee][Ss]$ ]]; then
    echo -e "${YELLOW}Download cancelled by user.${NC}"
    exit 0
fi

# Download using curl (HTTP API)
echo -e "${BLUE}📥 Downloading data...${NC}"
echo

TEMP_FILE="${OUTPUT_FILE}.tmp"

curl -X GET "https://hist.databento.com/v0/timeseries.get_range" \
  -H "Authorization: Bearer ${DATABENTO_API_KEY}" \
  -H "Content-Type: application/json" \
  -d "{
    \"dataset\": \"${DATASET}\",
    \"symbols\": [\"${SYMBOL}\"],
    \"schema\": \"${SCHEMA}\",
    \"start\": \"${START_DATE}\",
    \"end\": \"${END_DATE}\",
    \"encoding\": \"dbn\",
    \"stype_in\": \"parent\"
  }" \
  --output "$TEMP_FILE" \
  --fail-with-body \
  --progress-bar

# Check if download succeeded
if [ -f "$TEMP_FILE" ]; then
    FILE_SIZE=$(stat -f%z "$TEMP_FILE" 2>/dev/null || stat -c%s "$TEMP_FILE" 2>/dev/null)
    
    if [ "$FILE_SIZE" -gt 0 ]; then
        mv "$TEMP_FILE" "$OUTPUT_FILE"
        
        echo
        echo -e "${GREEN}✅ Download successful!${NC}"
        echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
        echo "  File:         $OUTPUT_FILE"
        echo "  Size:         $FILE_SIZE bytes ($(numfmt --to=iec-i --suffix=B $FILE_SIZE 2>/dev/null || echo "${FILE_SIZE} bytes"))"
        
        # Calculate actual cost
        SIZE_GB=$(echo "scale=10; $FILE_SIZE / 1073741824" | bc)
        COST_LOW=$(echo "scale=6; $SIZE_GB * 0.50" | bc)
        COST_HIGH=$(echo "scale=6; $SIZE_GB * 2.00" | bc)
        
        echo "  Size (GB):    $SIZE_GB"
        echo "  Actual Cost:  \$$COST_LOW - \$$COST_HIGH"
        echo
        
        # Update cost tracking
        TRACKING_FILE="${OUTPUT_DIR}/COST_TRACKING.md"
        if [ ! -f "$TRACKING_FILE" ]; then
            cat > "$TRACKING_FILE" <<EOF
# Databento Download Cost Tracking

## Total Budget: \$125.00
## Starting Balance: \$125.00

---

EOF
        fi
        
        cat >> "$TRACKING_FILE" <<EOF
## Download $(date +"%Y-%m-%d %H:%M:%S")
- Symbol: $SYMBOL
- Schema: $SCHEMA
- Date Range: 2024-01-02
- File: $OUTPUT_FILE
- File Size: $FILE_SIZE bytes ($SIZE_GB GB)
- Estimated Cost: \$$COST_LOW - \$$COST_HIGH
- Credits Remaining: ~\$$(echo "125.00 - $COST_HIGH" | bc)

EOF
        
        echo -e "${GREEN}📝 Cost tracking updated: $TRACKING_FILE${NC}"
        echo
        
        # Next steps
        echo -e "${BLUE}🎯 Next Steps:${NC}"
        echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
        echo "1. Verify data format:"
        echo "   file $OUTPUT_FILE"
        echo
        echo "2. Parse DBN format (requires databento Rust crate):"
        echo "   cargo run --example parse_dbn -- $OUTPUT_FILE"
        echo
        echo "3. Convert to Parquet for backtesting:"
        echo "   cargo run --bin convert_dbn_to_parquet -- $OUTPUT_FILE"
        echo
        echo "4. Check actual cost in Databento portal:"
        echo "   https://databento.com/portal/billing"
        echo
        echo "5. If successful, scale up to:"
        echo "   - Multiple days (5-10 days)"
        echo "   - Additional symbols (NQ.FUT, SPY, QQQ)"
        echo "   - Richer schemas (trades, quotes)"
        echo
        
    else
        echo -e "${RED}❌ Download failed: Empty response${NC}"
        rm -f "$TEMP_FILE"
        exit 1
    fi
else
    echo -e "${RED}❌ Download failed: No output file created${NC}"
    exit 1
fi

echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e "${GREEN}   Download Complete!${NC}"
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"