#!/bin/bash
# Download MBP-10 (Market By Price, 10 levels) data from Databento
#
# Dataset: GLBX.MDP3 (CME Globex)
# Schema: mbp-10 (Level-2 order book)
# Symbol: ES.FUT (E-mini S&P 500)
# Date Range: 2024-01-02 to 2024-01-10 (7 trading days)

set -euo pipefail

API_KEY="${DATABENTO_API_KEY:-}"
if [ -z "$API_KEY" ]; then
    echo "Error: DATABENTO_API_KEY environment variable not set"
    exit 1
fi

BASE_URL="https://hist.databento.com/v0"
OUTPUT_DIR="test_data/mbp10"
OUTPUT_FILE="$OUTPUT_DIR/ES.FUT.mbp10.2024-01-02_to_2024-01-10.dbn.zst"

# Create output directory
mkdir -p "$OUTPUT_DIR"

echo "=== Databento MBP-10 Data Download ==="
echo ""
echo "Dataset: GLBX.MDP3"
echo "Schema: mbp-10 (Market By Price, 10 levels)"
echo "Symbol: ES.FUT"
echo "Date Range: 2024-01-02 to 2024-01-10 (7 trading days)"
echo "Output: $OUTPUT_FILE"
echo ""
echo "=================================================="

# Step 1: Submit batch job
echo ""
echo "[1/4] Submitting batch download job..."

JOB_RESPONSE=$(curl -s -X POST "$BASE_URL/batch.submit_job" \
  -H "Authorization: Bearer $API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "dataset": "GLBX.MDP3",
    "schema": "mbp-10",
    "symbols": ["ES.FUT"],
    "stype_in": "continuous",
    "start": "2024-01-02T00:00:00Z",
    "end": "2024-01-10T23:59:59Z",
    "encoding": "dbn",
    "compression": "zstd"
  }')

JOB_ID=$(echo "$JOB_RESPONSE" | grep -o '"job_id":"[^"]*"' | cut -d'"' -f4)

if [ -z "$JOB_ID" ]; then
    echo "✗ Failed to submit job"
    echo "Response: $JOB_RESPONSE"
    exit 1
fi

echo "✓ Job submitted successfully: $JOB_ID"

# Step 2: Poll for completion
echo ""
echo "[2/4] Waiting for job to complete..."

while true; do
    STATUS_RESPONSE=$(curl -s "$BASE_URL/batch.status?job_id=$JOB_ID" \
      -H "Authorization: Bearer $API_KEY")

    STATE=$(echo "$STATUS_RESPONSE" | grep -o '"state":"[^"]*"' | cut -d'"' -f4)

    if [ "$STATE" = "done" ]; then
        echo "✓ Job completed successfully"

        DOWNLOAD_URL=$(echo "$STATUS_RESPONSE" | grep -o '"download_url":"[^"]*"' | cut -d'"' -f4)
        SIZE=$(echo "$STATUS_RESPONSE" | grep -o '"size_bytes":[0-9]*' | cut -d':' -f2)
        RECORDS=$(echo "$STATUS_RESPONSE" | grep -o '"record_count":[0-9]*' | cut -d':' -f2)

        if [ -n "$SIZE" ]; then
            SIZE_MB=$(awk "BEGIN {print $SIZE / 1048576}")
            echo "  • Records: $RECORDS"
            echo "  • Size: ${SIZE_MB} MB"
        fi

        break
    elif [ "$STATE" = "error" ]; then
        echo "✗ Job failed with error"
        echo "Response: $STATUS_RESPONSE"
        exit 1
    else
        echo -n "."
        sleep 5
    fi
done

# Step 3: Download file
echo ""
echo "[3/4] Downloading data file..."

if [ -z "$DOWNLOAD_URL" ]; then
    echo "✗ No download URL found"
    exit 1
fi

curl -# -L "$DOWNLOAD_URL" \
  -H "Authorization: Bearer $API_KEY" \
  -o "$OUTPUT_FILE"

echo "✓ Download completed"

# Step 4: Validate
echo ""
echo "[4/4] Validating downloaded file..."

FILE_SIZE=$(stat -f%z "$OUTPUT_FILE" 2>/dev/null || stat -c%s "$OUTPUT_FILE")
FILE_SIZE_MB=$(awk "BEGIN {print $FILE_SIZE / 1048576}")

echo "  • File size: ${FILE_SIZE_MB} MB"

if [ "$FILE_SIZE" -lt 1048576 ]; then
    echo "✗ File is suspiciously small (< 1 MB)"
    exit 1
fi

echo "✓ File validated successfully"

echo ""
echo "=================================================="
echo "SUCCESS: MBP-10 data downloaded to $OUTPUT_FILE"
echo ""
echo "Next Steps:"
echo "1. Decompress: zstd -d $OUTPUT_FILE"
echo "2. Validate schema with DBN parser"
echo "3. Implement MBP-10 data loader for TLOB training"