Download 360 DBN files (36.3 MB) using Rust databento client
- Created data/examples/download_ml_training_data.rs using reqwest + Databento HTTP API - Downloaded 90 days × 4 symbols (ES.FUT, NQ.FUT, ZN.FUT, 6E.FUT) - Files saved to test_data/real/databento/ml_training/ - Total: 360 files, 15 MB compressed DBN format - Used existing Rust pattern from download_nq_fut.rs - API key loaded from .env file - 100% success rate (360/360 files) - Ready for ML training benchmarks Next: Create simplified training benchmark for RTX 3050 Ti GPU measurements
This commit is contained in:
jgrusewski
183
scripts/databento_minimal_download.sh
Executable file
183
scripts/databento_minimal_download.sh
Executable file
@@ -0,0 +1,183 @@
|
||||
#!/bin/bash
|
||||
# Databento Minimal Test Download Script
|
||||
#
|
||||
# Purpose: Download smallest possible dataset to validate API and measure actual cost
|
||||
# Budget Impact: ~$0.00002 (0.00002% of $125 budget)
|
||||
# Safety: Explicit confirmation required before downloading
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
echo -e "${BLUE} Databento Minimal Test Download Script${NC}"
|
||||
echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
echo
|
||||
|
||||
# Check for API key
|
||||
if [ -z "${DATABENTO_API_KEY:-}" ]; then
|
||||
echo -e "${RED}❌ ERROR: DATABENTO_API_KEY not set${NC}"
|
||||
echo
|
||||
echo "Please set your API key:"
|
||||
echo " export DATABENTO_API_KEY=db-95LEt9gtDRPJfc55NVUB5KL3A3uf6"
|
||||
echo
|
||||
echo "Or load from .env:"
|
||||
echo " source .env"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}✅ API Key found: ${DATABENTO_API_KEY:0:10}...${DATABENTO_API_KEY: -10}${NC}"
|
||||
echo
|
||||
|
||||
# Download parameters
|
||||
SYMBOL="ES.FUT"
|
||||
DATASET="GLBX.MDP3"
|
||||
SCHEMA="ohlcv-1m"
|
||||
START_DATE="2024-01-02T00:00:00Z"
|
||||
END_DATE="2024-01-02T23:59:59Z"
|
||||
OUTPUT_DIR="test_data/real/databento"
|
||||
OUTPUT_FILE="${OUTPUT_DIR}/${SYMBOL//\//_}_${SCHEMA}_2024-01-02.dbn"
|
||||
|
||||
# Create output directory
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
# Display download plan
|
||||
echo -e "${YELLOW}📋 Download Plan:${NC}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " Symbol: $SYMBOL (E-mini S&P 500 Futures)"
|
||||
echo " Dataset: $DATASET (CME Group MDP 3.0)"
|
||||
echo " Schema: $SCHEMA (1-minute OHLCV bars)"
|
||||
echo " Date: 2024-01-02 (single trading day)"
|
||||
echo " Output: $OUTPUT_FILE"
|
||||
echo
|
||||
echo -e "${GREEN}💰 Cost Estimate:${NC}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " Expected Size: ~12 KB (390 bars × 32 bytes)"
|
||||
echo " Cost per GB: $0.50 - $2.00"
|
||||
echo " Estimated Cost: $0.00001 - $0.00002"
|
||||
echo " Budget Impact: 0.00002% of $125"
|
||||
echo " Credits After: ~$124.99998"
|
||||
echo
|
||||
|
||||
# Confirmation prompt
|
||||
echo -e "${YELLOW}⚠️ This will download data and consume credits.${NC}"
|
||||
echo
|
||||
read -p "Continue with download? (yes/no): " -r
|
||||
echo
|
||||
if [[ ! $REPLY =~ ^[Yy][Ee][Ss]$ ]]; then
|
||||
echo -e "${YELLOW}Download cancelled by user.${NC}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Download using curl (HTTP API)
|
||||
echo -e "${BLUE}📥 Downloading data...${NC}"
|
||||
echo
|
||||
|
||||
TEMP_FILE="${OUTPUT_FILE}.tmp"
|
||||
|
||||
curl -X GET "https://hist.databento.com/v0/timeseries.get_range" \
|
||||
-H "Authorization: Bearer ${DATABENTO_API_KEY}" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"dataset\": \"${DATASET}\",
|
||||
\"symbols\": [\"${SYMBOL}\"],
|
||||
\"schema\": \"${SCHEMA}\",
|
||||
\"start\": \"${START_DATE}\",
|
||||
\"end\": \"${END_DATE}\",
|
||||
\"encoding\": \"dbn\",
|
||||
\"stype_in\": \"parent\"
|
||||
}" \
|
||||
--output "$TEMP_FILE" \
|
||||
--fail-with-body \
|
||||
--progress-bar
|
||||
|
||||
# Check if download succeeded
|
||||
if [ -f "$TEMP_FILE" ]; then
|
||||
FILE_SIZE=$(stat -f%z "$TEMP_FILE" 2>/dev/null || stat -c%s "$TEMP_FILE" 2>/dev/null)
|
||||
|
||||
if [ "$FILE_SIZE" -gt 0 ]; then
|
||||
mv "$TEMP_FILE" "$OUTPUT_FILE"
|
||||
|
||||
echo
|
||||
echo -e "${GREEN}✅ Download successful!${NC}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo " File: $OUTPUT_FILE"
|
||||
echo " Size: $FILE_SIZE bytes ($(numfmt --to=iec-i --suffix=B $FILE_SIZE 2>/dev/null || echo "${FILE_SIZE} bytes"))"
|
||||
|
||||
# Calculate actual cost
|
||||
SIZE_GB=$(echo "scale=10; $FILE_SIZE / 1073741824" | bc)
|
||||
COST_LOW=$(echo "scale=6; $SIZE_GB * 0.50" | bc)
|
||||
COST_HIGH=$(echo "scale=6; $SIZE_GB * 2.00" | bc)
|
||||
|
||||
echo " Size (GB): $SIZE_GB"
|
||||
echo " Actual Cost: \$$COST_LOW - \$$COST_HIGH"
|
||||
echo
|
||||
|
||||
# Update cost tracking
|
||||
TRACKING_FILE="${OUTPUT_DIR}/COST_TRACKING.md"
|
||||
if [ ! -f "$TRACKING_FILE" ]; then
|
||||
cat > "$TRACKING_FILE" <<EOF
|
||||
# Databento Download Cost Tracking
|
||||
|
||||
## Total Budget: \$125.00
|
||||
## Starting Balance: \$125.00
|
||||
|
||||
---
|
||||
|
||||
EOF
|
||||
fi
|
||||
|
||||
cat >> "$TRACKING_FILE" <<EOF
|
||||
## Download $(date +"%Y-%m-%d %H:%M:%S")
|
||||
- Symbol: $SYMBOL
|
||||
- Schema: $SCHEMA
|
||||
- Date Range: 2024-01-02
|
||||
- File: $OUTPUT_FILE
|
||||
- File Size: $FILE_SIZE bytes ($SIZE_GB GB)
|
||||
- Estimated Cost: \$$COST_LOW - \$$COST_HIGH
|
||||
- Credits Remaining: ~\$$(echo "125.00 - $COST_HIGH" | bc)
|
||||
|
||||
EOF
|
||||
|
||||
echo -e "${GREEN}📝 Cost tracking updated: $TRACKING_FILE${NC}"
|
||||
echo
|
||||
|
||||
# Next steps
|
||||
echo -e "${BLUE}🎯 Next Steps:${NC}"
|
||||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
|
||||
echo "1. Verify data format:"
|
||||
echo " file $OUTPUT_FILE"
|
||||
echo
|
||||
echo "2. Parse DBN format (requires databento Rust crate):"
|
||||
echo " cargo run --example parse_dbn -- $OUTPUT_FILE"
|
||||
echo
|
||||
echo "3. Convert to Parquet for backtesting:"
|
||||
echo " cargo run --bin convert_dbn_to_parquet -- $OUTPUT_FILE"
|
||||
echo
|
||||
echo "4. Check actual cost in Databento portal:"
|
||||
echo " https://databento.com/portal/billing"
|
||||
echo
|
||||
echo "5. If successful, scale up to:"
|
||||
echo " - Multiple days (5-10 days)"
|
||||
echo " - Additional symbols (NQ.FUT, SPY, QQQ)"
|
||||
echo " - Richer schemas (trades, quotes)"
|
||||
echo
|
||||
|
||||
else
|
||||
echo -e "${RED}❌ Download failed: Empty response${NC}"
|
||||
rm -f "$TEMP_FILE"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo -e "${RED}❌ Download failed: No output file created${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
echo -e "${GREEN} Download Complete!${NC}"
|
||||
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
|
||||
140
scripts/databento_test.rs
Normal file
140
scripts/databento_test.rs
Normal file
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env cargo +nightly -Zscript
|
||||
//! Databento API Test & Balance Check Script
|
||||
//!
|
||||
//! Purpose: Verify API key, check credit balance, and test minimal data download
|
||||
//!
|
||||
//! Usage:
|
||||
//! cargo run --bin databento_test
|
||||
//!
|
||||
//! Requirements:
|
||||
//! - DATABENTO_API_KEY environment variable set
|
||||
//! - Network connectivity to Databento API
|
||||
//!
|
||||
//! Safety:
|
||||
//! - NO automatic downloads
|
||||
//! - Displays costs BEFORE any data download
|
||||
//! - Requires explicit confirmation
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::error::Error;
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct AccountInfo {
|
||||
balance: f64,
|
||||
currency: String,
|
||||
organization: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct DatasetInfo {
|
||||
dataset: String,
|
||||
schema: String,
|
||||
cost_per_gb: f64,
|
||||
available_symbols: Vec<String>,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn Error>> {
|
||||
println!("🔍 Databento API Test & Balance Checker");
|
||||
println!("========================================\n");
|
||||
|
||||
// 1. Check for API key
|
||||
let api_key = std::env::var("DATABENTO_API_KEY")
|
||||
.map_err(|_| "DATABENTO_API_KEY not set in environment")?;
|
||||
|
||||
println!("✅ API Key found: {}...{}",
|
||||
&api_key[..10],
|
||||
&api_key[api_key.len()-10..]);
|
||||
|
||||
// 2. Verify API key and get account info
|
||||
println!("\n📊 Checking account balance...");
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
// Note: Databento API endpoint for account info
|
||||
// This is a placeholder - actual endpoint needs to be verified from docs
|
||||
let account_url = "https://api.databento.com/v0/account";
|
||||
|
||||
let response = client
|
||||
.get(account_url)
|
||||
.header("Authorization", format!("Bearer {}", api_key))
|
||||
.send()
|
||||
.await;
|
||||
|
||||
match response {
|
||||
Ok(resp) => {
|
||||
if resp.status().is_success() {
|
||||
println!("✅ API key is valid!");
|
||||
|
||||
// Try to parse account info
|
||||
if let Ok(text) = resp.text().await {
|
||||
println!("\n📋 Account Info:");
|
||||
println!("{}", text);
|
||||
}
|
||||
} else {
|
||||
let status = resp.status();
|
||||
let error_text = resp.text().await.unwrap_or_default();
|
||||
println!("❌ API key validation failed:");
|
||||
println!(" Status: {}", status);
|
||||
println!(" Error: {}", error_text);
|
||||
return Err(format!("Invalid API key: {}", status).into());
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
println!("❌ Network error connecting to Databento API:");
|
||||
println!(" {}", e);
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Display pricing information
|
||||
println!("\n💰 Dataset Pricing Information:");
|
||||
println!("================================");
|
||||
println!("\nFrom Databento documentation:");
|
||||
println!("• $125 FREE credits for historical data");
|
||||
println!("• Pricing is per GB consumed");
|
||||
println!("• OHLCV bars (cheapest): ~$0.50-$2.00 per GB");
|
||||
println!("• Trades: ~$5-$15 per GB");
|
||||
println!("• L2 Order Book (MBP): ~$10-$30 per GB");
|
||||
println!("• L3 Order Book (MBO): ~$30-$100 per GB");
|
||||
|
||||
// 4. Recommended minimal test download
|
||||
println!("\n🎯 Recommended Minimal Test Download:");
|
||||
println!("====================================");
|
||||
println!("Symbol: ES.FUT (E-mini S&P 500 futures)");
|
||||
println!("Dataset: GLBX.MDP3 (CME MDP 3.0)");
|
||||
println!("Schema: ohlcv-1m (1-minute bars)");
|
||||
println!("Date Range: 1 day (e.g., 2024-01-02)");
|
||||
println!("Est. Size: ~5-20 MB");
|
||||
println!("Est. Cost: ~$0.01-$0.05 (well under $125 limit)");
|
||||
|
||||
println!("\nAlternative (even cheaper):");
|
||||
println!("Symbol: SPY (S&P 500 ETF)");
|
||||
println!("Dataset: XNAS.ITCH (Nasdaq)");
|
||||
println!("Schema: ohlcv-1m");
|
||||
println!("Date Range: 1 day");
|
||||
println!("Est. Cost: ~$0.005-$0.02");
|
||||
|
||||
// 5. Cost estimation tool
|
||||
println!("\n📐 Cost Estimation Formula:");
|
||||
println!("===========================");
|
||||
println!("Estimated GB = (symbols × days × data_points × bytes_per_point) / 1GB");
|
||||
println!(" OHLCV-1m: ~390 bars/day × 32 bytes = ~12 KB per symbol per day");
|
||||
println!(" Trades: ~10,000 trades/day × 24 bytes = ~240 KB per symbol per day");
|
||||
println!(" MBP-1: ~100,000 updates/day × 48 bytes = ~4.8 MB per symbol per day");
|
||||
|
||||
println!("\n⚠️ IMPORTANT: NO DATA DOWNLOADED YET!");
|
||||
println!(" This script only checks your account status.");
|
||||
println!(" Use the Databento Python/Rust client to actually download data.");
|
||||
println!(" Always check the cost estimate BEFORE downloading!");
|
||||
|
||||
println!("\n📚 Next Steps:");
|
||||
println!("==============");
|
||||
println!("1. Review the pricing information above");
|
||||
println!("2. Use Databento's cost estimator: https://databento.com/pricing");
|
||||
println!("3. Start with OHLCV-1m data (cheapest option)");
|
||||
println!("4. Download 1-2 days for 1-2 symbols first");
|
||||
println!("5. Validate data quality before scaling up");
|
||||
println!("6. Monitor your credit balance regularly");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
411
scripts/plan_databento_download.sh
Executable file
411
scripts/plan_databento_download.sh
Executable file
@@ -0,0 +1,411 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Databento Download Planning Tool
|
||||
#
|
||||
# Purpose: Estimate costs, check credit balance, and generate download plan
|
||||
# before actually downloading data from Databento.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/plan_databento_download.sh \
|
||||
# --symbols "ES.FUT,NQ.FUT" \
|
||||
# --days 5 \
|
||||
# --schema ohlcv-1m \
|
||||
# [--start-date 2024-01-02]
|
||||
#
|
||||
# Features:
|
||||
# - Cost estimation based on historical data
|
||||
# - Credit balance check (80%, 90% alerts)
|
||||
# - Download command generation
|
||||
# - Budget safety checks
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ============================================================================
|
||||
# Configuration
|
||||
# ============================================================================
|
||||
|
||||
COST_TRACKING_FILE="${COST_TRACKING_FILE:-./COST_TRACKING.md}"
|
||||
TOTAL_CREDITS="${TOTAL_CREDITS:-125.00}"
|
||||
WARN_THRESHOLD_PCT="${WARN_THRESHOLD_PCT:-80}"
|
||||
CRITICAL_THRESHOLD_PCT="${CRITICAL_THRESHOLD_PCT:-90}"
|
||||
|
||||
# Schema cost estimates (per day, per symbol)
|
||||
declare -A SCHEMA_COSTS=(
|
||||
["ohlcv-1m"]="0.00002"
|
||||
["ohlcv-1s"]="0.0004"
|
||||
["tbbo"]="0.0001"
|
||||
["trades"]="0.0012"
|
||||
["mbp-1"]="0.05"
|
||||
["mbp-10"]="1.0"
|
||||
["mbo"]="3.0"
|
||||
)
|
||||
|
||||
# Schema file size estimates (per day, per symbol, in KB)
|
||||
declare -A SCHEMA_SIZES=(
|
||||
["ohlcv-1m"]="90"
|
||||
["ohlcv-1s"]="1400"
|
||||
["tbbo"]="50"
|
||||
["trades"]="240"
|
||||
["mbp-1"]="4800"
|
||||
["mbp-10"]="48000"
|
||||
["mbo"]="100000"
|
||||
)
|
||||
|
||||
# ============================================================================
|
||||
# Color Codes
|
||||
# ============================================================================
|
||||
|
||||
RED='\033[0;31m'
|
||||
YELLOW='\033[1;33m'
|
||||
GREEN='\033[0;32m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# ============================================================================
|
||||
# Functions
|
||||
# ============================================================================
|
||||
|
||||
usage() {
|
||||
cat << EOF
|
||||
Usage: $0 [OPTIONS]
|
||||
|
||||
Databento Download Planning Tool - Estimate costs before downloading
|
||||
|
||||
OPTIONS:
|
||||
--symbols SYMBOLS Comma-separated list of symbols (e.g., "ES.FUT,NQ.FUT")
|
||||
--days DAYS Number of days to download
|
||||
--schema SCHEMA Schema type: ohlcv-1m, ohlcv-1s, tbbo, trades, mbp-1, mbp-10, mbo
|
||||
--start-date DATE (Optional) Start date in YYYY-MM-DD format
|
||||
--parent (Optional) Use parent symbol type (WARNING: increases cost 10-60x)
|
||||
--help Show this help message
|
||||
|
||||
EXAMPLES:
|
||||
# Single symbol, 5 days, OHLCV-1m
|
||||
$0 --symbols "ES.FUT" --days 5 --schema ohlcv-1m
|
||||
|
||||
# Multiple symbols, 10 days, trades
|
||||
$0 --symbols "ES.FUT,NQ.FUT,RTY.FUT" --days 10 --schema trades
|
||||
|
||||
# With specific start date
|
||||
$0 --symbols "ES.FUT" --days 7 --schema ohlcv-1m --start-date 2024-01-02
|
||||
|
||||
SUPPORTED SCHEMAS:
|
||||
ohlcv-1m 1-minute OHLCV bars (CHEAPEST, ~\$0.00002/day)
|
||||
ohlcv-1s 1-second OHLCV bars (~\$0.0004/day)
|
||||
tbbo Top of book quotes (~\$0.0001/day)
|
||||
trades All trades (~\$0.0012/day)
|
||||
mbp-1 Level 2, 1 level (~\$0.05/day)
|
||||
mbp-10 Level 2, 10 levels (~\$1.00/day)
|
||||
mbo Level 3, full depth (MOST EXPENSIVE, ~\$3.00/day)
|
||||
|
||||
EOF
|
||||
exit 1
|
||||
}
|
||||
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $*"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $*"
|
||||
}
|
||||
|
||||
log_warn() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $*"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $*"
|
||||
}
|
||||
|
||||
# Calculate credits used from COST_TRACKING.md
|
||||
calculate_credits_used() {
|
||||
if [ ! -f "$COST_TRACKING_FILE" ]; then
|
||||
log_warn "Cost tracking file not found: $COST_TRACKING_FILE"
|
||||
echo "0.00"
|
||||
return
|
||||
fi
|
||||
|
||||
# Extract "Estimated Cost: $X.XX - $Y.YY" lines and sum the high estimates
|
||||
local total
|
||||
total=$(grep -oP 'Estimated Cost: \$[\d.]+ - \$\K[\d.]+' "$COST_TRACKING_FILE" 2>/dev/null | \
|
||||
awk '{sum+=$1} END {printf "%.4f", sum}')
|
||||
|
||||
# If no matches, return 0
|
||||
if [ -z "$total" ]; then
|
||||
echo "0.00"
|
||||
else
|
||||
echo "$total"
|
||||
fi
|
||||
}
|
||||
|
||||
# Check if credits are available for this download
|
||||
check_credit_budget() {
|
||||
local estimated_cost=$1
|
||||
local credits_used
|
||||
local credits_remaining
|
||||
local usage_pct
|
||||
|
||||
credits_used=$(calculate_credits_used)
|
||||
credits_remaining=$(echo "$TOTAL_CREDITS - $credits_used" | bc -l)
|
||||
usage_pct=$(echo "scale=2; ($credits_used / $TOTAL_CREDITS) * 100" | bc -l)
|
||||
|
||||
echo ""
|
||||
echo "======================================================================"
|
||||
echo " CREDIT BUDGET STATUS"
|
||||
echo "======================================================================"
|
||||
printf "%-30s: \$%.4f\n" "Total Credits" "$TOTAL_CREDITS"
|
||||
printf "%-30s: \$%.4f (%.1f%%)\n" "Credits Used" "$credits_used" "$usage_pct"
|
||||
printf "%-30s: \$%.4f (%.1f%%)\n" "Credits Remaining" "$credits_remaining" "$(echo "100 - $usage_pct" | bc -l)"
|
||||
printf "%-30s: \$%.4f\n" "Estimated Cost (This Download)" "$estimated_cost"
|
||||
echo "======================================================================"
|
||||
|
||||
# Check alert thresholds
|
||||
if (( $(echo "$usage_pct >= $CRITICAL_THRESHOLD_PCT" | bc -l) )); then
|
||||
log_error "🚨 RED ALERT: Credit usage at ${usage_pct}% (>${CRITICAL_THRESHOLD_PCT}%)"
|
||||
log_error "Emergency halt recommended. Executive approval required."
|
||||
return 2
|
||||
elif (( $(echo "$usage_pct >= $WARN_THRESHOLD_PCT" | bc -l) )); then
|
||||
log_warn "🟠 ORANGE ALERT: Credit usage at ${usage_pct}% (>${WARN_THRESHOLD_PCT}%)"
|
||||
log_warn "Review download necessity before proceeding."
|
||||
return 1
|
||||
else
|
||||
log_success "✅ GREEN: Credit usage at ${usage_pct}% (budget healthy)"
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
# Estimate cost for download
|
||||
estimate_cost() {
|
||||
local symbols=$1
|
||||
local days=$2
|
||||
local schema=$3
|
||||
local use_parent=$4
|
||||
|
||||
# Count symbols
|
||||
IFS=',' read -ra symbol_array <<< "$symbols"
|
||||
local num_symbols=${#symbol_array[@]}
|
||||
|
||||
# Get base cost per day per symbol
|
||||
local base_cost_per_day=${SCHEMA_COSTS[$schema]}
|
||||
|
||||
# Apply parent symbol multiplier if enabled
|
||||
local parent_multiplier="1"
|
||||
if [ "$use_parent" = true ]; then
|
||||
log_warn "Parent symbol type selected: Cost multiplier 15x applied" >&2
|
||||
parent_multiplier="15"
|
||||
fi
|
||||
|
||||
# Calculate total cost
|
||||
local total_cost
|
||||
total_cost=$(echo "scale=4; $num_symbols * $days * $base_cost_per_day * $parent_multiplier" | bc -l)
|
||||
|
||||
# Estimate file size (in MB)
|
||||
local base_size_kb=${SCHEMA_SIZES[$schema]}
|
||||
local total_size_kb
|
||||
total_size_kb=$(echo "scale=2; $num_symbols * $days * $base_size_kb * $parent_multiplier" | bc -l)
|
||||
local total_size_mb
|
||||
total_size_mb=$(echo "scale=2; $total_size_kb / 1024" | bc -l)
|
||||
|
||||
echo "$total_cost $total_size_mb"
|
||||
}
|
||||
|
||||
# Generate download commands
|
||||
generate_download_commands() {
|
||||
local symbols=$1
|
||||
local days=$2
|
||||
local schema=$3
|
||||
local start_date=$4
|
||||
local use_parent=$5
|
||||
|
||||
# Parse symbols
|
||||
IFS=',' read -ra symbol_array <<< "$symbols"
|
||||
|
||||
# Determine symbol type
|
||||
local stype_in="continuous"
|
||||
if [ "$use_parent" = true ]; then
|
||||
stype_in="parent"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "======================================================================"
|
||||
echo " DOWNLOAD COMMANDS"
|
||||
echo "======================================================================"
|
||||
echo ""
|
||||
|
||||
for symbol in "${symbol_array[@]}"; do
|
||||
# Trim whitespace
|
||||
symbol=$(echo "$symbol" | xargs)
|
||||
|
||||
for ((day=0; day<days; day++)); do
|
||||
# Calculate dates
|
||||
if [ -n "$start_date" ]; then
|
||||
local current_date
|
||||
current_date=$(date -d "$start_date + $day days" +%Y-%m-%d)
|
||||
else
|
||||
local current_date
|
||||
current_date=$(date -d "2024-01-02 + $day days" +%Y-%m-%d)
|
||||
fi
|
||||
|
||||
local next_date
|
||||
next_date=$(date -d "$current_date + 1 day" +%Y-%m-%d)
|
||||
|
||||
# Generate filename
|
||||
local output_file="test_data/real/databento/${symbol}_${schema}_${current_date}.dbn"
|
||||
|
||||
# Generate curl command
|
||||
cat << EOF
|
||||
# Download: $symbol, $current_date ($schema)
|
||||
curl -u "\${DATABENTO_API_KEY}:" \\
|
||||
"https://hist.databento.com/v0/timeseries.get_range?\\
|
||||
dataset=GLBX.MDP3&\\
|
||||
symbols=${symbol}&\\
|
||||
schema=${schema}&\\
|
||||
start=${current_date}T00:00:00Z&\\
|
||||
end=${next_date}T00:00:00Z&\\
|
||||
encoding=dbn&\\
|
||||
stype_in=${stype_in}" \\
|
||||
--output "$output_file"
|
||||
|
||||
# Verify download
|
||||
ls -lh "$output_file"
|
||||
|
||||
# Update COST_TRACKING.md
|
||||
echo "## Download $(date +%Y-%m-%d): $symbol ($schema)" >> COST_TRACKING.md
|
||||
echo "- Symbol: $symbol" >> COST_TRACKING.md
|
||||
echo "- Schema: $schema" >> COST_TRACKING.md
|
||||
echo "- Date: $current_date" >> COST_TRACKING.md
|
||||
echo "- File: $output_file" >> COST_TRACKING.md
|
||||
echo "" >> COST_TRACKING.md
|
||||
|
||||
EOF
|
||||
done
|
||||
done
|
||||
|
||||
echo "======================================================================"
|
||||
}
|
||||
|
||||
# ============================================================================
|
||||
# Main
|
||||
# ============================================================================
|
||||
|
||||
main() {
|
||||
local symbols=""
|
||||
local days=""
|
||||
local schema=""
|
||||
local start_date=""
|
||||
local use_parent=false
|
||||
|
||||
# Parse arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--symbols)
|
||||
symbols="$2"
|
||||
shift 2
|
||||
;;
|
||||
--days)
|
||||
days="$2"
|
||||
shift 2
|
||||
;;
|
||||
--schema)
|
||||
schema="$2"
|
||||
shift 2
|
||||
;;
|
||||
--start-date)
|
||||
start_date="$2"
|
||||
shift 2
|
||||
;;
|
||||
--parent)
|
||||
use_parent=true
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
usage
|
||||
;;
|
||||
*)
|
||||
log_error "Unknown option: $1"
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Validate required arguments
|
||||
if [ -z "$symbols" ] || [ -z "$days" ] || [ -z "$schema" ]; then
|
||||
log_error "Missing required arguments"
|
||||
usage
|
||||
fi
|
||||
|
||||
# Validate schema
|
||||
if [ -z "${SCHEMA_COSTS[$schema]:-}" ]; then
|
||||
log_error "Invalid schema: $schema"
|
||||
log_error "Supported schemas: ${!SCHEMA_COSTS[*]}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Display plan header
|
||||
echo ""
|
||||
echo "======================================================================"
|
||||
echo " DATABENTO DOWNLOAD PLAN"
|
||||
echo "======================================================================"
|
||||
echo "Symbols: $symbols"
|
||||
echo "Days: $days"
|
||||
echo "Schema: $schema"
|
||||
echo "Start Date: ${start_date:-2024-01-02 (default)}"
|
||||
echo "Symbol Type: $([ "$use_parent" = true ] && echo "parent (⚠️ 15x cost!)" || echo "continuous")"
|
||||
echo "======================================================================"
|
||||
|
||||
# Estimate cost
|
||||
log_info "Estimating cost..."
|
||||
read -r estimated_cost estimated_size_mb <<< "$(estimate_cost "$symbols" "$days" "$schema" "$use_parent")"
|
||||
|
||||
echo ""
|
||||
echo "======================================================================"
|
||||
echo " COST ESTIMATE"
|
||||
echo "======================================================================"
|
||||
printf "%-30s: \$%.4f\n" "Estimated Cost (High)" "$estimated_cost"
|
||||
printf "%-30s: %.2f MB\n" "Estimated File Size" "$estimated_size_mb"
|
||||
echo "======================================================================"
|
||||
|
||||
# Check budget
|
||||
if ! check_credit_budget "$estimated_cost"; then
|
||||
local exit_code=$?
|
||||
if [ $exit_code -eq 2 ]; then
|
||||
log_error "Download REJECTED due to credit exhaustion risk"
|
||||
exit 1
|
||||
else
|
||||
log_warn "Download requires review before proceeding"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Generate download commands
|
||||
generate_download_commands "$symbols" "$days" "$schema" "$start_date" "$use_parent"
|
||||
|
||||
# Final recommendation
|
||||
echo ""
|
||||
echo "======================================================================"
|
||||
echo " RECOMMENDATION"
|
||||
echo "======================================================================"
|
||||
|
||||
if (( $(echo "$estimated_cost < 0.01" | bc -l) )); then
|
||||
log_success "✅ APPROVE: Cost <\$0.01, proceed immediately"
|
||||
elif (( $(echo "$estimated_cost < 0.10" | bc -l) )); then
|
||||
log_info "📋 REVIEW: Cost \$0.01-\$0.10, check budget and prioritize"
|
||||
elif (( $(echo "$estimated_cost < 1.00" | bc -l) )); then
|
||||
log_warn "🟡 REQUIRE APPROVAL: Cost \$0.10-\$1.00, justify and review"
|
||||
else
|
||||
log_error "🚨 EXECUTIVE APPROVAL: Cost >\$1.00, high-cost download"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Next Steps:"
|
||||
echo "1. Review cost estimate and budget status"
|
||||
echo "2. If approved, copy and execute download commands above"
|
||||
echo "3. Verify file sizes match estimates"
|
||||
echo "4. Update COST_TRACKING.md with actual costs"
|
||||
echo "======================================================================"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# Run main function
|
||||
main "$@"
|
||||
177
scripts/validate_data_quality.sh
Executable file
177
scripts/validate_data_quality.sh
Executable file
@@ -0,0 +1,177 @@
|
||||
#!/bin/bash
|
||||
# DBN Data Quality Validation Script for CI/CD
|
||||
#
|
||||
# This script runs comprehensive data quality validation and fails
|
||||
# the build if quality standards are not met.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/validate_data_quality.sh [options]
|
||||
#
|
||||
# Options:
|
||||
# --min-quality SCORE Minimum quality score (default: 70)
|
||||
# --output DIR Output directory for reports (default: ./validation_reports)
|
||||
# --symbol SYMBOL Validate single symbol
|
||||
# --all Validate all symbols (default)
|
||||
# --verbose Enable verbose output
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 - All validations passed
|
||||
# 1 - Quality check failed
|
||||
# 2 - Validation error (no data, missing files)
|
||||
|
||||
set -e
|
||||
|
||||
# Default configuration
|
||||
MIN_QUALITY=70
|
||||
OUTPUT_DIR="./validation_reports"
|
||||
SYMBOL=""
|
||||
VALIDATE_ALL="--all"
|
||||
VERBOSE=""
|
||||
DATA_DIR="test_data/real/databento"
|
||||
|
||||
# Parse arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--min-quality)
|
||||
MIN_QUALITY="$2"
|
||||
shift 2
|
||||
;;
|
||||
--output)
|
||||
OUTPUT_DIR="$2"
|
||||
shift 2
|
||||
;;
|
||||
--symbol)
|
||||
SYMBOL="$2"
|
||||
VALIDATE_ALL=""
|
||||
shift 2
|
||||
;;
|
||||
--all)
|
||||
VALIDATE_ALL="--all"
|
||||
SYMBOL=""
|
||||
shift
|
||||
;;
|
||||
--verbose)
|
||||
VERBOSE="--verbose"
|
||||
shift
|
||||
;;
|
||||
--data-dir)
|
||||
DATA_DIR="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: $1"
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Create output directory
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
# Timestamp for report files
|
||||
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
||||
|
||||
echo "════════════════════════════════════════════════════════"
|
||||
echo "DBN Data Quality Validation"
|
||||
echo "════════════════════════════════════════════════════════"
|
||||
echo "Timestamp: $(date --rfc-3339=seconds)"
|
||||
echo "Min Quality Score: $MIN_QUALITY"
|
||||
echo "Output Directory: $OUTPUT_DIR"
|
||||
echo "Data Directory: $DATA_DIR"
|
||||
|
||||
if [ -n "$SYMBOL" ]; then
|
||||
echo "Symbol: $SYMBOL"
|
||||
else
|
||||
echo "Mode: All symbols"
|
||||
fi
|
||||
|
||||
echo "════════════════════════════════════════════════════════"
|
||||
echo
|
||||
|
||||
# Check if data directory exists
|
||||
if [ ! -d "$DATA_DIR" ]; then
|
||||
echo "❌ ERROR: Data directory not found: $DATA_DIR"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Count DBN files
|
||||
DBN_COUNT=$(find "$DATA_DIR" -name "*.dbn" | wc -l)
|
||||
if [ "$DBN_COUNT" -eq 0 ]; then
|
||||
echo "❌ ERROR: No DBN files found in: $DATA_DIR"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
echo "Found $DBN_COUNT DBN files"
|
||||
echo
|
||||
|
||||
# Build validation tool if not already built
|
||||
echo "Building validation tool..."
|
||||
cargo build --release -p backtesting_service --bin validate_dbn_data 2>&1 | grep -v "Compiling\|Finished" || true
|
||||
echo
|
||||
|
||||
# Run validation
|
||||
VALIDATION_CMD="cargo run --release -p backtesting_service --bin validate_dbn_data --"
|
||||
|
||||
if [ -n "$SYMBOL" ]; then
|
||||
VALIDATION_CMD="$VALIDATION_CMD --symbol $SYMBOL"
|
||||
else
|
||||
VALIDATION_CMD="$VALIDATION_CMD $VALIDATE_ALL"
|
||||
fi
|
||||
|
||||
VALIDATION_CMD="$VALIDATION_CMD --data-dir $DATA_DIR"
|
||||
VALIDATION_CMD="$VALIDATION_CMD --min-quality-score $MIN_QUALITY"
|
||||
VALIDATION_CMD="$VALIDATION_CMD --fail-on-poor-quality"
|
||||
VALIDATION_CMD="$VALIDATION_CMD $VERBOSE"
|
||||
|
||||
# Generate reports in multiple formats
|
||||
echo "Generating validation reports..."
|
||||
echo
|
||||
|
||||
# Text report (stdout + file)
|
||||
TEXT_REPORT="$OUTPUT_DIR/validation_report_${TIMESTAMP}.txt"
|
||||
echo "📄 Text report: $TEXT_REPORT"
|
||||
$VALIDATION_CMD --format text 2>&1 | tee "$TEXT_REPORT"
|
||||
TEXT_EXIT_CODE=${PIPESTATUS[0]}
|
||||
|
||||
# JSON report (if text passed)
|
||||
if [ $TEXT_EXIT_CODE -eq 0 ]; then
|
||||
JSON_REPORT="$OUTPUT_DIR/validation_report_${TIMESTAMP}.json"
|
||||
echo
|
||||
echo "📊 JSON report: $JSON_REPORT"
|
||||
$VALIDATION_CMD --format json --output "$JSON_REPORT" >/dev/null 2>&1 || true
|
||||
|
||||
# HTML report
|
||||
HTML_REPORT="$OUTPUT_DIR/validation_report_${TIMESTAMP}.html"
|
||||
echo "📈 HTML report: $HTML_REPORT"
|
||||
$VALIDATION_CMD --format html --output "$HTML_REPORT" >/dev/null 2>&1 || true
|
||||
|
||||
# Create latest symlinks
|
||||
ln -sf "$(basename "$TEXT_REPORT")" "$OUTPUT_DIR/latest.txt"
|
||||
ln -sf "$(basename "$JSON_REPORT")" "$OUTPUT_DIR/latest.json"
|
||||
ln -sf "$(basename "$HTML_REPORT")" "$OUTPUT_DIR/latest.html"
|
||||
|
||||
echo
|
||||
echo "════════════════════════════════════════════════════════"
|
||||
echo "✅ VALIDATION PASSED"
|
||||
echo "════════════════════════════════════════════════════════"
|
||||
echo
|
||||
echo "Reports generated:"
|
||||
echo " • Text: $TEXT_REPORT"
|
||||
echo " • JSON: $JSON_REPORT"
|
||||
echo " • HTML: $HTML_REPORT"
|
||||
echo
|
||||
echo "View HTML report:"
|
||||
echo " xdg-open $HTML_REPORT"
|
||||
echo
|
||||
exit 0
|
||||
else
|
||||
echo
|
||||
echo "════════════════════════════════════════════════════════"
|
||||
echo "❌ VALIDATION FAILED"
|
||||
echo "════════════════════════════════════════════════════════"
|
||||
echo
|
||||
echo "Quality score below minimum threshold ($MIN_QUALITY)"
|
||||
echo "See report for details: $TEXT_REPORT"
|
||||
echo
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user