#!/usr/bin/env bash # Download Databento data via Argo Workflows. # # Usage: # ./scripts/argo-download.sh mbp-10 # MBP-10 with defaults # ./scripts/argo-download.sh trades --output /data/futures-baseline-trades # ./scripts/argo-download.sh ohlcv-1s --parallel 4 --pool platform # ./scripts/argo-download.sh mbp-10 --watch # follow logs # # Schemas: ohlcv-1m, ohlcv-1s, trades, mbp-10 # # Requires: argo CLI set -euo pipefail TEMPLATE="databento-download" PARALLEL="" OUTPUT="" POOL="" WATCH=false usage() { cat < [OPTIONS] Schemas: ohlcv-1m OHLCV 1-minute bars ohlcv-1s OHLCV 1-second bars trades Individual trade executions mbp-10 10-level order book snapshots Options: --output Output dir on PVC (default: schema-based) --parallel Parallel quarterly downloads (default: 9) --pool Node pool (default: ci-compile-cpu) --watch Follow workflow logs after submission -h, --help Show this help EOF exit 0 } [[ $# -eq 0 ]] && { echo "Error: schema argument required"; usage; } [[ "$1" == "-h" || "$1" == "--help" ]] && usage SCHEMA="$1"; shift # Validate schema case "$SCHEMA" in ohlcv-1m|ohlcv-1s|trades|mbp-10) ;; *) echo "Error: unknown schema '$SCHEMA'"; usage ;; esac while [[ $# -gt 0 ]]; do case $1 in --output) OUTPUT="$2"; shift 2 ;; --parallel) PARALLEL="$2"; shift 2 ;; --pool) POOL="$2"; shift 2 ;; --watch) WATCH=true; shift ;; -h|--help) usage ;; *) echo "Unknown option: $1"; usage ;; esac done # Default output dir based on schema if [[ -z "$OUTPUT" ]]; then case "$SCHEMA" in ohlcv-1m) OUTPUT="/data/futures-baseline" ;; ohlcv-1s) OUTPUT="/data/futures-baseline-1s" ;; trades) OUTPUT="/data/futures-baseline-trades" ;; mbp-10) OUTPUT="/data/futures-baseline-mbp10" ;; esac fi # Build argo submit command CMD="argo submit -n foxhunt --from=wftmpl/$TEMPLATE" CMD="$CMD -p schema=$SCHEMA" CMD="$CMD -p output-dir=$OUTPUT" [[ -n "$PARALLEL" ]] && CMD="$CMD -p parallel=$PARALLEL" [[ -n "$POOL" ]] && CMD="$CMD -p node-pool=$POOL" if $WATCH; then CMD="$CMD --watch" fi echo "Submitting Databento download workflow..." echo " schema: $SCHEMA" echo " output: $OUTPUT" [[ -n "$PARALLEL" ]] && echo " parallel: $PARALLEL" [[ -n "$POOL" ]] && echo " pool: $POOL" echo "" eval "$CMD"