cb8afe3caf
Replace hardcoded K8s Job with reusable Argo WorkflowTemplate (databento-download) parameterized by schema, output-dir, parallel, and node-pool. Add argo-download.sh CLI wrapper matching the argo-train.sh pattern. Remove old streaming job file. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
92 lines
2.4 KiB
Bash
Executable File
92 lines
2.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Download Databento data via Argo Workflows.
|
|
#
|
|
# Usage:
|
|
# ./scripts/argo-download.sh mbp-10 # MBP-10 with defaults
|
|
# ./scripts/argo-download.sh trades --output /data/futures-baseline-trades
|
|
# ./scripts/argo-download.sh ohlcv-1s --parallel 4 --pool platform
|
|
# ./scripts/argo-download.sh mbp-10 --watch # follow logs
|
|
#
|
|
# Schemas: ohlcv-1m, ohlcv-1s, trades, mbp-10
|
|
#
|
|
# Requires: argo CLI
|
|
set -euo pipefail
|
|
|
|
TEMPLATE="databento-download"
|
|
PARALLEL=""
|
|
OUTPUT=""
|
|
POOL=""
|
|
WATCH=false
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
Usage: $(basename "$0") <schema> [OPTIONS]
|
|
|
|
Schemas:
|
|
ohlcv-1m OHLCV 1-minute bars
|
|
ohlcv-1s OHLCV 1-second bars
|
|
trades Individual trade executions
|
|
mbp-10 10-level order book snapshots
|
|
|
|
Options:
|
|
--output <dir> Output dir on PVC (default: schema-based)
|
|
--parallel <n> Parallel quarterly downloads (default: 9)
|
|
--pool <name> Node pool (default: ci-compile-cpu)
|
|
--watch Follow workflow logs after submission
|
|
-h, --help Show this help
|
|
EOF
|
|
exit 0
|
|
}
|
|
|
|
[[ $# -eq 0 ]] && { echo "Error: schema argument required"; usage; }
|
|
[[ "$1" == "-h" || "$1" == "--help" ]] && usage
|
|
|
|
SCHEMA="$1"; shift
|
|
|
|
# Validate schema
|
|
case "$SCHEMA" in
|
|
ohlcv-1m|ohlcv-1s|trades|mbp-10) ;;
|
|
*) echo "Error: unknown schema '$SCHEMA'"; usage ;;
|
|
esac
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
--output) OUTPUT="$2"; shift 2 ;;
|
|
--parallel) PARALLEL="$2"; shift 2 ;;
|
|
--pool) POOL="$2"; shift 2 ;;
|
|
--watch) WATCH=true; shift ;;
|
|
-h|--help) usage ;;
|
|
*) echo "Unknown option: $1"; usage ;;
|
|
esac
|
|
done
|
|
|
|
# Default output dir based on schema
|
|
if [[ -z "$OUTPUT" ]]; then
|
|
case "$SCHEMA" in
|
|
ohlcv-1m) OUTPUT="/data/futures-baseline" ;;
|
|
ohlcv-1s) OUTPUT="/data/futures-baseline-1s" ;;
|
|
trades) OUTPUT="/data/futures-baseline-trades" ;;
|
|
mbp-10) OUTPUT="/data/futures-baseline-mbp10" ;;
|
|
esac
|
|
fi
|
|
|
|
# Build argo submit command
|
|
CMD="argo submit -n foxhunt --from=wftmpl/$TEMPLATE"
|
|
CMD="$CMD -p schema=$SCHEMA"
|
|
CMD="$CMD -p output-dir=$OUTPUT"
|
|
|
|
[[ -n "$PARALLEL" ]] && CMD="$CMD -p parallel=$PARALLEL"
|
|
[[ -n "$POOL" ]] && CMD="$CMD -p node-pool=$POOL"
|
|
|
|
if $WATCH; then
|
|
CMD="$CMD --watch"
|
|
fi
|
|
|
|
echo "Submitting Databento download workflow..."
|
|
echo " schema: $SCHEMA"
|
|
echo " output: $OUTPUT"
|
|
[[ -n "$PARALLEL" ]] && echo " parallel: $PARALLEL"
|
|
[[ -n "$POOL" ]] && echo " pool: $POOL"
|
|
echo ""
|
|
eval "$CMD"
|