diff --git a/infra/k8s/jobs/databento-download-streaming.yaml b/infra/k8s/jobs/databento-download-streaming.yaml new file mode 100644 index 000000000..c7fcc37ea --- /dev/null +++ b/infra/k8s/jobs/databento-download-streaming.yaml @@ -0,0 +1,148 @@ +# Databento data download via streaming API (get_range). +# +# Binary pre-uploaded to /data/bin/download_baseline on training-data-pvc. +# Universe configs loaded from configmap. +# +# Test: kubectl apply -f infra/k8s/jobs/databento-download-streaming.yaml +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: databento-download + namespace: foxhunt + labels: + app.kubernetes.io/part-of: foxhunt +spec: + podSelector: + matchLabels: + app.kubernetes.io/component: databento-download + policyTypes: + - Egress + egress: + - ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + - ports: + - port: 80 + protocol: TCP + - port: 443 + protocol: TCP + - ports: + - port: 443 + protocol: TCP + - port: 6443 + protocol: TCP + to: + - ipBlock: + cidr: 10.32.0.0/16 + - ipBlock: + cidr: 172.16.0.4/32 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: databento-download-all + namespace: foxhunt + labels: + app.kubernetes.io/name: databento-download + app.kubernetes.io/part-of: foxhunt +spec: + backoffLimit: 3 + activeDeadlineSeconds: 86400 + ttlSecondsAfterFinished: 3600 + template: + metadata: + labels: + app.kubernetes.io/name: databento-download + app.kubernetes.io/part-of: foxhunt + app.kubernetes.io/component: databento-download + spec: + restartPolicy: OnFailure + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + nodeSelector: + k8s.scaleway.com/pool-name: platform + containers: + - name: downloader + image: ubuntu:24.04 + command: ["/bin/bash", "-c"] + env: + - name: RUST_LOG + value: debug + - name: DATABENTO_API_KEY + valueFrom: + secretKeyRef: + name: databento-credentials + key: api-key + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: "2" + memory: 2Gi + volumeMounts: + - name: training-data + mountPath: /data + - name: universe-configs + mountPath: /configs + args: + - | + set -e + apt-get update -qq > /dev/null 2>&1 && apt-get install -y -qq ca-certificates > /dev/null 2>&1 + BINARY=/data/bin/download_baseline + chmod +x "$BINARY" + + # Download 4 schemas: OHLCV-1m, Trades, OHLCV-1s, MBP-10 + # Uses universe configs from the repo (mounted via configmap) + + for SCHEMA in ohlcv-1m trades ohlcv-1s mbp-10; do + case $SCHEMA in + ohlcv-1m) DIR=/data/futures-baseline; CONFIG_NAME=ohlcv-1m ;; + trades) DIR=/data/futures-baseline-trades; CONFIG_NAME=trades ;; + ohlcv-1s) DIR=/data/futures-baseline-1s; CONFIG_NAME=ohlcv-1s ;; + mbp-10) DIR=/data/futures-baseline-mbp10; CONFIG_NAME=mbp-10 ;; + esac + + cat > /tmp/universe-${CONFIG_NAME}.toml << TOML + [universe] + name = "es-${CONFIG_NAME}" + description = "ES.FUT ${SCHEMA}" + date_range_start = "2024-03-01" + date_range_end = "2026-03-30" + bar_size = "tick" + databento_dataset = "GLBX.MDP3" + databento_schema = "${SCHEMA}" + + [[symbols]] + symbol = "ES.FUT" + exchange = "GLBX.MDP3" + asset_class = "Future" + TOML + + echo "" + echo "=== Downloading ${SCHEMA} to ${DIR} ===" + mkdir -p "${DIR}/ES.FUT" + $BINARY \ + --universe-config /tmp/universe-${CONFIG_NAME}.toml \ + --output-dir "${DIR}/ES.FUT" \ + --yes || echo "WARN: ${SCHEMA} had failures (will retry on next run)" + echo "" + du -sh "${DIR}" 2>/dev/null + done + + echo "" + echo "=== Final PVC summary ===" + du -sh /data/*/ + df -h /data + volumes: + - name: training-data + persistentVolumeClaim: + claimName: training-data-pvc + - name: universe-configs + configMap: + name: databento-universe-configs