fix: Databento streaming download — CA certs + all 4 schemas
Root cause: ubuntu:24.04 has no CA certificates, rustls-native-certs fails TLS handshake. Fix: apt-get install ca-certificates before running download binary. Also add port 80 to network policy for apt. Downloads OHLCV-1m, Trades, OHLCV-1s, MBP-10 via streaming API (get_range) — parallel quarterly downloads, individual .dbn.zst files. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
148
infra/k8s/jobs/databento-download-streaming.yaml
Normal file
148
infra/k8s/jobs/databento-download-streaming.yaml
Normal file
@@ -0,0 +1,148 @@
|
||||
# Databento data download via streaming API (get_range).
|
||||
#
|
||||
# Binary pre-uploaded to /data/bin/download_baseline on training-data-pvc.
|
||||
# Universe configs loaded from configmap.
|
||||
#
|
||||
# Test: kubectl apply -f infra/k8s/jobs/databento-download-streaming.yaml
|
||||
---
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: databento-download
|
||||
namespace: foxhunt
|
||||
labels:
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/component: databento-download
|
||||
policyTypes:
|
||||
- Egress
|
||||
egress:
|
||||
- ports:
|
||||
- port: 53
|
||||
protocol: UDP
|
||||
- port: 53
|
||||
protocol: TCP
|
||||
- ports:
|
||||
- port: 80
|
||||
protocol: TCP
|
||||
- port: 443
|
||||
protocol: TCP
|
||||
- ports:
|
||||
- port: 443
|
||||
protocol: TCP
|
||||
- port: 6443
|
||||
protocol: TCP
|
||||
to:
|
||||
- ipBlock:
|
||||
cidr: 10.32.0.0/16
|
||||
- ipBlock:
|
||||
cidr: 172.16.0.4/32
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: databento-download-all
|
||||
namespace: foxhunt
|
||||
labels:
|
||||
app.kubernetes.io/name: databento-download
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
spec:
|
||||
backoffLimit: 3
|
||||
activeDeadlineSeconds: 86400
|
||||
ttlSecondsAfterFinished: 3600
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: databento-download
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
app.kubernetes.io/component: databento-download
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
securityContext:
|
||||
runAsUser: 1000
|
||||
runAsGroup: 1000
|
||||
fsGroup: 1000
|
||||
nodeSelector:
|
||||
k8s.scaleway.com/pool-name: platform
|
||||
containers:
|
||||
- name: downloader
|
||||
image: ubuntu:24.04
|
||||
command: ["/bin/bash", "-c"]
|
||||
env:
|
||||
- name: RUST_LOG
|
||||
value: debug
|
||||
- name: DATABENTO_API_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: databento-credentials
|
||||
key: api-key
|
||||
resources:
|
||||
requests:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "2"
|
||||
memory: 2Gi
|
||||
volumeMounts:
|
||||
- name: training-data
|
||||
mountPath: /data
|
||||
- name: universe-configs
|
||||
mountPath: /configs
|
||||
args:
|
||||
- |
|
||||
set -e
|
||||
apt-get update -qq > /dev/null 2>&1 && apt-get install -y -qq ca-certificates > /dev/null 2>&1
|
||||
BINARY=/data/bin/download_baseline
|
||||
chmod +x "$BINARY"
|
||||
|
||||
# Download 4 schemas: OHLCV-1m, Trades, OHLCV-1s, MBP-10
|
||||
# Uses universe configs from the repo (mounted via configmap)
|
||||
|
||||
for SCHEMA in ohlcv-1m trades ohlcv-1s mbp-10; do
|
||||
case $SCHEMA in
|
||||
ohlcv-1m) DIR=/data/futures-baseline; CONFIG_NAME=ohlcv-1m ;;
|
||||
trades) DIR=/data/futures-baseline-trades; CONFIG_NAME=trades ;;
|
||||
ohlcv-1s) DIR=/data/futures-baseline-1s; CONFIG_NAME=ohlcv-1s ;;
|
||||
mbp-10) DIR=/data/futures-baseline-mbp10; CONFIG_NAME=mbp-10 ;;
|
||||
esac
|
||||
|
||||
cat > /tmp/universe-${CONFIG_NAME}.toml << TOML
|
||||
[universe]
|
||||
name = "es-${CONFIG_NAME}"
|
||||
description = "ES.FUT ${SCHEMA}"
|
||||
date_range_start = "2024-03-01"
|
||||
date_range_end = "2026-03-30"
|
||||
bar_size = "tick"
|
||||
databento_dataset = "GLBX.MDP3"
|
||||
databento_schema = "${SCHEMA}"
|
||||
|
||||
[[symbols]]
|
||||
symbol = "ES.FUT"
|
||||
exchange = "GLBX.MDP3"
|
||||
asset_class = "Future"
|
||||
TOML
|
||||
|
||||
echo ""
|
||||
echo "=== Downloading ${SCHEMA} to ${DIR} ==="
|
||||
mkdir -p "${DIR}/ES.FUT"
|
||||
$BINARY \
|
||||
--universe-config /tmp/universe-${CONFIG_NAME}.toml \
|
||||
--output-dir "${DIR}/ES.FUT" \
|
||||
--yes || echo "WARN: ${SCHEMA} had failures (will retry on next run)"
|
||||
echo ""
|
||||
du -sh "${DIR}" 2>/dev/null
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== Final PVC summary ==="
|
||||
du -sh /data/*/
|
||||
df -h /data
|
||||
volumes:
|
||||
- name: training-data
|
||||
persistentVolumeClaim:
|
||||
claimName: training-data-pvc
|
||||
- name: universe-configs
|
||||
configMap:
|
||||
name: databento-universe-configs
|
||||
Reference in New Issue
Block a user