fix: Databento streaming download — CA certs + all 4 schemas

Root cause: ubuntu:24.04 has no CA certificates, rustls-native-certs
fails TLS handshake. Fix: apt-get install ca-certificates before
running download binary. Also add port 80 to network policy for apt.

Downloads OHLCV-1m, Trades, OHLCV-1s, MBP-10 via streaming API
(get_range) — parallel quarterly downloads, individual .dbn.zst files.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-31 21:48:28 +02:00
parent 3b77359e74
commit 67af21449e

View File

@@ -0,0 +1,148 @@
# Databento data download via streaming API (get_range).
#
# Binary pre-uploaded to /data/bin/download_baseline on training-data-pvc.
# Universe configs loaded from configmap.
#
# Test: kubectl apply -f infra/k8s/jobs/databento-download-streaming.yaml
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: databento-download
namespace: foxhunt
labels:
app.kubernetes.io/part-of: foxhunt
spec:
podSelector:
matchLabels:
app.kubernetes.io/component: databento-download
policyTypes:
- Egress
egress:
- ports:
- port: 53
protocol: UDP
- port: 53
protocol: TCP
- ports:
- port: 80
protocol: TCP
- port: 443
protocol: TCP
- ports:
- port: 443
protocol: TCP
- port: 6443
protocol: TCP
to:
- ipBlock:
cidr: 10.32.0.0/16
- ipBlock:
cidr: 172.16.0.4/32
---
apiVersion: batch/v1
kind: Job
metadata:
name: databento-download-all
namespace: foxhunt
labels:
app.kubernetes.io/name: databento-download
app.kubernetes.io/part-of: foxhunt
spec:
backoffLimit: 3
activeDeadlineSeconds: 86400
ttlSecondsAfterFinished: 3600
template:
metadata:
labels:
app.kubernetes.io/name: databento-download
app.kubernetes.io/part-of: foxhunt
app.kubernetes.io/component: databento-download
spec:
restartPolicy: OnFailure
securityContext:
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
nodeSelector:
k8s.scaleway.com/pool-name: platform
containers:
- name: downloader
image: ubuntu:24.04
command: ["/bin/bash", "-c"]
env:
- name: RUST_LOG
value: debug
- name: DATABENTO_API_KEY
valueFrom:
secretKeyRef:
name: databento-credentials
key: api-key
resources:
requests:
cpu: 500m
memory: 512Mi
limits:
cpu: "2"
memory: 2Gi
volumeMounts:
- name: training-data
mountPath: /data
- name: universe-configs
mountPath: /configs
args:
- |
set -e
apt-get update -qq > /dev/null 2>&1 && apt-get install -y -qq ca-certificates > /dev/null 2>&1
BINARY=/data/bin/download_baseline
chmod +x "$BINARY"
# Download 4 schemas: OHLCV-1m, Trades, OHLCV-1s, MBP-10
# Uses universe configs from the repo (mounted via configmap)
for SCHEMA in ohlcv-1m trades ohlcv-1s mbp-10; do
case $SCHEMA in
ohlcv-1m) DIR=/data/futures-baseline; CONFIG_NAME=ohlcv-1m ;;
trades) DIR=/data/futures-baseline-trades; CONFIG_NAME=trades ;;
ohlcv-1s) DIR=/data/futures-baseline-1s; CONFIG_NAME=ohlcv-1s ;;
mbp-10) DIR=/data/futures-baseline-mbp10; CONFIG_NAME=mbp-10 ;;
esac
cat > /tmp/universe-${CONFIG_NAME}.toml << TOML
[universe]
name = "es-${CONFIG_NAME}"
description = "ES.FUT ${SCHEMA}"
date_range_start = "2024-03-01"
date_range_end = "2026-03-30"
bar_size = "tick"
databento_dataset = "GLBX.MDP3"
databento_schema = "${SCHEMA}"
[[symbols]]
symbol = "ES.FUT"
exchange = "GLBX.MDP3"
asset_class = "Future"
TOML
echo ""
echo "=== Downloading ${SCHEMA} to ${DIR} ==="
mkdir -p "${DIR}/ES.FUT"
$BINARY \
--universe-config /tmp/universe-${CONFIG_NAME}.toml \
--output-dir "${DIR}/ES.FUT" \
--yes || echo "WARN: ${SCHEMA} had failures (will retry on next run)"
echo ""
du -sh "${DIR}" 2>/dev/null
done
echo ""
echo "=== Final PVC summary ==="
du -sh /data/*/
df -h /data
volumes:
- name: training-data
persistentVolumeClaim:
claimName: training-data-pvc
- name: universe-configs
configMap:
name: databento-universe-configs