diff --git a/infra/k8s/jobs/databento-download-streaming.yaml b/infra/k8s/jobs/databento-download-streaming.yaml index c7fcc37ea..a66fcba18 100644 --- a/infra/k8s/jobs/databento-download-streaming.yaml +++ b/infra/k8s/jobs/databento-download-streaming.yaml @@ -61,8 +61,6 @@ spec: spec: restartPolicy: OnFailure securityContext: - runAsUser: 1000 - runAsGroup: 1000 fsGroup: 1000 nodeSelector: k8s.scaleway.com/pool-name: platform @@ -97,41 +95,22 @@ spec: BINARY=/data/bin/download_baseline chmod +x "$BINARY" - # Download 4 schemas: OHLCV-1m, Trades, OHLCV-1s, MBP-10 - # Uses universe configs from the repo (mounted via configmap) - + # Download 4 schemas from configmap-mounted TOML configs for SCHEMA in ohlcv-1m trades ohlcv-1s mbp-10; do case $SCHEMA in - ohlcv-1m) DIR=/data/futures-baseline; CONFIG_NAME=ohlcv-1m ;; - trades) DIR=/data/futures-baseline-trades; CONFIG_NAME=trades ;; - ohlcv-1s) DIR=/data/futures-baseline-1s; CONFIG_NAME=ohlcv-1s ;; - mbp-10) DIR=/data/futures-baseline-mbp10; CONFIG_NAME=mbp-10 ;; + ohlcv-1m) DIR=/data/futures-baseline ;; + trades) DIR=/data/futures-baseline-trades ;; + ohlcv-1s) DIR=/data/futures-baseline-1s ;; + mbp-10) DIR=/data/futures-baseline-mbp10 ;; esac - cat > /tmp/universe-${CONFIG_NAME}.toml << TOML - [universe] - name = "es-${CONFIG_NAME}" - description = "ES.FUT ${SCHEMA}" - date_range_start = "2024-03-01" - date_range_end = "2026-03-30" - bar_size = "tick" - databento_dataset = "GLBX.MDP3" - databento_schema = "${SCHEMA}" - - [[symbols]] - symbol = "ES.FUT" - exchange = "GLBX.MDP3" - asset_class = "Future" - TOML - echo "" - echo "=== Downloading ${SCHEMA} to ${DIR} ===" - mkdir -p "${DIR}/ES.FUT" + echo "=== Downloading ${SCHEMA} ===" + mkdir -p "${DIR}" $BINARY \ - --universe-config /tmp/universe-${CONFIG_NAME}.toml \ - --output-dir "${DIR}/ES.FUT" \ - --yes || echo "WARN: ${SCHEMA} had failures (will retry on next run)" - echo "" + --universe-config /configs/${SCHEMA}.toml \ + --output-dir "${DIR}" \ + --yes || echo "WARN: ${SCHEMA} had failures" du -sh "${DIR}" 2>/dev/null done @@ -145,4 +124,4 @@ spec: claimName: training-data-pvc - name: universe-configs configMap: - name: databento-universe-configs + name: databento-universe-all