gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: Databento download — run as root for apt, fix output-dir path

Browse Source 
Remove runAsUser:1000 so apt-get install ca-certificates works.
Fix double-nested ES.FUT path (output-dir is the schema root, not ES.FUT subdir).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-31 21:58:19 +02:00
parent 67af21449e
commit fda01452c9
1 changed files with 11 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
infra/k8s/jobs/databento-download-streaming.yaml
View File

@@ -61,8 +61,6 @@ spec:
spec:
restartPolicy: OnFailure
securityContext:
runAsUser: 1000
runAsGroup: 1000
fsGroup: 1000
nodeSelector:
k8s.scaleway.com/pool-name: platform
@@ -97,41 +95,22 @@ spec:
BINARY=/data/bin/download_baseline
chmod +x "$BINARY"
# Download 4 schemas: OHLCV-1m, Trades, OHLCV-1s, MBP-10
# Uses universe configs from the repo (mounted via configmap)
# Download 4 schemas from configmap-mounted TOML configs
for SCHEMA in ohlcv-1m trades ohlcv-1s mbp-10; do
case $SCHEMA in
ohlcv-1m) DIR=/data/futures-baseline; CONFIG_NAME=ohlcv-1m ;;
trades) DIR=/data/futures-baseline-trades; CONFIG_NAME=trades ;;
ohlcv-1s) DIR=/data/futures-baseline-1s; CONFIG_NAME=ohlcv-1s ;;
mbp-10) DIR=/data/futures-baseline-mbp10; CONFIG_NAME=mbp-10 ;;
ohlcv-1m) DIR=/data/futures-baseline ;;
trades) DIR=/data/futures-baseline-trades ;;
ohlcv-1s) DIR=/data/futures-baseline-1s ;;
mbp-10) DIR=/data/futures-baseline-mbp10 ;;
esac
cat > /tmp/universe-${CONFIG_NAME}.toml << TOML
[universe]
name = "es-${CONFIG_NAME}"
description = "ES.FUT ${SCHEMA}"
date_range_start = "2024-03-01"
date_range_end = "2026-03-30"
bar_size = "tick"
databento_dataset = "GLBX.MDP3"
databento_schema = "${SCHEMA}"
[[symbols]]
symbol = "ES.FUT"
exchange = "GLBX.MDP3"
asset_class = "Future"
TOML
echo ""
echo "=== Downloading ${SCHEMA} to ${DIR} ==="
mkdir -p "${DIR}/ES.FUT"
echo "=== Downloading ${SCHEMA} ==="
mkdir -p "${DIR}"
$BINARY \
--universe-config /tmp/universe-${CONFIG_NAME}.toml \
--output-dir "${DIR}/ES.FUT" \
--yes || echo "WARN: ${SCHEMA} had failures (will retry on next run)"
echo ""
--universe-config /configs/${SCHEMA}.toml \
--output-dir "${DIR}" \
--yes || echo "WARN: ${SCHEMA} had failures"
du -sh "${DIR}" 2>/dev/null
done
@@ -145,4 +124,4 @@ spec:
claimName: training-data-pvc
- name: universe-configs
configMap:
name: databento-universe-configs
name: databento-universe-all