1ad79493dc
- Expand training-data-pvc from 10Gi → 200Gi (OHLCV + MBP-10 + headroom) - Add data-sync-job: rclone sync from MinIO → PVC (delta-only, fast repeats) - Add sync-training-data init container to training job template (auto-syncs data from MinIO before each training run) - Add training-job + data-sync-job NetworkPolicies (MinIO, Tempo, Pushgateway) - Add app.kubernetes.io/part-of: foxhunt to training pod template - download_baseline: add --parallel N flag for concurrent Databento downloads - download_baseline: delete local files after MinIO upload (saves scratch space) - Bump download job scratch volume to 100Gi Architecture: Databento → MinIO (source of truth) → PVC (local cache). First sync pulls everything; subsequent runs only sync deltas (seconds). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
114 lines
3.7 KiB
YAML
114 lines
3.7 KiB
YAML
# Sync training data from MinIO → training-data PVC.
|
|
# Runs rclone sync (only copies new/changed files).
|
|
#
|
|
# Usage:
|
|
# kubectl apply -f infra/k8s/training/data-sync-job.yaml
|
|
#
|
|
# First run: full sync (~50GB, ~10 min).
|
|
# Subsequent runs: delta only (seconds).
|
|
#
|
|
# Data layout on PVC:
|
|
# /data/futures-baseline/ — OHLCV-1m bars (ES.FUT, NQ.FUT, etc.)
|
|
# /data/futures-baseline-mbp10/ — MBP-10 order book (ES.FUT)
|
|
apiVersion: batch/v1
|
|
kind: Job
|
|
metadata:
|
|
generateName: data-sync-
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: data-sync
|
|
app.kubernetes.io/part-of: foxhunt
|
|
foxhunt/job-type: data-sync
|
|
spec:
|
|
backoffLimit: 2
|
|
activeDeadlineSeconds: 7200 # 2 hours
|
|
ttlSecondsAfterFinished: 300
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app.kubernetes.io/name: data-sync
|
|
app.kubernetes.io/part-of: foxhunt
|
|
foxhunt/job-type: data-sync
|
|
spec:
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: platform
|
|
imagePullSecrets:
|
|
- name: gitlab-registry
|
|
restartPolicy: Never
|
|
containers:
|
|
- name: sync
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
|
|
command: ["/bin/sh", "-c"]
|
|
args:
|
|
- |
|
|
set -e
|
|
|
|
echo "=== Training Data Sync: MinIO → PVC ==="
|
|
echo "Syncing from MinIO buckets to /data/..."
|
|
|
|
# Sync OHLCV data
|
|
echo ""
|
|
echo "--- OHLCV (futures-baseline) ---"
|
|
rclone sync \
|
|
":s3:foxhunt-training-data/futures-baseline" \
|
|
/data/futures-baseline \
|
|
--s3-provider=Minio \
|
|
--s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \
|
|
--s3-access-key-id="${MINIO_ACCESS_KEY}" \
|
|
--s3-secret-access-key="${MINIO_SECRET_KEY}" \
|
|
--s3-no-check-bucket \
|
|
--progress \
|
|
--stats-one-line \
|
|
-v
|
|
echo "OHLCV sync complete."
|
|
|
|
# Sync MBP-10 data
|
|
echo ""
|
|
echo "--- MBP-10 (futures-baseline-mbp10) ---"
|
|
rclone sync \
|
|
":s3:foxhunt-training-data/futures-baseline-mbp10" \
|
|
/data/futures-baseline-mbp10 \
|
|
--s3-provider=Minio \
|
|
--s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \
|
|
--s3-access-key-id="${MINIO_ACCESS_KEY}" \
|
|
--s3-secret-access-key="${MINIO_SECRET_KEY}" \
|
|
--s3-no-check-bucket \
|
|
--progress \
|
|
--stats-one-line \
|
|
-v
|
|
echo "MBP-10 sync complete."
|
|
|
|
echo ""
|
|
echo "=== Sync Summary ==="
|
|
echo "OHLCV:"
|
|
du -sh /data/futures-baseline/ 2>/dev/null || echo " (empty)"
|
|
echo "MBP-10:"
|
|
du -sh /data/futures-baseline-mbp10/ 2>/dev/null || echo " (empty)"
|
|
echo "Total:"
|
|
du -sh /data/
|
|
env:
|
|
- name: MINIO_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-credentials
|
|
key: access-key
|
|
- name: MINIO_SECRET_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-credentials
|
|
key: secret-key
|
|
volumeMounts:
|
|
- name: training-data
|
|
mountPath: /data
|
|
resources:
|
|
requests:
|
|
cpu: "1"
|
|
memory: 512Mi
|
|
limits:
|
|
cpu: "2"
|
|
memory: 1Gi
|
|
volumes:
|
|
- name: training-data
|
|
persistentVolumeClaim:
|
|
claimName: training-data-pvc
|