diff --git a/infra/k8s/training/populate-test-data-job.yaml b/infra/k8s/training/populate-test-data-job.yaml new file mode 100644 index 000000000..9f0ec58bb --- /dev/null +++ b/infra/k8s/training/populate-test-data-job.yaml @@ -0,0 +1,89 @@ +# infra/k8s/training/populate-test-data-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + generateName: populate-test-data- + namespace: foxhunt + labels: + app.kubernetes.io/name: populate-test-data + app.kubernetes.io/part-of: foxhunt +spec: + backoffLimit: 1 + ttlSecondsAfterFinished: 300 + template: + spec: + restartPolicy: Never + nodeSelector: + k8s.scaleway.com/pool-name: ci-training-h100 + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + containers: + - name: populate + image: busybox:1.37 + command: ["/bin/sh", "-c"] + args: + - | + set -e + echo "=== Populating test data PVC ===" + + # Target directories + mkdir -p /test-data/ohlcv/ES.FUT + mkdir -p /test-data/mbp10/ES.FUT + mkdir -p /test-data/trades/ES.FUT + + # Copy one quarter per schema (2025-Q2) + echo "Copying OHLCV..." + cp /training-data/futures-baseline/ES.FUT/ES.FUT_2025-Q2.dbn.zst \ + /test-data/ohlcv/ES.FUT/ 2>/dev/null || \ + cp /training-data/futures-baseline/ES.FUT/*2025*Q2* \ + /test-data/ohlcv/ES.FUT/ 2>/dev/null || \ + { echo "WARN: No 2025-Q2 OHLCV found, copying first available"; \ + cp /training-data/futures-baseline/ES.FUT/*.dbn.zst \ + /test-data/ohlcv/ES.FUT/ 2>/dev/null | head -1; } + + echo "Copying MBP-10..." + cp /training-data/futures-baseline-mbp10/ES.FUT/ES.FUT_2025-Q2.dbn.zst \ + /test-data/mbp10/ES.FUT/ 2>/dev/null || \ + cp /training-data/futures-baseline-mbp10/ES.FUT/*2025*Q2* \ + /test-data/mbp10/ES.FUT/ 2>/dev/null || \ + echo "WARN: No MBP-10 2025-Q2 found" + + echo "Copying trades..." + cp /training-data/futures-baseline-trades/ES.FUT/ES.FUT_2025-Q2.dbn.zst \ + /test-data/trades/ES.FUT/ 2>/dev/null || \ + cp /training-data/futures-baseline-trades/ES.FUT/*2025*Q2* \ + /test-data/trades/ES.FUT/ 2>/dev/null || \ + echo "WARN: No trades 2025-Q2 found" + + echo "=== Test data contents ===" + find /test-data -type f -exec ls -lh {} \; + echo "Total: $(du -sh /test-data | cut -f1)" + echo "=== Done ===" + resources: + requests: + nvidia.com/gpu: "1" + cpu: 100m + memory: 128Mi + limits: + nvidia.com/gpu: "1" + cpu: 500m + memory: 256Mi + volumeMounts: + - name: training-data + mountPath: /training-data + readOnly: true + - name: test-data + mountPath: /test-data + volumes: + - name: training-data + persistentVolumeClaim: + claimName: training-data-pvc + readOnly: true + - name: test-data + persistentVolumeClaim: + claimName: test-data-pvc diff --git a/scripts/refresh-test-data.sh b/scripts/refresh-test-data.sh new file mode 100755 index 000000000..60d1a32bf --- /dev/null +++ b/scripts/refresh-test-data.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Re-populate the test-data-pvc from training-data-pvc. +# Usage: ./scripts/refresh-test-data.sh +set -euo pipefail + +echo "Submitting populate-test-data job..." +kubectl -n foxhunt create -f infra/k8s/training/populate-test-data-job.yaml + +echo "Waiting for completion..." +kubectl -n foxhunt wait --for=condition=complete job \ + -l app.kubernetes.io/name=populate-test-data \ + --timeout=300s + +echo "Done. Test data PVC refreshed."