feat(infra): replace PVC training data with MinIO S3 download

Training data PVCs (ReadWriteOnce) don't work with ephemeral GPU nodes that autoscale across availability zones. Replace with rclone download from MinIO to /tmp at the start of each GPU step. - Remove training-data-pvc parameter and volume mount - Add rclone download from s3:foxhunt-training-data/futures-baseline/ to /tmp/futures-baseline/ in hyperopt, train-best, and evaluate steps - Add MINIO_ACCESS_KEY/SECRET_KEY env vars to all GPU steps - Add cuda-compute-cap parameter (default "90") - Change data-dir default to /tmp/futures-baseline - 36 .dbn.zst files (53MB total) uploaded to MinIO bucket Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-05 13:14:32 +01:00
parent a31512643b
commit 32e7ba2507
1 changed files with 74 additions and 24 deletions
--- a/infra/k8s/argo/training-workflow-template.yaml
+++ b/infra/k8s/argo/training-workflow-template.yaml
@@ -27,8 +27,6 @@ spec:
      - name: model
      - name: gpu-pool
        value: ci-training-h100
-      - name: training-data-pvc
-        value: training-data-h100-pvc
      - name: hyperopt-trials
        value: "20"
      - name: hyperopt-epochs
@@ -38,7 +36,7 @@ spec:
      - name: symbol
        value: ES.FUT
      - name: data-dir
-        value: /data/futures-baseline
+        value: /tmp/futures-baseline
      - name: tx-cost-bps
        value: "0.1"
      - name: tick-size
@@ -47,8 +45,8 @@ spec:
        value: "1.0"
      - name: initial-capital
        value: "35000"
-      - name: training-data-pvc
-        value: training-data-h100-pvc
+      - name: cuda-compute-cap
+        value: "90"

  # Ephemeral shared storage for passing data between DAG steps.
  # Auto-deleted when workflow completes (all artifacts uploaded to MinIO).
@@ -62,11 +60,6 @@ spec:
          requests:
            storage: 5Gi

-  volumes:
-    - name: training-data
-      persistentVolumeClaim:
-        claimName: "{{workflow.parameters.training-data-pvc}}"
-
  templates:
    - name: train-model
      dag:
@@ -90,7 +83,7 @@ spec:
      nodeSelector:
        k8s.scaleway.com/pool-name: platform
      container:
-        image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest
+        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
        command: ["/bin/sh", "-c"]
        args:
          - |
@@ -140,7 +133,7 @@ spec:
          operator: Exists
          effect: NoSchedule
      container:
-        image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-training-runtime:latest
+        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
        command: ["/bin/sh", "-c"]
        args:
          - |
@@ -148,6 +141,19 @@ spec:
            export PATH="/workspace/bin:$PATH"
            export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
            nvidia-smi
+
+            # Download training data from MinIO → /tmp (no PVC needed)
+            echo "=== Downloading training data from MinIO ==="
+            rclone copy :s3:foxhunt-training-data/futures-baseline/ /tmp/futures-baseline/ \
+              --s3-provider=Minio \
+              --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \
+              --s3-access-key-id="${MINIO_ACCESS_KEY}" \
+              --s3-secret-access-key="${MINIO_SECRET_KEY}" \
+              --s3-no-check-bucket \
+              --no-check-certificate \
+              --transfers=8
+            echo "Training data: $(find /tmp/futures-baseline -name '*.dbn.zst' | wc -l) files"
+
            mkdir -p /workspace/output/hyperopt

            MODEL="{{workflow.parameters.model}}"
@@ -189,12 +195,19 @@ spec:
            value: "http://tempo.foxhunt.svc.cluster.local:4317"
          - name: CUBLAS_WORKSPACE_CONFIG
            value: ":4096:8"
+          - name: MINIO_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                name: minio-credentials
+                key: access-key
+          - name: MINIO_SECRET_KEY
+            valueFrom:
+              secretKeyRef:
+                name: minio-credentials
+                key: secret-key
        volumeMounts:
          - name: workspace
            mountPath: /workspace
-          - name: training-data
-            mountPath: /data
-            readOnly: true
        resources:
          requests:
            nvidia.com/gpu: "1"
@@ -216,7 +229,7 @@ spec:
          operator: Exists
          effect: NoSchedule
      container:
-        image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-training-runtime:latest
+        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
        command: ["/bin/sh", "-c"]
        args:
          - |
@@ -225,6 +238,18 @@ spec:
            export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
            nvidia-smi

+            # Download training data from MinIO → /tmp
+            echo "=== Downloading training data from MinIO ==="
+            rclone copy :s3:foxhunt-training-data/futures-baseline/ /tmp/futures-baseline/ \
+              --s3-provider=Minio \
+              --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \
+              --s3-access-key-id="${MINIO_ACCESS_KEY}" \
+              --s3-secret-access-key="${MINIO_SECRET_KEY}" \
+              --s3-no-check-bucket \
+              --no-check-certificate \
+              --transfers=8
+            echo "Training data: $(find /tmp/futures-baseline -name '*.dbn.zst' | wc -l) files"
+
            MODEL="{{workflow.parameters.model}}"
            case "$MODEL" in
              dqn|ppo) BINARY=train_baseline_rl ;;
@@ -260,12 +285,19 @@ spec:
            value: "http://tempo.foxhunt.svc.cluster.local:4317"
          - name: CUBLAS_WORKSPACE_CONFIG
            value: ":4096:8"
+          - name: MINIO_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                name: minio-credentials
+                key: access-key
+          - name: MINIO_SECRET_KEY
+            valueFrom:
+              secretKeyRef:
+                name: minio-credentials
+                key: secret-key
        volumeMounts:
          - name: workspace
            mountPath: /workspace
-          - name: training-data
-            mountPath: /data
-            readOnly: true
        resources:
          requests:
            nvidia.com/gpu: "1"
@@ -287,7 +319,7 @@ spec:
          operator: Exists
          effect: NoSchedule
      container:
-        image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-training-runtime:latest
+        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
        command: ["/bin/sh", "-c"]
        args:
          - |
@@ -295,6 +327,17 @@ spec:
            export PATH="/workspace/bin:$PATH"
            export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')

+            # Download training data from MinIO → /tmp
+            echo "=== Downloading training data from MinIO ==="
+            rclone copy :s3:foxhunt-training-data/futures-baseline/ /tmp/futures-baseline/ \
+              --s3-provider=Minio \
+              --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \
+              --s3-access-key-id="${MINIO_ACCESS_KEY}" \
+              --s3-secret-access-key="${MINIO_SECRET_KEY}" \
+              --s3-no-check-bucket \
+              --no-check-certificate \
+              --transfers=8
+
            MODEL="{{workflow.parameters.model}}"
            HYPEROPT_FILE="/workspace/output/${MODEL}_hyperopt_results.json"
            HYPEROPT_FLAG=""
@@ -322,12 +365,19 @@ spec:
            value: info
          - name: SQLX_OFFLINE
            value: "true"
+          - name: MINIO_ACCESS_KEY
+            valueFrom:
+              secretKeyRef:
+                name: minio-credentials
+                key: access-key
+          - name: MINIO_SECRET_KEY
+            valueFrom:
+              secretKeyRef:
+                name: minio-credentials
+                key: secret-key
        volumeMounts:
          - name: workspace
            mountPath: /workspace
-          - name: training-data
-            mountPath: /data
-            readOnly: true
        resources:
          requests:
            nvidia.com/gpu: "1"
@@ -342,7 +392,7 @@ spec:
      nodeSelector:
        k8s.scaleway.com/pool-name: platform
      container:
-        image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest
+        image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
        command: ["/bin/sh", "-c"]
        args:
          - |