fix(ci): replace MinIO S3 with PVC + CI artifacts for training binaries

MinIO pod was removed but compile-training still uploaded to it via rclone, causing 403 Forbidden failures. Now: - compile-training: saves binaries as CI artifacts (3 day TTL) - deploy stage: copies training binaries to foxhunt-binaries PVC via binary-writer - CI training jobs: use compile-training CI artifacts directly - job-template.yaml: mounts foxhunt-binaries PVC (no initContainer) - Removed MinIO deploy from deploy stage (no longer needed) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-02 16:06:00 +01:00
parent 118ceca4d5
commit 81237e1bae
2 changed files with 26 additions and 136 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -499,20 +499,11 @@ compile-training:
      cp $TARGET_DIR/training_uploader build-out/training/
      strip build-out/training/training_uploader
    - ls -lh build-out/training/
-    # Upload stripped binaries to MinIO (in-cluster S3)
-    # Trust MinIO CA — rclone (Go) uses system CAs, not RCLONE_CA_CERT for S3.
-    # Container runs non-root, so merge system + MinIO CA into writable /tmp bundle.
-    - cp /etc/ssl/certs/ca-certificates.crt /tmp/ca-bundle.crt 2>/dev/null || true
-    - cat /etc/ssl/minio-ca/ca.crt >> /tmp/ca-bundle.crt 2>/dev/null || true
-    - export SSL_CERT_FILE=/tmp/ca-bundle.crt
-    - export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=https://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
-    - export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
-    - |
-      for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised training_uploader; do
-        rclone copyto build-out/training/$bin :s3:foxhunt-binaries/latest/training/$bin
-      done
-    - rclone copy build-out/training/ :s3:foxhunt-binaries/archive/${CI_COMMIT_SHA}/training/
-    - echo "Training binaries uploaded to MinIO"
+    - echo "Training binaries ready as CI artifacts"
+  artifacts:
+    paths:
+      - build-out/training/
+    expire_in: 3 days

 # --------------------------------------------------------------------------
 # Stage 3c: Build web-dashboard static assets (Vite)
@@ -581,15 +572,8 @@ build-web-dashboard:
    - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
    - nvidia-smi
    - mkdir -p ${CI_PROJECT_DIR}/output ${CI_PROJECT_DIR}/bin
-    # Fetch training binaries from MinIO (in-cluster S3)
-    # Trust MinIO CA — rclone (Go) uses system CAs, not RCLONE_CA_CERT for S3.
-    # Container runs non-root, so merge system + MinIO CA into writable /tmp bundle.
-    - cp /etc/ssl/certs/ca-certificates.crt /tmp/ca-bundle.crt 2>/dev/null || true
-    - cat /etc/ssl/minio-ca/ca.crt >> /tmp/ca-bundle.crt 2>/dev/null || true
-    - export SSL_CERT_FILE=/tmp/ca-bundle.crt
-    - export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=https://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
-    - export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
-    - rclone sync :s3:foxhunt-binaries/latest/training/ ${CI_PROJECT_DIR}/bin/
+    # Training binaries come from compile-training CI artifacts
+    - cp build-out/training/* ${CI_PROJECT_DIR}/bin/ 2>/dev/null || true
    - chmod +x ${CI_PROJECT_DIR}/bin/*
    - export PATH="${CI_PROJECT_DIR}/bin:$PATH"

@@ -627,15 +611,8 @@ build-web-dashboard:
    - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
    - nvidia-smi
    - mkdir -p ${CI_PROJECT_DIR}/output ${CI_PROJECT_DIR}/bin
-    # Fetch training binaries from MinIO (in-cluster S3)
-    # Trust MinIO CA — rclone (Go) uses system CAs, not RCLONE_CA_CERT for S3.
-    # Container runs non-root, so merge system + MinIO CA into writable /tmp bundle.
-    - cp /etc/ssl/certs/ca-certificates.crt /tmp/ca-bundle.crt 2>/dev/null || true
-    - cat /etc/ssl/minio-ca/ca.crt >> /tmp/ca-bundle.crt 2>/dev/null || true
-    - export SSL_CERT_FILE=/tmp/ca-bundle.crt
-    - export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=https://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
-    - export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
-    - rclone sync :s3:foxhunt-binaries/latest/training/ ${CI_PROJECT_DIR}/bin/
+    # Training binaries come from compile-training CI artifacts
+    - cp build-out/training/* ${CI_PROJECT_DIR}/bin/ 2>/dev/null || true
    - chmod +x ${CI_PROJECT_DIR}/bin/*
    - export PATH="${CI_PROJECT_DIR}/bin:$PATH"

@@ -1486,14 +1463,6 @@ deploy:
      else
        echo "WARNING: postgres pod not found, skipping migrations"
      fi
-    # Deploy MinIO (in-cluster S3 for compiled binaries and trained models)
-    # Must be ready BEFORE service rollout — initContainers fetch binaries from MinIO.
-    - kubectl apply -k infra/k8s/minio/
-    - |
-      echo "Waiting for MinIO to be ready..."
-      kubectl -n foxhunt rollout status deployment/minio --timeout=120s
-      echo "Waiting for MinIO init-buckets job..."
-      kubectl -n foxhunt wait --for=condition=complete job/minio-init-buckets --timeout=60s 2>/dev/null || true
    # Deploy monitoring stack (Loki, Tempo, Promtail, node-exporter, kube-state-metrics)
    - kubectl apply -f infra/k8s/monitoring/loki.yaml -f infra/k8s/monitoring/tempo.yaml -f infra/k8s/monitoring/promtail.yaml -f infra/k8s/monitoring/node-exporter.yaml -f infra/k8s/monitoring/kube-state-metrics.yaml
    # Deploy Grafana dashboards as ConfigMaps (sidecar auto-reloads)
@@ -1592,6 +1561,19 @@ deploy:
          kubectl cp build-out/services/$bin foxhunt/binary-writer:/binaries/$bin
          echo "  Copied $bin ($(stat -c%s build-out/services/$bin) bytes)"
        done
+        # Copy training binaries (if compile-training ran)
+        if [ -d "build-out/training" ]; then
+          kubectl exec -n foxhunt binary-writer -- mkdir -p /binaries/training
+          for bin in train_baseline_rl train_baseline_supervised evaluate_baseline \
+                     evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised \
+                     training_uploader; do
+            if [ -f "build-out/training/$bin" ]; then
+              kubectl cp build-out/training/$bin foxhunt/binary-writer:/binaries/training/$bin
+              echo "  Copied training/$bin ($(stat -c%s build-out/training/$bin) bytes)"
+            fi
+          done
+          kubectl exec -n foxhunt binary-writer -- chmod +x /binaries/training/*
+        fi
        # Copy web dashboard static assets (if build-web-dashboard ran)
        if [ -d "web-dashboard/dist" ]; then
          kubectl exec -n foxhunt binary-writer -- mkdir -p /binaries/static
--- a/infra/k8s/training/job-template.yaml
+++ b/infra/k8s/training/job-template.yaml
@@ -34,105 +34,16 @@ spec:
      imagePullSecrets:
        - name: scw-registry
      restartPolicy: Never
-      # Native sidecar (K8s 1.28+) — uploads artifacts on training completion
-      initContainers:
-        # Fetch training binaries from S3
-        - name: fetch-binaries
-          securityContext:
-            runAsUser: 0
-          image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-training-runtime:latest
-          command: ["/bin/sh", "-c"]
-          args:
-            - |
-              set -e
-              export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=https://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1 RCLONE_CA_CERT=/etc/ssl/minio-ca/ca.crt
-              rclone sync :s3:foxhunt-binaries/latest/training/ /binaries/
-              chmod +x /binaries/*
-              ls -lh /binaries/
-              echo "Fetched training binaries from MinIO"
-          env:
-            - name: RCLONE_S3_ACCESS_KEY_ID
-              valueFrom:
-                secretKeyRef:
-                  name: minio-credentials
-                  key: access-key
-            - name: RCLONE_S3_SECRET_ACCESS_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: minio-credentials
-                  key: secret-key
-          volumeMounts:
-            - name: binaries
-              mountPath: /binaries
-            - name: minio-ca
-              mountPath: /etc/ssl/minio-ca
-              readOnly: true
-          resources:
-            requests:
-              cpu: 100m
-              memory: 64Mi
-            limits:
-              cpu: 500m
-              memory: 256Mi
-        - name: uploader
-          image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-training-runtime:latest
-          restartPolicy: Always
-          command: ["/binaries/training_uploader"]
-          env:
-            - name: JOB_ID
-              value: "PLACEHOLDER"
-            - name: S3_BUCKET
-              value: foxhunt-models
-            - name: S3_ENDPOINT
-              value: https://minio.foxhunt.svc.cluster.local:9000
-            - name: MINIO_CA_CERT_PATH
-              value: /etc/ssl/minio-ca/ca.crt
-            - name: S3_REGION
-              value: us-east-1
-            - name: S3_PREFIX
-              value: "models/PLACEHOLDER"
-            - name: CALLBACK_ENDPOINT
-              value: http://ml-training-service.foxhunt.svc.cluster.local:50053
-            - name: RUST_LOG
-              value: info
-            - name: AWS_ACCESS_KEY_ID
-              valueFrom:
-                secretKeyRef:
-                  name: minio-credentials
-                  key: access-key
-            - name: AWS_SECRET_ACCESS_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: minio-credentials
-                  key: secret-key
-          volumeMounts:
-            - name: output
-              mountPath: /output
-              readOnly: true
-            - name: binaries
-              mountPath: /binaries
-              readOnly: true
-            - name: minio-ca
-              mountPath: /etc/ssl/minio-ca
-              readOnly: true
-          resources:
-            requests:
-              cpu: 100m
-              memory: 128Mi
-            limits:
-              cpu: 500m
-              memory: 256Mi
      containers:
        - name: training
          image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-training-runtime:latest
-          # Available binaries in /usr/local/bin/:
+          # Available binaries in /binaries/training/:
          #   train_baseline_rl (for dqn, ppo)
          #   train_baseline_supervised (for tft, mamba2, tggn, tlob, liquid, kan, xlstm, diffusion)
          #   evaluate_baseline
          #   hyperopt_baseline_rl (for dqn, ppo)
          #   hyperopt_baseline_supervised (for tft, mamba2)
-          #   training_uploader (sidecar binary)
-          command: ["/binaries/$(TRAINING_BINARY)"]
+          command: ["/binaries/training/$(TRAINING_BINARY)"]
          args:
            - "--symbol=ES.FUT"
            - "--data-dir=/data/futures-baseline"
@@ -172,8 +83,5 @@ spec:
          emptyDir:
            sizeLimit: 2Gi
        - name: binaries
-          emptyDir:
-            sizeLimit: 500Mi
-        - name: minio-ca
-          configMap:
-            name: minio-ca-cert
+          persistentVolumeClaim:
+            claimName: foxhunt-binaries