From 81237e1bae0f86ff387644b4cec2d9b6a7953104 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 2 Mar 2026 16:06:00 +0100 Subject: [PATCH] fix(ci): replace MinIO S3 with PVC + CI artifacts for training binaries MinIO pod was removed but compile-training still uploaded to it via rclone, causing 403 Forbidden failures. Now: - compile-training: saves binaries as CI artifacts (3 day TTL) - deploy stage: copies training binaries to foxhunt-binaries PVC via binary-writer - CI training jobs: use compile-training CI artifacts directly - job-template.yaml: mounts foxhunt-binaries PVC (no initContainer) - Removed MinIO deploy from deploy stage (no longer needed) Co-Authored-By: Claude Opus 4.6 --- .gitlab-ci.yml | 62 ++++++----------- infra/k8s/training/job-template.yaml | 100 ++------------------------- 2 files changed, 26 insertions(+), 136 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a65473f4c..99fc9229e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -499,20 +499,11 @@ compile-training: cp $TARGET_DIR/training_uploader build-out/training/ strip build-out/training/training_uploader - ls -lh build-out/training/ - # Upload stripped binaries to MinIO (in-cluster S3) - # Trust MinIO CA — rclone (Go) uses system CAs, not RCLONE_CA_CERT for S3. - # Container runs non-root, so merge system + MinIO CA into writable /tmp bundle. - - cp /etc/ssl/certs/ca-certificates.crt /tmp/ca-bundle.crt 2>/dev/null || true - - cat /etc/ssl/minio-ca/ca.crt >> /tmp/ca-bundle.crt 2>/dev/null || true - - export SSL_CERT_FILE=/tmp/ca-bundle.crt - - export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=https://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1 - - export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY - - | - for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised training_uploader; do - rclone copyto build-out/training/$bin :s3:foxhunt-binaries/latest/training/$bin - done - - rclone copy build-out/training/ :s3:foxhunt-binaries/archive/${CI_COMMIT_SHA}/training/ - - echo "Training binaries uploaded to MinIO" + - echo "Training binaries ready as CI artifacts" + artifacts: + paths: + - build-out/training/ + expire_in: 3 days # -------------------------------------------------------------------------- # Stage 3c: Build web-dashboard static assets (Vite) @@ -581,15 +572,8 @@ build-web-dashboard: - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') - nvidia-smi - mkdir -p ${CI_PROJECT_DIR}/output ${CI_PROJECT_DIR}/bin - # Fetch training binaries from MinIO (in-cluster S3) - # Trust MinIO CA — rclone (Go) uses system CAs, not RCLONE_CA_CERT for S3. - # Container runs non-root, so merge system + MinIO CA into writable /tmp bundle. - - cp /etc/ssl/certs/ca-certificates.crt /tmp/ca-bundle.crt 2>/dev/null || true - - cat /etc/ssl/minio-ca/ca.crt >> /tmp/ca-bundle.crt 2>/dev/null || true - - export SSL_CERT_FILE=/tmp/ca-bundle.crt - - export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=https://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1 - - export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY - - rclone sync :s3:foxhunt-binaries/latest/training/ ${CI_PROJECT_DIR}/bin/ + # Training binaries come from compile-training CI artifacts + - cp build-out/training/* ${CI_PROJECT_DIR}/bin/ 2>/dev/null || true - chmod +x ${CI_PROJECT_DIR}/bin/* - export PATH="${CI_PROJECT_DIR}/bin:$PATH" @@ -627,15 +611,8 @@ build-web-dashboard: - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') - nvidia-smi - mkdir -p ${CI_PROJECT_DIR}/output ${CI_PROJECT_DIR}/bin - # Fetch training binaries from MinIO (in-cluster S3) - # Trust MinIO CA — rclone (Go) uses system CAs, not RCLONE_CA_CERT for S3. - # Container runs non-root, so merge system + MinIO CA into writable /tmp bundle. - - cp /etc/ssl/certs/ca-certificates.crt /tmp/ca-bundle.crt 2>/dev/null || true - - cat /etc/ssl/minio-ca/ca.crt >> /tmp/ca-bundle.crt 2>/dev/null || true - - export SSL_CERT_FILE=/tmp/ca-bundle.crt - - export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=https://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1 - - export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY - - rclone sync :s3:foxhunt-binaries/latest/training/ ${CI_PROJECT_DIR}/bin/ + # Training binaries come from compile-training CI artifacts + - cp build-out/training/* ${CI_PROJECT_DIR}/bin/ 2>/dev/null || true - chmod +x ${CI_PROJECT_DIR}/bin/* - export PATH="${CI_PROJECT_DIR}/bin:$PATH" @@ -1486,14 +1463,6 @@ deploy: else echo "WARNING: postgres pod not found, skipping migrations" fi - # Deploy MinIO (in-cluster S3 for compiled binaries and trained models) - # Must be ready BEFORE service rollout — initContainers fetch binaries from MinIO. - - kubectl apply -k infra/k8s/minio/ - - | - echo "Waiting for MinIO to be ready..." - kubectl -n foxhunt rollout status deployment/minio --timeout=120s - echo "Waiting for MinIO init-buckets job..." - kubectl -n foxhunt wait --for=condition=complete job/minio-init-buckets --timeout=60s 2>/dev/null || true # Deploy monitoring stack (Loki, Tempo, Promtail, node-exporter, kube-state-metrics) - kubectl apply -f infra/k8s/monitoring/loki.yaml -f infra/k8s/monitoring/tempo.yaml -f infra/k8s/monitoring/promtail.yaml -f infra/k8s/monitoring/node-exporter.yaml -f infra/k8s/monitoring/kube-state-metrics.yaml # Deploy Grafana dashboards as ConfigMaps (sidecar auto-reloads) @@ -1592,6 +1561,19 @@ deploy: kubectl cp build-out/services/$bin foxhunt/binary-writer:/binaries/$bin echo " Copied $bin ($(stat -c%s build-out/services/$bin) bytes)" done + # Copy training binaries (if compile-training ran) + if [ -d "build-out/training" ]; then + kubectl exec -n foxhunt binary-writer -- mkdir -p /binaries/training + for bin in train_baseline_rl train_baseline_supervised evaluate_baseline \ + evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised \ + training_uploader; do + if [ -f "build-out/training/$bin" ]; then + kubectl cp build-out/training/$bin foxhunt/binary-writer:/binaries/training/$bin + echo " Copied training/$bin ($(stat -c%s build-out/training/$bin) bytes)" + fi + done + kubectl exec -n foxhunt binary-writer -- chmod +x /binaries/training/* + fi # Copy web dashboard static assets (if build-web-dashboard ran) if [ -d "web-dashboard/dist" ]; then kubectl exec -n foxhunt binary-writer -- mkdir -p /binaries/static diff --git a/infra/k8s/training/job-template.yaml b/infra/k8s/training/job-template.yaml index 6e56a9431..19be5295c 100644 --- a/infra/k8s/training/job-template.yaml +++ b/infra/k8s/training/job-template.yaml @@ -34,105 +34,16 @@ spec: imagePullSecrets: - name: scw-registry restartPolicy: Never - # Native sidecar (K8s 1.28+) — uploads artifacts on training completion - initContainers: - # Fetch training binaries from S3 - - name: fetch-binaries - securityContext: - runAsUser: 0 - image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-training-runtime:latest - command: ["/bin/sh", "-c"] - args: - - | - set -e - export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=https://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1 RCLONE_CA_CERT=/etc/ssl/minio-ca/ca.crt - rclone sync :s3:foxhunt-binaries/latest/training/ /binaries/ - chmod +x /binaries/* - ls -lh /binaries/ - echo "Fetched training binaries from MinIO" - env: - - name: RCLONE_S3_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: minio-credentials - key: access-key - - name: RCLONE_S3_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: secret-key - volumeMounts: - - name: binaries - mountPath: /binaries - - name: minio-ca - mountPath: /etc/ssl/minio-ca - readOnly: true - resources: - requests: - cpu: 100m - memory: 64Mi - limits: - cpu: 500m - memory: 256Mi - - name: uploader - image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-training-runtime:latest - restartPolicy: Always - command: ["/binaries/training_uploader"] - env: - - name: JOB_ID - value: "PLACEHOLDER" - - name: S3_BUCKET - value: foxhunt-models - - name: S3_ENDPOINT - value: https://minio.foxhunt.svc.cluster.local:9000 - - name: MINIO_CA_CERT_PATH - value: /etc/ssl/minio-ca/ca.crt - - name: S3_REGION - value: us-east-1 - - name: S3_PREFIX - value: "models/PLACEHOLDER" - - name: CALLBACK_ENDPOINT - value: http://ml-training-service.foxhunt.svc.cluster.local:50053 - - name: RUST_LOG - value: info - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: minio-credentials - key: access-key - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: secret-key - volumeMounts: - - name: output - mountPath: /output - readOnly: true - - name: binaries - mountPath: /binaries - readOnly: true - - name: minio-ca - mountPath: /etc/ssl/minio-ca - readOnly: true - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 500m - memory: 256Mi containers: - name: training image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-training-runtime:latest - # Available binaries in /usr/local/bin/: + # Available binaries in /binaries/training/: # train_baseline_rl (for dqn, ppo) # train_baseline_supervised (for tft, mamba2, tggn, tlob, liquid, kan, xlstm, diffusion) # evaluate_baseline # hyperopt_baseline_rl (for dqn, ppo) # hyperopt_baseline_supervised (for tft, mamba2) - # training_uploader (sidecar binary) - command: ["/binaries/$(TRAINING_BINARY)"] + command: ["/binaries/training/$(TRAINING_BINARY)"] args: - "--symbol=ES.FUT" - "--data-dir=/data/futures-baseline" @@ -172,8 +83,5 @@ spec: emptyDir: sizeLimit: 2Gi - name: binaries - emptyDir: - sizeLimit: 500Mi - - name: minio-ca - configMap: - name: minio-ca-cert + persistentVolumeClaim: + claimName: foxhunt-binaries