diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a99a2e931..8314a94f2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -390,15 +390,15 @@ compile-services: strip build-out/services/$bin done - ls -lh build-out/services/ - # Upload stripped binaries to S3 - - export RCLONE_S3_PROVIDER=Scaleway RCLONE_S3_ENDPOINT=s3.fr-par.scw.cloud RCLONE_S3_REGION=fr-par - - export RCLONE_S3_ACCESS_KEY_ID=$SCW_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$SCW_SECRET_KEY + # Upload stripped binaries to MinIO (in-cluster S3) + - export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1 + - export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY - | for bin in trading_service api_gateway broker_gateway_service ml_training_service backtesting_service trading_agent_service data_acquisition_service web-gateway; do rclone copyto build-out/services/$bin :s3:foxhunt-binaries/latest/services/$bin done - rclone copy build-out/services/ :s3:foxhunt-binaries/archive/${CI_COMMIT_SHA}/services/ - - echo "Service binaries uploaded to S3" + - echo "Service binaries uploaded to MinIO" # -------------------------------------------------------------------------- # Stage 3a: Compile training binaries (CPU node, CUDA stubs from ci-builder image) @@ -451,15 +451,15 @@ compile-training: cp $TARGET_DIR/training_uploader build-out/training/ strip build-out/training/training_uploader - ls -lh build-out/training/ - # Upload stripped binaries to S3 - - export RCLONE_S3_PROVIDER=Scaleway RCLONE_S3_ENDPOINT=s3.fr-par.scw.cloud RCLONE_S3_REGION=fr-par - - export RCLONE_S3_ACCESS_KEY_ID=$SCW_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$SCW_SECRET_KEY + # Upload stripped binaries to MinIO (in-cluster S3) + - export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1 + - export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY - | for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised training_uploader; do rclone copyto build-out/training/$bin :s3:foxhunt-binaries/latest/training/$bin done - rclone copy build-out/training/ :s3:foxhunt-binaries/archive/${CI_COMMIT_SHA}/training/ - - echo "Training binaries uploaded to S3" + - echo "Training binaries uploaded to MinIO" # -------------------------------------------------------------------------- # Stage 3c: Build web-dashboard static assets (Vite) @@ -489,10 +489,10 @@ build-web-dashboard: - cd web-dashboard - npm ci - npm run build - - export RCLONE_S3_PROVIDER=Scaleway RCLONE_S3_ENDPOINT=s3.fr-par.scw.cloud RCLONE_S3_REGION=fr-par - - export RCLONE_S3_ACCESS_KEY_ID=$SCW_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$SCW_SECRET_KEY + - export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1 + - export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY - rclone sync dist/ :s3:foxhunt-binaries/latest/web-dashboard/dist/ - - echo "Web dashboard assets uploaded to S3" + - echo "Web dashboard assets uploaded to MinIO" # -------------------------------------------------------------------------- # Stage 3d: Write manifest after all compiles complete @@ -517,8 +517,8 @@ write-manifest: - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api" script: - - export RCLONE_S3_PROVIDER=Scaleway RCLONE_S3_ENDPOINT=s3.fr-par.scw.cloud RCLONE_S3_REGION=fr-par - - export RCLONE_S3_ACCESS_KEY_ID=$SCW_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$SCW_SECRET_KEY + - export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1 + - export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY - echo "{\"sha\":\"${CI_COMMIT_SHA}\",\"ts\":\"$(date -Iseconds)\"}" | rclone rcat :s3:foxhunt-binaries/latest/MANIFEST.json - echo "Manifest written for ${CI_COMMIT_SHA}" @@ -556,9 +556,9 @@ write-manifest: - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') - nvidia-smi - mkdir -p ${CI_PROJECT_DIR}/output ${CI_PROJECT_DIR}/bin - # Fetch training binaries from S3 - - export RCLONE_S3_PROVIDER=Scaleway RCLONE_S3_ENDPOINT=s3.fr-par.scw.cloud RCLONE_S3_REGION=fr-par - - export RCLONE_S3_ACCESS_KEY_ID=$SCW_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$SCW_SECRET_KEY + # Fetch training binaries from MinIO (in-cluster S3) + - export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1 + - export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY - rclone sync :s3:foxhunt-binaries/latest/training/ ${CI_PROJECT_DIR}/bin/ - chmod +x ${CI_PROJECT_DIR}/bin/* - export PATH="${CI_PROJECT_DIR}/bin:$PATH" @@ -595,9 +595,9 @@ write-manifest: - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') - nvidia-smi - mkdir -p ${CI_PROJECT_DIR}/output ${CI_PROJECT_DIR}/bin - # Fetch training binaries from S3 - - export RCLONE_S3_PROVIDER=Scaleway RCLONE_S3_ENDPOINT=s3.fr-par.scw.cloud RCLONE_S3_REGION=fr-par - - export RCLONE_S3_ACCESS_KEY_ID=$SCW_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$SCW_SECRET_KEY + # Fetch training binaries from MinIO (in-cluster S3) + - export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1 + - export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY - rclone sync :s3:foxhunt-binaries/latest/training/ ${CI_PROJECT_DIR}/bin/ - chmod +x ${CI_PROJECT_DIR}/bin/* - export PATH="${CI_PROJECT_DIR}/bin:$PATH" @@ -1445,15 +1445,66 @@ deploy: else echo "WARNING: postgres pod not found, skipping migrations" fi + # Deploy MinIO (in-cluster S3 for compiled binaries and trained models) + # Must be ready BEFORE service rollout — initContainers fetch binaries from MinIO. + - kubectl apply -f infra/k8s/minio/ + - | + echo "Waiting for MinIO to be ready..." + kubectl -n foxhunt rollout status deployment/minio --timeout=120s + echo "Waiting for MinIO init-buckets job..." + kubectl -n foxhunt wait --for=condition=complete job/minio-init-buckets --timeout=60s 2>/dev/null || true # Deploy monitoring stack (Loki, Tempo, Promtail, node-exporter, kube-state-metrics) - kubectl apply -f infra/k8s/monitoring/loki.yaml -f infra/k8s/monitoring/tempo.yaml -f infra/k8s/monitoring/promtail.yaml -f infra/k8s/monitoring/node-exporter.yaml -f infra/k8s/monitoring/kube-state-metrics.yaml + # Deploy Grafana dashboards as ConfigMaps (sidecar auto-reloads) + # Resolve datasource variable placeholders to actual UIDs before applying + - | + PROM_UID="PBFA97CFB590B2093" + LOKI_UID="P8E80F9AEF21F6940" + TEMPO_UID="tempo" + DASH_DIR="infra/k8s/monitoring/dashboards" + TMP_DIR=$(mktemp -d) + for f in "$DASH_DIR"/*.json; do + python3 -c " + import json, sys + with open('$f') as fh: + d = json.load(fh) + d.pop('__inputs', None) + d.pop('__requires', None) + raw = json.dumps(d) + raw = raw.replace('\${DS_PROMETHEUS}', '${PROM_UID}') + raw = raw.replace('\${DS_LOKI}', '${LOKI_UID}') + raw = raw.replace('\${DS_TEMPO}', '${TEMPO_UID}') + print(raw) + " > "$TMP_DIR/$(basename "$f")" + done + # Build ConfigMaps grouped by category + declare -A GROUPS=( + [grafana-dashboards-foxhunt]="foxhunt-logs.json foxhunt-overview.json foxhunt-services.json foxhunt-cockpit.json foxhunt-trading-cockpit.json foxhunt-training-cockpit.json foxhunt-traces.json" + [grafana-dashboards-infra]="cluster-overview.json gpu-overview.json foxhunt-gpu-training.json foxhunt-infrastructure.json" + [grafana-dashboards-cicd]="foxhunt-ci-pipelines.json gitlab-services.json" + ) + for cm_name in "${!GROUPS[@]}"; do + args="" + for df in ${GROUPS[$cm_name]}; do + [ -f "$TMP_DIR/$df" ] && args="$args --from-file=$df=$TMP_DIR/$df" + done + if [ -n "$args" ]; then + kubectl create configmap "$cm_name" -n foxhunt $args \ + --dry-run=client -o yaml | \ + kubectl label -f - --dry-run=client -o yaml --local grafana_dashboard=1 | \ + kubectl apply -f - + echo " Applied ConfigMap: $cm_name" + fi + done + rm -rf "$TMP_DIR" + echo "Dashboard ConfigMaps deployed" # Apply proxy (picks up nginx config changes for dashboard.fxhnt.ai routing) - kubectl apply -f infra/k8s/gitlab/tailscale-proxy.yaml # Apply service deployments and binary cache PVCs (idempotent — picks up any YAML changes) # Skip training/job-template.yaml (placeholder names, not directly applicable) # Skip storage/training-data-pv.yaml (PVC spec immutable after creation) - kubectl apply -f infra/k8s/services/ -f infra/k8s/storage/binary-cache-pvcs.yaml - # Rolling restart triggers initContainers to fetch latest binaries from S3 + # Rolling restart triggers initContainers to fetch latest binaries from MinIO - | for svc in trading-service api-gateway broker-gateway \ ml-training-service backtesting-service \ @@ -1467,4 +1518,4 @@ deploy: trading-agent-service data-acquisition-service web-gateway; do kubectl -n foxhunt rollout status deployment/$svc --timeout=300s done - - echo "All services deployed with S3 binaries" + - echo "All services deployed with MinIO binaries" diff --git a/infra/k8s/minio/kustomization.yaml b/infra/k8s/minio/kustomization.yaml new file mode 100644 index 000000000..6a5695f9c --- /dev/null +++ b/infra/k8s/minio/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - minio.yaml diff --git a/infra/k8s/minio/minio.yaml b/infra/k8s/minio/minio.yaml new file mode 100644 index 000000000..206d600db --- /dev/null +++ b/infra/k8s/minio/minio.yaml @@ -0,0 +1,178 @@ +# MinIO — in-cluster S3 for compiled binaries and trained models +# Replaces Scaleway public S3 to keep proprietary code/IP in-cluster +apiVersion: v1 +kind: Secret +metadata: + name: minio-credentials + namespace: foxhunt + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/part-of: foxhunt +type: Opaque +stringData: + root-user: foxhunt-minio + root-password: CHANGE_ME_BEFORE_DEPLOY + # rclone-compatible keys (same values, different key names for initContainers) + access-key: foxhunt-minio + secret-key: CHANGE_ME_BEFORE_DEPLOY +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio-data + namespace: foxhunt + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/part-of: foxhunt +spec: + accessModes: + - ReadWriteOnce + storageClassName: scw-bssd + resources: + requests: + storage: 20Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: foxhunt + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/part-of: foxhunt +spec: + replicas: 1 + strategy: + type: Recreate # single-node MinIO — no rolling update + selector: + matchLabels: + app.kubernetes.io/name: minio + template: + metadata: + labels: + app.kubernetes.io/name: minio + spec: + nodeSelector: + k8s.scaleway.com/pool-name: services + containers: + - name: minio + image: minio/minio:latest + args: ["server", "/data", "--console-address", ":9001"] + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-user + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-password + volumeMounts: + - name: data + mountPath: /data + readinessProbe: + httpGet: + path: /minio/health/ready + port: 9000 + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /minio/health/live + port: 9000 + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 5 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + volumes: + - name: data + persistentVolumeClaim: + claimName: minio-data +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: foxhunt + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/part-of: foxhunt +spec: + selector: + app.kubernetes.io/name: minio + ports: + - port: 9000 + targetPort: 9000 + name: api + - port: 9001 + targetPort: 9001 + name: console +--- +# Create buckets on first deploy +apiVersion: batch/v1 +kind: Job +metadata: + name: minio-init-buckets + namespace: foxhunt + labels: + app.kubernetes.io/name: minio + app.kubernetes.io/part-of: foxhunt +spec: + backoffLimit: 5 + ttlSecondsAfterFinished: 300 + template: + metadata: + labels: + app.kubernetes.io/name: minio-init + spec: + nodeSelector: + k8s.scaleway.com/pool-name: services + restartPolicy: OnFailure + containers: + - name: mc + image: minio/mc:latest + command: ["/bin/sh", "-c"] + args: + - | + set -e + echo "Waiting for MinIO to be ready..." + until mc alias set foxhunt http://minio.foxhunt.svc.cluster.local:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" 2>/dev/null; do + echo "MinIO not ready, retrying in 3s..." + sleep 3 + done + echo "Creating buckets..." + mc mb --ignore-existing foxhunt/foxhunt-binaries + mc mb --ignore-existing foxhunt/foxhunt-models + echo "Buckets created:" + mc ls foxhunt/ + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-user + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: minio-credentials + key: root-password + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 200m + memory: 64Mi diff --git a/infra/k8s/services/web-gateway.yaml b/infra/k8s/services/web-gateway.yaml index c09bed1ee..626f4f1e4 100644 --- a/infra/k8s/services/web-gateway.yaml +++ b/infra/k8s/services/web-gateway.yaml @@ -52,20 +52,20 @@ spec: fi env: - name: RCLONE_S3_PROVIDER - value: Scaleway + value: Minio - name: RCLONE_S3_ENDPOINT - value: s3.fr-par.scw.cloud + value: minio.foxhunt.svc.cluster.local:9000 - name: RCLONE_S3_REGION - value: fr-par + value: us-east-1 - name: RCLONE_S3_ACCESS_KEY_ID valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: access-key - name: RCLONE_S3_SECRET_ACCESS_KEY valueFrom: secretKeyRef: - name: s3-credentials + name: minio-credentials key: secret-key volumeMounts: - name: binaries diff --git a/infra/k8s/training/s3-credentials.secret.example b/infra/k8s/training/s3-credentials.secret.example index afad5339c..b57c30e24 100644 --- a/infra/k8s/training/s3-credentials.secret.example +++ b/infra/k8s/training/s3-credentials.secret.example @@ -1,17 +1,24 @@ -# S3 credentials for rclone output sync. -# Apply with actual values (do NOT use this file directly): -# kubectl -n foxhunt create secret generic s3-credentials \ -# --from-literal=access-key= \ -# --from-literal=secret-key= +# MinIO credentials for rclone output sync in training jobs. +# No longer needed as a standalone secret — training jobs now use minio-credentials +# (deployed via infra/k8s/minio/minio.yaml). +# +# If you need to recreate manually: +# kubectl -n foxhunt create secret generic minio-credentials \ +# --from-literal=access-key= \ +# --from-literal=secret-key= \ +# --from-literal=root-user= \ +# --from-literal=root-password= apiVersion: v1 kind: Secret metadata: - name: s3-credentials + name: minio-credentials namespace: foxhunt labels: - app.kubernetes.io/name: s3-credentials + app.kubernetes.io/name: minio-credentials app.kubernetes.io/part-of: foxhunt type: Opaque stringData: access-key: REPLACE_ME secret-key: REPLACE_ME + root-user: REPLACE_ME + root-password: REPLACE_ME diff --git a/infra/scripts/train.sh b/infra/scripts/train.sh index bfc77b8d0..2cc4aeece 100755 --- a/infra/scripts/train.sh +++ b/infra/scripts/train.sh @@ -26,8 +26,8 @@ IMAGE="rg.fr-par.scw.cloud/foxhunt/training:latest" MODEL="" PRESET="" TIMESTAMP="$(date +%s)" -S3_BUCKET="foxhunt-artifacts" -S3_ENDPOINT="https://s3.fr-par.scw.cloud" +S3_BUCKET="foxhunt-models" +MINIO_ENDPOINT="http://minio.foxhunt.svc.cluster.local:9000" RUN_ID="$(date +%Y%m%d-%H%M%S)" ALL_MODELS=(dqn ppo tft mamba2 tggn tlob liquid kan xlstm diffusion) @@ -196,16 +196,15 @@ spec: nvidia-smi || echo "WARNING: nvidia-smi not available" echo "=== Starting training ===" ${train_args} - echo "=== Training complete, syncing output to S3 ===" - RCLONE_CONFIG_SCW_TYPE=s3 \\ - RCLONE_CONFIG_SCW_PROVIDER=Scaleway \\ - RCLONE_CONFIG_SCW_ACCESS_KEY_ID=\$(cat /secrets/access-key) \\ - RCLONE_CONFIG_SCW_SECRET_ACCESS_KEY=\$(cat /secrets/secret-key) \\ - RCLONE_CONFIG_SCW_ENDPOINT=${S3_ENDPOINT} \\ - RCLONE_CONFIG_SCW_REGION=fr-par \\ - RCLONE_CONFIG_SCW_ACL=private \\ - rclone sync /output/ "scw:${S3_BUCKET}/runs/${RUN_ID}/${model}/" -v - echo "=== Output synced to s3://${S3_BUCKET}/runs/${RUN_ID}/${model}/ ===" + echo "=== Training complete, syncing output to MinIO ===" + RCLONE_CONFIG_MINIO_TYPE=s3 \\ + RCLONE_CONFIG_MINIO_PROVIDER=Minio \\ + RCLONE_CONFIG_MINIO_ACCESS_KEY_ID=\$(cat /secrets/access-key) \\ + RCLONE_CONFIG_MINIO_SECRET_ACCESS_KEY=\$(cat /secrets/secret-key) \\ + RCLONE_CONFIG_MINIO_ENDPOINT=${MINIO_ENDPOINT} \\ + RCLONE_CONFIG_MINIO_REGION=us-east-1 \\ + rclone sync /output/ "minio:${S3_BUCKET}/runs/${RUN_ID}/${model}/" -v + echo "=== Output synced to minio://${S3_BUCKET}/runs/${RUN_ID}/${model}/ ===" env: - name: RUST_LOG value: info @@ -226,7 +225,7 @@ spec: readOnly: true - name: output mountPath: /output - - name: s3-credentials + - name: minio-credentials mountPath: /secrets readOnly: true volumes: @@ -235,9 +234,9 @@ spec: claimName: training-data-pvc - name: output emptyDir: {} - - name: s3-credentials + - name: minio-credentials secret: - secretName: s3-credentials + secretName: minio-credentials EOF } @@ -280,28 +279,26 @@ spec: set -e echo "=== Evaluation (run: ${RUN_ID}) ===" mkdir -p /output/models - echo "Fetching checkpoints from s3://${S3_BUCKET}/runs/${RUN_ID}/ ..." - RCLONE_CONFIG_SCW_TYPE=s3 \\ - RCLONE_CONFIG_SCW_PROVIDER=Scaleway \\ - RCLONE_CONFIG_SCW_ACCESS_KEY_ID=\$(cat /secrets/access-key) \\ - RCLONE_CONFIG_SCW_SECRET_ACCESS_KEY=\$(cat /secrets/secret-key) \\ - RCLONE_CONFIG_SCW_ENDPOINT=${S3_ENDPOINT} \\ - RCLONE_CONFIG_SCW_REGION=fr-par \\ - RCLONE_CONFIG_SCW_ACL=private \\ - rclone sync "scw:${S3_BUCKET}/runs/${RUN_ID}/" /output/models/ -v + echo "Fetching checkpoints from minio://${S3_BUCKET}/runs/${RUN_ID}/ ..." + RCLONE_CONFIG_MINIO_TYPE=s3 \\ + RCLONE_CONFIG_MINIO_PROVIDER=Minio \\ + RCLONE_CONFIG_MINIO_ACCESS_KEY_ID=\$(cat /secrets/access-key) \\ + RCLONE_CONFIG_MINIO_SECRET_ACCESS_KEY=\$(cat /secrets/secret-key) \\ + RCLONE_CONFIG_MINIO_ENDPOINT=${MINIO_ENDPOINT} \\ + RCLONE_CONFIG_MINIO_REGION=us-east-1 \\ + rclone sync "minio:${S3_BUCKET}/runs/${RUN_ID}/" /output/models/ -v echo "=== Running evaluation ===" ${EVAL_BINARY} --model both \\ --data-dir ${DATA_DIR} \\ --models-dir /output/models echo "=== Syncing evaluation results ===" - RCLONE_CONFIG_SCW_TYPE=s3 \\ - RCLONE_CONFIG_SCW_PROVIDER=Scaleway \\ - RCLONE_CONFIG_SCW_ACCESS_KEY_ID=\$(cat /secrets/access-key) \\ - RCLONE_CONFIG_SCW_SECRET_ACCESS_KEY=\$(cat /secrets/secret-key) \\ - RCLONE_CONFIG_SCW_ENDPOINT=${S3_ENDPOINT} \\ - RCLONE_CONFIG_SCW_REGION=fr-par \\ - RCLONE_CONFIG_SCW_ACL=private \\ - rclone sync /output/ "scw:${S3_BUCKET}/runs/${RUN_ID}/eval/" -v + RCLONE_CONFIG_MINIO_TYPE=s3 \\ + RCLONE_CONFIG_MINIO_PROVIDER=Minio \\ + RCLONE_CONFIG_MINIO_ACCESS_KEY_ID=\$(cat /secrets/access-key) \\ + RCLONE_CONFIG_MINIO_SECRET_ACCESS_KEY=\$(cat /secrets/secret-key) \\ + RCLONE_CONFIG_MINIO_ENDPOINT=${MINIO_ENDPOINT} \\ + RCLONE_CONFIG_MINIO_REGION=us-east-1 \\ + rclone sync /output/ "minio:${S3_BUCKET}/runs/${RUN_ID}/eval/" -v echo "=== Done ===" env: - name: RUST_LOG @@ -323,7 +320,7 @@ spec: readOnly: true - name: output mountPath: /output - - name: s3-credentials + - name: minio-credentials mountPath: /secrets readOnly: true volumes: @@ -332,9 +329,9 @@ spec: claimName: training-data-pvc - name: output emptyDir: {} - - name: s3-credentials + - name: minio-credentials secret: - secretName: s3-credentials + secretName: minio-credentials EOF } @@ -358,7 +355,7 @@ echo " Symbol : ${SYMBOL}" echo " Image : ${IMAGE}" echo " NS : ${NAMESPACE}" echo " Run ID : ${RUN_ID}" -echo " S3 : s3://${S3_BUCKET}/runs/${RUN_ID}/" +echo " MinIO : minio://${S3_BUCKET}/runs/${RUN_ID}/" echo "======================================================================" echo ""