diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c34111b8d..2397568fe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -43,13 +43,12 @@ variables: JWT_SECRET: "CiTestSecret_Kx7mP9nR2sW5vY8bC3fG6jH1kL4pQ7tZ0uN9dM5eV8xS2wT6yA4zB_64chars_min" # Training data on block storage PVC (mounted at /mnt/training-data by runner) FOXHUNT_DATA_DIR: "/mnt/training-data/futures-baseline" - # sccache backed by MinIO S3 — shared across all nodes (no PVC needed) + # sccache backed by MinIO S3 (HTTP) — shared across all nodes (no PVC needed) SCCACHE_BUCKET: foxhunt-sccache - SCCACHE_ENDPOINT: https://minio.foxhunt.svc.cluster.local:9000/ + SCCACHE_ENDPOINT: http://minio.foxhunt.svc.cluster.local:9000 SCCACHE_S3_KEY_PREFIX: cpu - SCCACHE_S3_USE_SSL: "true" - SCCACHE_S3_NO_CREDENTIALS: "" - SCCACHE_NO_CERT_VERIFICATION: "true" + SCCACHE_S3_USE_SSL: "false" + SCCACHE_S3_NO_CREDENTIALS: "false" RUSTC_WRAPPER: /usr/local/bin/sccache # Build profile: "dev-release" (thin LTO, fast) is default for quick iteration # Set DEV_RELEASE=false in pipeline vars to use full "release" (fat LTO) profile @@ -727,11 +726,10 @@ build-web-dashboard: echo "Fetching training binaries from MinIO..." rclone copy :s3:foxhunt-binaries/training/ ${CI_PROJECT_DIR}/training-bin/ \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ --s3-no-check-bucket \ - --no-check-certificate echo "Fetched $(ls ${CI_PROJECT_DIR}/training-bin/ | wc -l) training binaries from MinIO" fi - chmod +x ${CI_PROJECT_DIR}/training-bin/* 2>/dev/null || true @@ -783,11 +781,10 @@ build-web-dashboard: echo "Fetching training binaries from MinIO..." rclone copy :s3:foxhunt-binaries/training/ ${CI_PROJECT_DIR}/training-bin/ \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ --s3-no-check-bucket \ - --no-check-certificate echo "Fetched $(ls ${CI_PROJECT_DIR}/training-bin/ | wc -l) training binaries from MinIO" fi - chmod +x ${CI_PROJECT_DIR}/training-bin/* 2>/dev/null || true @@ -1617,7 +1614,7 @@ infra-drift-check: script: - export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}" - export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}" - - S3="aws s3 --endpoint-url https://minio.foxhunt.svc.cluster.local:9000 --no-verify-ssl" + - S3="aws s3 --endpoint-url http://minio.foxhunt.svc.cluster.local:9000" - echo "Deploying ${SERVICE_PACKAGE}..." - $S3 cp build-out/services/${SERVICE_PACKAGE} s3://foxhunt-binaries/services/${SERVICE_PACKAGE} - kubectl apply -f infra/k8s/services/${SERVICE_PACKAGE}.yaml @@ -1704,7 +1701,7 @@ deploy-training: script: - export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}" - export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}" - - S3="aws s3 --endpoint-url https://minio.foxhunt.svc.cluster.local:9000 --no-verify-ssl" + - S3="aws s3 --endpoint-url http://minio.foxhunt.svc.cluster.local:9000" - $S3 cp build-out/training/ s3://foxhunt-binaries/training/ --recursive - echo "Training binaries uploaded to MinIO" @@ -1724,7 +1721,7 @@ deploy-web-dashboard: script: - export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}" - export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}" - - S3="aws s3 --endpoint-url https://minio.foxhunt.svc.cluster.local:9000 --no-verify-ssl" + - S3="aws s3 --endpoint-url http://minio.foxhunt.svc.cluster.local:9000" - $S3 cp web-dashboard/dist/ s3://foxhunt-binaries/static/ --recursive - echo "Web dashboard assets uploaded to MinIO" diff --git a/infra/k8s/argo/ci-pipeline-template.yaml b/infra/k8s/argo/ci-pipeline-template.yaml index 07b2ea03a..856cb3691 100644 --- a/infra/k8s/argo/ci-pipeline-template.yaml +++ b/infra/k8s/argo/ci-pipeline-template.yaml @@ -30,9 +30,6 @@ spec: items: - key: .dockerconfigjson path: config.json - - name: minio-ca-cert - configMap: - name: minio-ca-cert arguments: parameters: - name: commit-sha @@ -261,14 +258,9 @@ spec: - name: git-ssh-key mountPath: /etc/git-ssh readOnly: true - - name: minio-ca-cert - mountPath: /etc/ssl/minio-ca - readOnly: true args: - | set -e - cat /etc/ssl/minio-ca/ca.crt >> /etc/ssl/certs/ca-certificates.crt - SHA="{{workflow.parameters.commit-sha}}" # Git clone @@ -298,11 +290,10 @@ spec: echo "=== Uploading to MinIO ===" rclone copy dist/ :s3:foxhunt-binaries/web-dashboard/ \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket \ - --no-check-certificate + --s3-no-check-bucket echo "=== Web dashboard build + upload done ===" @@ -327,13 +318,13 @@ spec: - name: SCCACHE_BUCKET value: foxhunt-sccache - name: SCCACHE_ENDPOINT - value: https://minio.foxhunt.svc.cluster.local:9000/ + value: http://minio.foxhunt.svc.cluster.local:9000 - name: SCCACHE_REGION value: us-east-1 - name: SCCACHE_S3_KEY_PREFIX value: cpu - name: SCCACHE_S3_USE_SSL - value: "true" + value: "false" - name: SCCACHE_S3_NO_CREDENTIALS value: "false" - name: AWS_ACCESS_KEY_ID @@ -367,15 +358,9 @@ spec: - name: git-ssh-key mountPath: /etc/git-ssh readOnly: true - - name: minio-ca-cert - mountPath: /etc/ssl/minio-ca - readOnly: true args: - | set -e - # Trust MinIO self-signed CA - cat /etc/ssl/minio-ca/ca.crt >> /etc/ssl/certs/ca-certificates.crt - SHA="{{workflow.parameters.commit-sha}}" # Git clone (self-contained) @@ -426,11 +411,10 @@ spec: echo "=== Uploading service binaries ===" rclone copy "$WORKSPACE/bin/services/" :s3:foxhunt-binaries/services/ \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket \ - --no-check-certificate + --s3-no-check-bucket echo "=== Service compile + upload done ===" @@ -455,11 +439,11 @@ spec: - name: SCCACHE_BUCKET value: foxhunt-sccache - name: SCCACHE_ENDPOINT - value: https://minio.foxhunt.svc.cluster.local:9000/ + value: http://minio.foxhunt.svc.cluster.local:9000 - name: SCCACHE_REGION value: us-east-1 - name: SCCACHE_S3_USE_SSL - value: "true" + value: "false" - name: SCCACHE_S3_NO_CREDENTIALS value: "false" - name: AWS_ACCESS_KEY_ID @@ -493,15 +477,9 @@ spec: - name: git-ssh-key mountPath: /etc/git-ssh readOnly: true - - name: minio-ca-cert - mountPath: /etc/ssl/minio-ca - readOnly: true args: - | set -e - # Trust MinIO self-signed CA - cat /etc/ssl/minio-ca/ca.crt >> /etc/ssl/certs/ca-certificates.crt - SHA="{{workflow.parameters.commit-sha}}" # Git clone (self-contained — ephemeral node) @@ -554,11 +532,10 @@ spec: echo "=== Uploading training binaries ===" rclone copy "$WORKSPACE/bin/training/" :s3:foxhunt-binaries/training/ \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket \ - --no-check-certificate + --s3-no-check-bucket echo "=== Training compile + upload done ===" diff --git a/infra/k8s/argo/training-workflow-template.yaml b/infra/k8s/argo/training-workflow-template.yaml index 1c00bc220..54c3fdadf 100644 --- a/infra/k8s/argo/training-workflow-template.yaml +++ b/infra/k8s/argo/training-workflow-template.yaml @@ -128,11 +128,10 @@ spec: echo "Fetching training binaries from MinIO..." rclone copy :s3:foxhunt-binaries/training/ /workspace/bin/ \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ --s3-no-check-bucket \ - --no-check-certificate chmod +x /workspace/bin/* echo "Fetched binaries:" ls -lh /workspace/bin/ @@ -183,11 +182,10 @@ spec: echo "=== Downloading training data from MinIO ===" rclone copy :s3:foxhunt-training-data/futures-baseline/ /tmp/futures-baseline/ \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ --s3-no-check-bucket \ - --no-check-certificate \ --transfers=8 echo "Training data: $(find /tmp/futures-baseline -name '*.dbn.zst' | wc -l) files" @@ -280,11 +278,10 @@ spec: echo "=== Downloading training data from MinIO ===" rclone copy :s3:foxhunt-training-data/futures-baseline/ /tmp/futures-baseline/ \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ --s3-no-check-bucket \ - --no-check-certificate \ --transfers=8 echo "Training data: $(find /tmp/futures-baseline -name '*.dbn.zst' | wc -l) files" @@ -371,11 +368,10 @@ spec: echo "=== Downloading training data from MinIO ===" rclone copy :s3:foxhunt-training-data/futures-baseline/ /tmp/futures-baseline/ \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ --s3-no-check-bucket \ - --no-check-certificate \ --transfers=8 MODEL="{{workflow.parameters.model}}" @@ -448,20 +444,18 @@ spec: echo " checkpoints (*.safetensors), norm stats, hyperopt results, eval reports" rclone copy /workspace/output/ ":${DEST}" \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ --s3-no-check-bucket \ - --no-check-certificate echo "=== Upload complete ===" rclone ls ":${DEST}" \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ --s3-no-check-bucket \ - --no-check-certificate env: - name: MINIO_ACCESS_KEY valueFrom: diff --git a/infra/k8s/gitlab/runner-values.yaml b/infra/k8s/gitlab/runner-values.yaml index 078322f72..408aed731 100644 --- a/infra/k8s/gitlab/runner-values.yaml +++ b/infra/k8s/gitlab/runner-values.yaml @@ -63,11 +63,6 @@ runners: name = "training-data-pvc" mount_path = "/mnt/training-data" read_only = true - # MinIO CA cert for HTTPS S3 access (rclone uses RCLONE_CA_CERT env) - [[runners.kubernetes.volumes.config_map]] - name = "minio-ca-cert" - mount_path = "/etc/ssl/minio-ca" - read_only = true # Runner tags for job matching tags: "kapsule,rust,docker,gpu" diff --git a/infra/k8s/gitlab/tailscale-proxy.yaml b/infra/k8s/gitlab/tailscale-proxy.yaml index 940737644..d866488fa 100644 --- a/infra/k8s/gitlab/tailscale-proxy.yaml +++ b/infra/k8s/gitlab/tailscale-proxy.yaml @@ -269,7 +269,7 @@ data: client_max_body_size 0; location / { - proxy_pass https://minio.foxhunt.svc.cluster.local:9000; + proxy_pass http://minio.foxhunt.svc.cluster.local:9000; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; diff --git a/infra/k8s/gpu-overlays/ml-training-service-gpu.yaml b/infra/k8s/gpu-overlays/ml-training-service-gpu.yaml index 4e31ab100..2650bad28 100644 --- a/infra/k8s/gpu-overlays/ml-training-service-gpu.yaml +++ b/infra/k8s/gpu-overlays/ml-training-service-gpu.yaml @@ -58,11 +58,10 @@ spec: ":s3:foxhunt-binaries/services/ml-training-service" \ "/binaries/ml-training-service" \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket \ - --no-check-certificate + --s3-no-check-bucket chmod +x /binaries/ml-training-service echo "Fetched ml-training-service ($(stat -c%s /binaries/ml-training-service) bytes)" env: @@ -120,11 +119,9 @@ spec: - name: JWT_AUDIENCE value: foxhunt-services - name: S3_ENDPOINT - value: "https://minio.foxhunt.svc.cluster.local:9000" + value: "http://minio.foxhunt.svc.cluster.local:9000" - name: S3_BUCKET value: foxhunt-models - - name: MINIO_CA_CERT_PATH - value: /etc/ssl/minio-ca/ca.crt - name: ENABLE_GPU value: "true" - name: RUST_LOG @@ -138,9 +135,6 @@ spec: - name: tls-certs mountPath: /app/certs/ml-training-service readOnly: true - - name: minio-ca - mountPath: /etc/ssl/minio-ca - readOnly: true - name: tmp mountPath: /tmp readinessProbe: @@ -170,9 +164,6 @@ spec: - name: tls-certs secret: secretName: ml-training-tls - - name: minio-ca - configMap: - name: minio-ca-cert - name: tmp emptyDir: sizeLimit: 50Mi diff --git a/infra/k8s/gpu-overlays/trading-service-gpu.yaml b/infra/k8s/gpu-overlays/trading-service-gpu.yaml index 1806ac817..52dcf2ed1 100644 --- a/infra/k8s/gpu-overlays/trading-service-gpu.yaml +++ b/infra/k8s/gpu-overlays/trading-service-gpu.yaml @@ -39,7 +39,7 @@ spec: seccompProfile: type: RuntimeDefault nodeSelector: - k8s.scaleway.com/pool-name: gpu-inference + k8s.scaleway.com/pool-name: l40s tolerations: - key: nvidia.com/gpu operator: Exists @@ -57,11 +57,10 @@ spec: ":s3:foxhunt-binaries/services/trading-service" \ "/binaries/trading-service" \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket \ - --no-check-certificate + --s3-no-check-bucket chmod +x /binaries/trading-service echo "Fetched trading-service ($(stat -c%s /binaries/trading-service) bytes)" env: diff --git a/infra/k8s/minio/minio.yaml b/infra/k8s/minio/minio.yaml index 673d0e1a0..e9ca047ab 100644 --- a/infra/k8s/minio/minio.yaml +++ b/infra/k8s/minio/minio.yaml @@ -1,5 +1,6 @@ # MinIO — in-cluster S3 for compiled binaries and trained models # Replaces Scaleway public S3 to keep proprietary code/IP in-cluster +# Plain HTTP only — internal cluster traffic, no TLS needed apiVersion: v1 kind: Secret metadata: @@ -30,7 +31,7 @@ spec: storageClassName: scw-bssd resources: requests: - storage: 20Gi + storage: 100Gi --- apiVersion: apps/v1 kind: Deployment @@ -54,7 +55,7 @@ spec: app.kubernetes.io/part-of: foxhunt spec: nodeSelector: - k8s.scaleway.com/pool-name: infra + k8s.scaleway.com/pool-name: platform containers: - name: minio image: minio/minio:latest @@ -78,16 +79,15 @@ spec: volumeMounts: - name: data mountPath: /data - - name: tls-certs - mountPath: /root/.minio/certs - readOnly: true readinessProbe: - tcpSocket: + httpGet: + path: /minio/health/live port: 9000 initialDelaySeconds: 5 periodSeconds: 10 livenessProbe: - tcpSocket: + httpGet: + path: /minio/health/live port: 9000 initialDelaySeconds: 10 periodSeconds: 15 @@ -103,14 +103,6 @@ spec: - name: data persistentVolumeClaim: claimName: minio-data-new - - name: tls-certs - secret: - secretName: minio-tls - items: - - key: tls.crt - path: public.crt - - key: tls.key - path: private.key --- apiVersion: v1 kind: Service @@ -149,7 +141,7 @@ spec: app.kubernetes.io/name: minio-init spec: nodeSelector: - k8s.scaleway.com/pool-name: infra + k8s.scaleway.com/pool-name: platform restartPolicy: OnFailure containers: - name: mc @@ -158,11 +150,8 @@ spec: args: - | set -e - # mc trusts CAs placed in ~/.mc/certs/CAs/ - mkdir -p /root/.mc/certs/CAs - cp /etc/ssl/minio-ca/ca.crt /root/.mc/certs/CAs/minio-ca.crt echo "Waiting for MinIO to be ready..." - until mc alias set foxhunt https://minio.foxhunt.svc.cluster.local:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" 2>/dev/null; do + until mc alias set foxhunt http://minio.foxhunt.svc.cluster.local:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" 2>/dev/null; do echo "MinIO not ready, retrying in 3s..." sleep 3 done @@ -170,6 +159,8 @@ spec: mc mb --ignore-existing foxhunt/foxhunt-binaries mc mb --ignore-existing foxhunt/foxhunt-models mc mb --ignore-existing foxhunt/foxhunt-training-results + mc mb --ignore-existing foxhunt/foxhunt-sccache + mc mb --ignore-existing foxhunt/foxhunt-training-data echo "Buckets created:" mc ls foxhunt/ env: @@ -183,10 +174,6 @@ spec: secretKeyRef: name: minio-credentials key: root-password - volumeMounts: - - name: ca-cert - mountPath: /etc/ssl/minio-ca - readOnly: true resources: requests: cpu: 50m @@ -194,7 +181,3 @@ spec: limits: cpu: 200m memory: 64Mi - volumes: - - name: ca-cert - configMap: - name: minio-ca-cert diff --git a/infra/k8s/services/api.yaml b/infra/k8s/services/api.yaml index 9d6dda1f0..55d439c8e 100644 --- a/infra/k8s/services/api.yaml +++ b/infra/k8s/services/api.yaml @@ -37,7 +37,7 @@ spec: imagePullSecrets: - name: gitlab-registry nodeSelector: - k8s.scaleway.com/pool-name: foxhunt + k8s.scaleway.com/pool-name: platform initContainers: - name: fetch-binary image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest @@ -49,11 +49,10 @@ spec: ":s3:foxhunt-binaries/services/api" \ "/binaries/api" \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket \ - --no-check-certificate + --s3-no-check-bucket chmod +x /binaries/api echo "Fetched api ($(stat -c%s /binaries/api) bytes)" env: diff --git a/infra/k8s/services/backtesting-service.yaml b/infra/k8s/services/backtesting-service.yaml index e0c1a0377..c261d2645 100644 --- a/infra/k8s/services/backtesting-service.yaml +++ b/infra/k8s/services/backtesting-service.yaml @@ -36,7 +36,7 @@ spec: imagePullSecrets: - name: gitlab-registry nodeSelector: - k8s.scaleway.com/pool-name: foxhunt + k8s.scaleway.com/pool-name: platform initContainers: - name: fetch-binary image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest @@ -48,11 +48,10 @@ spec: ":s3:foxhunt-binaries/services/backtesting-service" \ "/binaries/backtesting-service" \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket \ - --no-check-certificate + --s3-no-check-bucket chmod +x /binaries/backtesting-service echo "Fetched backtesting-service ($(stat -c%s /binaries/backtesting-service) bytes)" env: diff --git a/infra/k8s/services/broker-gateway.yaml b/infra/k8s/services/broker-gateway.yaml index d629ae9ca..ead15a5fe 100644 --- a/infra/k8s/services/broker-gateway.yaml +++ b/infra/k8s/services/broker-gateway.yaml @@ -36,7 +36,7 @@ spec: imagePullSecrets: - name: gitlab-registry nodeSelector: - k8s.scaleway.com/pool-name: foxhunt + k8s.scaleway.com/pool-name: platform initContainers: - name: fetch-binary image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest @@ -48,11 +48,10 @@ spec: ":s3:foxhunt-binaries/services/broker-gateway" \ "/binaries/broker-gateway" \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket \ - --no-check-certificate + --s3-no-check-bucket chmod +x /binaries/broker-gateway echo "Fetched broker-gateway ($(stat -c%s /binaries/broker-gateway) bytes)" env: diff --git a/infra/k8s/services/data-acquisition-service.yaml b/infra/k8s/services/data-acquisition-service.yaml index 682d34445..9413c2803 100644 --- a/infra/k8s/services/data-acquisition-service.yaml +++ b/infra/k8s/services/data-acquisition-service.yaml @@ -36,7 +36,7 @@ spec: imagePullSecrets: - name: gitlab-registry nodeSelector: - k8s.scaleway.com/pool-name: foxhunt + k8s.scaleway.com/pool-name: platform initContainers: - name: fetch-binary image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest @@ -48,11 +48,10 @@ spec: ":s3:foxhunt-binaries/services/data-acquisition-service" \ "/binaries/data-acquisition-service" \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket \ - --no-check-certificate + --s3-no-check-bucket chmod +x /binaries/data-acquisition-service echo "Fetched data-acquisition-service ($(stat -c%s /binaries/data-acquisition-service) bytes)" env: diff --git a/infra/k8s/services/ib-gateway.yaml b/infra/k8s/services/ib-gateway.yaml index 4235d4a08..670fc892f 100644 --- a/infra/k8s/services/ib-gateway.yaml +++ b/infra/k8s/services/ib-gateway.yaml @@ -25,7 +25,7 @@ spec: imagePullSecrets: - name: gitlab-registry nodeSelector: - k8s.scaleway.com/pool-name: foxhunt + k8s.scaleway.com/pool-name: platform containers: - name: ib-gateway image: ghcr.io/gnzsnz/ib-gateway:stable diff --git a/infra/k8s/services/ml-training-service.yaml b/infra/k8s/services/ml-training-service.yaml index f7a314758..5749cc1a3 100644 --- a/infra/k8s/services/ml-training-service.yaml +++ b/infra/k8s/services/ml-training-service.yaml @@ -79,7 +79,7 @@ spec: imagePullSecrets: - name: gitlab-registry nodeSelector: - k8s.scaleway.com/pool-name: foxhunt + k8s.scaleway.com/pool-name: platform initContainers: - name: fetch-binary image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest @@ -91,11 +91,10 @@ spec: ":s3:foxhunt-binaries/services/ml-training-service" \ "/binaries/ml-training-service" \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket \ - --no-check-certificate + --s3-no-check-bucket chmod +x /binaries/ml-training-service echo "Fetched ml-training-service ($(stat -c%s /binaries/ml-training-service) bytes)" env: @@ -153,11 +152,9 @@ spec: - name: JWT_AUDIENCE value: foxhunt-services - name: S3_ENDPOINT - value: "https://minio.foxhunt.svc.cluster.local:9000" + value: "http://minio.foxhunt.svc.cluster.local:9000" - name: S3_BUCKET value: foxhunt-models - - name: MINIO_CA_CERT_PATH - value: /etc/ssl/minio-ca/ca.crt - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: @@ -181,9 +178,6 @@ spec: - name: tls-certs mountPath: /app/certs/ml_training_service readOnly: true - - name: minio-ca - mountPath: /etc/ssl/minio-ca - readOnly: true - name: tmp mountPath: /tmp readinessProbe: @@ -211,9 +205,6 @@ spec: - name: tls-certs secret: secretName: ml-training-tls - - name: minio-ca - configMap: - name: minio-ca-cert - name: tmp emptyDir: sizeLimit: 50Mi diff --git a/infra/k8s/services/trading-agent-service.yaml b/infra/k8s/services/trading-agent-service.yaml index 6f0327375..8cac12bed 100644 --- a/infra/k8s/services/trading-agent-service.yaml +++ b/infra/k8s/services/trading-agent-service.yaml @@ -36,7 +36,7 @@ spec: imagePullSecrets: - name: gitlab-registry nodeSelector: - k8s.scaleway.com/pool-name: foxhunt + k8s.scaleway.com/pool-name: platform initContainers: - name: fetch-binary image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest @@ -48,11 +48,10 @@ spec: ":s3:foxhunt-binaries/services/trading-agent-service" \ "/binaries/trading-agent-service" \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket \ - --no-check-certificate + --s3-no-check-bucket chmod +x /binaries/trading-agent-service echo "Fetched trading-agent-service ($(stat -c%s /binaries/trading-agent-service) bytes)" env: diff --git a/infra/k8s/services/trading-service.yaml b/infra/k8s/services/trading-service.yaml index 20a247bd5..1e721735e 100644 --- a/infra/k8s/services/trading-service.yaml +++ b/infra/k8s/services/trading-service.yaml @@ -36,7 +36,7 @@ spec: imagePullSecrets: - name: gitlab-registry nodeSelector: - k8s.scaleway.com/pool-name: foxhunt + k8s.scaleway.com/pool-name: platform initContainers: - name: fetch-binary image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest @@ -48,11 +48,10 @@ spec: ":s3:foxhunt-binaries/services/trading-service" \ "/binaries/trading-service" \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket \ - --no-check-certificate + --s3-no-check-bucket chmod +x /binaries/trading-service echo "Fetched trading-service ($(stat -c%s /binaries/trading-service) bytes)" env: diff --git a/infra/k8s/services/web-dashboard.yaml b/infra/k8s/services/web-dashboard.yaml index 8031227c9..ebd958162 100644 --- a/infra/k8s/services/web-dashboard.yaml +++ b/infra/k8s/services/web-dashboard.yaml @@ -37,11 +37,10 @@ spec: ":s3:foxhunt-binaries/web-dashboard/" \ "/usr/share/nginx/html/" \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket \ - --no-check-certificate + --s3-no-check-bucket echo "Fetched web-dashboard assets ($(find /usr/share/nginx/html -type f | wc -l) files)" env: - name: MINIO_ACCESS_KEY diff --git a/infra/k8s/training/job-template.yaml b/infra/k8s/training/job-template.yaml index 118e8aa69..5febf44fa 100644 --- a/infra/k8s/training/job-template.yaml +++ b/infra/k8s/training/job-template.yaml @@ -47,11 +47,10 @@ spec: ":s3:foxhunt-binaries/training/$(TRAINING_BINARY)" \ "/binaries/$(TRAINING_BINARY)" \ --s3-provider=Minio \ - --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ --s3-access-key-id="${MINIO_ACCESS_KEY}" \ --s3-secret-access-key="${MINIO_SECRET_KEY}" \ --s3-no-check-bucket \ - --no-check-certificate chmod +x "/binaries/$(TRAINING_BINARY)" echo "Fetched $(TRAINING_BINARY) ($(stat -c%s /binaries/$(TRAINING_BINARY)) bytes)" env: diff --git a/infra/scripts/train.sh b/infra/scripts/train.sh index 159df3420..540875f51 100755 --- a/infra/scripts/train.sh +++ b/infra/scripts/train.sh @@ -27,7 +27,7 @@ MODEL="" PRESET="" TIMESTAMP="$(date +%s)" S3_BUCKET="foxhunt-models" -MINIO_ENDPOINT="https://minio.foxhunt.svc.cluster.local:9000" +MINIO_ENDPOINT="http://minio.foxhunt.svc.cluster.local:9000" RUN_ID="$(date +%Y%m%d-%H%M%S)" ALL_MODELS=(dqn ppo tft mamba2 tggn tlob liquid kan xlstm diffusion) @@ -203,7 +203,6 @@ spec: RCLONE_CONFIG_MINIO_SECRET_ACCESS_KEY=\$(cat /secrets/secret-key) \\ RCLONE_CONFIG_MINIO_ENDPOINT=${MINIO_ENDPOINT} \\ RCLONE_CONFIG_MINIO_REGION=us-east-1 \\ - RCLONE_CA_CERT=/etc/ssl/minio-ca/ca.crt \\ rclone sync /output/ "minio:${S3_BUCKET}/runs/${RUN_ID}/${model}/" -v echo "=== Output synced to minio://${S3_BUCKET}/runs/${RUN_ID}/${model}/ ===" env: @@ -229,9 +228,6 @@ spec: - name: minio-credentials mountPath: /secrets readOnly: true - - name: minio-ca - mountPath: /etc/ssl/minio-ca - readOnly: true volumes: - name: training-data persistentVolumeClaim: @@ -241,9 +237,6 @@ spec: - name: minio-credentials secret: secretName: minio-credentials - - name: minio-ca - configMap: - name: minio-ca-cert EOF } @@ -293,7 +286,6 @@ spec: RCLONE_CONFIG_MINIO_SECRET_ACCESS_KEY=\$(cat /secrets/secret-key) \\ RCLONE_CONFIG_MINIO_ENDPOINT=${MINIO_ENDPOINT} \\ RCLONE_CONFIG_MINIO_REGION=us-east-1 \\ - RCLONE_CA_CERT=/etc/ssl/minio-ca/ca.crt \\ rclone sync "minio:${S3_BUCKET}/runs/${RUN_ID}/" /output/models/ -v echo "=== Running evaluation ===" ${EVAL_BINARY} --model both \\ @@ -306,7 +298,6 @@ spec: RCLONE_CONFIG_MINIO_SECRET_ACCESS_KEY=\$(cat /secrets/secret-key) \\ RCLONE_CONFIG_MINIO_ENDPOINT=${MINIO_ENDPOINT} \\ RCLONE_CONFIG_MINIO_REGION=us-east-1 \\ - RCLONE_CA_CERT=/etc/ssl/minio-ca/ca.crt \\ rclone sync /output/ "minio:${S3_BUCKET}/runs/${RUN_ID}/eval/" -v echo "=== Done ===" env: @@ -332,9 +323,6 @@ spec: - name: minio-credentials mountPath: /secrets readOnly: true - - name: minio-ca - mountPath: /etc/ssl/minio-ca - readOnly: true volumes: - name: training-data persistentVolumeClaim: @@ -344,9 +332,6 @@ spec: - name: minio-credentials secret: secretName: minio-credentials - - name: minio-ca - configMap: - name: minio-ca-cert EOF } diff --git a/services/ml_training_service/src/k8s_dispatcher.rs b/services/ml_training_service/src/k8s_dispatcher.rs index 453e0756a..5cba98904 100644 --- a/services/ml_training_service/src/k8s_dispatcher.rs +++ b/services/ml_training_service/src/k8s_dispatcher.rs @@ -9,7 +9,7 @@ use std::collections::BTreeMap; use anyhow::{Context, Result}; use k8s_openapi::api::batch::v1::{Job, JobSpec}; use k8s_openapi::api::core::v1::{ - ConfigMapVolumeSource, Container, EmptyDirVolumeSource, EnvFromSource, EnvVar, + Container, EmptyDirVolumeSource, EnvFromSource, EnvVar, LocalObjectReference, PersistentVolumeClaimVolumeSource, PodSpec, PodTemplateSpec, ResourceRequirements, SecretEnvSource, SecurityContext, Toleration, Volume, VolumeMount, }; @@ -46,24 +46,27 @@ impl Default for DispatcherConfig { } impl DispatcherConfig { - /// Reads configuration from environment variables, falling back to sensible defaults. + /// Reads configuration from environment variables. + /// + /// Required env vars: S3_ENDPOINT, TRAINING_RUNTIME_IMAGE, CALLBACK_ENDPOINT. + /// Falls back to defaults only for non-URL values (namespace, bucket names, PVC names). pub fn from_env() -> Self { Self { namespace: std::env::var("K8S_NAMESPACE").unwrap_or_else(|_| "foxhunt".to_string()), - runtime_image: std::env::var("TRAINING_RUNTIME_IMAGE").unwrap_or_else(|_| { - "foxhunt-training-runtime:latest".to_string() - }), - data_pvc: "training-data-pvc".to_string(), + runtime_image: std::env::var("TRAINING_RUNTIME_IMAGE") + .expect("TRAINING_RUNTIME_IMAGE env var required"), + data_pvc: std::env::var("TRAINING_DATA_PVC") + .unwrap_or_else(|_| "training-data-pvc".to_string()), s3_bucket: std::env::var("S3_BUCKET") .unwrap_or_else(|_| "foxhunt-models".to_string()), s3_endpoint: std::env::var("S3_ENDPOINT") - .unwrap_or_else(|_| "https://minio.foxhunt.svc.cluster.local:9000".to_string()), - s3_region: "fr-par".to_string(), + .expect("S3_ENDPOINT env var required"), + s3_region: std::env::var("S3_REGION") + .unwrap_or_else(|_| "fr-par".to_string()), binaries_bucket: std::env::var("BINARIES_BUCKET") .unwrap_or_else(|_| "foxhunt-binaries".to_string()), - callback_endpoint: std::env::var("CALLBACK_ENDPOINT").unwrap_or_else(|_| { - "http://ml-training-service.foxhunt.svc.cluster.local:50053".to_string() - }), + callback_endpoint: std::env::var("CALLBACK_ENDPOINT") + .expect("CALLBACK_ENDPOINT env var required"), } } } @@ -237,7 +240,7 @@ pub fn build_job_spec(config: &DispatcherConfig, params: &TrainingJobParams) -> command: Some(vec!["/bin/sh".to_string(), "-c".to_string()]), args: Some(vec![format!( "set -e\n\ - export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=fr-par RCLONE_CA_CERT=/etc/ssl/minio-ca/ca.crt\n\ + export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=$S3_ENDPOINT RCLONE_S3_REGION=$S3_REGION\n\ rclone sync :s3:{}/latest/training/ /binaries/\n\ chmod +x /binaries/*\n\ ls -lh /binaries/\n\ @@ -245,6 +248,8 @@ pub fn build_job_spec(config: &DispatcherConfig, params: &TrainingJobParams) -> config.binaries_bucket )]), env: Some(vec![ + env_var("S3_ENDPOINT", &config.s3_endpoint), + env_var("S3_REGION", &config.s3_region), EnvVar { name: "RCLONE_S3_ACCESS_KEY_ID".to_string(), value_from: Some(k8s_openapi::api::core::v1::EnvVarSource { @@ -276,12 +281,6 @@ pub fn build_job_spec(config: &DispatcherConfig, params: &TrainingJobParams) -> mount_path: "/binaries".to_string(), ..Default::default() }, - VolumeMount { - name: "minio-ca".to_string(), - mount_path: "/etc/ssl/minio-ca".to_string(), - read_only: Some(true), - ..Default::default() - }, ]), resources: Some(ResourceRequirements { requests: Some(BTreeMap::from([ @@ -310,7 +309,6 @@ pub fn build_job_spec(config: &DispatcherConfig, params: &TrainingJobParams) -> env_var("S3_REGION", &config.s3_region), env_var("S3_PREFIX", &format!("models/{}", params.job_id)), env_var("CALLBACK_ENDPOINT", &config.callback_endpoint), - env_var("MINIO_CA_CERT_PATH", "/etc/ssl/minio-ca/ca.crt"), env_var("RUST_LOG", "info"), ]), env_from: Some(vec![EnvFromSource { @@ -333,12 +331,6 @@ pub fn build_job_spec(config: &DispatcherConfig, params: &TrainingJobParams) -> read_only: Some(true), ..Default::default() }, - VolumeMount { - name: "minio-ca".to_string(), - mount_path: "/etc/ssl/minio-ca".to_string(), - read_only: Some(true), - ..Default::default() - }, ]), resources: Some(ResourceRequirements { requests: Some(BTreeMap::from([ @@ -438,14 +430,6 @@ pub fn build_job_spec(config: &DispatcherConfig, params: &TrainingJobParams) -> }), ..Default::default() }, - Volume { - name: "minio-ca".to_string(), - config_map: Some(ConfigMapVolumeSource { - name: "minio-ca-cert".to_string(), - ..Default::default() - }), - ..Default::default() - }, ]; // -- Node selector --------------------------------------------------------- @@ -679,14 +663,12 @@ mod tests { assert!(env_names.contains(&"S3_ENDPOINT")); assert!(env_names.contains(&"S3_PREFIX")); assert!(env_names.contains(&"CALLBACK_ENDPOINT")); - assert!(env_names.contains(&"MINIO_CA_CERT_PATH")); - // Sidecar volume mounts (output + binaries + minio-ca) + // Sidecar volume mounts (output + binaries) let sidecar_mounts = sidecar.volume_mounts.as_ref().unwrap(); - assert_eq!(sidecar_mounts.len(), 3); + assert_eq!(sidecar_mounts.len(), 2); assert!(sidecar_mounts.iter().any(|m| m.name == "output")); assert!(sidecar_mounts.iter().any(|m| m.name == "binaries")); - assert!(sidecar_mounts.iter().any(|m| m.name == "minio-ca")); // -- Main container -- let containers = &pod_spec.containers; @@ -722,9 +704,9 @@ mod tests { assert!(mounts.iter().any(|m| m.name == "output" && m.mount_path == "/output")); assert!(mounts.iter().any(|m| m.name == "binaries" && m.mount_path == "/binaries")); - // -- Volumes (PVC + 2 emptyDir + ConfigMap) -- + // -- Volumes (PVC + 2 emptyDir) -- let volumes = pod_spec.volumes.as_ref().unwrap(); - assert_eq!(volumes.len(), 4); + assert_eq!(volumes.len(), 3); let data_vol = volumes.iter().find(|v| v.name == "training-data").unwrap(); assert_eq!( @@ -737,12 +719,6 @@ mod tests { let binaries_vol = volumes.iter().find(|v| v.name == "binaries").unwrap(); assert!(binaries_vol.empty_dir.is_some(), "binaries should be emptyDir"); - - let ca_vol = volumes.iter().find(|v| v.name == "minio-ca").unwrap(); - assert_eq!( - ca_vol.config_map.as_ref().unwrap().name, - "minio-ca-cert" - ); } #[test] @@ -843,23 +819,25 @@ mod tests { #[test] fn test_from_env_defaults() { - // Clear env vars to test defaults (safe in single-threaded test). + // Set required env vars, clear optional ones to test defaults. + std::env::set_var("S3_ENDPOINT", "http://minio:9000"); + std::env::set_var("TRAINING_RUNTIME_IMAGE", "test-runtime:latest"); + std::env::set_var("CALLBACK_ENDPOINT", "http://callback:50053"); std::env::remove_var("K8S_NAMESPACE"); - std::env::remove_var("TRAINING_RUNTIME_IMAGE"); std::env::remove_var("S3_BUCKET"); - std::env::remove_var("S3_ENDPOINT"); std::env::remove_var("BINARIES_BUCKET"); - std::env::remove_var("CALLBACK_ENDPOINT"); + std::env::remove_var("TRAINING_DATA_PVC"); + std::env::remove_var("S3_REGION"); let cfg = DispatcherConfig::from_env(); assert_eq!(cfg.namespace, "foxhunt"); - assert!(cfg.runtime_image.contains("foxhunt-training-runtime")); + assert_eq!(cfg.runtime_image, "test-runtime:latest"); assert_eq!(cfg.data_pvc, "training-data-pvc"); assert_eq!(cfg.binaries_bucket, "foxhunt-binaries"); assert_eq!(cfg.s3_bucket, "foxhunt-models"); - assert!(cfg.s3_endpoint.contains("minio.foxhunt.svc.cluster.local")); + assert_eq!(cfg.s3_endpoint, "http://minio:9000"); assert_eq!(cfg.s3_region, "fr-par"); - assert!(cfg.callback_endpoint.contains("ml-training-service")); + assert_eq!(cfg.callback_endpoint, "http://callback:50053"); } }