gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(ci): trust MinIO CA in system store for rclone S3 uploads

Browse Source 
RCLONE_CA_CERT env var does not apply to rclone's S3 backend — Go's
crypto/tls reads from the system CA bundle. Append the MinIO CA cert
to /etc/ssl/certs/ca-certificates.crt before rclone commands. Fixes
compile-services, compile-training, web-dashboard, write-manifest,
and both training base templates.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-01 23:47:05 +01:00
parent 3a6def362f
commit f4e2ce852b
1 changed files with 18 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
.gitlab-ci.yml
View File

@@ -443,7 +443,9 @@ compile-services:
done
- ls -lh build-out/services/
# Upload stripped binaries to MinIO (in-cluster S3)
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=https://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1 RCLONE_CA_CERT=/etc/ssl/minio-ca/ca.crt
# Trust MinIO CA — rclone S3 backend uses Go's system CAs, not RCLONE_CA_CERT
- cat /etc/ssl/minio-ca/ca.crt >> /etc/ssl/certs/ca-certificates.crt 2>/dev/null || true
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=https://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
- export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
- |
for bin in trading_service api_gateway broker_gateway_service ml_training_service backtesting_service trading_agent_service data_acquisition_service web-gateway; do
@@ -504,7 +506,9 @@ compile-training:
strip build-out/training/training_uploader
- ls -lh build-out/training/
# Upload stripped binaries to MinIO (in-cluster S3)
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=https://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1 RCLONE_CA_CERT=/etc/ssl/minio-ca/ca.crt
# Trust MinIO CA — rclone S3 backend uses Go's system CAs, not RCLONE_CA_CERT
- cat /etc/ssl/minio-ca/ca.crt >> /etc/ssl/certs/ca-certificates.crt 2>/dev/null || true
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=https://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
- export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
- |
for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised training_uploader; do
@@ -542,7 +546,9 @@ build-web-dashboard:
- cd web-dashboard
- npm ci
- npm run build
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=https://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1 RCLONE_CA_CERT=/etc/ssl/minio-ca/ca.crt
# Trust MinIO CA — rclone S3 backend uses Go's system CAs, not RCLONE_CA_CERT
- cat /etc/ssl/minio-ca/ca.crt >> /etc/ssl/certs/ca-certificates.crt 2>/dev/null || true
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=https://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
- export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
- rclone sync dist/ :s3:foxhunt-binaries/latest/web-dashboard/dist/
- echo "Web dashboard assets uploaded to MinIO"
@@ -571,7 +577,9 @@ write-manifest:
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
script:
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=https://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1 RCLONE_CA_CERT=/etc/ssl/minio-ca/ca.crt
# Trust MinIO CA — rclone S3 backend uses Go's system CAs, not RCLONE_CA_CERT
- cat /etc/ssl/minio-ca/ca.crt >> /etc/ssl/certs/ca-certificates.crt 2>/dev/null || true
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=https://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
- export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
- echo "{\"sha\":\"${CI_COMMIT_SHA}\",\"ts\":\"$(date -Iseconds)\"}" | rclone rcat :s3:foxhunt-binaries/latest/MANIFEST.json
- echo "Manifest written for ${CI_COMMIT_SHA}"
@@ -612,7 +620,9 @@ write-manifest:
- nvidia-smi
- mkdir -p ${CI_PROJECT_DIR}/output ${CI_PROJECT_DIR}/bin
# Fetch training binaries from MinIO (in-cluster S3)
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=https://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1 RCLONE_CA_CERT=/etc/ssl/minio-ca/ca.crt
# Trust MinIO CA — rclone S3 backend uses Go's system CAs, not RCLONE_CA_CERT
- cat /etc/ssl/minio-ca/ca.crt >> /etc/ssl/certs/ca-certificates.crt 2>/dev/null || true
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=https://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
- export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
- rclone sync :s3:foxhunt-binaries/latest/training/ ${CI_PROJECT_DIR}/bin/
- chmod +x ${CI_PROJECT_DIR}/bin/*
@@ -652,7 +662,9 @@ write-manifest:
- nvidia-smi
- mkdir -p ${CI_PROJECT_DIR}/output ${CI_PROJECT_DIR}/bin
# Fetch training binaries from MinIO (in-cluster S3)
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=https://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1 RCLONE_CA_CERT=/etc/ssl/minio-ca/ca.crt
# Trust MinIO CA — rclone S3 backend uses Go's system CAs, not RCLONE_CA_CERT
- cat /etc/ssl/minio-ca/ca.crt >> /etc/ssl/certs/ca-certificates.crt 2>/dev/null || true
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=https://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
- export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
- rclone sync :s3:foxhunt-binaries/latest/training/ ${CI_PROJECT_DIR}/bin/
- chmod +x ${CI_PROJECT_DIR}/bin/*