gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(infra): add http:// scheme to MinIO endpoints, harden initContainer fallback

Browse Source 
Two bugs found during validation:

1. rclone defaults to HTTPS when endpoint has no scheme — MinIO serves
   plain HTTP. All RCLONE_S3_ENDPOINT values now include http:// prefix.

2. rclone copyto returns exit 0 for non-existent S3 keys (empty bucket).
   initContainers crashed on chmod of missing file instead of falling
   through to PVC cache. Added `&& [ -f /binaries/$SERVICE ]` guard.

Also: deploy stage uses `kubectl apply -k` (not -f) for minio/ dir
to properly handle kustomization.yaml.

Deleted stale s3-credentials secret from cluster.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-01 01:32:38 +01:00
parent 927551b33c
commit 1c4ab071ae
12 changed files with 27 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
.gitlab-ci.yml
View File

@@ -391,7 +391,7 @@ compile-services:
done
- ls -lh build-out/services/
# Upload stripped binaries to MinIO (in-cluster S3)
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=http://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
- export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
- |
for bin in trading_service api_gateway broker_gateway_service ml_training_service backtesting_service trading_agent_service data_acquisition_service web-gateway; do
@@ -452,7 +452,7 @@ compile-training:
strip build-out/training/training_uploader
- ls -lh build-out/training/
# Upload stripped binaries to MinIO (in-cluster S3)
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=http://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
- export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
- |
for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised training_uploader; do
@@ -489,7 +489,7 @@ build-web-dashboard:
- cd web-dashboard
- npm ci
- npm run build
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=http://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
- export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
- rclone sync dist/ :s3:foxhunt-binaries/latest/web-dashboard/dist/
- echo "Web dashboard assets uploaded to MinIO"
@@ -517,7 +517,7 @@ write-manifest:
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
- if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
script:
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=http://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
- export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
- echo "{\"sha\":\"${CI_COMMIT_SHA}\",\"ts\":\"$(date -Iseconds)\"}" | rclone rcat :s3:foxhunt-binaries/latest/MANIFEST.json
- echo "Manifest written for ${CI_COMMIT_SHA}"
@@ -557,7 +557,7 @@ write-manifest:
- nvidia-smi
- mkdir -p ${CI_PROJECT_DIR}/output ${CI_PROJECT_DIR}/bin
# Fetch training binaries from MinIO (in-cluster S3)
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=http://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
- export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
- rclone sync :s3:foxhunt-binaries/latest/training/ ${CI_PROJECT_DIR}/bin/
- chmod +x ${CI_PROJECT_DIR}/bin/*
@@ -596,7 +596,7 @@ write-manifest:
- nvidia-smi
- mkdir -p ${CI_PROJECT_DIR}/output ${CI_PROJECT_DIR}/bin
# Fetch training binaries from MinIO (in-cluster S3)
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
- export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=http://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
- export RCLONE_S3_ACCESS_KEY_ID=$MINIO_ACCESS_KEY RCLONE_S3_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
- rclone sync :s3:foxhunt-binaries/latest/training/ ${CI_PROJECT_DIR}/bin/
- chmod +x ${CI_PROJECT_DIR}/bin/*

4
infra/k8s/services/api-gateway.yaml
View File

@@ -39,7 +39,7 @@ spec:
- |
set -e
SERVICE=api_gateway
if rclone copyto :s3:foxhunt-binaries/latest/services/$SERVICE /binaries/$SERVICE 2>/dev/null; then
if rclone copyto :s3:foxhunt-binaries/latest/services/$SERVICE /binaries/$SERVICE 2>/dev/null && [ -f /binaries/$SERVICE ]; then
chmod +x /binaries/$SERVICE
cp /binaries/$SERVICE /cache/$SERVICE
echo "Fetched $SERVICE from S3 ($(stat -c%s /binaries/$SERVICE) bytes)"
@@ -55,7 +55,7 @@ spec:
- name: RCLONE_S3_PROVIDER
value: Minio
- name: RCLONE_S3_ENDPOINT
value: minio.foxhunt.svc.cluster.local:9000
value: "http://minio.foxhunt.svc.cluster.local:9000"
- name: RCLONE_S3_REGION
value: us-east-1
- name: RCLONE_S3_ACCESS_KEY_ID

4
infra/k8s/services/backtesting-service.yaml
View File

@@ -39,7 +39,7 @@ spec:
- |
set -e
SERVICE=backtesting_service
if rclone copyto :s3:foxhunt-binaries/latest/services/$SERVICE /binaries/$SERVICE 2>/dev/null; then
if rclone copyto :s3:foxhunt-binaries/latest/services/$SERVICE /binaries/$SERVICE 2>/dev/null && [ -f /binaries/$SERVICE ]; then
chmod +x /binaries/$SERVICE
cp /binaries/$SERVICE /cache/$SERVICE
echo "Fetched $SERVICE from S3 ($(stat -c%s /binaries/$SERVICE) bytes)"
@@ -55,7 +55,7 @@ spec:
- name: RCLONE_S3_PROVIDER
value: Minio
- name: RCLONE_S3_ENDPOINT
value: minio.foxhunt.svc.cluster.local:9000
value: "http://minio.foxhunt.svc.cluster.local:9000"
- name: RCLONE_S3_REGION
value: us-east-1
- name: RCLONE_S3_ACCESS_KEY_ID

4
infra/k8s/services/broker-gateway.yaml
View File

@@ -39,7 +39,7 @@ spec:
- |
set -e
SERVICE=broker_gateway_service
if rclone copyto :s3:foxhunt-binaries/latest/services/$SERVICE /binaries/$SERVICE 2>/dev/null; then
if rclone copyto :s3:foxhunt-binaries/latest/services/$SERVICE /binaries/$SERVICE 2>/dev/null && [ -f /binaries/$SERVICE ]; then
chmod +x /binaries/$SERVICE
cp /binaries/$SERVICE /cache/$SERVICE
echo "Fetched $SERVICE from S3 ($(stat -c%s /binaries/$SERVICE) bytes)"
@@ -55,7 +55,7 @@ spec:
- name: RCLONE_S3_PROVIDER
value: Minio
- name: RCLONE_S3_ENDPOINT
value: minio.foxhunt.svc.cluster.local:9000
value: "http://minio.foxhunt.svc.cluster.local:9000"
- name: RCLONE_S3_REGION
value: us-east-1
- name: RCLONE_S3_ACCESS_KEY_ID

4
infra/k8s/services/data-acquisition-service.yaml
View File

@@ -39,7 +39,7 @@ spec:
- |
set -e
SERVICE=data_acquisition_service
if rclone copyto :s3:foxhunt-binaries/latest/services/$SERVICE /binaries/$SERVICE 2>/dev/null; then
if rclone copyto :s3:foxhunt-binaries/latest/services/$SERVICE /binaries/$SERVICE 2>/dev/null && [ -f /binaries/$SERVICE ]; then
chmod +x /binaries/$SERVICE
cp /binaries/$SERVICE /cache/$SERVICE
echo "Fetched $SERVICE from S3 ($(stat -c%s /binaries/$SERVICE) bytes)"
@@ -55,7 +55,7 @@ spec:
- name: RCLONE_S3_PROVIDER
value: Minio
- name: RCLONE_S3_ENDPOINT
value: minio.foxhunt.svc.cluster.local:9000
value: "http://minio.foxhunt.svc.cluster.local:9000"
- name: RCLONE_S3_REGION
value: us-east-1
- name: RCLONE_S3_ACCESS_KEY_ID

4
infra/k8s/services/ml-training-service-gpu.yaml
View File

@@ -47,7 +47,7 @@ spec:
- |
set -e
SERVICE=ml_training_service
if rclone copyto :s3:foxhunt-binaries/latest/services/$SERVICE /binaries/$SERVICE 2>/dev/null; then
if rclone copyto :s3:foxhunt-binaries/latest/services/$SERVICE /binaries/$SERVICE 2>/dev/null && [ -f /binaries/$SERVICE ]; then
chmod +x /binaries/$SERVICE
cp /binaries/$SERVICE /cache/$SERVICE
echo "Fetched $SERVICE from S3 ($(stat -c%s /binaries/$SERVICE) bytes)"
@@ -63,7 +63,7 @@ spec:
- name: RCLONE_S3_PROVIDER
value: Minio
- name: RCLONE_S3_ENDPOINT
value: minio.foxhunt.svc.cluster.local:9000
value: "http://minio.foxhunt.svc.cluster.local:9000"
- name: RCLONE_S3_REGION
value: us-east-1
- name: RCLONE_S3_ACCESS_KEY_ID

4
infra/k8s/services/ml-training-service.yaml
View File

@@ -82,7 +82,7 @@ spec:
- |
set -e
SERVICE=ml_training_service
if rclone copyto :s3:foxhunt-binaries/latest/services/$SERVICE /binaries/$SERVICE 2>/dev/null; then
if rclone copyto :s3:foxhunt-binaries/latest/services/$SERVICE /binaries/$SERVICE 2>/dev/null && [ -f /binaries/$SERVICE ]; then
chmod +x /binaries/$SERVICE
cp /binaries/$SERVICE /cache/$SERVICE
echo "Fetched $SERVICE from S3 ($(stat -c%s /binaries/$SERVICE) bytes)"
@@ -98,7 +98,7 @@ spec:
- name: RCLONE_S3_PROVIDER
value: Minio
- name: RCLONE_S3_ENDPOINT
value: minio.foxhunt.svc.cluster.local:9000
value: "http://minio.foxhunt.svc.cluster.local:9000"
- name: RCLONE_S3_REGION
value: us-east-1
- name: RCLONE_S3_ACCESS_KEY_ID

4
infra/k8s/services/trading-agent-service.yaml
View File

@@ -39,7 +39,7 @@ spec:
- |
set -e
SERVICE=trading_agent_service
if rclone copyto :s3:foxhunt-binaries/latest/services/$SERVICE /binaries/$SERVICE 2>/dev/null; then
if rclone copyto :s3:foxhunt-binaries/latest/services/$SERVICE /binaries/$SERVICE 2>/dev/null && [ -f /binaries/$SERVICE ]; then
chmod +x /binaries/$SERVICE
cp /binaries/$SERVICE /cache/$SERVICE
echo "Fetched $SERVICE from S3 ($(stat -c%s /binaries/$SERVICE) bytes)"
@@ -55,7 +55,7 @@ spec:
- name: RCLONE_S3_PROVIDER
value: Minio
- name: RCLONE_S3_ENDPOINT
value: minio.foxhunt.svc.cluster.local:9000
value: "http://minio.foxhunt.svc.cluster.local:9000"
- name: RCLONE_S3_REGION
value: us-east-1
- name: RCLONE_S3_ACCESS_KEY_ID

4
infra/k8s/services/trading-service-gpu.yaml
View File

@@ -46,7 +46,7 @@ spec:
- |
set -e
SERVICE=trading_service
if rclone copyto :s3:foxhunt-binaries/latest/services/$SERVICE /binaries/$SERVICE 2>/dev/null; then
if rclone copyto :s3:foxhunt-binaries/latest/services/$SERVICE /binaries/$SERVICE 2>/dev/null && [ -f /binaries/$SERVICE ]; then
chmod +x /binaries/$SERVICE
cp /binaries/$SERVICE /cache/$SERVICE
echo "Fetched $SERVICE from S3 ($(stat -c%s /binaries/$SERVICE) bytes)"
@@ -62,7 +62,7 @@ spec:
- name: RCLONE_S3_PROVIDER
value: Minio
- name: RCLONE_S3_ENDPOINT
value: minio.foxhunt.svc.cluster.local:9000
value: "http://minio.foxhunt.svc.cluster.local:9000"
- name: RCLONE_S3_REGION
value: us-east-1
- name: RCLONE_S3_ACCESS_KEY_ID

4
infra/k8s/services/trading-service.yaml
View File

@@ -39,7 +39,7 @@ spec:
- |
set -e
SERVICE=trading_service
if rclone copyto :s3:foxhunt-binaries/latest/services/$SERVICE /binaries/$SERVICE 2>/dev/null; then
if rclone copyto :s3:foxhunt-binaries/latest/services/$SERVICE /binaries/$SERVICE 2>/dev/null && [ -f /binaries/$SERVICE ]; then
chmod +x /binaries/$SERVICE
cp /binaries/$SERVICE /cache/$SERVICE
echo "Fetched $SERVICE from S3 ($(stat -c%s /binaries/$SERVICE) bytes)"
@@ -55,7 +55,7 @@ spec:
- name: RCLONE_S3_PROVIDER
value: Minio
- name: RCLONE_S3_ENDPOINT
value: minio.foxhunt.svc.cluster.local:9000
value: "http://minio.foxhunt.svc.cluster.local:9000"
- name: RCLONE_S3_REGION
value: us-east-1
- name: RCLONE_S3_ACCESS_KEY_ID

4
infra/k8s/services/web-gateway.yaml
View File

@@ -35,7 +35,7 @@ spec:
- |
set -e
SERVICE=web-gateway
if rclone copyto :s3:foxhunt-binaries/latest/services/$SERVICE /binaries/$SERVICE 2>/dev/null; then
if rclone copyto :s3:foxhunt-binaries/latest/services/$SERVICE /binaries/$SERVICE 2>/dev/null && [ -f /binaries/$SERVICE ]; then
chmod +x /binaries/$SERVICE
cp /binaries/$SERVICE /cache/$SERVICE
rclone sync :s3:foxhunt-binaries/latest/web-dashboard/dist/ /binaries/static/
@@ -54,7 +54,7 @@ spec:
- name: RCLONE_S3_PROVIDER
value: Minio
- name: RCLONE_S3_ENDPOINT
value: minio.foxhunt.svc.cluster.local:9000
value: "http://minio.foxhunt.svc.cluster.local:9000"
- name: RCLONE_S3_REGION
value: us-east-1
- name: RCLONE_S3_ACCESS_KEY_ID

2
infra/k8s/training/job-template.yaml
View File

@@ -45,7 +45,7 @@ spec:
args:
- |
set -e
export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
export RCLONE_S3_PROVIDER=Minio RCLONE_S3_ENDPOINT=http://minio.foxhunt.svc.cluster.local:9000 RCLONE_S3_REGION=us-east-1
rclone sync :s3:foxhunt-binaries/latest/training/ /binaries/
chmod +x /binaries/*
ls -lh /binaries/