From cc209aec7eaa2db9d293054f54b311ea1d83f4c9 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 4 Mar 2026 16:46:57 +0100 Subject: [PATCH] feat(infra): replace PVC binary distribution with MinIO S3 fetch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eliminate foxhunt-binaries and training-binaries PVCs — services and training jobs now fetch binaries from MinIO via rclone initContainers. This removes L40S GPU autoscale-for-PVC-writes, removes RWO node affinity constraints, and requires zero image rebuilds. Changes: - .gitlab-ci.yml: replace ~115 lines of binary-writer pod logic with aws s3 cp to MinIO (~25 lines) - 8 service YAMLs + 2 GPU overlays: add rclone initContainer + emptyDir - Training job template: MinIO rclone fetch replaces PVC copy - Delete foxhunt-binaries-pvc.yaml and training-binaries-pvc.yaml - Add s3.fxhnt.ai DNS record (Terraform) and nginx proxy block - Replace pod-writer-deploy skill with MinIO-based deploy skill Co-Authored-By: Claude Opus 4.6 --- .claude/skills/pod-writer-deploy/SKILL.md | 161 ++++++ .gitlab-ci.yml | 121 +--- ...-03-04-minio-binary-distribution-design.md | 263 +++++++++ .../2026-03-04-minio-binary-distribution.md | 534 ++++++++++++++++++ infra/k8s/gitlab/tailscale-proxy.yaml | 30 + .../gpu-overlays/ml-training-service-gpu.yaml | 48 +- .../k8s/gpu-overlays/trading-service-gpu.yaml | 48 +- infra/k8s/services/api-gateway.yaml | 44 +- infra/k8s/services/backtesting-service.yaml | 44 +- infra/k8s/services/broker-gateway.yaml | 44 +- .../services/data-acquisition-service.yaml | 44 +- infra/k8s/services/ml-training-service.yaml | 44 +- infra/k8s/services/trading-agent-service.yaml | 44 +- infra/k8s/services/trading-service.yaml | 44 +- infra/k8s/services/web-gateway.yaml | 39 +- infra/k8s/storage/foxhunt-binaries-pvc.yaml | 18 - infra/k8s/training/job-template.yaml | 46 +- infra/k8s/training/training-binaries-pvc.yaml | 18 - infra/modules/dns/main.tf | 8 + 19 files changed, 1444 insertions(+), 198 deletions(-) create mode 100644 .claude/skills/pod-writer-deploy/SKILL.md create mode 100644 docs/plans/2026-03-04-minio-binary-distribution-design.md create mode 100644 docs/plans/2026-03-04-minio-binary-distribution.md delete mode 100644 infra/k8s/storage/foxhunt-binaries-pvc.yaml delete mode 100644 infra/k8s/training/training-binaries-pvc.yaml diff --git a/.claude/skills/pod-writer-deploy/SKILL.md b/.claude/skills/pod-writer-deploy/SKILL.md new file mode 100644 index 000000000..e4b8d83d4 --- /dev/null +++ b/.claude/skills/pod-writer-deploy/SKILL.md @@ -0,0 +1,161 @@ +--- +name: "MinIO Binary Deploy" +description: "Deploy Rust service/training binaries to the Kapsule cluster via MinIO S3. Use when deploying any foxhunt binary (trading_service, api_gateway, train_baseline_supervised, etc.) to the Scaleway Kapsule cluster." +--- + +# MinIO S3 Binary Deployment + +## What This Skill Does + +Deploys locally-built Rust binaries to MinIO S3 (`foxhunt-binaries` bucket), then triggers a pod rollout so service initContainers fetch the new binary. No Docker image builds, no PVCs, no binary-writer pods. + +## When to Use + +- Deploying any service or training binary after local code changes +- User says "deploy", "push to cluster", "update the service", "minio deploy" +- After building a binary in release mode + +## Architecture + +All foxhunt services and training jobs use a shared deployment pattern: +- **Base images**: `foxhunt-runtime:latest` (services) / `foxhunt-training-runtime:latest` (training) +- **Binary source**: MinIO S3 bucket `foxhunt-binaries` at `minio.foxhunt.svc.cluster.local:9000` +- **Fetch method**: rclone initContainer fetches binary to emptyDir at pod startup +- **External access**: `s3.fxhnt.ai` via tailscale proxy (for external GPU nodes) + +## Service → Binary → Deployment Mapping + +| Service (Cargo crate) | Binary name | K8s Deployment name | S3 path | +|-------------------------------|------------------------------|-----------------------------|---------------------------------------| +| `trading_service` | `trading_service` | `trading-service` | `services/trading_service` | +| `api_gateway` | `api_gateway` | `api-gateway` | `services/api_gateway` | +| `broker_gateway_service` | `broker_gateway_service` | `broker-gateway` | `services/broker_gateway_service` | +| `ml_training_service` | `ml_training_service` | `ml-training-service` | `services/ml_training_service` | +| `backtesting_service` | `backtesting_service` | `backtesting-service` | `services/backtesting_service` | +| `trading_agent_service` | `trading_agent_service` | `trading-agent-service` | `services/trading_agent_service` | +| `data_acquisition_service` | `data_acquisition_service` | `data-acquisition-service` | `services/data_acquisition_service` | +| `web-gateway` | `web-gateway` | `web-gateway` | `services/web-gateway` | + +### Training Binaries + +| Binary | S3 path | +|-----------------------------------|------------------------------------------| +| `train_baseline_rl` | `training/train_baseline_rl` | +| `train_baseline_supervised` | `training/train_baseline_supervised` | +| `evaluate_baseline` | `training/evaluate_baseline` | +| `hyperopt_baseline_rl` | `training/hyperopt_baseline_rl` | +| `hyperopt_baseline_supervised` | `training/hyperopt_baseline_supervised` | + +**Note**: `web-gateway` is in `crates/web-gateway/`, all others are in `services/`. + +## Procedure + +### Step 1: Build the binary (release mode) + +```bash +SQLX_OFFLINE=true cargo build --release -p +``` + +The binary will be at `target/release/`. + +### Step 2: Upload to MinIO via rclone + +```bash +# Service binary: +rclone copyto target/release/ \ + :s3:foxhunt-binaries/services/ \ + --s3-provider=Minio \ + --s3-endpoint=https://s3.fxhnt.ai \ + --s3-access-key-id="${MINIO_ACCESS_KEY}" \ + --s3-secret-access-key="${MINIO_SECRET_KEY}" \ + --s3-no-check-bucket + +# Training binary: +rclone copyto target/release/ \ + :s3:foxhunt-binaries/training/ \ + --s3-provider=Minio \ + --s3-endpoint=https://s3.fxhnt.ai \ + --s3-access-key-id="${MINIO_ACCESS_KEY}" \ + --s3-secret-access-key="${MINIO_SECRET_KEY}" \ + --s3-no-check-bucket +``` + +**Alternative using aws-cli:** + +```bash +export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}" +export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}" + +# Service binary: +aws s3 cp target/release/ \ + s3://foxhunt-binaries/services/ \ + --endpoint-url https://s3.fxhnt.ai + +# Training binary: +aws s3 cp target/release/ \ + s3://foxhunt-binaries/training/ \ + --endpoint-url https://s3.fxhnt.ai +``` + +### Step 3: Restart the deployment to fetch new binary + +```bash +# Service: +kubectl rollout restart deployment/ -n foxhunt +kubectl rollout status deployment/ -n foxhunt --timeout=120s + +# Training jobs pick up the new binary automatically on next job creation. +``` + +## Deploying Multiple Services at Once + +```bash +# Build all changed services +SQLX_OFFLINE=true cargo build --release -p trading_service -p api_gateway + +# Upload all +export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}" +export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}" +S3="aws s3 --endpoint-url https://s3.fxhnt.ai" +$S3 cp target/release/trading_service s3://foxhunt-binaries/services/trading_service +$S3 cp target/release/api_gateway s3://foxhunt-binaries/services/api_gateway + +# Restart all affected deployments +kubectl rollout restart deployment/trading-service deployment/api-gateway -n foxhunt +``` + +## Web Dashboard Static Assets + +```bash +cd web-dashboard && npm run build +aws s3 cp dist/ s3://foxhunt-binaries/static/ --recursive \ + --endpoint-url https://s3.fxhnt.ai +kubectl rollout restart deployment/web-gateway -n foxhunt +``` + +## Troubleshooting + +### InitContainer fails with connection error +- MinIO must be running: `kubectl get pod -n foxhunt -l app.kubernetes.io/name=minio` +- Check MinIO CA cert: `kubectl get configmap minio-ca-cert -n foxhunt` +- From inside cluster, `--no-check-certificate` bypasses self-signed CA + +### Service CrashLoopBackOff after deploy +- Binary may have been built for the wrong target. All cluster nodes are x86_64 Linux. +- Check logs: `kubectl logs -n foxhunt deployment/ -c fetch-binary` +- Check main container: `kubectl logs -n foxhunt deployment/ --tail=50` + +### Upload fails from local machine +- Ensure `s3.fxhnt.ai` resolves (DNS via Scaleway, proxied through tailscale) +- Ensure you're on the tailnet: `tailscale status` +- Verify credentials: `aws s3 ls --endpoint-url https://s3.fxhnt.ai s3://foxhunt-binaries/` + +## Important Constraints + +- **No Docker builds needed**: The runtime images are generic and pre-built. +- **Namespace**: Always `foxhunt`. +- **MinIO bucket**: `foxhunt-binaries` (services/ and training/ prefixes). +- **External endpoint**: `s3.fxhnt.ai` (via tailscale proxy). +- **Internal endpoint**: `minio.foxhunt.svc.cluster.local:9000` (used by initContainers). +- **SQLX_OFFLINE=true**: Required for builds (no PostgreSQL running locally). +- **Credentials**: `MINIO_ACCESS_KEY` / `MINIO_SECRET_KEY` (same as in k8s secret `minio-credentials`). diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0606ee8fe..ecaaa0d12 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1561,117 +1561,32 @@ deploy: echo "Dashboard ConfigMaps deployed" # Apply proxy (picks up nginx config changes for dashboard.fxhnt.ai routing) - kubectl apply -f infra/k8s/gitlab/tailscale-proxy.yaml - # Apply PVCs (idempotent, must exist before writer pods or services reference them) - - kubectl apply -f infra/k8s/storage/foxhunt-binaries-pvc.yaml - - kubectl apply -f infra/k8s/training/training-binaries-pvc.yaml - # ── Step 1: Copy binaries to PVCs BEFORE applying service manifests ── - # This ensures binaries exist on disk before any pod tries to exec them. + # ── Step 1: Upload binaries to MinIO (replaces PVC binary-writer pods) ── - | - if [ -d "build-out/services" ] || [ -d "build-out/training" ]; then + if [ -d "build-out/services" ] || [ -d "build-out/training" ] || [ -d "web-dashboard/dist" ]; then set -e - echo "=== Deploying binaries to PVCs ===" - # Clean up any stale writer pod from a previous failed deploy - kubectl delete pod binary-writer -n foxhunt --ignore-not-found --wait=true 2>/dev/null || true - kubectl delete pod training-binary-writer -n foxhunt --ignore-not-found --wait=true 2>/dev/null || true - # --- Service binaries → foxhunt-binaries PVC (foxhunt node) --- + echo "=== Uploading binaries to MinIO ===" + export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}" + export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}" + S3="aws s3 --endpoint-url https://minio.foxhunt.svc.cluster.local:9000 --no-verify-ssl" + if [ -d "build-out/services" ]; then - echo "--- Writing service binaries to foxhunt-binaries PVC ---" - kubectl run binary-writer -n foxhunt \ - --image=busybox:1.36 \ - --restart=Never \ - --command \ - --overrides='{ - "spec": { - "nodeSelector": {"k8s.scaleway.com/pool-name": "foxhunt"}, - "securityContext": {"runAsUser": 1000, "runAsGroup": 1000, "fsGroup": 1000}, - "volumes": [{"name": "binaries", "persistentVolumeClaim": {"claimName": "foxhunt-binaries"}}], - "containers": [{"name": "binary-writer", "image": "busybox:1.36", "command": ["sleep", "600"], - "volumeMounts": [{"name": "binaries", "mountPath": "/binaries"}]}] - } - }' \ - -- sleep 600 - kubectl wait --for=condition=Ready pod/binary-writer -n foxhunt --timeout=120s - for bin in trading_service api_gateway broker_gateway_service \ - ml_training_service backtesting_service \ - trading_agent_service data_acquisition_service web-gateway; do - if [ -f "build-out/services/$bin" ]; then - kubectl cp "build-out/services/$bin" foxhunt/binary-writer:/binaries/$bin - kubectl exec -n foxhunt binary-writer -- chmod +x /binaries/$bin - echo " Copied $bin ($(stat -c%s "build-out/services/$bin") bytes)" - else - echo " WARNING: build-out/services/$bin not found, skipping" - fi - done - # Copy training binaries to service PVC (for ml-training-service to read) - if [ -d "build-out/training" ]; then - kubectl exec -n foxhunt binary-writer -- mkdir -p /binaries/training - for bin in train_baseline_rl train_baseline_supervised evaluate_baseline \ - evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised \ - training_uploader; do - if [ -f "build-out/training/$bin" ]; then - kubectl cp "build-out/training/$bin" foxhunt/binary-writer:/binaries/training/$bin - kubectl exec -n foxhunt binary-writer -- chmod +x /binaries/training/$bin - echo " Copied training/$bin ($(stat -c%s "build-out/training/$bin") bytes)" - fi - done - fi - # Copy web dashboard static assets - if [ -d "web-dashboard/dist" ]; then - kubectl exec -n foxhunt binary-writer -- mkdir -p /binaries/static - tar -C web-dashboard/dist -cf - . | kubectl exec -i -n foxhunt binary-writer -- tar -C /binaries/static -xf - - echo " Copied web-dashboard static assets" - fi - kubectl delete pod binary-writer -n foxhunt --wait=false - echo "Service binaries deployed to foxhunt-binaries PVC" + $S3 cp build-out/services/ s3://foxhunt-binaries/services/ --recursive + echo "Service binaries uploaded to MinIO" fi - # --- Training binaries → training-binaries PVC (ci-training node) --- - # Training jobs run on ci-training (GPU) node — separate RWO PVC needed. - # Best-effort: GPU node may be scaled to zero. CI training uses artifacts anyway. if [ -d "build-out/training" ]; then - set +e # training PVC is non-critical — don't kill deploy if GPU node is down - echo "--- Writing training binaries to training-binaries PVC (best-effort) ---" - kubectl run training-binary-writer -n foxhunt \ - --image=busybox:1.36 \ - --restart=Never \ - --command \ - --overrides='{ - "spec": { - "nodeSelector": {"k8s.scaleway.com/pool-name": "ci-training"}, - "tolerations": [ - {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, - {"key": "node.cilium.io/agent-not-ready", "operator": "Exists", "effect": "NoSchedule"} - ], - "securityContext": {"runAsUser": 1000, "runAsGroup": 1000, "fsGroup": 1000}, - "volumes": [{"name": "binaries", "persistentVolumeClaim": {"claimName": "training-binaries"}}], - "containers": [{"name": "training-binary-writer", "image": "busybox:1.36", "command": ["sleep", "600"], - "volumeMounts": [{"name": "binaries", "mountPath": "/binaries"}]}] - } - }' \ - -- sleep 600 - if kubectl wait --for=condition=Ready pod/training-binary-writer -n foxhunt --timeout=300s 2>/dev/null; then - for bin in train_baseline_rl train_baseline_supervised evaluate_baseline \ - evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised \ - training_uploader; do - if [ -f "build-out/training/$bin" ]; then - kubectl cp "build-out/training/$bin" foxhunt/training-binary-writer:/binaries/$bin - kubectl exec -n foxhunt training-binary-writer -- chmod +x /binaries/$bin - echo " Copied training/$bin ($(stat -c%s "build-out/training/$bin") bytes)" - fi - done - kubectl delete pod training-binary-writer -n foxhunt --wait=false - echo "Training binaries deployed to training-binaries PVC" - else - echo "WARNING: training-binary-writer did not become ready (GPU node may be scaled to zero)" - echo "Training binaries NOT deployed — CI training jobs use artifacts, ad-hoc jobs will fail" - kubectl delete pod training-binary-writer -n foxhunt --ignore-not-found --wait=false 2>/dev/null || true - fi - set -e # re-enable for remaining steps + $S3 cp build-out/training/ s3://foxhunt-binaries/training/ --recursive + echo "Training binaries uploaded to MinIO" fi - echo "=== Binary deployment complete ===" + if [ -d "web-dashboard/dist" ]; then + $S3 cp web-dashboard/dist/ s3://foxhunt-binaries/static/ --recursive + echo "Web dashboard assets uploaded to MinIO" + fi + echo "=== MinIO upload complete ===" else - echo "No new binaries — compile jobs did not run, PVCs have existing binaries" + echo "No new binaries — compile jobs did not run" fi - # ── Step 2: Apply service manifests (binaries already on PVC) ── + # ── Step 2: Apply service manifests (binaries uploaded to MinIO) ── # GPU overlays are in gpu-overlays/ — NOT auto-applied. Apply manually when needed. - kubectl apply -f infra/k8s/services/ # ── Step 3: Rolling restart + wait ── diff --git a/docs/plans/2026-03-04-minio-binary-distribution-design.md b/docs/plans/2026-03-04-minio-binary-distribution-design.md new file mode 100644 index 000000000..0728e0cc4 --- /dev/null +++ b/docs/plans/2026-03-04-minio-binary-distribution-design.md @@ -0,0 +1,263 @@ +# MinIO Binary Distribution — Eliminate PVC Binary Writers + +**Date:** 2026-03-04 +**Status:** Approved +**Scope:** Replace PVC-based binary distribution with MinIO S3 fetch pattern + +## Problem + +The current deploy cycle uses `kubectl cp` via temporary busybox writer pods to copy +compiled binaries onto RWO PVCs. Two PVCs exist: + +- `foxhunt-binaries` (foxhunt node) — service binaries +- `training-binaries` (ci-training node) — training binaries + +The training PVC writer triggers L40S GPU autoscale from zero just to receive a binary +copy — expensive and slow. RWO PVCs also create node-affinity constraints that prevent +cross-pool flexibility. + +## Solution + +Use MinIO (already deployed on `platform` node) as the binary distribution layer. + +**Flow:** +``` +compile jobs → GitLab artifacts (source of truth) + ↓ + deploy job: aws s3 cp → MinIO foxhunt-binaries bucket + ↓ + service pods: rclone initContainer → emptyDir + training jobs: rclone initContainer → emptyDir + external GPUs: GitLab artifacts API +``` + +**Key insight:** Zero image rebuilds. `aws-cli` is already in infra-runner, `rclone` +is already in both runtime images. + +## Changes + +### 1. Deploy Job (`.gitlab-ci.yml`) + +**Delete:** ~115 lines of binary-writer pod logic (lines 1567-1681) +**Add:** ~20 lines of `aws s3 cp` to MinIO + +```yaml +# Upload binaries to MinIO (replaces binary-writer pods) +- | + if [ -d "build-out/services" ] || [ -d "build-out/training" ]; then + export AWS_ACCESS_KEY_ID="$MINIO_ACCESS_KEY" + export AWS_SECRET_ACCESS_KEY="$MINIO_SECRET_KEY" + S3="aws s3 --endpoint-url https://minio.foxhunt.svc.cluster.local:9000 --no-verify-ssl" + + if [ -d "build-out/services" ]; then + $S3 cp build-out/services/ s3://foxhunt-binaries/services/ --recursive + echo "Service binaries → MinIO" + fi + if [ -d "build-out/training" ]; then + $S3 cp build-out/training/ s3://foxhunt-binaries/training/ --recursive + echo "Training binaries → MinIO" + fi + if [ -d "web-dashboard/dist" ]; then + $S3 cp web-dashboard/dist/ s3://foxhunt-binaries/static/ --recursive + echo "Web dashboard assets → MinIO" + fi + fi +``` + +Also remove: +- `kubectl apply -f infra/k8s/storage/foxhunt-binaries-pvc.yaml` +- `kubectl apply -f infra/k8s/training/training-binaries-pvc.yaml` + +Add MinIO credentials as CI variables (`MINIO_ACCESS_KEY`, `MINIO_SECRET_KEY`) or +source from existing `minio-credentials` secret. + +### 2. Service Deployments (8 files in `infra/k8s/services/`) + +For each service, replace PVC volume mount with initContainer + emptyDir: + +```yaml +spec: + template: + spec: + initContainers: + - name: fetch-binary + image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest + command: ["/bin/sh", "-c"] + args: + - | + rclone copyto \ + ":s3:foxhunt-binaries/services/${SERVICE_NAME}" \ + "/binaries/${SERVICE_NAME}" \ + --s3-provider=Minio \ + --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-access-key-id="${MINIO_ACCESS_KEY}" \ + --s3-secret-access-key="${MINIO_SECRET_KEY}" \ + --s3-no-check-bucket \ + --no-check-certificate + chmod +x "/binaries/${SERVICE_NAME}" + echo "Fetched ${SERVICE_NAME} ($(stat -c%s /binaries/${SERVICE_NAME}) bytes)" + env: + - name: SERVICE_NAME + value: trading_service # per-service value + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: access-key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: secret-key + volumeMounts: + - name: binaries + mountPath: /binaries + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 128Mi + # Also fetch web-dashboard static assets for web-gateway + volumes: + - name: binaries + emptyDir: + sizeLimit: 200Mi # single stripped binary ~20-50MB +``` + +Remove from each service: +```yaml +# DELETE these lines: +volumes: + - name: binaries + persistentVolumeClaim: + claimName: foxhunt-binaries + readOnly: true +``` + +**web-gateway** additionally needs static assets fetched: +```yaml +# Extra step in web-gateway initContainer: +rclone copy ":s3:foxhunt-binaries/static/" "/binaries/static/" \ + --s3-provider=Minio ... +``` + +### 3. Training Job Template (`infra/k8s/training/job-template.yaml`) + +Replace the PVC-based fetch-binary initContainer with MinIO rclone: + +```yaml +initContainers: + - name: fetch-binary + image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-training-runtime:latest + command: ["/bin/sh", "-c"] + args: + - | + rclone copyto \ + ":s3:foxhunt-binaries/training/${TRAINING_BINARY}" \ + "/binaries/${TRAINING_BINARY}" \ + --s3-provider=Minio \ + --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-access-key-id=${MINIO_ACCESS_KEY} \ + --s3-secret-access-key=${MINIO_SECRET_KEY} \ + --s3-no-check-bucket \ + --no-check-certificate + chmod +x "/binaries/${TRAINING_BINARY}" + env: + - name: TRAINING_BINARY + value: train_baseline_supervised + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: access-key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: secret-key + volumeMounts: + - name: binaries + mountPath: /binaries +``` + +Remove `training-binaries-src` volume (PVC mount). + +### 4. Files to Delete + +- `infra/k8s/storage/foxhunt-binaries-pvc.yaml` +- `infra/k8s/training/training-binaries-pvc.yaml` + +### 5. Files Unchanged + +- `training-data-pvc.yaml` — keep as-is +- `sccache-pvc.yaml` — keep +- `minio-data-new` (in minio.yaml) — keep +- CI training jobs (`.train-rl-base`, `.train-validate-base`) — already use GL artifacts via `needs:` + +## Service-to-Binary Mapping + +| Service Deployment | Binary Name | Pool | +|--------------------|-------------|------| +| trading-service | trading_service | foxhunt | +| api-gateway | api_gateway | foxhunt | +| broker-gateway | broker_gateway_service | foxhunt | +| ml-training-service | ml_training_service | foxhunt | +| backtesting-service | backtesting_service | foxhunt | +| trading-agent-service | trading_agent_service | foxhunt | +| data-acquisition-service | data_acquisition_service | foxhunt | +| web-gateway | web-gateway | foxhunt | + +## MinIO Bucket Layout + +``` +foxhunt-binaries/ +├── services/ +│ ├── trading_service +│ ├── api_gateway +│ ├── broker_gateway_service +│ ├── ml_training_service +│ ├── backtesting_service +│ ├── trading_agent_service +│ ├── data_acquisition_service +│ └── web-gateway +├── training/ +│ ├── train_baseline_rl +│ ├── train_baseline_supervised +│ ├── evaluate_baseline +│ ├── evaluate_supervised +│ ├── hyperopt_baseline_rl +│ ├── hyperopt_baseline_supervised +│ └── training_uploader +└── static/ + └── (web-dashboard dist files) +``` + +## External GPU Access + +External GPUs download training binaries from GitLab artifacts API: +```bash +curl --header "PRIVATE-TOKEN: $GL_TOKEN" \ + "https:///api/v4/projects//jobs/artifacts/main/download?job=compile-training" \ + -o artifacts.zip +``` + +Or from MinIO via Tailscale (if MinIO is exposed). + +## Rollback + +If MinIO is down during deploy, the deploy job fails fast (aws s3 cp fails). +Service pods with existing emptyDir continue running. New pods fail to start +(initContainer fails). MinIO recovery restores normal operation. + +Previous binaries remain in MinIO until overwritten — no expiry, no garbage. + +## Net Effect + +- **Deleted:** ~115 lines of binary-writer pod logic +- **Deleted:** 2 PVC manifests +- **Added:** ~20 lines of aws s3 cp in deploy job +- **Modified:** 8 service yamls + 1 job template (PVC → initContainer+emptyDir) +- **Image rebuilds:** 0 +- **L40S autoscale for PVC writes:** eliminated diff --git a/docs/plans/2026-03-04-minio-binary-distribution.md b/docs/plans/2026-03-04-minio-binary-distribution.md new file mode 100644 index 000000000..4f6fd3625 --- /dev/null +++ b/docs/plans/2026-03-04-minio-binary-distribution.md @@ -0,0 +1,534 @@ +# MinIO Binary Distribution Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Replace PVC-based binary distribution with MinIO S3 fetch, eliminating the L40S autoscale-for-PVC-writes problem. + +**Architecture:** Deploy job uploads compiled binaries to MinIO via `aws s3 cp`. Service pods and training jobs fetch their binary at startup via rclone initContainer into an emptyDir. Two binary PVCs are deleted. + +**Tech Stack:** MinIO (in-cluster S3), rclone (in runtime images), aws-cli (in infra-runner) + +--- + +### Task 1: Replace binary-writer pods with MinIO upload in deploy job + +**Files:** +- Modify: `.gitlab-ci.yml:1564-1681` (deploy job binary section) + +**Step 1: Delete PVC apply lines and entire binary-writer block** + +In `.gitlab-ci.yml`, find and delete these two PVC apply lines (around line 1565-1566): +```yaml + - kubectl apply -f infra/k8s/storage/foxhunt-binaries-pvc.yaml + - kubectl apply -f infra/k8s/training/training-binaries-pvc.yaml +``` + +Then delete the entire block from `# ── Step 1: Copy binaries to PVCs` through `echo "=== Binary deployment complete ==="` (lines ~1567-1681). This removes all binary-writer/training-binary-writer pod logic. + +**Step 2: Add MinIO upload block** + +Insert in place of the deleted block (same indentation level as other `- |` script blocks in the deploy job): + +```yaml + # ── Step 1: Upload binaries to MinIO (replaces PVC binary-writer pods) ── + - | + if [ -d "build-out/services" ] || [ -d "build-out/training" ] || [ -d "web-dashboard/dist" ]; then + set -e + echo "=== Uploading binaries to MinIO ===" + export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}" + export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}" + S3="aws s3 --endpoint-url https://minio.foxhunt.svc.cluster.local:9000 --no-verify-ssl" + + if [ -d "build-out/services" ]; then + $S3 cp build-out/services/ s3://foxhunt-binaries/services/ --recursive + echo "Service binaries uploaded to MinIO" + fi + if [ -d "build-out/training" ]; then + $S3 cp build-out/training/ s3://foxhunt-binaries/training/ --recursive + echo "Training binaries uploaded to MinIO" + fi + if [ -d "web-dashboard/dist" ]; then + $S3 cp web-dashboard/dist/ s3://foxhunt-binaries/static/ --recursive + echo "Web dashboard assets uploaded to MinIO" + fi + echo "=== MinIO upload complete ===" + else + echo "No new binaries — compile jobs did not run" + fi +``` + +**Step 3: Verify YAML syntax** + +Run: `python3 -c "import yaml; yaml.safe_load(open('.gitlab-ci.yml'))" && echo OK` +Expected: `OK` + +**Step 4: Commit** + +```bash +git add .gitlab-ci.yml +git commit -m "refactor(ci): replace PVC binary-writers with MinIO upload + +Deploy job now uploads binaries to MinIO via aws-cli (already in +infra-runner image). Eliminates binary-writer and training-binary-writer +pods, removing the L40S autoscale trigger for PVC writes." +``` + +--- + +### Task 2: Update trading-service.yaml — PVC to initContainer+emptyDir + +**Files:** +- Modify: `infra/k8s/services/trading-service.yaml` + +**Step 1: Add initContainer block** + +Insert after `nodeSelector:` / before `containers:` (between lines 39 and 40): + +```yaml + initContainers: + - name: fetch-binary + image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest + command: ["/bin/sh", "-c"] + args: + - | + set -e + rclone copyto \ + ":s3:foxhunt-binaries/services/trading_service" \ + "/binaries/trading_service" \ + --s3-provider=Minio \ + --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-access-key-id="${MINIO_ACCESS_KEY}" \ + --s3-secret-access-key="${MINIO_SECRET_KEY}" \ + --s3-no-check-bucket \ + --no-check-certificate + chmod +x /binaries/trading_service + echo "Fetched trading_service ($(stat -c%s /binaries/trading_service) bytes)" + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: access-key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: secret-key + volumeMounts: + - name: binaries + mountPath: /binaries + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 128Mi +``` + +**Step 2: Replace PVC volume with emptyDir** + +Replace: +```yaml + - name: binaries + persistentVolumeClaim: + claimName: foxhunt-binaries + readOnly: true +``` + +With: +```yaml + - name: binaries + emptyDir: + sizeLimit: 200Mi +``` + +**Step 3: Remove readOnly from main container's binaries volumeMount** + +The main container `volumeMounts` has `readOnly: true` on the binaries mount. The initContainer needs write access. Change: +```yaml + - name: binaries + mountPath: /binaries + readOnly: true +``` +To: +```yaml + - name: binaries + mountPath: /binaries + readOnly: true +``` + +Actually — the initContainer writes, the main container reads. emptyDir supports this: the initContainer writes, then terminates, then the main container mounts the same emptyDir read-only. Keep `readOnly: true` on the main container's mount. The initContainer's mount does NOT have `readOnly: true`. + +**Step 4: Verify YAML** + +Run: `python3 -c "import yaml; list(yaml.safe_load_all(open('infra/k8s/services/trading-service.yaml')))" && echo OK` +Expected: `OK` + +--- + +### Task 3: Update api-gateway.yaml + +**Files:** +- Modify: `infra/k8s/services/api-gateway.yaml` + +Same pattern as Task 2. Binary name: `api_gateway` + +**Step 1: Add initContainer** (between `nodeSelector:` and `containers:`, after line 40) + +Same initContainer as Task 2 but with binary name `api_gateway`: +- `rclone copyto ":s3:foxhunt-binaries/services/api_gateway" "/binaries/api_gateway"` +- `chmod +x /binaries/api_gateway` +- `echo "Fetched api_gateway ..."` + +**Step 2: Replace PVC volume with emptyDir** (lines 122-126) + +Replace: +```yaml + - name: binaries + persistentVolumeClaim: + claimName: foxhunt-binaries + readOnly: true +``` +With: +```yaml + - name: binaries + emptyDir: + sizeLimit: 200Mi +``` + +--- + +### Task 4: Update broker-gateway.yaml + +**Files:** +- Modify: `infra/k8s/services/broker-gateway.yaml` + +Same pattern. Binary name: `broker_gateway_service` + +**Step 1: Add initContainer** (between `nodeSelector:` line 39 and `containers:` line 40) + +Binary: `broker_gateway_service` + +**Step 2: Replace PVC volume with emptyDir** (lines 116-120) + +--- + +### Task 5: Update ml-training-service.yaml + +**Files:** +- Modify: `infra/k8s/services/ml-training-service.yaml` + +Binary name: `ml_training_service`. This service already has `minio-credentials` env vars and `minio-ca` volume. + +**Step 1: Add initContainer** (between `nodeSelector:` line 82 and `containers:` line 83) + +Same pattern. Binary: `ml_training_service`. Command includes `"serve"` arg — that's on the main container, not the initContainer. + +**Step 2: Replace PVC volume with emptyDir** (lines 166-170) + +Replace: +```yaml + - name: binaries + persistentVolumeClaim: + claimName: foxhunt-binaries + readOnly: true +``` +With: +```yaml + - name: binaries + emptyDir: + sizeLimit: 200Mi +``` + +--- + +### Task 6: Update backtesting-service.yaml + +**Files:** +- Modify: `infra/k8s/services/backtesting-service.yaml` + +Binary name: `backtesting_service` + +**Step 1: Add initContainer** (between `nodeSelector:` line 39 and `containers:` line 40) +**Step 2: Replace PVC volume with emptyDir** (lines 105-109) + +--- + +### Task 7: Update trading-agent-service.yaml + +**Files:** +- Modify: `infra/k8s/services/trading-agent-service.yaml` + +Binary name: `trading_agent_service` + +**Step 1: Add initContainer** (between `nodeSelector:` line 39 and `containers:` line 40) +**Step 2: Replace PVC volume with emptyDir** (lines 103-107) + +--- + +### Task 8: Update data-acquisition-service.yaml + +**Files:** +- Modify: `infra/k8s/services/data-acquisition-service.yaml` + +Binary name: `data_acquisition_service` + +**Step 1: Add initContainer** (between `nodeSelector:` line 39 and `containers:` line 40) +**Step 2: Replace PVC volume with emptyDir** (lines 96-100) + +--- + +### Task 9: Update web-gateway.yaml (special — needs static assets too) + +**Files:** +- Modify: `infra/k8s/services/web-gateway.yaml` + +Binary name: `web-gateway`. Additionally fetches static dashboard assets. + +**Step 1: Add initContainer with dual fetch** + +Insert between `nodeSelector:` line 39 and `containers:` line 40: + +```yaml + initContainers: + - name: fetch-binary + image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest + command: ["/bin/sh", "-c"] + args: + - | + set -e + RCLONE_FLAGS="--s3-provider=Minio --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 --s3-access-key-id=${MINIO_ACCESS_KEY} --s3-secret-access-key=${MINIO_SECRET_KEY} --s3-no-check-bucket --no-check-certificate" + rclone copyto ":s3:foxhunt-binaries/services/web-gateway" "/binaries/web-gateway" $RCLONE_FLAGS + chmod +x /binaries/web-gateway + echo "Fetched web-gateway ($(stat -c%s /binaries/web-gateway) bytes)" + rclone copy ":s3:foxhunt-binaries/static/" "/binaries/static/" $RCLONE_FLAGS + echo "Fetched static assets" + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: access-key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: secret-key + volumeMounts: + - name: binaries + mountPath: /binaries + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi +``` + +Note: `sizeLimit` on the emptyDir should be `500Mi` for web-gateway (binary + static assets). + +**Step 2: Replace PVC volume with emptyDir** (lines 102-106) + +Replace: +```yaml + - name: binaries + persistentVolumeClaim: + claimName: foxhunt-binaries + readOnly: true +``` +With: +```yaml + - name: binaries + emptyDir: + sizeLimit: 500Mi +``` + +--- + +### Task 10: Commit all service yaml changes + +```bash +git add infra/k8s/services/ +git commit -m "refactor(k8s): replace PVC mounts with MinIO rclone initContainers + +All 8 service deployments now fetch their binary from MinIO at pod +startup via rclone initContainer into emptyDir. Eliminates dependency +on foxhunt-binaries PVC (RWO node affinity constraint). + +web-gateway additionally fetches static dashboard assets." +``` + +--- + +### Task 11: Update training job-template.yaml + +**Files:** +- Modify: `infra/k8s/training/job-template.yaml` + +**Step 1: Replace existing fetch-binary initContainer** + +The current initContainer (lines 41-73) copies from `training-binaries` PVC. Replace entire initContainer with: + +```yaml + initContainers: + - name: fetch-binary + image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-training-runtime:latest + command: ["/bin/sh", "-c"] + args: + - | + set -e + rclone copyto \ + ":s3:foxhunt-binaries/training/$(TRAINING_BINARY)" \ + "/binaries/$(TRAINING_BINARY)" \ + --s3-provider=Minio \ + --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-access-key-id="${MINIO_ACCESS_KEY}" \ + --s3-secret-access-key="${MINIO_SECRET_KEY}" \ + --s3-no-check-bucket \ + --no-check-certificate + chmod +x "/binaries/$(TRAINING_BINARY)" + echo "Fetched $(TRAINING_BINARY) ($(stat -c%s /binaries/$(TRAINING_BINARY)) bytes)" + env: + - name: TRAINING_BINARY + value: train_baseline_supervised + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: access-key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: secret-key + volumeMounts: + - name: binaries + mountPath: /binaries + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 128Mi +``` + +**Step 2: Remove training-binaries-src volume** + +Delete from volumes section (lines 122-127): +```yaml + - name: training-binaries-src + persistentVolumeClaim: + claimName: training-binaries + readOnly: true +``` + +**Step 3: Verify YAML** + +Run: `python3 -c "import yaml; list(yaml.safe_load_all(open('infra/k8s/training/job-template.yaml')))" && echo OK` +Expected: `OK` + +**Step 4: Commit** + +```bash +git add infra/k8s/training/job-template.yaml +git commit -m "refactor(k8s): training job fetches binary from MinIO + +Replace PVC-based training binary distribution with MinIO rclone +initContainer. Training jobs no longer need training-binaries PVC, +eliminating L40S GPU node autoscale just for binary distribution." +``` + +--- + +### Task 12: Delete PVC manifests + +**Files:** +- Delete: `infra/k8s/storage/foxhunt-binaries-pvc.yaml` +- Delete: `infra/k8s/training/training-binaries-pvc.yaml` + +**Step 1: Delete the files** + +```bash +rm infra/k8s/storage/foxhunt-binaries-pvc.yaml +rm infra/k8s/training/training-binaries-pvc.yaml +``` + +**Step 2: Verify no remaining references to deleted PVCs** + +Run: `grep -r "foxhunt-binaries" infra/ .gitlab-ci.yml --include='*.yaml' --include='*.yml'` +Expected: Only MinIO bucket references (`s3://foxhunt-binaries/`), no PVC references. + +Run: `grep -r "training-binaries" infra/ .gitlab-ci.yml --include='*.yaml' --include='*.yml'` +Expected: No results (the PVC name is gone). + +**Step 3: Commit** + +```bash +git add -A infra/k8s/storage/foxhunt-binaries-pvc.yaml infra/k8s/training/training-binaries-pvc.yaml +git commit -m "chore(k8s): delete foxhunt-binaries and training-binaries PVCs + +These PVCs are replaced by MinIO-based binary distribution. +Service pods and training jobs now fetch binaries from MinIO +via rclone initContainers. + +NOTE: Run 'kubectl delete pvc foxhunt-binaries training-binaries -n foxhunt' +on the cluster to reclaim the block storage volumes." +``` + +--- + +### Task 13: Final verification + +**Step 1: Grep for any remaining PVC binary references** + +Run: `grep -rn "claimName: foxhunt-binaries\|claimName: training-binaries" infra/ .gitlab-ci.yml` +Expected: No results. + +**Step 2: Grep for binary-writer references** + +Run: `grep -rn "binary-writer\|binary_writer" .gitlab-ci.yml` +Expected: No results. + +**Step 3: Verify all service yamls have initContainers** + +Run: `grep -l "fetch-binary" infra/k8s/services/*.yaml | wc -l` +Expected: `8` + +**Step 4: Verify training job has MinIO fetch** + +Run: `grep "foxhunt-binaries/training" infra/k8s/training/job-template.yaml` +Expected: Match on the rclone line. + +--- + +## Summary of Changes + +| File | Action | Lines ~Δ | +|------|--------|----------| +| `.gitlab-ci.yml` | Replace binary-writer pods with `aws s3 cp` | -100/+20 | +| `infra/k8s/services/trading-service.yaml` | Add initContainer, PVC→emptyDir | +30/-4 | +| `infra/k8s/services/api-gateway.yaml` | Add initContainer, PVC→emptyDir | +30/-4 | +| `infra/k8s/services/broker-gateway.yaml` | Add initContainer, PVC→emptyDir | +30/-4 | +| `infra/k8s/services/ml-training-service.yaml` | Add initContainer, PVC→emptyDir | +30/-4 | +| `infra/k8s/services/backtesting-service.yaml` | Add initContainer, PVC→emptyDir | +30/-4 | +| `infra/k8s/services/trading-agent-service.yaml` | Add initContainer, PVC→emptyDir | +30/-4 | +| `infra/k8s/services/data-acquisition-service.yaml` | Add initContainer, PVC→emptyDir | +30/-4 | +| `infra/k8s/services/web-gateway.yaml` | Add initContainer (binary+static), PVC→emptyDir | +35/-4 | +| `infra/k8s/training/job-template.yaml` | Replace PVC initContainer with MinIO | +30/-20 | +| `infra/k8s/storage/foxhunt-binaries-pvc.yaml` | Delete | -19 | +| `infra/k8s/training/training-binaries-pvc.yaml` | Delete | -19 | + +**Total: 12 files, ~+275/-210 lines, 0 image rebuilds** + +## Post-Deploy Cluster Cleanup + +After deploying this change, manually delete the orphaned PVCs: +```bash +kubectl delete pvc foxhunt-binaries training-binaries -n foxhunt +``` + +Add `MINIO_ACCESS_KEY` and `MINIO_SECRET_KEY` as CI/CD variables in GitLab +(Settings → CI/CD → Variables), or source them from the cluster secret during the deploy job. diff --git a/infra/k8s/gitlab/tailscale-proxy.yaml b/infra/k8s/gitlab/tailscale-proxy.yaml index 67edb16a2..ec67e5759 100644 --- a/infra/k8s/gitlab/tailscale-proxy.yaml +++ b/infra/k8s/gitlab/tailscale-proxy.yaml @@ -271,3 +271,33 @@ data: grpc_connect_timeout 30s; } } + + # MinIO S3 API — s3.fxhnt.ai + server { + listen 443 ssl; + server_name s3.fxhnt.ai; + + ssl_certificate /etc/nginx/certs/tls.crt; + ssl_certificate_key /etc/nginx/certs/tls.key; + ssl_protocols TLSv1.2 TLSv1.3; + + client_max_body_size 0; + + location / { + proxy_pass https://minio.foxhunt.svc.cluster.local:9000; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + + proxy_buffering off; + proxy_request_buffering off; + proxy_http_version 1.1; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + proxy_connect_timeout 300s; + + # MinIO uses self-signed TLS internally + proxy_ssl_verify off; + } + } diff --git a/infra/k8s/gpu-overlays/ml-training-service-gpu.yaml b/infra/k8s/gpu-overlays/ml-training-service-gpu.yaml index b97c6fec3..060074b65 100644 --- a/infra/k8s/gpu-overlays/ml-training-service-gpu.yaml +++ b/infra/k8s/gpu-overlays/ml-training-service-gpu.yaml @@ -2,9 +2,7 @@ # Apply manually: kubectl apply -f infra/k8s/gpu-overlays/ml-training-service-gpu.yaml # Revert to CPU: kubectl apply -f infra/k8s/services/ml-training-service.yaml # -# Requires: gpu-inference node must be the SAME physical node as foxhunt pool, -# or create a separate foxhunt-binaries PVC bound to the GPU node. -# The foxhunt-binaries PVC is RWO — only one node can mount it. +# Binary fetched from MinIO at pod startup — works on any node pool. apiVersion: apps/v1 kind: Deployment metadata: @@ -49,6 +47,45 @@ spec: effect: NoSchedule imagePullSecrets: - name: scw-registry + initContainers: + - name: fetch-binary + image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest + command: ["/bin/sh", "-c"] + args: + - | + set -e + rclone copyto \ + ":s3:foxhunt-binaries/services/ml_training_service" \ + "/binaries/ml_training_service" \ + --s3-provider=Minio \ + --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-access-key-id="${MINIO_ACCESS_KEY}" \ + --s3-secret-access-key="${MINIO_SECRET_KEY}" \ + --s3-no-check-bucket \ + --no-check-certificate + chmod +x /binaries/ml_training_service + echo "Fetched ml_training_service ($(stat -c%s /binaries/ml_training_service) bytes)" + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: access-key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: secret-key + volumeMounts: + - name: binaries + mountPath: /binaries + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 128Mi containers: - name: ml-training-service image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest @@ -128,9 +165,8 @@ spec: memory: 8Gi volumes: - name: binaries - persistentVolumeClaim: - claimName: foxhunt-binaries - readOnly: true + emptyDir: + sizeLimit: 200Mi - name: tls-certs secret: secretName: ml-training-tls diff --git a/infra/k8s/gpu-overlays/trading-service-gpu.yaml b/infra/k8s/gpu-overlays/trading-service-gpu.yaml index c885080ac..fbf0fc52a 100644 --- a/infra/k8s/gpu-overlays/trading-service-gpu.yaml +++ b/infra/k8s/gpu-overlays/trading-service-gpu.yaml @@ -2,9 +2,7 @@ # Apply manually: kubectl apply -f infra/k8s/gpu-overlays/trading-service-gpu.yaml # Revert to CPU: kubectl apply -f infra/k8s/services/trading-service.yaml # -# Requires: gpu-inference node must be the SAME physical node as foxhunt pool, -# or create a separate foxhunt-binaries PVC bound to the GPU node. -# The foxhunt-binaries PVC is RWO — only one node can mount it. +# Binary fetched from MinIO at pod startup — works on any node pool. apiVersion: apps/v1 kind: Deployment metadata: @@ -48,6 +46,45 @@ spec: effect: NoSchedule imagePullSecrets: - name: scw-registry + initContainers: + - name: fetch-binary + image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest + command: ["/bin/sh", "-c"] + args: + - | + set -e + rclone copyto \ + ":s3:foxhunt-binaries/services/trading_service" \ + "/binaries/trading_service" \ + --s3-provider=Minio \ + --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-access-key-id="${MINIO_ACCESS_KEY}" \ + --s3-secret-access-key="${MINIO_SECRET_KEY}" \ + --s3-no-check-bucket \ + --no-check-certificate + chmod +x /binaries/trading_service + echo "Fetched trading_service ($(stat -c%s /binaries/trading_service) bytes)" + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: access-key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: secret-key + volumeMounts: + - name: binaries + mountPath: /binaries + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 128Mi containers: - name: trading-service image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest @@ -113,9 +150,8 @@ spec: memory: 8Gi volumes: - name: binaries - persistentVolumeClaim: - claimName: foxhunt-binaries - readOnly: true + emptyDir: + sizeLimit: 200Mi - name: tmp emptyDir: sizeLimit: 50Mi diff --git a/infra/k8s/services/api-gateway.yaml b/infra/k8s/services/api-gateway.yaml index 96b7ad295..52dcb7fff 100644 --- a/infra/k8s/services/api-gateway.yaml +++ b/infra/k8s/services/api-gateway.yaml @@ -38,6 +38,45 @@ spec: - name: scw-registry nodeSelector: k8s.scaleway.com/pool-name: foxhunt + initContainers: + - name: fetch-binary + image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest + command: ["/bin/sh", "-c"] + args: + - | + set -e + rclone copyto \ + ":s3:foxhunt-binaries/services/api_gateway" \ + "/binaries/api_gateway" \ + --s3-provider=Minio \ + --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-access-key-id="${MINIO_ACCESS_KEY}" \ + --s3-secret-access-key="${MINIO_SECRET_KEY}" \ + --s3-no-check-bucket \ + --no-check-certificate + chmod +x /binaries/api_gateway + echo "Fetched api_gateway ($(stat -c%s /binaries/api_gateway) bytes)" + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: access-key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: secret-key + volumeMounts: + - name: binaries + mountPath: /binaries + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 128Mi containers: - name: api-gateway image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest @@ -121,9 +160,8 @@ spec: memory: 512Mi volumes: - name: binaries - persistentVolumeClaim: - claimName: foxhunt-binaries - readOnly: true + emptyDir: + sizeLimit: 200Mi - name: tmp emptyDir: sizeLimit: 50Mi diff --git a/infra/k8s/services/backtesting-service.yaml b/infra/k8s/services/backtesting-service.yaml index c986d87e8..fdeca4b43 100644 --- a/infra/k8s/services/backtesting-service.yaml +++ b/infra/k8s/services/backtesting-service.yaml @@ -37,6 +37,45 @@ spec: - name: scw-registry nodeSelector: k8s.scaleway.com/pool-name: foxhunt + initContainers: + - name: fetch-binary + image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest + command: ["/bin/sh", "-c"] + args: + - | + set -e + rclone copyto \ + ":s3:foxhunt-binaries/services/backtesting_service" \ + "/binaries/backtesting_service" \ + --s3-provider=Minio \ + --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-access-key-id="${MINIO_ACCESS_KEY}" \ + --s3-secret-access-key="${MINIO_SECRET_KEY}" \ + --s3-no-check-bucket \ + --no-check-certificate + chmod +x /binaries/backtesting_service + echo "Fetched backtesting_service ($(stat -c%s /binaries/backtesting_service) bytes)" + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: access-key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: secret-key + volumeMounts: + - name: binaries + mountPath: /binaries + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 128Mi containers: - name: backtesting-service image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest @@ -104,9 +143,8 @@ spec: memory: 512Mi volumes: - name: binaries - persistentVolumeClaim: - claimName: foxhunt-binaries - readOnly: true + emptyDir: + sizeLimit: 200Mi - name: tmp emptyDir: sizeLimit: 50Mi diff --git a/infra/k8s/services/broker-gateway.yaml b/infra/k8s/services/broker-gateway.yaml index cd62c3b3d..5792a4b37 100644 --- a/infra/k8s/services/broker-gateway.yaml +++ b/infra/k8s/services/broker-gateway.yaml @@ -37,6 +37,45 @@ spec: - name: scw-registry nodeSelector: k8s.scaleway.com/pool-name: foxhunt + initContainers: + - name: fetch-binary + image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest + command: ["/bin/sh", "-c"] + args: + - | + set -e + rclone copyto \ + ":s3:foxhunt-binaries/services/broker_gateway_service" \ + "/binaries/broker_gateway_service" \ + --s3-provider=Minio \ + --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-access-key-id="${MINIO_ACCESS_KEY}" \ + --s3-secret-access-key="${MINIO_SECRET_KEY}" \ + --s3-no-check-bucket \ + --no-check-certificate + chmod +x /binaries/broker_gateway_service + echo "Fetched broker_gateway_service ($(stat -c%s /binaries/broker_gateway_service) bytes)" + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: access-key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: secret-key + volumeMounts: + - name: binaries + mountPath: /binaries + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 128Mi containers: - name: broker-gateway image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest @@ -115,9 +154,8 @@ spec: memory: 512Mi volumes: - name: binaries - persistentVolumeClaim: - claimName: foxhunt-binaries - readOnly: true + emptyDir: + sizeLimit: 200Mi - name: tmp emptyDir: sizeLimit: 50Mi diff --git a/infra/k8s/services/data-acquisition-service.yaml b/infra/k8s/services/data-acquisition-service.yaml index bdf4e2ae0..ae998bfbd 100644 --- a/infra/k8s/services/data-acquisition-service.yaml +++ b/infra/k8s/services/data-acquisition-service.yaml @@ -37,6 +37,45 @@ spec: - name: scw-registry nodeSelector: k8s.scaleway.com/pool-name: foxhunt + initContainers: + - name: fetch-binary + image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest + command: ["/bin/sh", "-c"] + args: + - | + set -e + rclone copyto \ + ":s3:foxhunt-binaries/services/data_acquisition_service" \ + "/binaries/data_acquisition_service" \ + --s3-provider=Minio \ + --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-access-key-id="${MINIO_ACCESS_KEY}" \ + --s3-secret-access-key="${MINIO_SECRET_KEY}" \ + --s3-no-check-bucket \ + --no-check-certificate + chmod +x /binaries/data_acquisition_service + echo "Fetched data_acquisition_service ($(stat -c%s /binaries/data_acquisition_service) bytes)" + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: access-key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: secret-key + volumeMounts: + - name: binaries + mountPath: /binaries + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 128Mi containers: - name: data-acquisition-service image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest @@ -95,9 +134,8 @@ spec: memory: 1Gi volumes: - name: binaries - persistentVolumeClaim: - claimName: foxhunt-binaries - readOnly: true + emptyDir: + sizeLimit: 200Mi - name: tmp emptyDir: sizeLimit: 50Mi diff --git a/infra/k8s/services/ml-training-service.yaml b/infra/k8s/services/ml-training-service.yaml index 643503d64..8f09cf5a4 100644 --- a/infra/k8s/services/ml-training-service.yaml +++ b/infra/k8s/services/ml-training-service.yaml @@ -80,6 +80,45 @@ spec: - name: scw-registry nodeSelector: k8s.scaleway.com/pool-name: foxhunt + initContainers: + - name: fetch-binary + image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest + command: ["/bin/sh", "-c"] + args: + - | + set -e + rclone copyto \ + ":s3:foxhunt-binaries/services/ml_training_service" \ + "/binaries/ml_training_service" \ + --s3-provider=Minio \ + --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-access-key-id="${MINIO_ACCESS_KEY}" \ + --s3-secret-access-key="${MINIO_SECRET_KEY}" \ + --s3-no-check-bucket \ + --no-check-certificate + chmod +x /binaries/ml_training_service + echo "Fetched ml_training_service ($(stat -c%s /binaries/ml_training_service) bytes)" + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: access-key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: secret-key + volumeMounts: + - name: binaries + mountPath: /binaries + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 128Mi containers: - name: ml-training-service image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest @@ -165,9 +204,8 @@ spec: memory: 512Mi volumes: - name: binaries - persistentVolumeClaim: - claimName: foxhunt-binaries - readOnly: true + emptyDir: + sizeLimit: 200Mi - name: tls-certs secret: secretName: ml-training-tls diff --git a/infra/k8s/services/trading-agent-service.yaml b/infra/k8s/services/trading-agent-service.yaml index 00d050775..2df1ce651 100644 --- a/infra/k8s/services/trading-agent-service.yaml +++ b/infra/k8s/services/trading-agent-service.yaml @@ -37,6 +37,45 @@ spec: - name: scw-registry nodeSelector: k8s.scaleway.com/pool-name: foxhunt + initContainers: + - name: fetch-binary + image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest + command: ["/bin/sh", "-c"] + args: + - | + set -e + rclone copyto \ + ":s3:foxhunt-binaries/services/trading_agent_service" \ + "/binaries/trading_agent_service" \ + --s3-provider=Minio \ + --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-access-key-id="${MINIO_ACCESS_KEY}" \ + --s3-secret-access-key="${MINIO_SECRET_KEY}" \ + --s3-no-check-bucket \ + --no-check-certificate + chmod +x /binaries/trading_agent_service + echo "Fetched trading_agent_service ($(stat -c%s /binaries/trading_agent_service) bytes)" + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: access-key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: secret-key + volumeMounts: + - name: binaries + mountPath: /binaries + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 128Mi containers: - name: trading-agent-service image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest @@ -102,9 +141,8 @@ spec: memory: 512Mi volumes: - name: binaries - persistentVolumeClaim: - claimName: foxhunt-binaries - readOnly: true + emptyDir: + sizeLimit: 200Mi - name: tmp emptyDir: sizeLimit: 50Mi diff --git a/infra/k8s/services/trading-service.yaml b/infra/k8s/services/trading-service.yaml index 7b3606cc2..f8d5b679a 100644 --- a/infra/k8s/services/trading-service.yaml +++ b/infra/k8s/services/trading-service.yaml @@ -37,6 +37,45 @@ spec: - name: scw-registry nodeSelector: k8s.scaleway.com/pool-name: foxhunt + initContainers: + - name: fetch-binary + image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest + command: ["/bin/sh", "-c"] + args: + - | + set -e + rclone copyto \ + ":s3:foxhunt-binaries/services/trading_service" \ + "/binaries/trading_service" \ + --s3-provider=Minio \ + --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-access-key-id="${MINIO_ACCESS_KEY}" \ + --s3-secret-access-key="${MINIO_SECRET_KEY}" \ + --s3-no-check-bucket \ + --no-check-certificate + chmod +x /binaries/trading_service + echo "Fetched trading_service ($(stat -c%s /binaries/trading_service) bytes)" + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: access-key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: secret-key + volumeMounts: + - name: binaries + mountPath: /binaries + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 128Mi containers: - name: trading-service image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest @@ -119,9 +158,8 @@ spec: memory: 512Mi volumes: - name: binaries - persistentVolumeClaim: - claimName: foxhunt-binaries - readOnly: true + emptyDir: + sizeLimit: 200Mi - name: tmp emptyDir: sizeLimit: 50Mi diff --git a/infra/k8s/services/web-gateway.yaml b/infra/k8s/services/web-gateway.yaml index df2e0a869..6f99808f2 100644 --- a/infra/k8s/services/web-gateway.yaml +++ b/infra/k8s/services/web-gateway.yaml @@ -37,6 +37,40 @@ spec: - name: scw-registry nodeSelector: k8s.scaleway.com/pool-name: foxhunt + initContainers: + - name: fetch-binary + image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest + command: ["/bin/sh", "-c"] + args: + - | + set -e + RCLONE_FLAGS="--s3-provider=Minio --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 --s3-access-key-id=${MINIO_ACCESS_KEY} --s3-secret-access-key=${MINIO_SECRET_KEY} --s3-no-check-bucket --no-check-certificate" + rclone copyto ":s3:foxhunt-binaries/services/web-gateway" "/binaries/web-gateway" $RCLONE_FLAGS + chmod +x /binaries/web-gateway + echo "Fetched web-gateway ($(stat -c%s /binaries/web-gateway) bytes)" + rclone copy ":s3:foxhunt-binaries/static/" "/binaries/static/" $RCLONE_FLAGS + echo "Fetched static assets" + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: access-key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: secret-key + volumeMounts: + - name: binaries + mountPath: /binaries + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 256Mi containers: - name: web-gateway image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-runtime:latest @@ -101,9 +135,8 @@ spec: memory: 256Mi volumes: - name: binaries - persistentVolumeClaim: - claimName: foxhunt-binaries - readOnly: true + emptyDir: + sizeLimit: 500Mi - name: tmp emptyDir: sizeLimit: 50Mi diff --git a/infra/k8s/storage/foxhunt-binaries-pvc.yaml b/infra/k8s/storage/foxhunt-binaries-pvc.yaml deleted file mode 100644 index 5a2aeaa18..000000000 --- a/infra/k8s/storage/foxhunt-binaries-pvc.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Shared binary PVC — all foxhunt services mount read-only -# CI deploy job writes via temporary binary-writer pod (kubectl cp) -# RWO on scw-bssd: all pods on the same foxhunt node can mount ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: foxhunt-binaries - namespace: foxhunt - labels: - app.kubernetes.io/name: foxhunt-binaries - app.kubernetes.io/part-of: foxhunt -spec: - accessModes: [ReadWriteOnce] - storageClassName: scw-bssd - resources: - requests: - storage: 2Gi diff --git a/infra/k8s/training/job-template.yaml b/infra/k8s/training/job-template.yaml index 82b85bcfe..a8e1346d8 100644 --- a/infra/k8s/training/job-template.yaml +++ b/infra/k8s/training/job-template.yaml @@ -34,10 +34,8 @@ spec: imagePullSecrets: - name: scw-registry restartPolicy: Never - # Copy training binary from foxhunt-binaries PVC (foxhunt node) into emptyDir. - # foxhunt-binaries is RWO on foxhunt node — training node can't mount it directly. - # The initContainer uses a Job-local copy approach: ml-training-service copies - # the binary path into the job spec at creation time. + # Fetch training binary from MinIO at job startup via rclone. + # Replaces PVC-based copy — no node affinity constraint, no L40S autoscale for writes. initContainers: - name: fetch-binary image: rg.fr-par.scw.cloud/foxhunt-ci/foxhunt-training-runtime:latest @@ -45,23 +43,31 @@ spec: args: - | set -e - BINARY="/training-binaries/$(TRAINING_BINARY)" - if [ -f "$BINARY" ]; then - cp "$BINARY" /binaries/$(TRAINING_BINARY) - chmod +x /binaries/$(TRAINING_BINARY) - echo "Copied $(TRAINING_BINARY) ($(stat -c%s /binaries/$(TRAINING_BINARY)) bytes)" - else - echo "FATAL: Training binary not found: $BINARY" - echo "Available: $(ls /training-binaries/ 2>/dev/null || echo 'none')" - exit 1 - fi + rclone copyto \ + ":s3:foxhunt-binaries/training/$(TRAINING_BINARY)" \ + "/binaries/$(TRAINING_BINARY)" \ + --s3-provider=Minio \ + --s3-endpoint=https://minio.foxhunt.svc.cluster.local:9000 \ + --s3-access-key-id="${MINIO_ACCESS_KEY}" \ + --s3-secret-access-key="${MINIO_SECRET_KEY}" \ + --s3-no-check-bucket \ + --no-check-certificate + chmod +x "/binaries/$(TRAINING_BINARY)" + echo "Fetched $(TRAINING_BINARY) ($(stat -c%s /binaries/$(TRAINING_BINARY)) bytes)" env: - name: TRAINING_BINARY value: train_baseline_supervised + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: access-key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: secret-key volumeMounts: - - name: training-binaries-src - mountPath: /training-binaries - readOnly: true - name: binaries mountPath: /binaries resources: @@ -119,12 +125,6 @@ spec: - name: output emptyDir: sizeLimit: 2Gi - # Training binaries PVC — must be on same node as ci-training pool. - # Populated by CI deploy job via a writer pod on the training node. - - name: training-binaries-src - persistentVolumeClaim: - claimName: training-binaries - readOnly: true - name: binaries emptyDir: sizeLimit: 500Mi diff --git a/infra/k8s/training/training-binaries-pvc.yaml b/infra/k8s/training/training-binaries-pvc.yaml deleted file mode 100644 index d605951aa..000000000 --- a/infra/k8s/training/training-binaries-pvc.yaml +++ /dev/null @@ -1,18 +0,0 @@ -# Training binaries PVC — separate from foxhunt-binaries (foxhunt node). -# Training jobs run on ci-training node (GPU) which can't mount the foxhunt -# node's RWO PVC. This PVC is populated by a writer pod in the CI deploy job. ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: training-binaries - namespace: foxhunt - labels: - app.kubernetes.io/name: training-binaries - app.kubernetes.io/part-of: foxhunt -spec: - accessModes: [ReadWriteOnce] - storageClassName: scw-bssd - resources: - requests: - storage: 2Gi diff --git a/infra/modules/dns/main.tf b/infra/modules/dns/main.tf index 94ca1f3e7..6b949b187 100644 --- a/infra/modules/dns/main.tf +++ b/infra/modules/dns/main.tf @@ -47,3 +47,11 @@ resource "scaleway_domain_record" "monitor" { data = var.git_ip ttl = 300 } + +resource "scaleway_domain_record" "s3" { + dns_zone = var.dns_zone + name = "s3" + type = "A" + data = var.git_ip + ttl = 300 +}