From 7a9683d5fb7c7749f9dee2f2eb6170debad53658 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 7 Mar 2026 20:38:41 +0100 Subject: [PATCH] feat(infra): GitLab releases with CalVer auto-versioning Replace MinIO binary distribution with GitLab Generic Package Registry. Every code push to main auto-creates a CalVer tag (vYYYY.MM.N), compiles, uploads binaries to GitLab packages, creates a Release with auto-generated notes, and deploys via deployment patching. - New CI templates: create-tag, upload-release - Modified: compile-services/training upload to GitLab packages - Modified: deploy-services patches FOXHUNT_RELEASE on deployments - All 7 service initContainers fetch from GitLab (curl, deploy token) - Training job-template binary fetch from GitLab (data stays MinIO) - MinIO retains: sccache, training data, model checkpoints Co-Authored-By: Claude Opus 4.6 --- .../2026-03-07-gitlab-releases-design.md | 182 ++++ docs/plans/2026-03-07-gitlab-releases-plan.md | 875 ++++++++++++++++++ infra/k8s/argo/ci-pipeline-template.yaml | 267 +++++- infra/k8s/services/api.yaml | 31 +- infra/k8s/services/backtesting-service.yaml | 31 +- infra/k8s/services/broker-gateway.yaml | 31 +- .../services/data-acquisition-service.yaml | 31 +- infra/k8s/services/ml-training-service.yaml | 31 +- infra/k8s/services/trading-agent-service.yaml | 31 +- infra/k8s/services/trading-service.yaml | 31 +- infra/k8s/training/job-template.yaml | 33 +- 11 files changed, 1381 insertions(+), 193 deletions(-) create mode 100644 docs/plans/2026-03-07-gitlab-releases-design.md create mode 100644 docs/plans/2026-03-07-gitlab-releases-plan.md diff --git a/docs/plans/2026-03-07-gitlab-releases-design.md b/docs/plans/2026-03-07-gitlab-releases-design.md new file mode 100644 index 000000000..951888923 --- /dev/null +++ b/docs/plans/2026-03-07-gitlab-releases-design.md @@ -0,0 +1,182 @@ +# GitLab Releases with CalVer Auto-Versioning + +**Date**: 2026-03-07 +**Status**: Approved + +## Summary + +Replace MinIO binary distribution with GitLab Generic Package Registry + Releases. +Every push to `main` that touches code auto-creates a CalVer tag, compiles, uploads +binaries as GitLab generic packages, creates a GitLab Release, and deploys. + +MinIO retains sccache, training data, and model checkpoints (not release artifacts). + +## Versioning + +**Scheme**: CalVer `vYYYY.MM.N` where N auto-increments per month. + +Examples: `v2026.03.1`, `v2026.03.2`, `v2026.04.1` + +**Increment logic**: Query GitLab tags API for latest `vYYYY.MM.*`, parse N, increment. +If no tag exists for current month, start at 1. + +**Cargo.toml**: Workspace `version` stays at `1.0.0` (Cargo version != release version). +Runtime version comes from `FOXHUNT_BUILD_VERSION` env var baked at compile time +via `option_env!()` in `common/src/build_info.rs` (existing design from 2026-03-04). + +## Branching Model + +Trunk-based. No develop/release branches. + +``` +feature/* --merge--> main --auto--> tag vYYYY.MM.N -> compile -> release -> deploy +``` + +## Pipeline Changes (ci-pipeline-template.yaml) + +### Current DAG + +``` +detect-changes -> [compile-services | compile-training | build-web-dashboard] -> deploy-services +``` + +### New DAG + +``` +detect-changes -> create-tag (if code changed) + | + +---------+---------+ + | | + compile-services compile-training + | | + +---------+---------+ + | + upload-release -> deploy-services +``` + +Non-code changes (docker images, dashboard) remain unchanged and don't create tags. + +### New template: create-tag + +- Runs on `platform` node (lightweight) +- Uses GitLab API via deploy token with `api` scope +- Computes version: `YYYY.MM` from date, queries `GET /api/v4/projects/:id/repository/tags?search=vYYYY.MM` +- Increments N, creates tag via `POST /api/v4/projects/:id/repository/tags` +- Outputs the tag name for downstream steps + +### Modified: compile-services / compile-training + +- `FOXHUNT_BUILD_VERSION` = tag from create-tag step (was `YYYY.MM.argo`) +- Upload destination changes from MinIO to GitLab Generic Package Registry: + ``` + PUT /api/v4/projects/:id/packages/generic/foxhunt-services// + PUT /api/v4/projects/:id/packages/generic/foxhunt-training// + ``` +- Removes rclone + MinIO credentials for binary upload +- Adds `GITLAB_API_TOKEN` from secret + +### New template: upload-release + +- Creates GitLab Release via `POST /api/v4/projects/:id/releases` +- Attaches package links as release assets +- Auto-generates release notes from commits since last tag: + `GET /api/v4/projects/:id/repository/changelog?version=` +- Runs after both compile steps complete + +### Modified: deploy-services + +- Sets `FOXHUNT_RELEASE` annotation on deployments to trigger rollout +- initContainers now know which version to fetch + +## Service Deployment Changes (infra/k8s/services/*.yaml) + +### initContainer: fetch-binary + +Before (MinIO): +```yaml +- name: fetch-binary + args: + - | + rclone copyto ":s3:foxhunt-binaries/services/trading-service" "/binaries/trading-service" \ + --s3-provider=Minio --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 ... + env: + - name: MINIO_ACCESS_KEY ... + - name: MINIO_SECRET_KEY ... +``` + +After (GitLab Generic Packages): +```yaml +- name: fetch-binary + args: + - | + set -e + curl -fSL -o /binaries/trading-service \ + --header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \ + "${GITLAB_API}/projects/1/packages/generic/foxhunt-services/${FOXHUNT_RELEASE}/trading-service" + chmod +x /binaries/trading-service + env: + - name: GITLAB_DEPLOY_TOKEN + valueFrom: + secretKeyRef: + name: gitlab-deploy-token + key: token + - name: GITLAB_API + value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" + - name: FOXHUNT_RELEASE + value: "v2026.03.1" # Updated by deploy-services step +``` + +### Training job-template.yaml + +Same pattern: replace rclone/MinIO with curl/GitLab for binary fetch. +Training data sync stays on MinIO (not a release artifact). + +## Secrets + +### New +- `gitlab-deploy-token`: GitLab deploy token with `read_package_registry` scope (for service pods) +- `gitlab-api-token`: GitLab PAT or deploy token with `api` scope (for CI tag/release creation) + +### Removed from binary path +- `minio-credentials` no longer needed in service deployments or compile upload steps +- MinIO credentials remain for: sccache, training data sync, model checkpoints + +## Rollback + +```bash +# Roll back to previous release +kubectl -n foxhunt set env deployment/trading-service FOXHUNT_RELEASE=v2026.03.1 +kubectl -n foxhunt rollout restart deployment/trading-service +``` + +All previous binaries remain available in GitLab's package registry indefinitely. + +## What stays on MinIO + +| Bucket | Purpose | Change | +|--------|---------|--------| +| foxhunt-sccache | Rust build cache | No change | +| foxhunt-training-data | DBN market data | No change | +| foxhunt-training-results | Training outputs | No change | +| foxhunt-models | Model checkpoints | No change | +| foxhunt-binaries | **Service/training binaries** | **Decommission after migration** | + +## Migration + +1. Deploy new CI pipeline with dual-write (MinIO + GitLab) for one release +2. Switch service initContainers to GitLab fetch +3. Verify all services boot correctly +4. Remove MinIO binary upload from CI +5. Delete `foxhunt-binaries` bucket contents (keep bucket for sccache) + +## Files Changed + +### New files +- `crates/common/src/build_info.rs` — version function (from 2026-03-04 design) + +### Modified files +- `infra/k8s/argo/ci-pipeline-template.yaml` — new create-tag + upload-release templates, modified compile steps +- `infra/k8s/services/*.yaml` (8 files) — initContainer fetch from GitLab +- `infra/k8s/training/job-template.yaml` — binary fetch from GitLab +- `infra/k8s/secrets/` — new gitlab-deploy-token, gitlab-api-token +- `crates/common/src/lib.rs` — re-export build_info diff --git a/docs/plans/2026-03-07-gitlab-releases-plan.md b/docs/plans/2026-03-07-gitlab-releases-plan.md new file mode 100644 index 000000000..d2d0886da --- /dev/null +++ b/docs/plans/2026-03-07-gitlab-releases-plan.md @@ -0,0 +1,875 @@ +# GitLab Releases with CalVer Auto-Versioning — Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Replace MinIO binary distribution with GitLab Generic Package Registry + Releases, with CalVer auto-tagging on every code push to main. + +**Architecture:** Argo CI pipeline creates a CalVer git tag, compiles binaries, uploads them to GitLab's Generic Package Registry, creates a GitLab Release with auto-generated notes, then patches service deployments to fetch from GitLab. MinIO retains sccache, training data, and model checkpoints. + +**Tech Stack:** Argo Workflows, GitLab CE API v4, Generic Package Registry, K8s strategic merge patches, curl + +--- + +## Context + +**GitLab internal URL:** `http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181` +**GitLab project ID:** `1` (root/foxhunt) +**Existing secrets:** `gitlab-pat` (auto-rotated PAT, monthly), `minio-credentials` +**Existing file:** `crates/common/src/build_info.rs` — already captures `FOXHUNT_BUILD_VERSION` at compile time +**Web dashboard:** stays on MinIO (not code, not part of release) + +**Services affected (7):** api, trading-service, ml-training-service, backtesting-service, trading-agent-service, broker-gateway, data-acquisition-service + +**Binary names from cargo build:** +- Services: `api`, `trading-service`, `ml-training-service`, `backtesting-service`, `trading-agent-service`, `broker-gateway`, `data-acquisition-service` +- Training: `train_baseline_rl`, `train_baseline_supervised`, `evaluate_baseline`, `evaluate_supervised`, `hyperopt_baseline_rl`, `hyperopt_baseline_supervised`, `training_uploader` + +--- + +### Task 1: Create GitLab deploy token and K8s secret + +This is a one-time manual setup. The deploy token lets service pods download packages from GitLab's Generic Package Registry (read-only). + +**Step 1: Create deploy token via GitLab API** + +Run from a machine with access to the cluster: + +```bash +# Use existing PAT to create a project deploy token +GITLAB="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181" +PAT=$(kubectl -n foxhunt get secret gitlab-pat -o jsonpath='{.data.token}' | base64 -d) + +curl -sf -X POST "${GITLAB}/api/v4/projects/1/deploy_tokens" \ + -H "PRIVATE-TOKEN: ${PAT}" \ + -d "name=foxhunt-package-reader" \ + -d "scopes[]=read_package_registry" | jq . +``` + +Save the returned `token` value. + +**Step 2: Create K8s secret** + +```bash +kubectl -n foxhunt create secret generic gitlab-deploy-token \ + --from-literal=token= +``` + +**Step 3: Verify token can read packages** + +```bash +# Should return 200 (empty list is fine — no packages yet) +curl -sf -o /dev/null -w "%{http_code}" \ + -H "DEPLOY-TOKEN: " \ + "${GITLAB}/api/v4/projects/1/packages" +``` + +Expected: `200` + +**Step 4: Verify existing gitlab-pat has api scope** + +```bash +curl -sf "${GITLAB}/api/v4/personal_access_tokens/self" \ + -H "PRIVATE-TOKEN: ${PAT}" | jq '.scopes' +``` + +Expected: scopes must include `api` (for creating tags, releases, uploading packages). If not, recreate the PAT with `api` scope. + +--- + +### Task 2: Add `needs-code` output to detect-changes + +**Files:** +- Modify: `infra/k8s/argo/ci-pipeline-template.yaml:191-220` (detect-changes script outputs) + +**Step 1: Add needs-code computation to detect-changes script** + +In the detect-changes shell script, after the `NEEDS_TRAINING` computation (line ~207) and before the `echo "=== Build decisions ==="` line, add: + +```sh +if [ "$NEEDS_SERVICES" = "true" ] || [ "$NEEDS_TRAINING" = "true" ]; then + NEEDS_CODE="true" +else + NEEDS_CODE="false" +fi +``` + +And after `echo -n "$DOCKER_IMAGES" > /tmp/outputs/docker-images`, add: + +```sh +echo -n "$NEEDS_CODE" > /tmp/outputs/needs-code +``` + +Also add the build decisions echo: + +```sh +echo "needs-code: $NEEDS_CODE" +``` + +**Step 2: Add needs-code output parameter** + +After the `docker-images` output parameter (line ~235), add: + +```yaml + - name: needs-code + valueFrom: + path: /tmp/outputs/needs-code +``` + +**Step 3: Verify YAML is valid** + +```bash +python3 -c "import yaml; yaml.safe_load(open('infra/k8s/argo/ci-pipeline-template.yaml'))" +``` + +Expected: no error + +--- + +### Task 3: New `create-tag` template + +**Files:** +- Modify: `infra/k8s/argo/ci-pipeline-template.yaml` (add template after detect-changes, before compile-services) + +**Step 1: Add create-tag template** + +Insert after the detect-changes template `outputs:` block (after line ~237) and before the `build-web-dashboard` template: + +```yaml + # ── create-tag: CalVer auto-tag on code changes ── + - name: create-tag + nodeSelector: + k8s.scaleway.com/pool-name: platform + container: + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest + command: ["/bin/sh", "-c"] + env: + - name: GITLAB_PAT + valueFrom: + secretKeyRef: + name: gitlab-pat + key: token + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + args: + - | + set -e + GITLAB="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181" + PROJECT_ID=1 + SHA="{{workflow.parameters.commit-sha}}" + + # Compute CalVer prefix: vYYYY.MM + PREFIX="v$(date +%Y.%m)" + + # Query existing tags for this month + TAGS=$(curl -sf \ + -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + "${GITLAB}/api/v4/projects/${PROJECT_ID}/repository/tags?search=${PREFIX}" \ + || echo "[]") + + # Parse highest N from vYYYY.MM.N tags + LAST_N=$(echo "$TAGS" | grep -oP "\"name\":\"${PREFIX}\.\K[0-9]+" | sort -n | tail -1) + if [ -z "$LAST_N" ]; then + NEXT_N=1 + else + NEXT_N=$((LAST_N + 1)) + fi + + TAG="${PREFIX}.${NEXT_N}" + echo "Creating tag: ${TAG} at ${SHA}" + + # Create the tag + RESULT=$(curl -sf -X POST \ + -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + -d "tag_name=${TAG}" \ + -d "ref=${SHA}" \ + "${GITLAB}/api/v4/projects/${PROJECT_ID}/repository/tags") + + echo "$RESULT" | grep -q "$TAG" || { echo "Tag creation failed: $RESULT"; exit 1; } + echo "Tag ${TAG} created successfully" + + mkdir -p /tmp/outputs + echo -n "$TAG" > /tmp/outputs/tag + outputs: + parameters: + - name: tag + valueFrom: + path: /tmp/outputs/tag +``` + +**Step 2: Add gitlab-pat volume to workflow spec** + +In the `spec.volumes` list (around line 22-32), add: + +```yaml + - name: gitlab-pat + secret: + secretName: gitlab-pat +``` + +**Step 3: Verify YAML** + +```bash +python3 -c "import yaml; yaml.safe_load(open('infra/k8s/argo/ci-pipeline-template.yaml'))" +``` + +Expected: no error + +--- + +### Task 4: Modify compile-services — version from tag + upload to GitLab packages + +**Files:** +- Modify: `infra/k8s/argo/ci-pipeline-template.yaml` (compile-services template, lines ~309-426) + +**Step 1: Replace version computation** + +Find and replace the 3 lines (around line 393-395): + +```sh +YEAR=$(date +%Y) +MONTH=$(date +%m) +export FOXHUNT_BUILD_VERSION="${YEAR}.${MONTH}.argo" +``` + +With: + +```sh +export FOXHUNT_BUILD_VERSION="{{tasks.create-tag.outputs.parameters.tag}}" +``` + +**Step 2: Replace MinIO upload with GitLab package upload** + +Find and replace the rclone upload block (around lines 418-424): + +```sh +echo "=== Uploading service binaries ===" +rclone copy "$WORKSPACE/bin/services/" :s3:foxhunt-binaries/services/ \ + --s3-provider=Minio \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ + --s3-access-key-id="${MINIO_ACCESS_KEY}" \ + --s3-secret-access-key="${MINIO_SECRET_KEY}" \ + --s3-no-check-bucket +``` + +With: + +```sh +echo "=== Uploading service binaries to GitLab packages ===" +GITLAB="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181" +TAG="${FOXHUNT_BUILD_VERSION}" +for bin in "$WORKSPACE/bin/services/"*; do + BIN_NAME=$(basename "$bin") + echo "Uploading ${BIN_NAME}..." + curl -f --upload-file "$bin" \ + -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + "${GITLAB}/api/v4/projects/1/packages/generic/foxhunt-services/${TAG}/${BIN_NAME}" +done +``` + +**Step 3: Add GITLAB_PAT env var, remove MINIO upload env vars** + +In the compile-services container `env:` list, add: + +```yaml + - name: GITLAB_PAT + valueFrom: + secretKeyRef: + name: gitlab-pat + key: token +``` + +Remove the `MINIO_ACCESS_KEY` and `MINIO_SECRET_KEY` env entries (keep `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` — they're needed by sccache). + +**Step 4: Verify YAML** + +```bash +python3 -c "import yaml; yaml.safe_load(open('infra/k8s/argo/ci-pipeline-template.yaml'))" +``` + +--- + +### Task 5: Modify compile-training — version from tag + upload to GitLab packages + +**Files:** +- Modify: `infra/k8s/argo/ci-pipeline-template.yaml` (compile-training template, lines ~430-547) + +**Step 1: Replace version computation** + +Find and replace (around line 513-514): + +```sh +YEAR=$(date +%Y) +MONTH=$(date +%m) +export FOXHUNT_BUILD_VERSION="${YEAR}.${MONTH}.argo" +``` + +With: + +```sh +export FOXHUNT_BUILD_VERSION="{{tasks.create-tag.outputs.parameters.tag}}" +``` + +**Step 2: Replace MinIO upload with GitLab package upload** + +Find and replace the rclone upload block (around lines 539-545): + +```sh +echo "=== Uploading training binaries ===" +rclone copy "$WORKSPACE/bin/training/" :s3:foxhunt-binaries/training/ \ + --s3-provider=Minio \ + --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ + --s3-access-key-id="${MINIO_ACCESS_KEY}" \ + --s3-secret-access-key="${MINIO_SECRET_KEY}" \ + --s3-no-check-bucket +``` + +With: + +```sh +echo "=== Uploading training binaries to GitLab packages ===" +GITLAB="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181" +TAG="${FOXHUNT_BUILD_VERSION}" +for bin in "$WORKSPACE/bin/training/"*; do + BIN_NAME=$(basename "$bin") + echo "Uploading ${BIN_NAME}..." + curl -f --upload-file "$bin" \ + -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + "${GITLAB}/api/v4/projects/1/packages/generic/foxhunt-training/${TAG}/${BIN_NAME}" +done +``` + +**Step 3: Add GITLAB_PAT env var, remove MINIO upload env vars** + +Same as Task 4 Step 3. Add `GITLAB_PAT` from `gitlab-pat` secret. Remove `MINIO_ACCESS_KEY` and `MINIO_SECRET_KEY` (keep `AWS_*` for sccache). + +**Step 4: Verify YAML** + +```bash +python3 -c "import yaml; yaml.safe_load(open('infra/k8s/argo/ci-pipeline-template.yaml'))" +``` + +--- + +### Task 6: New `upload-release` template + +**Files:** +- Modify: `infra/k8s/argo/ci-pipeline-template.yaml` (add template after compile-training, before gpu-warmup) + +**Step 1: Add upload-release template** + +```yaml + # ── upload-release: create GitLab Release with package links ── + - name: upload-release + nodeSelector: + k8s.scaleway.com/pool-name: platform + container: + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest + command: ["/bin/sh", "-c"] + env: + - name: GITLAB_PAT + valueFrom: + secretKeyRef: + name: gitlab-pat + key: token + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + args: + - | + set -e + GITLAB="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181" + PROJECT_ID=1 + TAG="{{tasks.create-tag.outputs.parameters.tag}}" + + echo "=== Creating GitLab Release ${TAG} ===" + + # Get commits since previous tag for release notes + PREV_TAG=$(curl -sf \ + -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + "${GITLAB}/api/v4/projects/${PROJECT_ID}/repository/tags?per_page=2" \ + | grep -oP '"name":"\K[^"]+' | sed -n '2p') + + if [ -n "$PREV_TAG" ]; then + COMMITS=$(curl -sf \ + -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + "${GITLAB}/api/v4/projects/${PROJECT_ID}/repository/compare?from=${PREV_TAG}&to=${TAG}" \ + | grep -oP '"title":"\K[^"]+' | head -20 \ + | sed 's/^/- /' || echo "- Release ${TAG}") + DESCRIPTION="## Changes since ${PREV_TAG}\n\n${COMMITS}" + else + DESCRIPTION="## Initial release\n\nFirst CalVer release." + fi + + # Create the release + RESPONSE=$(curl -sf -X POST \ + -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + -H "Content-Type: application/json" \ + -d "{ + \"tag_name\": \"${TAG}\", + \"name\": \"${TAG}\", + \"description\": \"$(printf '%s' "$DESCRIPTION" | sed 's/"/\\"/g')\" + }" \ + "${GITLAB}/api/v4/projects/${PROJECT_ID}/releases") || { + echo "WARN: Release creation failed (may already exist)" + } + + echo "Release ${TAG} created" + echo "$RESPONSE" | head -5 + + # Add package links as release assets + PKG_URL="${GITLAB}/api/v4/projects/${PROJECT_ID}/packages/generic" + for pkg_name in foxhunt-services foxhunt-training; do + # Check if package exists for this tag + PKG_CHECK=$(curl -sf \ + -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + "${GITLAB}/api/v4/projects/${PROJECT_ID}/packages?package_name=${pkg_name}&package_version=${TAG}" \ + || echo "[]") + + if echo "$PKG_CHECK" | grep -q "$TAG"; then + curl -sf -X POST \ + -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + -H "Content-Type: application/json" \ + -d "{ + \"name\": \"${pkg_name}\", + \"url\": \"${GITLAB}/-/packages?type=generic&search=${pkg_name}&version=${TAG}\", + \"link_type\": \"package\" + }" \ + "${GITLAB}/api/v4/projects/${PROJECT_ID}/releases/${TAG}/assets/links" || true + echo "Linked ${pkg_name} package to release" + fi + done + + echo "=== Release ${TAG} complete ===" +``` + +**Step 2: Verify YAML** + +```bash +python3 -c "import yaml; yaml.safe_load(open('infra/k8s/argo/ci-pipeline-template.yaml'))" +``` + +--- + +### Task 7: Modify deploy-services — patch FOXHUNT_RELEASE on deployments + +**Files:** +- Modify: `infra/k8s/argo/ci-pipeline-template.yaml` (deploy-services template, lines ~582-620) + +**Step 1: Replace deploy-services script** + +Replace the entire `args` block of deploy-services with: + +```sh +set -e +if ! command -v kubectl >/dev/null 2>&1; then + echo "=== Installing kubectl ===" + curl -sLo /tmp/kubectl "https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl" + chmod +x /tmp/kubectl + export PATH="/tmp:$PATH" +fi + +TAG="{{tasks.create-tag.outputs.parameters.tag}}" +echo "=== Deploying release ${TAG} ===" + +SERVICES="api trading-service ml-training-service backtesting-service trading-agent-service broker-gateway data-acquisition-service" +for svc in $SERVICES; do + echo "Patching $svc with FOXHUNT_RELEASE=${TAG}..." + kubectl -n foxhunt patch deployment "$svc" -p "{ + \"spec\":{\"template\":{ + \"metadata\":{\"annotations\":{\"foxhunt.io/release\":\"${TAG}\"}}, + \"spec\":{\"initContainers\":[{ + \"name\":\"fetch-binary\", + \"env\":[{\"name\":\"FOXHUNT_RELEASE\",\"value\":\"${TAG}\"}] + }]} + }} + }" || echo "WARN: $svc patch failed (may not exist)" +done + +echo "=== Waiting for rollouts ===" +for svc in $SERVICES; do + kubectl -n foxhunt rollout status deployment "$svc" --timeout=120s || echo "WARN: $svc rollout timeout" +done + +echo "=== Deploy ${TAG} complete ===" +``` + +**Step 2: Verify YAML** + +```bash +python3 -c "import yaml; yaml.safe_load(open('infra/k8s/argo/ci-pipeline-template.yaml'))" +``` + +--- + +### Task 8: Update DAG dependencies + +**Files:** +- Modify: `infra/k8s/argo/ci-pipeline-template.yaml` (DAG tasks, lines ~44-136) + +**Step 1: Replace the DAG tasks section** + +Replace the entire `dag.tasks` list with: + +```yaml + tasks: + - name: detect-changes + template: detect-changes + + # Create CalVer tag when code changed + - name: create-tag + dependencies: [detect-changes] + template: create-tag + when: "{{tasks.detect-changes.outputs.parameters.needs-code}} == true" + + # Parallel: each compile step is self-contained (own git clone) + # sccache backed by MinIO — no PVC needed, shared across all nodes + - name: compile-services + dependencies: [create-tag] + template: compile-services + when: "{{tasks.detect-changes.outputs.parameters.needs-services}} == true" + + - name: compile-training + dependencies: [create-tag] + template: compile-training + when: "{{tasks.detect-changes.outputs.parameters.needs-training}} == true" + + # GPU warmup: trigger GPU node autoscale during compile so the node + # is ready when we submit a training workflow after CI completes. + - name: gpu-warmup + dependencies: [detect-changes] + template: gpu-warmup + when: "{{tasks.detect-changes.outputs.parameters.needs-training}} == true" + + - name: build-web-dashboard + dependencies: [detect-changes] + template: build-web-dashboard + when: "{{tasks.detect-changes.outputs.parameters.needs-dashboard}} == true" + + # Create GitLab Release after all binaries are uploaded + - name: upload-release + dependencies: [compile-services, compile-training] + template: upload-release + when: "{{tasks.detect-changes.outputs.parameters.needs-code}} == true" + + # Deploy after release is created + - name: deploy-services + dependencies: [upload-release] + template: deploy-services + when: "{{tasks.detect-changes.outputs.parameters.needs-services}} == true" + + - name: rebuild-ci-builder + dependencies: [detect-changes] + templateRef: + name: build-ci-image + template: build + arguments: + parameters: + - name: commit-sha + value: "{{workflow.parameters.commit-sha}}" + - name: dockerfile + value: Dockerfile.ci-builder + - name: image-name + value: ci-builder + when: "{{tasks.detect-changes.outputs.parameters.docker-images}} == true" + + - name: rebuild-ci-builder-cpu + dependencies: [detect-changes] + templateRef: + name: build-ci-image + template: build + arguments: + parameters: + - name: commit-sha + value: "{{workflow.parameters.commit-sha}}" + - name: dockerfile + value: Dockerfile.ci-builder-cpu + - name: image-name + value: ci-builder-cpu + when: "{{tasks.detect-changes.outputs.parameters.docker-images}} == true" + + - name: rebuild-runtime + dependencies: [detect-changes] + templateRef: + name: build-ci-image + template: build + arguments: + parameters: + - name: commit-sha + value: "{{workflow.parameters.commit-sha}}" + - name: dockerfile + value: Dockerfile.foxhunt-runtime + - name: image-name + value: foxhunt-runtime + when: "{{tasks.detect-changes.outputs.parameters.docker-images}} == true" + + - name: rebuild-training-runtime + dependencies: [detect-changes] + templateRef: + name: build-ci-image + template: build + arguments: + parameters: + - name: commit-sha + value: "{{workflow.parameters.commit-sha}}" + - name: dockerfile + value: Dockerfile.foxhunt-training-runtime + - name: image-name + value: foxhunt-training-runtime + when: "{{tasks.detect-changes.outputs.parameters.docker-images}} == true" +``` + +**Step 2: Verify the full pipeline YAML** + +```bash +python3 -c "import yaml; yaml.safe_load(open('infra/k8s/argo/ci-pipeline-template.yaml'))" +``` + +--- + +### Task 9: Update service initContainers — fetch from GitLab + +**Files (7 service deployments):** +- Modify: `infra/k8s/services/api.yaml:41-78` +- Modify: `infra/k8s/services/trading-service.yaml:41-77` +- Modify: `infra/k8s/services/ml-training-service.yaml:83-120` (offset by SA/RBAC) +- Modify: `infra/k8s/services/backtesting-service.yaml` (same initContainer pattern) +- Modify: `infra/k8s/services/broker-gateway.yaml` (same initContainer pattern) +- Modify: `infra/k8s/services/data-acquisition-service.yaml` (same initContainer pattern) +- Modify: `infra/k8s/services/trading-agent-service.yaml` (same initContainer pattern) + +Each file has the same initContainer pattern. Replace the `fetch-binary` initContainer in all 7 files. + +**Step 1: Replace initContainer in each service** + +For each service, replace the `fetch-binary` initContainer block. The template (substitute `SERVICE_NAME` for each): + +```yaml + - name: fetch-binary + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest + command: ["/bin/sh", "-c"] + args: + - | + set -e + BINARY="SERVICE_NAME" + curl -fSL -o "/binaries/${BINARY}" \ + --header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \ + "${GITLAB_API}/projects/1/packages/generic/foxhunt-services/${FOXHUNT_RELEASE}/${BINARY}" + chmod +x "/binaries/${BINARY}" + echo "Fetched ${BINARY} ${FOXHUNT_RELEASE} ($(stat -c%s /binaries/${BINARY}) bytes)" + env: + - name: GITLAB_DEPLOY_TOKEN + valueFrom: + secretKeyRef: + name: gitlab-deploy-token + key: token + - name: GITLAB_API + value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" + - name: FOXHUNT_RELEASE + value: "latest" + volumeMounts: + - name: binaries + mountPath: /binaries + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 128Mi +``` + +Substitutions per file: +- `api.yaml`: `SERVICE_NAME` = `api` +- `trading-service.yaml`: `SERVICE_NAME` = `trading-service` +- `ml-training-service.yaml`: `SERVICE_NAME` = `ml-training-service` +- `backtesting-service.yaml`: `SERVICE_NAME` = `backtesting-service` +- `trading-agent-service.yaml`: `SERVICE_NAME` = `trading-agent-service` +- `broker-gateway.yaml`: `SERVICE_NAME` = `broker-gateway` +- `data-acquisition-service.yaml`: `SERVICE_NAME` = `data-acquisition-service` + +Note: The `FOXHUNT_RELEASE` value `"latest"` is a placeholder — the deploy-services CI step patches it to the actual tag via `kubectl patch`. + +**Step 2: Verify all 7 service YAMLs** + +```bash +for f in infra/k8s/services/{api,trading-service,ml-training-service,backtesting-service,trading-agent-service,broker-gateway,data-acquisition-service}.yaml; do + python3 -c "import yaml; list(yaml.safe_load_all(open('$f')))" && echo "$f OK" || echo "$f FAILED" +done +``` + +Expected: all OK + +--- + +### Task 10: Update training job-template — binary fetch from GitLab + +**Files:** +- Modify: `infra/k8s/training/job-template.yaml:38-78` (fetch-binary initContainer only; sync-training-data stays MinIO) + +**Step 1: Replace fetch-binary initContainer** + +Replace the first initContainer (`fetch-binary`, lines 39-78) with: + +```yaml + - name: fetch-binary + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest + command: ["/bin/sh", "-c"] + args: + - | + set -e + BINARY="$(TRAINING_BINARY)" + curl -fSL -o "/binaries/${BINARY}" \ + --header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \ + "${GITLAB_API}/projects/1/packages/generic/foxhunt-training/${FOXHUNT_RELEASE}/${BINARY}" + chmod +x "/binaries/${BINARY}" + echo "Fetched ${BINARY} ${FOXHUNT_RELEASE} ($(stat -c%s /binaries/${BINARY}) bytes)" + env: + - name: TRAINING_BINARY + value: train_baseline_supervised + - name: GITLAB_DEPLOY_TOKEN + valueFrom: + secretKeyRef: + name: gitlab-deploy-token + key: token + - name: GITLAB_API + value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" + - name: FOXHUNT_RELEASE + value: "latest" + volumeMounts: + - name: binaries + mountPath: /binaries + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 500m + memory: 128Mi +``` + +Note: `sync-training-data` initContainer stays unchanged (MinIO). Only binary fetch moves to GitLab. + +**Step 2: Verify YAML** + +```bash +python3 -c "import yaml; yaml.safe_load(open('infra/k8s/training/job-template.yaml'))" +``` + +--- + +### Task 11: Commit and apply to cluster + +**Step 1: Commit all changes** + +```bash +git add infra/k8s/argo/ci-pipeline-template.yaml \ + infra/k8s/services/api.yaml \ + infra/k8s/services/trading-service.yaml \ + infra/k8s/services/ml-training-service.yaml \ + infra/k8s/services/backtesting-service.yaml \ + infra/k8s/services/trading-agent-service.yaml \ + infra/k8s/services/broker-gateway.yaml \ + infra/k8s/services/data-acquisition-service.yaml \ + infra/k8s/training/job-template.yaml + +git commit -m "feat(infra): GitLab releases with CalVer auto-versioning + +Replace MinIO binary distribution with GitLab Generic Package Registry. +Every code push to main auto-creates a CalVer tag (vYYYY.MM.N), +compiles, uploads binaries to GitLab packages, creates a Release +with auto-generated notes, and deploys via deployment patching. + +- New CI templates: create-tag, upload-release +- Modified: compile-services/training upload to GitLab packages +- Modified: deploy-services patches FOXHUNT_RELEASE on deployments +- All 7 service initContainers fetch from GitLab (curl, deploy token) +- Training job-template binary fetch from GitLab (data stays MinIO) +- MinIO retains: sccache, training data, model checkpoints" +``` + +**Step 2: Apply CI pipeline template** + +```bash +kubectl apply -f infra/k8s/argo/ci-pipeline-template.yaml +``` + +**Step 3: Apply service deployments (will NOT restart — just updates spec)** + +```bash +kubectl apply -f infra/k8s/services/api.yaml +kubectl apply -f infra/k8s/services/trading-service.yaml +kubectl apply -f infra/k8s/services/ml-training-service.yaml +kubectl apply -f infra/k8s/services/backtesting-service.yaml +kubectl apply -f infra/k8s/services/trading-agent-service.yaml +kubectl apply -f infra/k8s/services/broker-gateway.yaml +kubectl apply -f infra/k8s/services/data-acquisition-service.yaml +``` + +Note: Don't apply job-template.yaml — it's a template that is applied per-job, not a persistent resource. + +**Step 4: Push to trigger first release** + +```bash +git push origin main +``` + +**Step 5: Monitor the Argo workflow** + +```bash +# Watch the workflow +kubectl -n foxhunt get workflows -w + +# Check tag was created +kubectl -n foxhunt logs -l workflows.argoproj.io/workflow -c main --tail=50 | grep "Creating tag" +``` + +Expected: workflow creates tag `v2026.03.1`, compiles, uploads to GitLab packages, creates release, patches deployments. + +**Step 6: Verify services restarted with new binary** + +```bash +kubectl -n foxhunt get pods -o wide +kubectl -n foxhunt logs deployment/api -c fetch-binary +``` + +Expected: initContainer logs show `Fetched api v2026.03.1 (... bytes)` + +--- + +### Task 12: Cleanup — remove MinIO binary bucket + +Only do this after Task 11 is verified working. + +**Step 1: Verify GitLab packages exist** + +```bash +GITLAB="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181" +PAT=$(kubectl -n foxhunt get secret gitlab-pat -o jsonpath='{.data.token}' | base64 -d) + +curl -sf -H "PRIVATE-TOKEN: ${PAT}" \ + "${GITLAB}/api/v4/projects/1/packages?package_type=generic" | python3 -m json.tool +``` + +Expected: lists foxhunt-services and foxhunt-training packages with the tag version. + +**Step 2: Remove binary bucket contents from MinIO** + +```bash +kubectl -n foxhunt exec deployment/minio -- mc rm --recursive --force /data/foxhunt-binaries/services/ +kubectl -n foxhunt exec deployment/minio -- mc rm --recursive --force /data/foxhunt-binaries/training/ +``` + +Keep the bucket itself (other paths may reference it). + +**Step 3: Commit cleanup note** + +No code changes needed — the MinIO upload code was already removed in Task 4/5. diff --git a/infra/k8s/argo/ci-pipeline-template.yaml b/infra/k8s/argo/ci-pipeline-template.yaml index b314e9f20..c0257ca69 100644 --- a/infra/k8s/argo/ci-pipeline-template.yaml +++ b/infra/k8s/argo/ci-pipeline-template.yaml @@ -30,6 +30,9 @@ spec: items: - key: .dockerconfigjson path: config.json + - name: gitlab-pat + secret: + secretName: gitlab-pat arguments: parameters: - name: commit-sha @@ -46,20 +49,21 @@ spec: - name: detect-changes template: detect-changes - # Parallel: each compile step is self-contained (own git clone) - # sccache backed by MinIO — no PVC needed, shared across all nodes - - name: compile-services + - name: create-tag dependencies: [detect-changes] + template: create-tag + when: "{{tasks.detect-changes.outputs.parameters.needs-code}} == true" + + - name: compile-services + dependencies: [create-tag] template: compile-services when: "{{tasks.detect-changes.outputs.parameters.needs-services}} == true" - name: compile-training - dependencies: [detect-changes] + dependencies: [create-tag] template: compile-training when: "{{tasks.detect-changes.outputs.parameters.needs-training}} == true" - # GPU warmup: trigger GPU node autoscale during compile so the node - # is ready when we submit a training workflow after CI completes. - name: gpu-warmup dependencies: [detect-changes] template: gpu-warmup @@ -70,8 +74,13 @@ spec: template: build-web-dashboard when: "{{tasks.detect-changes.outputs.parameters.needs-dashboard}} == true" + - name: upload-release + dependencies: [compile-services, compile-training] + template: upload-release + when: "{{tasks.detect-changes.outputs.parameters.needs-code}} == true" + - name: deploy-services - dependencies: [compile-services] + dependencies: [upload-release] template: deploy-services when: "{{tasks.detect-changes.outputs.parameters.needs-services}} == true" @@ -206,9 +215,16 @@ spec: NEEDS_TRAINING="false" fi + if [ "$NEEDS_SERVICES" = "true" ] || [ "$NEEDS_TRAINING" = "true" ]; then + NEEDS_CODE="true" + else + NEEDS_CODE="false" + fi + echo "=== Build decisions ===" echo "needs-services: $NEEDS_SERVICES" echo "needs-training: $NEEDS_TRAINING" + echo "needs-code: $NEEDS_CODE" echo "needs-dashboard: $NEEDS_DASHBOARD" echo "docker-images: $DOCKER_IMAGES" echo "======================" @@ -218,6 +234,7 @@ spec: echo -n "$NEEDS_TRAINING" > /tmp/outputs/needs-training echo -n "$NEEDS_DASHBOARD" > /tmp/outputs/needs-dashboard echo -n "$DOCKER_IMAGES" > /tmp/outputs/docker-images + echo -n "$NEEDS_CODE" > /tmp/outputs/needs-code SCRIPT chmod +x /tmp/detect.sh /tmp/detect.sh @@ -235,6 +252,74 @@ spec: - name: docker-images valueFrom: path: /tmp/outputs/docker-images + - name: needs-code + valueFrom: + path: /tmp/outputs/needs-code + + # ── create-tag: CalVer auto-tag on code changes ── + - name: create-tag + nodeSelector: + k8s.scaleway.com/pool-name: platform + container: + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest + command: ["/bin/sh", "-c"] + env: + - name: GITLAB_PAT + valueFrom: + secretKeyRef: + name: gitlab-pat + key: token + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + args: + - | + set -e + GITLAB="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181" + PROJECT_ID=1 + SHA="{{workflow.parameters.commit-sha}}" + + # Compute CalVer prefix: vYYYY.MM + PREFIX="v$(date +%Y.%m)" + + # Query existing tags for this month + TAGS=$(curl -sf \ + -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + "${GITLAB}/api/v4/projects/${PROJECT_ID}/repository/tags?search=${PREFIX}" \ + || echo "[]") + + # Parse highest N from vYYYY.MM.N tags + LAST_N=$(echo "$TAGS" | grep -oP "\"name\":\"${PREFIX}\.\K[0-9]+" | sort -n | tail -1) + if [ -z "$LAST_N" ]; then + NEXT_N=1 + else + NEXT_N=$((LAST_N + 1)) + fi + + TAG="${PREFIX}.${NEXT_N}" + echo "Creating tag: ${TAG} at ${SHA}" + + # Create the tag + RESULT=$(curl -sf -X POST \ + -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + -d "tag_name=${TAG}" \ + -d "ref=${SHA}" \ + "${GITLAB}/api/v4/projects/${PROJECT_ID}/repository/tags") + + echo "$RESULT" | grep -q "$TAG" || { echo "Tag creation failed: $RESULT"; exit 1; } + echo "Tag ${TAG} created successfully" + + mkdir -p /tmp/outputs + echo -n "$TAG" > /tmp/outputs/tag + outputs: + parameters: + - name: tag + valueFrom: + path: /tmp/outputs/tag # ── build-web-dashboard: npm build + upload to MinIO ── - name: build-web-dashboard @@ -344,16 +429,11 @@ spec: secretKeyRef: name: minio-credentials key: secret-key - - name: MINIO_ACCESS_KEY + - name: GITLAB_PAT valueFrom: secretKeyRef: - name: minio-credentials - key: access-key - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: secret-key + name: gitlab-pat + key: token resources: requests: cpu: "14" @@ -390,9 +470,7 @@ spec: cd "$WORKSPACE" echo "Checked out $(git rev-parse --short HEAD)" - YEAR=$(date +%Y) - MONTH=$(date +%m) - export FOXHUNT_BUILD_VERSION="${YEAR}.${MONTH}.argo" + export FOXHUNT_BUILD_VERSION="{{tasks.create-tag.outputs.parameters.tag}}" export RUSTC_WRAPPER=sccache sccache --zero-stats || true @@ -415,13 +493,16 @@ spec: ls -lh "$WORKSPACE/bin/services/" sccache --show-stats || true - echo "=== Uploading service binaries ===" - rclone copy "$WORKSPACE/bin/services/" :s3:foxhunt-binaries/services/ \ - --s3-provider=Minio \ - --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ - --s3-access-key-id="${MINIO_ACCESS_KEY}" \ - --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket + echo "=== Uploading service binaries to GitLab packages ===" + GITLAB="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181" + TAG="${FOXHUNT_BUILD_VERSION}" + for bin in "$WORKSPACE/bin/services/"*; do + BIN_NAME=$(basename "$bin") + echo "Uploading ${BIN_NAME}..." + curl -f --upload-file "$bin" \ + -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + "${GITLAB}/api/v4/projects/1/packages/generic/foxhunt-services/${TAG}/${BIN_NAME}" + done echo "=== Service compile + upload done ===" @@ -463,16 +544,11 @@ spec: secretKeyRef: name: minio-credentials key: secret-key - - name: MINIO_ACCESS_KEY + - name: GITLAB_PAT valueFrom: secretKeyRef: - name: minio-credentials - key: access-key - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: secret-key + name: gitlab-pat + key: token resources: requests: cpu: "14" @@ -509,9 +585,7 @@ spec: cd "$WORKSPACE" echo "Checked out $(git rev-parse --short HEAD)" - YEAR=$(date +%Y) - MONTH=$(date +%m) - export FOXHUNT_BUILD_VERSION="${YEAR}.${MONTH}.argo" + export FOXHUNT_BUILD_VERSION="{{tasks.create-tag.outputs.parameters.tag}}" export RUSTC_WRAPPER=sccache export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}} export SCCACHE_S3_KEY_PREFIX="cuda/sm_${CUDA_COMPUTE_CAP}" @@ -536,16 +610,104 @@ spec: ls -lh "$WORKSPACE/bin/training/" sccache --show-stats || true - echo "=== Uploading training binaries ===" - rclone copy "$WORKSPACE/bin/training/" :s3:foxhunt-binaries/training/ \ - --s3-provider=Minio \ - --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ - --s3-access-key-id="${MINIO_ACCESS_KEY}" \ - --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket + echo "=== Uploading training binaries to GitLab packages ===" + GITLAB="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181" + TAG="${FOXHUNT_BUILD_VERSION}" + for bin in "$WORKSPACE/bin/training/"*; do + BIN_NAME=$(basename "$bin") + echo "Uploading ${BIN_NAME}..." + curl -f --upload-file "$bin" \ + -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + "${GITLAB}/api/v4/projects/1/packages/generic/foxhunt-training/${TAG}/${BIN_NAME}" + done echo "=== Training compile + upload done ===" + # ── upload-release: create GitLab Release with package links ── + - name: upload-release + nodeSelector: + k8s.scaleway.com/pool-name: platform + container: + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest + command: ["/bin/sh", "-c"] + env: + - name: GITLAB_PAT + valueFrom: + secretKeyRef: + name: gitlab-pat + key: token + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + args: + - | + set -e + GITLAB="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181" + PROJECT_ID=1 + TAG="{{tasks.create-tag.outputs.parameters.tag}}" + + echo "=== Creating GitLab Release ${TAG} ===" + + # Get commits since previous tag for release notes + PREV_TAG=$(curl -sf \ + -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + "${GITLAB}/api/v4/projects/${PROJECT_ID}/repository/tags?per_page=2" \ + | grep -oP '"name":"\K[^"]+' | sed -n '2p') + + if [ -n "$PREV_TAG" ]; then + COMMITS=$(curl -sf \ + -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + "${GITLAB}/api/v4/projects/${PROJECT_ID}/repository/compare?from=${PREV_TAG}&to=${TAG}" \ + | grep -oP '"title":"\K[^"]+' | head -20 \ + | sed 's/^/- /' || echo "- Release ${TAG}") + DESCRIPTION="## Changes since ${PREV_TAG}\n\n${COMMITS}" + else + DESCRIPTION="## Initial release\n\nFirst CalVer release." + fi + + # Create the release + RESPONSE=$(curl -sf -X POST \ + -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + -H "Content-Type: application/json" \ + -d "{ + \"tag_name\": \"${TAG}\", + \"name\": \"${TAG}\", + \"description\": \"$(printf '%s' "$DESCRIPTION" | sed 's/"/\\"/g')\" + }" \ + "${GITLAB}/api/v4/projects/${PROJECT_ID}/releases") || { + echo "WARN: Release creation failed (may already exist)" + } + + echo "Release ${TAG} created" + echo "$RESPONSE" | head -5 + + # Add package links as release assets + for pkg_name in foxhunt-services foxhunt-training; do + PKG_CHECK=$(curl -sf \ + -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + "${GITLAB}/api/v4/projects/${PROJECT_ID}/packages?package_name=${pkg_name}&package_version=${TAG}" \ + || echo "[]") + + if echo "$PKG_CHECK" | grep -q "$TAG"; then + curl -sf -X POST \ + -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \ + -H "Content-Type: application/json" \ + -d "{ + \"name\": \"${pkg_name}\", + \"url\": \"${GITLAB}/-/packages?type=generic&search=${pkg_name}&version=${TAG}\", + \"link_type\": \"package\" + }" \ + "${GITLAB}/api/v4/projects/${PROJECT_ID}/releases/${TAG}/assets/links" || true + echo "Linked ${pkg_name} package to release" + fi + done + + echo "=== Release ${TAG} complete ===" + # ── gpu-warmup: trigger GPU node autoscale during compile ── # Runs in parallel with compile-training. By the time compile finishes # (~5.5 min) the GPU node is autoscaled, driver mounted, and ready. @@ -596,7 +758,6 @@ spec: args: - | set -e - # kubectl baked into foxhunt-runtime image; fallback to /tmp for non-root if ! command -v kubectl >/dev/null 2>&1; then echo "=== Installing kubectl ===" curl -sLo /tmp/kubectl "https://dl.k8s.io/release/v1.31.4/bin/linux/amd64/kubectl" @@ -604,11 +765,21 @@ spec: export PATH="/tmp:$PATH" fi - echo "=== Rolling restart of service deployments ===" + TAG="{{tasks.create-tag.outputs.parameters.tag}}" + echo "=== Deploying release ${TAG} ===" + SERVICES="api trading-service ml-training-service backtesting-service trading-agent-service broker-gateway data-acquisition-service" for svc in $SERVICES; do - echo "Restarting $svc..." - kubectl -n foxhunt rollout restart deployment "$svc" || echo "WARN: $svc restart failed (may not exist)" + echo "Patching $svc with FOXHUNT_RELEASE=${TAG}..." + kubectl -n foxhunt patch deployment "$svc" -p "{ + \"spec\":{\"template\":{ + \"metadata\":{\"annotations\":{\"foxhunt.io/release\":\"${TAG}\"}}, + \"spec\":{\"initContainers\":[{ + \"name\":\"fetch-binary\", + \"env\":[{\"name\":\"FOXHUNT_RELEASE\",\"value\":\"${TAG}\"}] + }]} + }} + }" || echo "WARN: $svc patch failed (may not exist)" done echo "=== Waiting for rollouts ===" @@ -616,4 +787,4 @@ spec: kubectl -n foxhunt rollout status deployment "$svc" --timeout=120s || echo "WARN: $svc rollout timeout" done - echo "=== Deploy complete ===" + echo "=== Deploy ${TAG} complete ===" diff --git a/infra/k8s/services/api.yaml b/infra/k8s/services/api.yaml index 55d439c8e..5abcc7243 100644 --- a/infra/k8s/services/api.yaml +++ b/infra/k8s/services/api.yaml @@ -45,27 +45,22 @@ spec: args: - | set -e - rclone copyto \ - ":s3:foxhunt-binaries/services/api" \ - "/binaries/api" \ - --s3-provider=Minio \ - --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ - --s3-access-key-id="${MINIO_ACCESS_KEY}" \ - --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket - chmod +x /binaries/api - echo "Fetched api ($(stat -c%s /binaries/api) bytes)" + BINARY="api" + curl -fSL -o "/binaries/${BINARY}" \ + --header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \ + "${GITLAB_API}/projects/1/packages/generic/foxhunt-services/${FOXHUNT_RELEASE}/${BINARY}" + chmod +x "/binaries/${BINARY}" + echo "Fetched ${BINARY} ${FOXHUNT_RELEASE} ($(stat -c%s /binaries/${BINARY}) bytes)" env: - - name: MINIO_ACCESS_KEY + - name: GITLAB_DEPLOY_TOKEN valueFrom: secretKeyRef: - name: minio-credentials - key: access-key - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: secret-key + name: gitlab-deploy-token + key: token + - name: GITLAB_API + value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" + - name: FOXHUNT_RELEASE + value: "latest" volumeMounts: - name: binaries mountPath: /binaries diff --git a/infra/k8s/services/backtesting-service.yaml b/infra/k8s/services/backtesting-service.yaml index c261d2645..ce3285ed3 100644 --- a/infra/k8s/services/backtesting-service.yaml +++ b/infra/k8s/services/backtesting-service.yaml @@ -44,27 +44,22 @@ spec: args: - | set -e - rclone copyto \ - ":s3:foxhunt-binaries/services/backtesting-service" \ - "/binaries/backtesting-service" \ - --s3-provider=Minio \ - --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ - --s3-access-key-id="${MINIO_ACCESS_KEY}" \ - --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket - chmod +x /binaries/backtesting-service - echo "Fetched backtesting-service ($(stat -c%s /binaries/backtesting-service) bytes)" + BINARY="backtesting-service" + curl -fSL -o "/binaries/${BINARY}" \ + --header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \ + "${GITLAB_API}/projects/1/packages/generic/foxhunt-services/${FOXHUNT_RELEASE}/${BINARY}" + chmod +x "/binaries/${BINARY}" + echo "Fetched ${BINARY} ${FOXHUNT_RELEASE} ($(stat -c%s /binaries/${BINARY}) bytes)" env: - - name: MINIO_ACCESS_KEY + - name: GITLAB_DEPLOY_TOKEN valueFrom: secretKeyRef: - name: minio-credentials - key: access-key - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: secret-key + name: gitlab-deploy-token + key: token + - name: GITLAB_API + value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" + - name: FOXHUNT_RELEASE + value: "latest" volumeMounts: - name: binaries mountPath: /binaries diff --git a/infra/k8s/services/broker-gateway.yaml b/infra/k8s/services/broker-gateway.yaml index ead15a5fe..b6cb916ff 100644 --- a/infra/k8s/services/broker-gateway.yaml +++ b/infra/k8s/services/broker-gateway.yaml @@ -44,27 +44,22 @@ spec: args: - | set -e - rclone copyto \ - ":s3:foxhunt-binaries/services/broker-gateway" \ - "/binaries/broker-gateway" \ - --s3-provider=Minio \ - --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ - --s3-access-key-id="${MINIO_ACCESS_KEY}" \ - --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket - chmod +x /binaries/broker-gateway - echo "Fetched broker-gateway ($(stat -c%s /binaries/broker-gateway) bytes)" + BINARY="broker-gateway" + curl -fSL -o "/binaries/${BINARY}" \ + --header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \ + "${GITLAB_API}/projects/1/packages/generic/foxhunt-services/${FOXHUNT_RELEASE}/${BINARY}" + chmod +x "/binaries/${BINARY}" + echo "Fetched ${BINARY} ${FOXHUNT_RELEASE} ($(stat -c%s /binaries/${BINARY}) bytes)" env: - - name: MINIO_ACCESS_KEY + - name: GITLAB_DEPLOY_TOKEN valueFrom: secretKeyRef: - name: minio-credentials - key: access-key - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: secret-key + name: gitlab-deploy-token + key: token + - name: GITLAB_API + value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" + - name: FOXHUNT_RELEASE + value: "latest" volumeMounts: - name: binaries mountPath: /binaries diff --git a/infra/k8s/services/data-acquisition-service.yaml b/infra/k8s/services/data-acquisition-service.yaml index 9413c2803..8cdd23fd0 100644 --- a/infra/k8s/services/data-acquisition-service.yaml +++ b/infra/k8s/services/data-acquisition-service.yaml @@ -44,27 +44,22 @@ spec: args: - | set -e - rclone copyto \ - ":s3:foxhunt-binaries/services/data-acquisition-service" \ - "/binaries/data-acquisition-service" \ - --s3-provider=Minio \ - --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ - --s3-access-key-id="${MINIO_ACCESS_KEY}" \ - --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket - chmod +x /binaries/data-acquisition-service - echo "Fetched data-acquisition-service ($(stat -c%s /binaries/data-acquisition-service) bytes)" + BINARY="data-acquisition-service" + curl -fSL -o "/binaries/${BINARY}" \ + --header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \ + "${GITLAB_API}/projects/1/packages/generic/foxhunt-services/${FOXHUNT_RELEASE}/${BINARY}" + chmod +x "/binaries/${BINARY}" + echo "Fetched ${BINARY} ${FOXHUNT_RELEASE} ($(stat -c%s /binaries/${BINARY}) bytes)" env: - - name: MINIO_ACCESS_KEY + - name: GITLAB_DEPLOY_TOKEN valueFrom: secretKeyRef: - name: minio-credentials - key: access-key - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: secret-key + name: gitlab-deploy-token + key: token + - name: GITLAB_API + value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" + - name: FOXHUNT_RELEASE + value: "latest" volumeMounts: - name: binaries mountPath: /binaries diff --git a/infra/k8s/services/ml-training-service.yaml b/infra/k8s/services/ml-training-service.yaml index 5749cc1a3..e473e9a83 100644 --- a/infra/k8s/services/ml-training-service.yaml +++ b/infra/k8s/services/ml-training-service.yaml @@ -87,27 +87,22 @@ spec: args: - | set -e - rclone copyto \ - ":s3:foxhunt-binaries/services/ml-training-service" \ - "/binaries/ml-training-service" \ - --s3-provider=Minio \ - --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ - --s3-access-key-id="${MINIO_ACCESS_KEY}" \ - --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket - chmod +x /binaries/ml-training-service - echo "Fetched ml-training-service ($(stat -c%s /binaries/ml-training-service) bytes)" + BINARY="ml-training-service" + curl -fSL -o "/binaries/${BINARY}" \ + --header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \ + "${GITLAB_API}/projects/1/packages/generic/foxhunt-services/${FOXHUNT_RELEASE}/${BINARY}" + chmod +x "/binaries/${BINARY}" + echo "Fetched ${BINARY} ${FOXHUNT_RELEASE} ($(stat -c%s /binaries/${BINARY}) bytes)" env: - - name: MINIO_ACCESS_KEY + - name: GITLAB_DEPLOY_TOKEN valueFrom: secretKeyRef: - name: minio-credentials - key: access-key - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: secret-key + name: gitlab-deploy-token + key: token + - name: GITLAB_API + value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" + - name: FOXHUNT_RELEASE + value: "latest" volumeMounts: - name: binaries mountPath: /binaries diff --git a/infra/k8s/services/trading-agent-service.yaml b/infra/k8s/services/trading-agent-service.yaml index 8cac12bed..5123180de 100644 --- a/infra/k8s/services/trading-agent-service.yaml +++ b/infra/k8s/services/trading-agent-service.yaml @@ -44,27 +44,22 @@ spec: args: - | set -e - rclone copyto \ - ":s3:foxhunt-binaries/services/trading-agent-service" \ - "/binaries/trading-agent-service" \ - --s3-provider=Minio \ - --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ - --s3-access-key-id="${MINIO_ACCESS_KEY}" \ - --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket - chmod +x /binaries/trading-agent-service - echo "Fetched trading-agent-service ($(stat -c%s /binaries/trading-agent-service) bytes)" + BINARY="trading-agent-service" + curl -fSL -o "/binaries/${BINARY}" \ + --header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \ + "${GITLAB_API}/projects/1/packages/generic/foxhunt-services/${FOXHUNT_RELEASE}/${BINARY}" + chmod +x "/binaries/${BINARY}" + echo "Fetched ${BINARY} ${FOXHUNT_RELEASE} ($(stat -c%s /binaries/${BINARY}) bytes)" env: - - name: MINIO_ACCESS_KEY + - name: GITLAB_DEPLOY_TOKEN valueFrom: secretKeyRef: - name: minio-credentials - key: access-key - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: secret-key + name: gitlab-deploy-token + key: token + - name: GITLAB_API + value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" + - name: FOXHUNT_RELEASE + value: "latest" volumeMounts: - name: binaries mountPath: /binaries diff --git a/infra/k8s/services/trading-service.yaml b/infra/k8s/services/trading-service.yaml index 1e721735e..bb6711a9f 100644 --- a/infra/k8s/services/trading-service.yaml +++ b/infra/k8s/services/trading-service.yaml @@ -44,27 +44,22 @@ spec: args: - | set -e - rclone copyto \ - ":s3:foxhunt-binaries/services/trading-service" \ - "/binaries/trading-service" \ - --s3-provider=Minio \ - --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ - --s3-access-key-id="${MINIO_ACCESS_KEY}" \ - --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket - chmod +x /binaries/trading-service - echo "Fetched trading-service ($(stat -c%s /binaries/trading-service) bytes)" + BINARY="trading-service" + curl -fSL -o "/binaries/${BINARY}" \ + --header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \ + "${GITLAB_API}/projects/1/packages/generic/foxhunt-services/${FOXHUNT_RELEASE}/${BINARY}" + chmod +x "/binaries/${BINARY}" + echo "Fetched ${BINARY} ${FOXHUNT_RELEASE} ($(stat -c%s /binaries/${BINARY}) bytes)" env: - - name: MINIO_ACCESS_KEY + - name: GITLAB_DEPLOY_TOKEN valueFrom: secretKeyRef: - name: minio-credentials - key: access-key - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: secret-key + name: gitlab-deploy-token + key: token + - name: GITLAB_API + value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" + - name: FOXHUNT_RELEASE + value: "latest" volumeMounts: - name: binaries mountPath: /binaries diff --git a/infra/k8s/training/job-template.yaml b/infra/k8s/training/job-template.yaml index e7134c042..a4d04333b 100644 --- a/infra/k8s/training/job-template.yaml +++ b/infra/k8s/training/job-template.yaml @@ -36,36 +36,31 @@ spec: - name: gitlab-registry restartPolicy: Never initContainers: - # 1. Fetch training binary from MinIO + # 1. Fetch training binary from GitLab Generic Package Registry - name: fetch-binary image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest command: ["/bin/sh", "-c"] args: - | set -e - rclone copyto \ - ":s3:foxhunt-binaries/training/$(TRAINING_BINARY)" \ - "/binaries/$(TRAINING_BINARY)" \ - --s3-provider=Minio \ - --s3-endpoint=http://minio.foxhunt.svc.cluster.local:9000 \ - --s3-access-key-id="${MINIO_ACCESS_KEY}" \ - --s3-secret-access-key="${MINIO_SECRET_KEY}" \ - --s3-no-check-bucket - chmod +x "/binaries/$(TRAINING_BINARY)" - echo "Fetched $(TRAINING_BINARY) ($(stat -c%s /binaries/$(TRAINING_BINARY)) bytes)" + BINARY="$(TRAINING_BINARY)" + curl -fSL -o "/binaries/${BINARY}" \ + --header "DEPLOY-TOKEN: ${GITLAB_DEPLOY_TOKEN}" \ + "${GITLAB_API}/projects/1/packages/generic/foxhunt-training/${FOXHUNT_RELEASE}/${BINARY}" + chmod +x "/binaries/${BINARY}" + echo "Fetched ${BINARY} ${FOXHUNT_RELEASE} ($(stat -c%s /binaries/${BINARY}) bytes)" env: - name: TRAINING_BINARY value: train_baseline_supervised - - name: MINIO_ACCESS_KEY + - name: GITLAB_DEPLOY_TOKEN valueFrom: secretKeyRef: - name: minio-credentials - key: access-key - - name: MINIO_SECRET_KEY - valueFrom: - secretKeyRef: - name: minio-credentials - key: secret-key + name: gitlab-deploy-token + key: token + - name: GITLAB_API + value: "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181/api/v4" + - name: FOXHUNT_RELEASE + value: "latest" volumeMounts: - name: binaries mountPath: /binaries