Merge branch 'worktree-training-deploy' (CPU pipeline split)

2026-02-28 00:23:20 +01:00
parent c328e3eb3a 829d05e870
commit fc48395ad1
2 changed files with 211 additions and 44 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,8 +1,9 @@
 # GitLab CI/CD — Foxhunt
 # Two runners: CPU (compile/build/deploy) and GPU (training)
-# CI builder image: CUDA 12.4 + Rust 1.89 + protoc + sccache (hosted on Scaleway CR)
+# CI builder images:
+#   ci-builder:     CUDA 12.4 + Rust 1.89 + protoc + sccache (for training binaries)
+#   ci-builder-cpu: Rust 1.89 + protoc + sccache, no CUDA (for service binaries)
 # Service images: pushed to Scaleway Container Registry (Kaniko)
-# Note: .gitlab-ci.yml itself is NOT in changes: filters — CI-only edits skip build
 #
 # Runner routing (by tags):
 #   kapsule,rust,docker → CPU runner → ci-compile-cpu pool (POP2-32C-128G, 32 vCPU)
@@ -50,6 +51,7 @@ variables:
  AWS_SECRET_ACCESS_KEY: $SCW_SECRET_KEY
  # CI builder image on Scaleway Container Registry (reachable by Kapsule nodes)
  CI_BUILDER_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/ci-builder:latest
+  CI_BUILDER_CPU_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/ci-builder-cpu:latest
  # Scaleway Container Registry for service images
  REGISTRY: rg.fr-par.scw.cloud/foxhunt-ci
  # IaC runner image on Scaleway Container Registry
@@ -87,6 +89,38 @@ build-ci-builder:
      --cache=true --cache-repo="${REGISTRY}/cache"
      --destination "${CI_BUILDER_IMAGE}"

+# --------------------------------------------------------------------------
+# Stage 0a: Build CPU-only CI builder image → push to Scaleway CR
+# --------------------------------------------------------------------------
+build-ci-builder-cpu:
+  stage: prepare
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
+  tags:
+    - kapsule
+    - docker
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "push"
+      changes:
+        - infra/docker/Dockerfile.ci-builder-cpu
+      when: on_success
+    - when: manual
+      allow_failure: true
+  before_script:
+    - mkdir -p /kaniko/.docker
+    - >-
+      echo "{\"auths\":{
+      \"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
+      \"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
+      }}" > /kaniko/.docker/config.json
+  script:
+    - /kaniko/executor
+      --context "${CI_PROJECT_DIR}"
+      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.ci-builder-cpu"
+      --cache=true --cache-repo="${REGISTRY}/cache"
+      --destination "${CI_BUILDER_CPU_IMAGE}"
+
 # --------------------------------------------------------------------------
 # Stage 0b: Build devcontainer image → push to internal GitLab registry
 # --------------------------------------------------------------------------
@@ -181,6 +215,29 @@ build-infra-runner:
        echo "mold not found, falling back to lld via ld.mold wrapper"
      fi

+# Base template for CPU-only Rust jobs — lightweight builder without CUDA
+# Uses ci-builder-cpu image (rust:1.89-slim, ~2-3GB, no CUDA toolkit)
+.rust-base-cpu:
+  image: ${CI_BUILDER_CPU_IMAGE}
+  tags:
+    - kapsule
+    - rust
+  variables:
+    # No CUDA_COMPUTE_CAP — CPU-only builds don't compile CUDA kernels
+    CARGO_BUILD_JOBS: "30"
+    KUBERNETES_CPU_REQUEST: "28000m"
+    KUBERNETES_CPU_LIMIT: "31000m"
+    KUBERNETES_MEMORY_REQUEST: "24Gi"
+    KUBERNETES_MEMORY_LIMIT: "56Gi"
+  before_script:
+    # Mold linker fallback — same as .rust-base
+    - |
+      if ! command -v mold &>/dev/null; then
+        printf '#!/bin/sh\nexec ld.lld "$@"\n' > /usr/local/bin/ld.mold
+        chmod +x /usr/local/bin/ld.mold
+        echo "mold not found, falling back to lld via ld.mold wrapper"
+      fi
+
 # --------------------------------------------------------------------------
 # Stage 2: tests
 # --------------------------------------------------------------------------
@@ -228,14 +285,13 @@ test:
    - sccache --show-stats || true

 # --------------------------------------------------------------------------
-# Stage 3a: Compile all service binaries (single job, PVC sccache)
+# Stage 3a: Compile service binaries (CPU-only, no CUDA)
 # --------------------------------------------------------------------------
 compile-services:
-  extends: .rust-base
+  extends: .rust-base-cpu
  stage: compile
  needs: []
  rules:
-    # Only recompile when Rust source or Dockerfiles change — infra-only pushes skip
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
      changes:
        - Cargo.toml
@@ -246,16 +302,10 @@ compile-services:
        - infra/docker/Dockerfile.*
    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
  script:
-    # Compile uses CUDA stubs — no nvidia runtime needed, CUDA_COMPUTE_CAP from variables
-    - echo "CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
-    - export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}"
-    - mkdir -p "$SCCACHE_DIR"
-    - sccache --zero-stats || true
-    # Select build profile: dev-release (fast, thin LTO) or release (full, fat LTO)
+    - mkdir -p "$SCCACHE_DIR" && sccache --zero-stats || true
    - PROFILE=${DEV_RELEASE:+dev-release}; PROFILE=${PROFILE:-release}
-    - echo "Building with --profile $PROFILE"
+    - echo "Building services with --profile $PROFILE (CPU-only, no CUDA)"
    - TARGET_DIR="target/$PROFILE"
-    # 1) Build service binaries (no CUDA feature)
    - cargo build --profile $PROFILE
        -p trading_service
        -p api_gateway
@@ -265,7 +315,47 @@ compile-services:
        -p trading_agent_service
        -p data_acquisition_service
        -p web-gateway
-    # 2) Build training binaries with CUDA (sccache already warm from step 1)
+    - sccache --show-stats || true
+    - mkdir -p build-out/services
+    - |
+      for bin in trading_service api_gateway broker_gateway_service ml_training_service backtesting_service trading_agent_service data_acquisition_service web-gateway; do
+        cp $TARGET_DIR/$bin build-out/services/
+        strip build-out/services/$bin
+      done
+    - ls -lh build-out/services/
+  artifacts:
+    paths:
+      - build-out/services/
+    expire_in: 1 hour
+
+# --------------------------------------------------------------------------
+# Stage 3a: Compile training binaries (CUDA stubs for candle GPU kernels)
+# --------------------------------------------------------------------------
+compile-training:
+  extends: .rust-base
+  stage: compile
+  needs: []
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
+      changes:
+        - Cargo.toml
+        - Cargo.lock
+        - crates/ml/**
+        - crates/trading_engine/**
+        - crates/common/**
+        - crates/risk/**
+        - crates/data/**
+        - crates/config/**
+        - crates/storage/**
+        - services/training_uploader/**
+    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
+  script:
+    - echo "CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP"
+    - export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}"
+    - mkdir -p "$SCCACHE_DIR" && sccache --zero-stats || true
+    - PROFILE=${DEV_RELEASE:+dev-release}; PROFILE=${PROFILE:-release}
+    - echo "Building training binaries with --profile $PROFILE (CUDA stubs)"
+    - TARGET_DIR="target/$PROFILE"
    - cargo build --profile $PROFILE -p ml --features ml/cuda
        --example train_baseline_rl
        --example train_baseline_supervised
@@ -273,31 +363,28 @@ compile-services:
        --example evaluate_supervised
        --example hyperopt_baseline_rl
        --example hyperopt_baseline_supervised
-    # 3) Build training uploader sidecar (no CUDA needed)
    - cargo build --profile $PROFILE -p training_uploader
    - sccache --show-stats || true
-    - mkdir -p build-out
+    - mkdir -p build-out/training
    - |
-      for bin in trading_service api_gateway broker_gateway_service ml_training_service backtesting_service trading_agent_service data_acquisition_service web-gateway; do
-        cp $TARGET_DIR/$bin build-out/
-        strip build-out/$bin
-      done
      for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised; do
-        cp $TARGET_DIR/examples/$bin build-out/
-        strip build-out/$bin
+        cp $TARGET_DIR/examples/$bin build-out/training/
+        strip build-out/training/$bin
      done
-      cp $TARGET_DIR/training_uploader build-out/
-      strip build-out/training_uploader
-    - ls -lh build-out/
+      cp $TARGET_DIR/training_uploader build-out/training/
+      strip build-out/training/training_uploader
+    - ls -lh build-out/training/
  artifacts:
    paths:
-      - build-out/
+      - build-out/training/
    expire_in: 1 hour

 # --------------------------------------------------------------------------
 # Stage 3b: Package service images (Kaniko → Scaleway CR, uses pre-built binaries)
 # --------------------------------------------------------------------------
-.kaniko-base:
+
+# Kaniko base for service images (depends on compile-services)
+.kaniko-service-base:
  stage: build
  needs:
    - job: compile-services
@@ -332,72 +419,105 @@ compile-services:
      \"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
      }}" > /kaniko/.docker/config.json

+# Kaniko base for training image (depends on compile-training)
+.kaniko-training-base:
+  stage: build
+  needs:
+    - job: compile-training
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
+  tags:
+    - kapsule
+    - docker
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
+      changes:
+        - Cargo.toml
+        - Cargo.lock
+        - crates/ml/**
+        - crates/trading_engine/**
+        - crates/common/**
+        - crates/risk/**
+        - crates/data/**
+        - crates/config/**
+        - crates/storage/**
+        - services/training_uploader/**
+    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
+  before_script:
+    - mkdir -p /kaniko/.docker
+    - >-
+      echo "{\"auths\":{
+      \"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"},
+      \"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"}
+      }}" > /kaniko/.docker/config.json
+
 # 7 services: pre-built binary → Dockerfile.runtime (no Rust toolchain, ~30s each)
 build-trading-service:
-  extends: .kaniko-base
+  extends: .kaniko-service-base
  script:
    - /kaniko/executor
-      --context dir://${CI_PROJECT_DIR}/build-out
+      --context dir://${CI_PROJECT_DIR}/build-out/services
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
      --build-arg SERVICE=trading_service
      --destination "${REGISTRY}/trading_service:${CI_COMMIT_SHA}"
      --destination "${REGISTRY}/trading_service:latest"

 build-api-gateway:
-  extends: .kaniko-base
+  extends: .kaniko-service-base
  script:
    - /kaniko/executor
-      --context dir://${CI_PROJECT_DIR}/build-out
+      --context dir://${CI_PROJECT_DIR}/build-out/services
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
      --build-arg SERVICE=api_gateway
      --destination "${REGISTRY}/api_gateway:${CI_COMMIT_SHA}"
      --destination "${REGISTRY}/api_gateway:latest"

 build-broker-gateway:
-  extends: .kaniko-base
+  extends: .kaniko-service-base
  script:
    - /kaniko/executor
-      --context dir://${CI_PROJECT_DIR}/build-out
+      --context dir://${CI_PROJECT_DIR}/build-out/services
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
      --build-arg SERVICE=broker_gateway_service
      --destination "${REGISTRY}/broker_gateway_service:${CI_COMMIT_SHA}"
      --destination "${REGISTRY}/broker_gateway_service:latest"

 build-ml-training:
-  extends: .kaniko-base
+  extends: .kaniko-service-base
  script:
    - /kaniko/executor
-      --context dir://${CI_PROJECT_DIR}/build-out
+      --context dir://${CI_PROJECT_DIR}/build-out/services
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
      --build-arg SERVICE=ml_training_service
      --destination "${REGISTRY}/ml_training_service:${CI_COMMIT_SHA}"
      --destination "${REGISTRY}/ml_training_service:latest"

 build-backtesting:
-  extends: .kaniko-base
+  extends: .kaniko-service-base
  script:
    - /kaniko/executor
-      --context dir://${CI_PROJECT_DIR}/build-out
+      --context dir://${CI_PROJECT_DIR}/build-out/services
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
      --build-arg SERVICE=backtesting_service
      --destination "${REGISTRY}/backtesting_service:${CI_COMMIT_SHA}"
      --destination "${REGISTRY}/backtesting_service:latest"

 build-trading-agent:
-  extends: .kaniko-base
+  extends: .kaniko-service-base
  script:
    - /kaniko/executor
-      --context dir://${CI_PROJECT_DIR}/build-out
+      --context dir://${CI_PROJECT_DIR}/build-out/services
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
      --build-arg SERVICE=trading_agent_service
      --destination "${REGISTRY}/trading_agent_service:${CI_COMMIT_SHA}"
      --destination "${REGISTRY}/trading_agent_service:latest"

 build-data-acquisition:
-  extends: .kaniko-base
+  extends: .kaniko-service-base
  script:
    - /kaniko/executor
-      --context dir://${CI_PROJECT_DIR}/build-out
+      --context dir://${CI_PROJECT_DIR}/build-out/services
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime"
      --build-arg SERVICE=data_acquisition_service
      --destination "${REGISTRY}/data_acquisition_service:${CI_COMMIT_SHA}"
@@ -405,7 +525,7 @@ build-data-acquisition:

 # web-gateway: Node.js dashboard build + pre-built Rust binary
 build-web-gateway:
-  extends: .kaniko-base
+  extends: .kaniko-service-base
  script:
    - /kaniko/executor
      --context "${CI_PROJECT_DIR}"
@@ -416,10 +536,10 @@ build-web-gateway:

 # training: pre-built CUDA binaries → CUDA runtime image (no recompilation)
 build-training:
-  extends: .kaniko-base
+  extends: .kaniko-training-base
  script:
    - /kaniko/executor
-      --context dir://${CI_PROJECT_DIR}/build-out
+      --context dir://${CI_PROJECT_DIR}/build-out/training
      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.training-runtime"
      --destination "${REGISTRY}/training:${CI_COMMIT_SHA}"
      --destination "${REGISTRY}/training:latest"
--- a/infra/docker/Dockerfile.ci-builder-cpu
+++ b/infra/docker/Dockerfile.ci-builder-cpu
@@ -0,0 +1,47 @@
+# CUDA-free CI builder image for Foxhunt service compilation
+# ~2-3GB vs ~8GB for the CUDA devel variant (Dockerfile.ci-builder)
+# Contains: Rust 1.89, protoc, sccache, git, OpenSSL, clang, mold, lld, make
+# Build: docker build -f infra/docker/Dockerfile.ci-builder-cpu -t foxhunt-ci-builder-cpu .
+# Push:  docker tag foxhunt-ci-builder-cpu rg.fr-par.scw.cloud/foxhunt-ci/ci-builder-cpu:latest
+#        docker push rg.fr-par.scw.cloud/foxhunt-ci/ci-builder-cpu:latest
+
+FROM rust:1.89-slim-bookworm
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    curl \
+    ca-certificates \
+    pkg-config \
+    libssl-dev \
+    lld \
+    clang \
+    make \
+    perl \
+    build-essential \
+    libfontconfig1-dev \
+    unzip \
+    && rm -rf /var/lib/apt/lists/*
+
+# mold linker — 2-5x faster than lld for final link step on large Rust binaries
+RUN curl -fsSL https://github.com/rui314/mold/releases/download/v2.35.1/mold-2.35.1-x86_64-linux.tar.gz \
+    | tar xz -C /usr/local --strip-components=1 \
+    && chmod +x /usr/local/bin/mold
+
+# protoc 28.3 (proto3 optional support, matches local dev)
+RUN curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v28.3/protoc-28.3-linux-x86_64.zip -o /tmp/protoc.zip \
+    && unzip -o /tmp/protoc.zip -d /usr/local bin/protoc 'include/*' \
+    && rm /tmp/protoc.zip \
+    && chmod +x /usr/local/bin/protoc
+
+# sccache for S3-backed build caching
+RUN curl -fsSL https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz \
+    | tar xz -C /usr/local/bin --strip-components=1 sccache-v0.10.0-x86_64-unknown-linux-musl/sccache \
+    && chmod +x /usr/local/bin/sccache
+
+# clippy pre-installed
+RUN rustup component add clippy
+
+# Verify
+RUN rustc --version && cargo --version && git --version && protoc --version && sccache --version && make --version && mold --version