From d4bd3d9465bcb07a13710f382f7fb709fd00256f Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 28 Feb 2026 00:16:28 +0100 Subject: [PATCH 1/2] feat(ci): add CUDA-free CI builder image for service compilation Based on rust:1.89-slim-bookworm (~2-3GB vs ~8GB CUDA devel). Same toolchain: mold 2.35, protoc 28.3, sccache 0.10, clang, lld. Co-Authored-By: Claude Opus 4.6 --- infra/docker/Dockerfile.ci-builder-cpu | 47 ++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 infra/docker/Dockerfile.ci-builder-cpu diff --git a/infra/docker/Dockerfile.ci-builder-cpu b/infra/docker/Dockerfile.ci-builder-cpu new file mode 100644 index 000000000..32a96d812 --- /dev/null +++ b/infra/docker/Dockerfile.ci-builder-cpu @@ -0,0 +1,47 @@ +# CUDA-free CI builder image for Foxhunt service compilation +# ~2-3GB vs ~8GB for the CUDA devel variant (Dockerfile.ci-builder) +# Contains: Rust 1.89, protoc, sccache, git, OpenSSL, clang, mold, lld, make +# Build: docker build -f infra/docker/Dockerfile.ci-builder-cpu -t foxhunt-ci-builder-cpu . +# Push: docker tag foxhunt-ci-builder-cpu rg.fr-par.scw.cloud/foxhunt-ci/ci-builder-cpu:latest +# docker push rg.fr-par.scw.cloud/foxhunt-ci/ci-builder-cpu:latest + +FROM rust:1.89-slim-bookworm + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + curl \ + ca-certificates \ + pkg-config \ + libssl-dev \ + lld \ + clang \ + make \ + perl \ + build-essential \ + libfontconfig1-dev \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +# mold linker — 2-5x faster than lld for final link step on large Rust binaries +RUN curl -fsSL https://github.com/rui314/mold/releases/download/v2.35.1/mold-2.35.1-x86_64-linux.tar.gz \ + | tar xz -C /usr/local --strip-components=1 \ + && chmod +x /usr/local/bin/mold + +# protoc 28.3 (proto3 optional support, matches local dev) +RUN curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v28.3/protoc-28.3-linux-x86_64.zip -o /tmp/protoc.zip \ + && unzip -o /tmp/protoc.zip -d /usr/local bin/protoc 'include/*' \ + && rm /tmp/protoc.zip \ + && chmod +x /usr/local/bin/protoc + +# sccache for S3-backed build caching +RUN curl -fsSL https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz \ + | tar xz -C /usr/local/bin --strip-components=1 sccache-v0.10.0-x86_64-unknown-linux-musl/sccache \ + && chmod +x /usr/local/bin/sccache + +# clippy pre-installed +RUN rustup component add clippy + +# Verify +RUN rustc --version && cargo --version && git --version && protoc --version && sccache --version && make --version && mold --version From 829d05e870652c41ae6ebd5a05b40f1e264c2bdb Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 28 Feb 2026 00:21:23 +0100 Subject: [PATCH 2/2] feat(ci): split compile pipeline into CPU services and CUDA training - Add CI_BUILDER_CPU_IMAGE variable and build-ci-builder-cpu prepare job - Add .rust-base-cpu template (no CUDA_COMPUTE_CAP) - Split compile-services into compile-services (CPU) and compile-training (CUDA) - Split .kaniko-base into .kaniko-service-base and .kaniko-training-base - Service builds use build-out/services/, training uses build-out/training/ - Each compile job has conditional changes: rules for its source paths Co-Authored-By: Claude Opus 4.6 --- .gitlab-ci.yml | 208 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 164 insertions(+), 44 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b4158b66b..597e64e52 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,9 @@ # GitLab CI/CD — Foxhunt # Two runners: CPU (compile/build/deploy) and GPU (training) -# CI builder image: CUDA 12.4 + Rust 1.89 + protoc + sccache (hosted on Scaleway CR) +# CI builder images: +# ci-builder: CUDA 12.4 + Rust 1.89 + protoc + sccache (for training binaries) +# ci-builder-cpu: Rust 1.89 + protoc + sccache, no CUDA (for service binaries) # Service images: pushed to Scaleway Container Registry (Kaniko) -# Note: .gitlab-ci.yml itself is NOT in changes: filters — CI-only edits skip build # # Runner routing (by tags): # kapsule,rust,docker → CPU runner → ci-compile-cpu pool (POP2-32C-128G, 32 vCPU) @@ -49,6 +50,7 @@ variables: AWS_SECRET_ACCESS_KEY: $SCW_SECRET_KEY # CI builder image on Scaleway Container Registry (reachable by Kapsule nodes) CI_BUILDER_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/ci-builder:latest + CI_BUILDER_CPU_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/ci-builder-cpu:latest # Scaleway Container Registry for service images REGISTRY: rg.fr-par.scw.cloud/foxhunt-ci # IaC runner image on Scaleway Container Registry @@ -86,6 +88,38 @@ build-ci-builder: --cache=true --cache-repo="${REGISTRY}/cache" --destination "${CI_BUILDER_IMAGE}" +# -------------------------------------------------------------------------- +# Stage 0a: Build CPU-only CI builder image → push to Scaleway CR +# -------------------------------------------------------------------------- +build-ci-builder-cpu: + stage: prepare + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] + tags: + - kapsule + - docker + rules: + - if: $CI_PIPELINE_SOURCE == "push" + changes: + - infra/docker/Dockerfile.ci-builder-cpu + when: on_success + - when: manual + allow_failure: true + before_script: + - mkdir -p /kaniko/.docker + - >- + echo "{\"auths\":{ + \"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"}, + \"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"} + }}" > /kaniko/.docker/config.json + script: + - /kaniko/executor + --context "${CI_PROJECT_DIR}" + --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.ci-builder-cpu" + --cache=true --cache-repo="${REGISTRY}/cache" + --destination "${CI_BUILDER_CPU_IMAGE}" + # -------------------------------------------------------------------------- # Stage 0b: Build devcontainer image → push to internal GitLab registry # -------------------------------------------------------------------------- @@ -180,6 +214,29 @@ build-infra-runner: echo "mold not found, falling back to lld via ld.mold wrapper" fi +# Base template for CPU-only Rust jobs — lightweight builder without CUDA +# Uses ci-builder-cpu image (rust:1.89-slim, ~2-3GB, no CUDA toolkit) +.rust-base-cpu: + image: ${CI_BUILDER_CPU_IMAGE} + tags: + - kapsule + - rust + variables: + # No CUDA_COMPUTE_CAP — CPU-only builds don't compile CUDA kernels + CARGO_BUILD_JOBS: "30" + KUBERNETES_CPU_REQUEST: "28000m" + KUBERNETES_CPU_LIMIT: "31000m" + KUBERNETES_MEMORY_REQUEST: "24Gi" + KUBERNETES_MEMORY_LIMIT: "56Gi" + before_script: + # Mold linker fallback — same as .rust-base + - | + if ! command -v mold &>/dev/null; then + printf '#!/bin/sh\nexec ld.lld "$@"\n' > /usr/local/bin/ld.mold + chmod +x /usr/local/bin/ld.mold + echo "mold not found, falling back to lld via ld.mold wrapper" + fi + # -------------------------------------------------------------------------- # Stage 2: tests # -------------------------------------------------------------------------- @@ -227,14 +284,13 @@ test: - sccache --show-stats || true # -------------------------------------------------------------------------- -# Stage 3a: Compile all service binaries (single job, PVC sccache) +# Stage 3a: Compile service binaries (CPU-only, no CUDA) # -------------------------------------------------------------------------- compile-services: - extends: .rust-base + extends: .rust-base-cpu stage: compile needs: [] rules: - # Only recompile when Rust source or Dockerfiles change — infra-only pushes skip - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" changes: - Cargo.toml @@ -245,16 +301,10 @@ compile-services: - infra/docker/Dockerfile.* - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api" script: - # Compile uses CUDA stubs — no nvidia runtime needed, CUDA_COMPUTE_CAP from variables - - echo "CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP" - - export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}" - - mkdir -p "$SCCACHE_DIR" - - sccache --zero-stats || true - # Select build profile: dev-release (fast, thin LTO) or release (full, fat LTO) + - mkdir -p "$SCCACHE_DIR" && sccache --zero-stats || true - PROFILE=${DEV_RELEASE:+dev-release}; PROFILE=${PROFILE:-release} - - echo "Building with --profile $PROFILE" + - echo "Building services with --profile $PROFILE (CPU-only, no CUDA)" - TARGET_DIR="target/$PROFILE" - # 1) Build service binaries (no CUDA feature) - cargo build --profile $PROFILE -p trading_service -p api_gateway @@ -264,7 +314,47 @@ compile-services: -p trading_agent_service -p data_acquisition_service -p web-gateway - # 2) Build training binaries with CUDA (sccache already warm from step 1) + - sccache --show-stats || true + - mkdir -p build-out/services + - | + for bin in trading_service api_gateway broker_gateway_service ml_training_service backtesting_service trading_agent_service data_acquisition_service web-gateway; do + cp $TARGET_DIR/$bin build-out/services/ + strip build-out/services/$bin + done + - ls -lh build-out/services/ + artifacts: + paths: + - build-out/services/ + expire_in: 1 hour + +# -------------------------------------------------------------------------- +# Stage 3a: Compile training binaries (CUDA stubs for candle GPU kernels) +# -------------------------------------------------------------------------- +compile-training: + extends: .rust-base + stage: compile + needs: [] + rules: + - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" + changes: + - Cargo.toml + - Cargo.lock + - crates/ml/** + - crates/trading_engine/** + - crates/common/** + - crates/risk/** + - crates/data/** + - crates/config/** + - crates/storage/** + - services/training_uploader/** + - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api" + script: + - echo "CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP" + - export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}" + - mkdir -p "$SCCACHE_DIR" && sccache --zero-stats || true + - PROFILE=${DEV_RELEASE:+dev-release}; PROFILE=${PROFILE:-release} + - echo "Building training binaries with --profile $PROFILE (CUDA stubs)" + - TARGET_DIR="target/$PROFILE" - cargo build --profile $PROFILE -p ml --features ml/cuda --example train_baseline_rl --example train_baseline_supervised @@ -272,31 +362,28 @@ compile-services: --example evaluate_supervised --example hyperopt_baseline_rl --example hyperopt_baseline_supervised - # 3) Build training uploader sidecar (no CUDA needed) - cargo build --profile $PROFILE -p training_uploader - sccache --show-stats || true - - mkdir -p build-out + - mkdir -p build-out/training - | - for bin in trading_service api_gateway broker_gateway_service ml_training_service backtesting_service trading_agent_service data_acquisition_service web-gateway; do - cp $TARGET_DIR/$bin build-out/ - strip build-out/$bin - done for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised; do - cp $TARGET_DIR/examples/$bin build-out/ - strip build-out/$bin + cp $TARGET_DIR/examples/$bin build-out/training/ + strip build-out/training/$bin done - cp $TARGET_DIR/training_uploader build-out/ - strip build-out/training_uploader - - ls -lh build-out/ + cp $TARGET_DIR/training_uploader build-out/training/ + strip build-out/training/training_uploader + - ls -lh build-out/training/ artifacts: paths: - - build-out/ + - build-out/training/ expire_in: 1 hour # -------------------------------------------------------------------------- # Stage 3b: Package service images (Kaniko → Scaleway CR, uses pre-built binaries) # -------------------------------------------------------------------------- -.kaniko-base: + +# Kaniko base for service images (depends on compile-services) +.kaniko-service-base: stage: build needs: - job: compile-services @@ -324,72 +411,105 @@ compile-services: \"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"} }}" > /kaniko/.docker/config.json +# Kaniko base for training image (depends on compile-training) +.kaniko-training-base: + stage: build + needs: + - job: compile-training + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] + tags: + - kapsule + - docker + rules: + - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" + changes: + - Cargo.toml + - Cargo.lock + - crates/ml/** + - crates/trading_engine/** + - crates/common/** + - crates/risk/** + - crates/data/** + - crates/config/** + - crates/storage/** + - services/training_uploader/** + - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api" + before_script: + - mkdir -p /kaniko/.docker + - >- + echo "{\"auths\":{ + \"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"}, + \"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"} + }}" > /kaniko/.docker/config.json + # 7 services: pre-built binary → Dockerfile.runtime (no Rust toolchain, ~30s each) build-trading-service: - extends: .kaniko-base + extends: .kaniko-service-base script: - /kaniko/executor - --context dir://${CI_PROJECT_DIR}/build-out + --context dir://${CI_PROJECT_DIR}/build-out/services --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime" --build-arg SERVICE=trading_service --destination "${REGISTRY}/trading_service:${CI_COMMIT_SHA}" --destination "${REGISTRY}/trading_service:latest" build-api-gateway: - extends: .kaniko-base + extends: .kaniko-service-base script: - /kaniko/executor - --context dir://${CI_PROJECT_DIR}/build-out + --context dir://${CI_PROJECT_DIR}/build-out/services --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime" --build-arg SERVICE=api_gateway --destination "${REGISTRY}/api_gateway:${CI_COMMIT_SHA}" --destination "${REGISTRY}/api_gateway:latest" build-broker-gateway: - extends: .kaniko-base + extends: .kaniko-service-base script: - /kaniko/executor - --context dir://${CI_PROJECT_DIR}/build-out + --context dir://${CI_PROJECT_DIR}/build-out/services --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime" --build-arg SERVICE=broker_gateway_service --destination "${REGISTRY}/broker_gateway_service:${CI_COMMIT_SHA}" --destination "${REGISTRY}/broker_gateway_service:latest" build-ml-training: - extends: .kaniko-base + extends: .kaniko-service-base script: - /kaniko/executor - --context dir://${CI_PROJECT_DIR}/build-out + --context dir://${CI_PROJECT_DIR}/build-out/services --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime" --build-arg SERVICE=ml_training_service --destination "${REGISTRY}/ml_training_service:${CI_COMMIT_SHA}" --destination "${REGISTRY}/ml_training_service:latest" build-backtesting: - extends: .kaniko-base + extends: .kaniko-service-base script: - /kaniko/executor - --context dir://${CI_PROJECT_DIR}/build-out + --context dir://${CI_PROJECT_DIR}/build-out/services --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime" --build-arg SERVICE=backtesting_service --destination "${REGISTRY}/backtesting_service:${CI_COMMIT_SHA}" --destination "${REGISTRY}/backtesting_service:latest" build-trading-agent: - extends: .kaniko-base + extends: .kaniko-service-base script: - /kaniko/executor - --context dir://${CI_PROJECT_DIR}/build-out + --context dir://${CI_PROJECT_DIR}/build-out/services --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime" --build-arg SERVICE=trading_agent_service --destination "${REGISTRY}/trading_agent_service:${CI_COMMIT_SHA}" --destination "${REGISTRY}/trading_agent_service:latest" build-data-acquisition: - extends: .kaniko-base + extends: .kaniko-service-base script: - /kaniko/executor - --context dir://${CI_PROJECT_DIR}/build-out + --context dir://${CI_PROJECT_DIR}/build-out/services --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime" --build-arg SERVICE=data_acquisition_service --destination "${REGISTRY}/data_acquisition_service:${CI_COMMIT_SHA}" @@ -397,7 +517,7 @@ build-data-acquisition: # web-gateway: Node.js dashboard build + pre-built Rust binary build-web-gateway: - extends: .kaniko-base + extends: .kaniko-service-base script: - /kaniko/executor --context "${CI_PROJECT_DIR}" @@ -408,10 +528,10 @@ build-web-gateway: # training: pre-built CUDA binaries → CUDA runtime image (no recompilation) build-training: - extends: .kaniko-base + extends: .kaniko-training-base script: - /kaniko/executor - --context dir://${CI_PROJECT_DIR}/build-out + --context dir://${CI_PROJECT_DIR}/build-out/training --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.training-runtime" --destination "${REGISTRY}/training:${CI_COMMIT_SHA}" --destination "${REGISTRY}/training:latest"