diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4fc975e79..f829e6f23 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,8 +1,9 @@ # GitLab CI/CD — Foxhunt # Two runners: CPU (compile/build/deploy) and GPU (training) -# CI builder image: CUDA 12.4 + Rust 1.89 + protoc + sccache (hosted on Scaleway CR) +# CI builder images: +# ci-builder: CUDA 12.4 + Rust 1.89 + protoc + sccache (for training binaries) +# ci-builder-cpu: Rust 1.89 + protoc + sccache, no CUDA (for service binaries) # Service images: pushed to Scaleway Container Registry (Kaniko) -# Note: .gitlab-ci.yml itself is NOT in changes: filters — CI-only edits skip build # # Runner routing (by tags): # kapsule,rust,docker → CPU runner → ci-compile-cpu pool (POP2-32C-128G, 32 vCPU) @@ -50,6 +51,7 @@ variables: AWS_SECRET_ACCESS_KEY: $SCW_SECRET_KEY # CI builder image on Scaleway Container Registry (reachable by Kapsule nodes) CI_BUILDER_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/ci-builder:latest + CI_BUILDER_CPU_IMAGE: rg.fr-par.scw.cloud/foxhunt-ci/ci-builder-cpu:latest # Scaleway Container Registry for service images REGISTRY: rg.fr-par.scw.cloud/foxhunt-ci # IaC runner image on Scaleway Container Registry @@ -87,6 +89,38 @@ build-ci-builder: --cache=true --cache-repo="${REGISTRY}/cache" --destination "${CI_BUILDER_IMAGE}" +# -------------------------------------------------------------------------- +# Stage 0a: Build CPU-only CI builder image → push to Scaleway CR +# -------------------------------------------------------------------------- +build-ci-builder-cpu: + stage: prepare + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] + tags: + - kapsule + - docker + rules: + - if: $CI_PIPELINE_SOURCE == "push" + changes: + - infra/docker/Dockerfile.ci-builder-cpu + when: on_success + - when: manual + allow_failure: true + before_script: + - mkdir -p /kaniko/.docker + - >- + echo "{\"auths\":{ + \"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"}, + \"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"} + }}" > /kaniko/.docker/config.json + script: + - /kaniko/executor + --context "${CI_PROJECT_DIR}" + --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.ci-builder-cpu" + --cache=true --cache-repo="${REGISTRY}/cache" + --destination "${CI_BUILDER_CPU_IMAGE}" + # -------------------------------------------------------------------------- # Stage 0b: Build devcontainer image → push to internal GitLab registry # -------------------------------------------------------------------------- @@ -181,6 +215,29 @@ build-infra-runner: echo "mold not found, falling back to lld via ld.mold wrapper" fi +# Base template for CPU-only Rust jobs — lightweight builder without CUDA +# Uses ci-builder-cpu image (rust:1.89-slim, ~2-3GB, no CUDA toolkit) +.rust-base-cpu: + image: ${CI_BUILDER_CPU_IMAGE} + tags: + - kapsule + - rust + variables: + # No CUDA_COMPUTE_CAP — CPU-only builds don't compile CUDA kernels + CARGO_BUILD_JOBS: "30" + KUBERNETES_CPU_REQUEST: "28000m" + KUBERNETES_CPU_LIMIT: "31000m" + KUBERNETES_MEMORY_REQUEST: "24Gi" + KUBERNETES_MEMORY_LIMIT: "56Gi" + before_script: + # Mold linker fallback — same as .rust-base + - | + if ! command -v mold &>/dev/null; then + printf '#!/bin/sh\nexec ld.lld "$@"\n' > /usr/local/bin/ld.mold + chmod +x /usr/local/bin/ld.mold + echo "mold not found, falling back to lld via ld.mold wrapper" + fi + # -------------------------------------------------------------------------- # Stage 2: tests # -------------------------------------------------------------------------- @@ -228,14 +285,13 @@ test: - sccache --show-stats || true # -------------------------------------------------------------------------- -# Stage 3a: Compile all service binaries (single job, PVC sccache) +# Stage 3a: Compile service binaries (CPU-only, no CUDA) # -------------------------------------------------------------------------- compile-services: - extends: .rust-base + extends: .rust-base-cpu stage: compile needs: [] rules: - # Only recompile when Rust source or Dockerfiles change — infra-only pushes skip - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" changes: - Cargo.toml @@ -246,16 +302,10 @@ compile-services: - infra/docker/Dockerfile.* - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api" script: - # Compile uses CUDA stubs — no nvidia runtime needed, CUDA_COMPUTE_CAP from variables - - echo "CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP" - - export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}" - - mkdir -p "$SCCACHE_DIR" - - sccache --zero-stats || true - # Select build profile: dev-release (fast, thin LTO) or release (full, fat LTO) + - mkdir -p "$SCCACHE_DIR" && sccache --zero-stats || true - PROFILE=${DEV_RELEASE:+dev-release}; PROFILE=${PROFILE:-release} - - echo "Building with --profile $PROFILE" + - echo "Building services with --profile $PROFILE (CPU-only, no CUDA)" - TARGET_DIR="target/$PROFILE" - # 1) Build service binaries (no CUDA feature) - cargo build --profile $PROFILE -p trading_service -p api_gateway @@ -265,7 +315,47 @@ compile-services: -p trading_agent_service -p data_acquisition_service -p web-gateway - # 2) Build training binaries with CUDA (sccache already warm from step 1) + - sccache --show-stats || true + - mkdir -p build-out/services + - | + for bin in trading_service api_gateway broker_gateway_service ml_training_service backtesting_service trading_agent_service data_acquisition_service web-gateway; do + cp $TARGET_DIR/$bin build-out/services/ + strip build-out/services/$bin + done + - ls -lh build-out/services/ + artifacts: + paths: + - build-out/services/ + expire_in: 1 hour + +# -------------------------------------------------------------------------- +# Stage 3a: Compile training binaries (CUDA stubs for candle GPU kernels) +# -------------------------------------------------------------------------- +compile-training: + extends: .rust-base + stage: compile + needs: [] + rules: + - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" + changes: + - Cargo.toml + - Cargo.lock + - crates/ml/** + - crates/trading_engine/** + - crates/common/** + - crates/risk/** + - crates/data/** + - crates/config/** + - crates/storage/** + - services/training_uploader/** + - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api" + script: + - echo "CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP" + - export SCCACHE_DIR="/mnt/sccache/sm_${CUDA_COMPUTE_CAP}" + - mkdir -p "$SCCACHE_DIR" && sccache --zero-stats || true + - PROFILE=${DEV_RELEASE:+dev-release}; PROFILE=${PROFILE:-release} + - echo "Building training binaries with --profile $PROFILE (CUDA stubs)" + - TARGET_DIR="target/$PROFILE" - cargo build --profile $PROFILE -p ml --features ml/cuda --example train_baseline_rl --example train_baseline_supervised @@ -273,31 +363,28 @@ compile-services: --example evaluate_supervised --example hyperopt_baseline_rl --example hyperopt_baseline_supervised - # 3) Build training uploader sidecar (no CUDA needed) - cargo build --profile $PROFILE -p training_uploader - sccache --show-stats || true - - mkdir -p build-out + - mkdir -p build-out/training - | - for bin in trading_service api_gateway broker_gateway_service ml_training_service backtesting_service trading_agent_service data_acquisition_service web-gateway; do - cp $TARGET_DIR/$bin build-out/ - strip build-out/$bin - done for bin in train_baseline_rl train_baseline_supervised evaluate_baseline evaluate_supervised hyperopt_baseline_rl hyperopt_baseline_supervised; do - cp $TARGET_DIR/examples/$bin build-out/ - strip build-out/$bin + cp $TARGET_DIR/examples/$bin build-out/training/ + strip build-out/training/$bin done - cp $TARGET_DIR/training_uploader build-out/ - strip build-out/training_uploader - - ls -lh build-out/ + cp $TARGET_DIR/training_uploader build-out/training/ + strip build-out/training/training_uploader + - ls -lh build-out/training/ artifacts: paths: - - build-out/ + - build-out/training/ expire_in: 1 hour # -------------------------------------------------------------------------- # Stage 3b: Package service images (Kaniko → Scaleway CR, uses pre-built binaries) # -------------------------------------------------------------------------- -.kaniko-base: + +# Kaniko base for service images (depends on compile-services) +.kaniko-service-base: stage: build needs: - job: compile-services @@ -332,72 +419,105 @@ compile-services: \"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"} }}" > /kaniko/.docker/config.json +# Kaniko base for training image (depends on compile-training) +.kaniko-training-base: + stage: build + needs: + - job: compile-training + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] + tags: + - kapsule + - docker + rules: + - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" + changes: + - Cargo.toml + - Cargo.lock + - crates/ml/** + - crates/trading_engine/** + - crates/common/** + - crates/risk/** + - crates/data/** + - crates/config/** + - crates/storage/** + - services/training_uploader/** + - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api" + before_script: + - mkdir -p /kaniko/.docker + - >- + echo "{\"auths\":{ + \"rg.fr-par.scw.cloud\":{\"username\":\"nologin\",\"password\":\"${SCW_SECRET_KEY}\"}, + \"https://index.docker.io/v1/\":{\"username\":\"${DOCKERHUB_USERNAME}\",\"password\":\"${DOCKERHUB_TOKEN}\"} + }}" > /kaniko/.docker/config.json + # 7 services: pre-built binary → Dockerfile.runtime (no Rust toolchain, ~30s each) build-trading-service: - extends: .kaniko-base + extends: .kaniko-service-base script: - /kaniko/executor - --context dir://${CI_PROJECT_DIR}/build-out + --context dir://${CI_PROJECT_DIR}/build-out/services --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime" --build-arg SERVICE=trading_service --destination "${REGISTRY}/trading_service:${CI_COMMIT_SHA}" --destination "${REGISTRY}/trading_service:latest" build-api-gateway: - extends: .kaniko-base + extends: .kaniko-service-base script: - /kaniko/executor - --context dir://${CI_PROJECT_DIR}/build-out + --context dir://${CI_PROJECT_DIR}/build-out/services --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime" --build-arg SERVICE=api_gateway --destination "${REGISTRY}/api_gateway:${CI_COMMIT_SHA}" --destination "${REGISTRY}/api_gateway:latest" build-broker-gateway: - extends: .kaniko-base + extends: .kaniko-service-base script: - /kaniko/executor - --context dir://${CI_PROJECT_DIR}/build-out + --context dir://${CI_PROJECT_DIR}/build-out/services --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime" --build-arg SERVICE=broker_gateway_service --destination "${REGISTRY}/broker_gateway_service:${CI_COMMIT_SHA}" --destination "${REGISTRY}/broker_gateway_service:latest" build-ml-training: - extends: .kaniko-base + extends: .kaniko-service-base script: - /kaniko/executor - --context dir://${CI_PROJECT_DIR}/build-out + --context dir://${CI_PROJECT_DIR}/build-out/services --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime" --build-arg SERVICE=ml_training_service --destination "${REGISTRY}/ml_training_service:${CI_COMMIT_SHA}" --destination "${REGISTRY}/ml_training_service:latest" build-backtesting: - extends: .kaniko-base + extends: .kaniko-service-base script: - /kaniko/executor - --context dir://${CI_PROJECT_DIR}/build-out + --context dir://${CI_PROJECT_DIR}/build-out/services --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime" --build-arg SERVICE=backtesting_service --destination "${REGISTRY}/backtesting_service:${CI_COMMIT_SHA}" --destination "${REGISTRY}/backtesting_service:latest" build-trading-agent: - extends: .kaniko-base + extends: .kaniko-service-base script: - /kaniko/executor - --context dir://${CI_PROJECT_DIR}/build-out + --context dir://${CI_PROJECT_DIR}/build-out/services --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime" --build-arg SERVICE=trading_agent_service --destination "${REGISTRY}/trading_agent_service:${CI_COMMIT_SHA}" --destination "${REGISTRY}/trading_agent_service:latest" build-data-acquisition: - extends: .kaniko-base + extends: .kaniko-service-base script: - /kaniko/executor - --context dir://${CI_PROJECT_DIR}/build-out + --context dir://${CI_PROJECT_DIR}/build-out/services --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.runtime" --build-arg SERVICE=data_acquisition_service --destination "${REGISTRY}/data_acquisition_service:${CI_COMMIT_SHA}" @@ -405,7 +525,7 @@ build-data-acquisition: # web-gateway: Node.js dashboard build + pre-built Rust binary build-web-gateway: - extends: .kaniko-base + extends: .kaniko-service-base script: - /kaniko/executor --context "${CI_PROJECT_DIR}" @@ -416,10 +536,10 @@ build-web-gateway: # training: pre-built CUDA binaries → CUDA runtime image (no recompilation) build-training: - extends: .kaniko-base + extends: .kaniko-training-base script: - /kaniko/executor - --context dir://${CI_PROJECT_DIR}/build-out + --context dir://${CI_PROJECT_DIR}/build-out/training --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.training-runtime" --destination "${REGISTRY}/training:${CI_COMMIT_SHA}" --destination "${REGISTRY}/training:latest" diff --git a/infra/docker/Dockerfile.ci-builder-cpu b/infra/docker/Dockerfile.ci-builder-cpu new file mode 100644 index 000000000..32a96d812 --- /dev/null +++ b/infra/docker/Dockerfile.ci-builder-cpu @@ -0,0 +1,47 @@ +# CUDA-free CI builder image for Foxhunt service compilation +# ~2-3GB vs ~8GB for the CUDA devel variant (Dockerfile.ci-builder) +# Contains: Rust 1.89, protoc, sccache, git, OpenSSL, clang, mold, lld, make +# Build: docker build -f infra/docker/Dockerfile.ci-builder-cpu -t foxhunt-ci-builder-cpu . +# Push: docker tag foxhunt-ci-builder-cpu rg.fr-par.scw.cloud/foxhunt-ci/ci-builder-cpu:latest +# docker push rg.fr-par.scw.cloud/foxhunt-ci/ci-builder-cpu:latest + +FROM rust:1.89-slim-bookworm + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + curl \ + ca-certificates \ + pkg-config \ + libssl-dev \ + lld \ + clang \ + make \ + perl \ + build-essential \ + libfontconfig1-dev \ + unzip \ + && rm -rf /var/lib/apt/lists/* + +# mold linker — 2-5x faster than lld for final link step on large Rust binaries +RUN curl -fsSL https://github.com/rui314/mold/releases/download/v2.35.1/mold-2.35.1-x86_64-linux.tar.gz \ + | tar xz -C /usr/local --strip-components=1 \ + && chmod +x /usr/local/bin/mold + +# protoc 28.3 (proto3 optional support, matches local dev) +RUN curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v28.3/protoc-28.3-linux-x86_64.zip -o /tmp/protoc.zip \ + && unzip -o /tmp/protoc.zip -d /usr/local bin/protoc 'include/*' \ + && rm /tmp/protoc.zip \ + && chmod +x /usr/local/bin/protoc + +# sccache for S3-backed build caching +RUN curl -fsSL https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz \ + | tar xz -C /usr/local/bin --strip-components=1 sccache-v0.10.0-x86_64-unknown-linux-musl/sccache \ + && chmod +x /usr/local/bin/sccache + +# clippy pre-installed +RUN rustup component add clippy + +# Verify +RUN rustc --version && cargo --version && git --version && protoc --version && sccache --version && make --version && mold --version