From ac7b6e6d200c3aaf17d5bce97f252bc094895d90 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 4 Mar 2026 22:10:25 +0100 Subject: [PATCH] ci: replace compile-services with 8 per-service compile jobs Each service gets its own compile job with dependency-aware changes: filters so only affected services recompile on push. Deploy job selectively restarts only services whose binary was compiled. - Add .compile-service-base template (4 CPUs, 8Gi per job) - 8 jobs: api-gateway, trading-service, ml-training-service, backtesting-service, trading-agent-service, broker-gateway, data-acquisition-service, web-gateway - Deploy needs: all 8 compile jobs (optional: true, artifacts: true) - Rollout restart loop checks build-out/services/ for actual binaries Co-Authored-By: Claude Opus 4.6 --- .gitlab-ci.yml | 276 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 215 insertions(+), 61 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 109b883b8..38d5fc6fb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -307,7 +307,7 @@ scan-runtime-images: # Base template for Rust jobs — pre-baked CI builder from Scaleway CR # Image pre-exists in SCR; rebuild via build-ci-builder when Dockerfile changes # Runs on ci-compile-cpu pool (POP2-32C-128G: 32 vCPU, 128GB RAM) via CPU runner -# CPU budget: 14 CPUs per job → compile-services + compile-training run in parallel on 1 node +# CPU budget: 14 CPUs per job for tests; per-service compile jobs use 4 CPUs each # (Scaleway quota: 1× POP2-32C-128G, so both jobs must share) # Base template for jobs needing CUDA builder (test stage only — compile uses .rust-base-cpu) .rust-base: @@ -335,7 +335,7 @@ scan-runtime-images: # Base template for CPU-only Rust jobs — lightweight builder without CUDA # Uses ci-builder-cpu image (rust:1.89-slim, ~2-3GB, no CUDA toolkit) -# CPU budget: 14 CPUs per job → runs in parallel with compile-training on same node +# Per-service compile jobs override CPU/memory to 4 CPUs each (see .compile-service-base) .rust-base-cpu: image: ${CI_BUILDER_CPU_IMAGE} tags: @@ -359,6 +359,44 @@ scan-runtime-images: echo "mold not found, falling back to lld via ld.mold wrapper" fi +# Base template for per-service compile jobs (CPU-only, no CUDA) +# Each service gets its own job with dependency-aware changes: filters. +# CARGO_BUILD_JOBS=4 (down from 10) since up to 8 jobs may run on the same node. +.compile-service-base: + extends: .rust-base-cpu + stage: compile + needs: [] + interruptible: true + variables: + CARGO_BUILD_JOBS: "4" + KUBERNETES_CPU_REQUEST: "4000m" + KUBERNETES_CPU_LIMIT: "6000m" + KUBERNETES_MEMORY_REQUEST: "8Gi" + KUBERNETES_MEMORY_LIMIT: "16Gi" + script: + - mkdir -p "$SCCACHE_DIR" && sccache --zero-stats || true + - PROFILE=${DEV_RELEASE:+dev-release}; PROFILE=${PROFILE:-release} + - | + # CalVer: YYYY.MM. + YEAR=$(date -d "$CI_COMMIT_TIMESTAMP" +%Y 2>/dev/null || date +%Y) + MONTH=$(date -d "$CI_COMMIT_TIMESTAMP" +%m 2>/dev/null || date +%m) + export FOXHUNT_BUILD_VERSION="${YEAR}.${MONTH}.${CI_PIPELINE_IID:-0}" + echo "Build version: $FOXHUNT_BUILD_VERSION" + - echo "Building ${SERVICE_PACKAGE} with --profile $PROFILE (CPU-only, no CUDA)" + - TARGET_DIR="target/$PROFILE" + - cargo build --profile $PROFILE -p ${SERVICE_PACKAGE} + - sccache --show-stats || true + - mkdir -p build-out/services + - | + cp $TARGET_DIR/${SERVICE_PACKAGE} build-out/services/ + strip build-out/services/${SERVICE_PACKAGE} + - ls -lh build-out/services/ + - echo "${SERVICE_PACKAGE} binary ready as CI artifact" + artifacts: + paths: + - build-out/services/ + expire_in: 1 day + # -------------------------------------------------------------------------- # Stage 2: tests # -------------------------------------------------------------------------- @@ -406,57 +444,146 @@ test: - sccache --show-stats || true # -------------------------------------------------------------------------- -# Stage 3a: Compile service binaries (CPU-only, no CUDA) +# Stage 3a: Per-service compile jobs (CPU-only, no CUDA) +# Each service compiled independently with dependency-aware changes: filters. +# sccache shared across jobs — incremental crate compilation is cached. # -------------------------------------------------------------------------- -compile-services: - extends: .rust-base-cpu - stage: compile - needs: [] - resource_group: compile-services - interruptible: true +compile-api-gateway: + extends: .compile-service-base + variables: + SERVICE_PACKAGE: api-gateway rules: - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" changes: + - services/api_gateway/** + - crates/common/** + - crates/config/** + - crates/trading_engine/** + - bin/fxt/** + - Cargo.toml + - Cargo.lock + - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api" + +compile-trading-service: + extends: .compile-service-base + variables: + SERVICE_PACKAGE: trading-service + rules: + - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" + changes: + - services/trading_service/** + - services/api_gateway/** + - crates/common/** + - crates/config/** + - crates/data/** + - crates/database/** + - crates/ml/** + - crates/risk/** + - crates/storage/** + - crates/trading_engine/** + - Cargo.toml + - Cargo.lock + - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api" + +compile-ml-training-service: + extends: .compile-service-base + variables: + SERVICE_PACKAGE: ml-training-service + rules: + - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" + changes: + - services/ml_training_service/** + - crates/common/** + - crates/config/** + - crates/data/** + - crates/ml/** + - crates/risk/** + - crates/storage/** + - crates/trading_engine/** + - Cargo.toml + - Cargo.lock + - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api" + +compile-backtesting-service: + extends: .compile-service-base + variables: + SERVICE_PACKAGE: backtesting-service + rules: + - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" + changes: + - services/backtesting_service/** + - crates/common/** + - crates/config/** + - crates/data/** + - crates/ml/** + - crates/model_loader/** + - crates/risk/** + - crates/storage/** + - crates/trading_engine/** + - bin/fxt/** + - Cargo.toml + - Cargo.lock + - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api" + +compile-trading-agent-service: + extends: .compile-service-base + variables: + SERVICE_PACKAGE: trading-agent-service + rules: + - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" + changes: + - services/trading_agent_service/** + - crates/common/** + - crates/config/** + - crates/ml/** + - crates/risk/** + - Cargo.toml + - Cargo.lock + - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api" + +compile-broker-gateway: + extends: .compile-service-base + variables: + SERVICE_PACKAGE: broker-gateway + rules: + - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" + changes: + - services/broker_gateway_service/** + - crates/common/** + - crates/config/** + - crates/trading_engine/** + - vendor/ctrader-openapi/** + - Cargo.toml + - Cargo.lock + - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api" + +compile-data-acquisition-service: + extends: .compile-service-base + variables: + SERVICE_PACKAGE: data-acquisition-service + rules: + - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" + changes: + - services/data_acquisition_service/** + - crates/common/** + - crates/config/** + - crates/storage/** + - Cargo.toml + - Cargo.lock + - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api" + +compile-web-gateway: + extends: .compile-service-base + variables: + SERVICE_PACKAGE: web-gateway + rules: + - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" + changes: + - crates/web-gateway/** + - crates/common/** - Cargo.toml - Cargo.lock - - crates/** - - bin/** - - services/** - - infra/docker/Dockerfile.* - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api" - script: - - mkdir -p "$SCCACHE_DIR" && sccache --zero-stats || true - - PROFILE=${DEV_RELEASE:+dev-release}; PROFILE=${PROFILE:-release} - - | - # CalVer: YYYY.MM. - YEAR=$(date -d "$CI_COMMIT_TIMESTAMP" +%Y 2>/dev/null || date +%Y) - MONTH=$(date -d "$CI_COMMIT_TIMESTAMP" +%m 2>/dev/null || date +%m) - export FOXHUNT_BUILD_VERSION="${YEAR}.${MONTH}.${CI_PIPELINE_IID:-0}" - echo "Build version: $FOXHUNT_BUILD_VERSION" - - echo "Building services with --profile $PROFILE (CPU-only, no CUDA)" - - TARGET_DIR="target/$PROFILE" - - cargo build --profile $PROFILE - -p trading-service - -p api-gateway - -p broker-gateway - -p ml-training-service - -p backtesting-service - -p trading-agent-service - -p data-acquisition-service - -p web-gateway - - sccache --show-stats || true - - mkdir -p build-out/services - - | - for bin in trading-service api-gateway broker-gateway ml-training-service backtesting-service trading-agent-service data-acquisition-service web-gateway; do - cp $TARGET_DIR/$bin build-out/services/ - strip build-out/services/$bin - done - - ls -lh build-out/services/ - - echo "Service binaries ready as CI artifacts" - artifacts: - paths: - - build-out/services/ - expire_in: 1 day # -------------------------------------------------------------------------- # Stage 3a: Compile training binaries (CPU node, CUDA stubs from ci-builder image) @@ -465,7 +592,7 @@ compile-training: extends: .rust-base-cpu stage: compile image: ${CI_BUILDER_IMAGE} - needs: [] # parallel with compile-services — both share 1 POP2-32C-128G node (10 CPUs each, burstable) + needs: [] # parallel with per-service compile jobs — shares 1 POP2-32C-128G node resource_group: compile-training interruptible: true variables: @@ -1472,7 +1599,28 @@ deploy: KUBERNETES_MEMORY_REQUEST: "256Mi" KUBERNETES_MEMORY_LIMIT: "512Mi" needs: - - job: compile-services + - job: compile-api-gateway + optional: true + artifacts: true + - job: compile-trading-service + optional: true + artifacts: true + - job: compile-ml-training-service + optional: true + artifacts: true + - job: compile-backtesting-service + optional: true + artifacts: true + - job: compile-trading-agent-service + optional: true + artifacts: true + - job: compile-broker-gateway + optional: true + artifacts: true + - job: compile-data-acquisition-service + optional: true + artifacts: true + - job: compile-web-gateway optional: true artifacts: true - job: compile-training @@ -1622,18 +1770,24 @@ deploy: # ── Step 2: Apply service manifests (binaries uploaded to MinIO) ── # GPU overlays are in gpu-overlays/ — NOT auto-applied. Apply manually when needed. - kubectl apply -f infra/k8s/services/ - # ── Step 3: Rolling restart + wait ── + # ── Step 3: Selective rolling restart — only services whose binary was compiled ── - | - for svc in trading-service api-gateway broker-gateway \ - ml-training-service backtesting-service \ - trading-agent-service data-acquisition-service web-gateway; do - kubectl -n foxhunt rollout restart deployment/$svc - echo "Restarted $svc" + DEPLOYED="" + for bin in build-out/services/*; do + [ -f "$bin" ] || continue + name=$(basename "$bin") + DEPLOYED="$DEPLOYED $name" done - - | - for svc in trading-service api-gateway broker-gateway \ - ml-training-service backtesting-service \ - trading-agent-service data-acquisition-service web-gateway; do - kubectl -n foxhunt rollout status deployment/$svc --timeout=300s - done - - echo "All services deployed" + + if [ -n "$DEPLOYED" ]; then + for svc in $DEPLOYED; do + kubectl -n foxhunt rollout restart deployment/$svc + echo "Restarted $svc" + done + for svc in $DEPLOYED; do + kubectl -n foxhunt rollout status deployment/$svc --timeout=300s + done + echo "Deployed:$DEPLOYED" + else + echo "No service binaries changed — manifests-only deploy" + fi