ci: replace compile-services with 8 per-service compile jobs

Each service gets its own compile job with dependency-aware changes: filters so only affected services recompile on push. Deploy job selectively restarts only services whose binary was compiled. - Add .compile-service-base template (4 CPUs, 8Gi per job) - 8 jobs: api-gateway, trading-service, ml-training-service, backtesting-service, trading-agent-service, broker-gateway, data-acquisition-service, web-gateway - Deploy needs: all 8 compile jobs (optional: true, artifacts: true) - Rollout restart loop checks build-out/services/ for actual binaries Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-03-04 22:10:25 +01:00
parent dd88f21bdd
commit ac7b6e6d20
1 changed files with 215 additions and 61 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -307,7 +307,7 @@ scan-runtime-images:
 # Base template for Rust jobs — pre-baked CI builder from Scaleway CR
 # Image pre-exists in SCR; rebuild via build-ci-builder when Dockerfile changes
 # Runs on ci-compile-cpu pool (POP2-32C-128G: 32 vCPU, 128GB RAM) via CPU runner
-# CPU budget: 14 CPUs per job → compile-services + compile-training run in parallel on 1 node
+# CPU budget: 14 CPUs per job for tests; per-service compile jobs use 4 CPUs each
 # (Scaleway quota: 1× POP2-32C-128G, so both jobs must share)
 # Base template for jobs needing CUDA builder (test stage only — compile uses .rust-base-cpu)
 .rust-base:
@@ -335,7 +335,7 @@ scan-runtime-images:

 # Base template for CPU-only Rust jobs — lightweight builder without CUDA
 # Uses ci-builder-cpu image (rust:1.89-slim, ~2-3GB, no CUDA toolkit)
-# CPU budget: 14 CPUs per job → runs in parallel with compile-training on same node
+# Per-service compile jobs override CPU/memory to 4 CPUs each (see .compile-service-base)
 .rust-base-cpu:
  image: ${CI_BUILDER_CPU_IMAGE}
  tags:
@@ -359,6 +359,44 @@ scan-runtime-images:
        echo "mold not found, falling back to lld via ld.mold wrapper"
      fi

+# Base template for per-service compile jobs (CPU-only, no CUDA)
+# Each service gets its own job with dependency-aware changes: filters.
+# CARGO_BUILD_JOBS=4 (down from 10) since up to 8 jobs may run on the same node.
+.compile-service-base:
+  extends: .rust-base-cpu
+  stage: compile
+  needs: []
+  interruptible: true
+  variables:
+    CARGO_BUILD_JOBS: "4"
+    KUBERNETES_CPU_REQUEST: "4000m"
+    KUBERNETES_CPU_LIMIT: "6000m"
+    KUBERNETES_MEMORY_REQUEST: "8Gi"
+    KUBERNETES_MEMORY_LIMIT: "16Gi"
+  script:
+    - mkdir -p "$SCCACHE_DIR" && sccache --zero-stats || true
+    - PROFILE=${DEV_RELEASE:+dev-release}; PROFILE=${PROFILE:-release}
+    - |
+      # CalVer: YYYY.MM.<pipeline_iid>
+      YEAR=$(date -d "$CI_COMMIT_TIMESTAMP" +%Y 2>/dev/null || date +%Y)
+      MONTH=$(date -d "$CI_COMMIT_TIMESTAMP" +%m 2>/dev/null || date +%m)
+      export FOXHUNT_BUILD_VERSION="${YEAR}.${MONTH}.${CI_PIPELINE_IID:-0}"
+      echo "Build version: $FOXHUNT_BUILD_VERSION"
+    - echo "Building ${SERVICE_PACKAGE} with --profile $PROFILE (CPU-only, no CUDA)"
+    - TARGET_DIR="target/$PROFILE"
+    - cargo build --profile $PROFILE -p ${SERVICE_PACKAGE}
+    - sccache --show-stats || true
+    - mkdir -p build-out/services
+    - |
+      cp $TARGET_DIR/${SERVICE_PACKAGE} build-out/services/
+      strip build-out/services/${SERVICE_PACKAGE}
+    - ls -lh build-out/services/
+    - echo "${SERVICE_PACKAGE} binary ready as CI artifact"
+  artifacts:
+    paths:
+      - build-out/services/
+    expire_in: 1 day
+
 # --------------------------------------------------------------------------
 # Stage 2: tests
 # --------------------------------------------------------------------------
@@ -406,57 +444,146 @@ test:
    - sccache --show-stats || true

 # --------------------------------------------------------------------------
-# Stage 3a: Compile service binaries (CPU-only, no CUDA)
+# Stage 3a: Per-service compile jobs (CPU-only, no CUDA)
+# Each service compiled independently with dependency-aware changes: filters.
+# sccache shared across jobs — incremental crate compilation is cached.
 # --------------------------------------------------------------------------
-compile-services:
-  extends: .rust-base-cpu
-  stage: compile
-  needs: []
-  resource_group: compile-services
-  interruptible: true
+compile-api-gateway:
+  extends: .compile-service-base
+  variables:
+    SERVICE_PACKAGE: api-gateway
  rules:
    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
      changes:
+        - services/api_gateway/**
+        - crates/common/**
+        - crates/config/**
+        - crates/trading_engine/**
+        - bin/fxt/**
+        - Cargo.toml
+        - Cargo.lock
+    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
+
+compile-trading-service:
+  extends: .compile-service-base
+  variables:
+    SERVICE_PACKAGE: trading-service
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
+      changes:
+        - services/trading_service/**
+        - services/api_gateway/**
+        - crates/common/**
+        - crates/config/**
+        - crates/data/**
+        - crates/database/**
+        - crates/ml/**
+        - crates/risk/**
+        - crates/storage/**
+        - crates/trading_engine/**
+        - Cargo.toml
+        - Cargo.lock
+    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
+
+compile-ml-training-service:
+  extends: .compile-service-base
+  variables:
+    SERVICE_PACKAGE: ml-training-service
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
+      changes:
+        - services/ml_training_service/**
+        - crates/common/**
+        - crates/config/**
+        - crates/data/**
+        - crates/ml/**
+        - crates/risk/**
+        - crates/storage/**
+        - crates/trading_engine/**
+        - Cargo.toml
+        - Cargo.lock
+    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
+
+compile-backtesting-service:
+  extends: .compile-service-base
+  variables:
+    SERVICE_PACKAGE: backtesting-service
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
+      changes:
+        - services/backtesting_service/**
+        - crates/common/**
+        - crates/config/**
+        - crates/data/**
+        - crates/ml/**
+        - crates/model_loader/**
+        - crates/risk/**
+        - crates/storage/**
+        - crates/trading_engine/**
+        - bin/fxt/**
+        - Cargo.toml
+        - Cargo.lock
+    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
+
+compile-trading-agent-service:
+  extends: .compile-service-base
+  variables:
+    SERVICE_PACKAGE: trading-agent-service
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
+      changes:
+        - services/trading_agent_service/**
+        - crates/common/**
+        - crates/config/**
+        - crates/ml/**
+        - crates/risk/**
+        - Cargo.toml
+        - Cargo.lock
+    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
+
+compile-broker-gateway:
+  extends: .compile-service-base
+  variables:
+    SERVICE_PACKAGE: broker-gateway
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
+      changes:
+        - services/broker_gateway_service/**
+        - crates/common/**
+        - crates/config/**
+        - crates/trading_engine/**
+        - vendor/ctrader-openapi/**
+        - Cargo.toml
+        - Cargo.lock
+    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
+
+compile-data-acquisition-service:
+  extends: .compile-service-base
+  variables:
+    SERVICE_PACKAGE: data-acquisition-service
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
+      changes:
+        - services/data_acquisition_service/**
+        - crates/common/**
+        - crates/config/**
+        - crates/storage/**
+        - Cargo.toml
+        - Cargo.lock
+    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
+
+compile-web-gateway:
+  extends: .compile-service-base
+  variables:
+    SERVICE_PACKAGE: web-gateway
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
+      changes:
+        - crates/web-gateway/**
+        - crates/common/**
        - Cargo.toml
        - Cargo.lock
-        - crates/**
-        - bin/**
-        - services/**
-        - infra/docker/Dockerfile.*
    - if: $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "api"
-  script:
-    - mkdir -p "$SCCACHE_DIR" && sccache --zero-stats || true
-    - PROFILE=${DEV_RELEASE:+dev-release}; PROFILE=${PROFILE:-release}
-    - |
-      # CalVer: YYYY.MM.<pipeline_iid>
-      YEAR=$(date -d "$CI_COMMIT_TIMESTAMP" +%Y 2>/dev/null || date +%Y)
-      MONTH=$(date -d "$CI_COMMIT_TIMESTAMP" +%m 2>/dev/null || date +%m)
-      export FOXHUNT_BUILD_VERSION="${YEAR}.${MONTH}.${CI_PIPELINE_IID:-0}"
-      echo "Build version: $FOXHUNT_BUILD_VERSION"
-    - echo "Building services with --profile $PROFILE (CPU-only, no CUDA)"
-    - TARGET_DIR="target/$PROFILE"
-    - cargo build --profile $PROFILE
-        -p trading-service
-        -p api-gateway
-        -p broker-gateway
-        -p ml-training-service
-        -p backtesting-service
-        -p trading-agent-service
-        -p data-acquisition-service
-        -p web-gateway
-    - sccache --show-stats || true
-    - mkdir -p build-out/services
-    - |
-      for bin in trading-service api-gateway broker-gateway ml-training-service backtesting-service trading-agent-service data-acquisition-service web-gateway; do
-        cp $TARGET_DIR/$bin build-out/services/
-        strip build-out/services/$bin
-      done
-    - ls -lh build-out/services/
-    - echo "Service binaries ready as CI artifacts"
-  artifacts:
-    paths:
-      - build-out/services/
-    expire_in: 1 day

 # --------------------------------------------------------------------------
 # Stage 3a: Compile training binaries (CPU node, CUDA stubs from ci-builder image)
@@ -465,7 +592,7 @@ compile-training:
  extends: .rust-base-cpu
  stage: compile
  image: ${CI_BUILDER_IMAGE}
-  needs: []  # parallel with compile-services — both share 1 POP2-32C-128G node (10 CPUs each, burstable)
+  needs: []  # parallel with per-service compile jobs — shares 1 POP2-32C-128G node
  resource_group: compile-training
  interruptible: true
  variables:
@@ -1472,7 +1599,28 @@ deploy:
    KUBERNETES_MEMORY_REQUEST: "256Mi"
    KUBERNETES_MEMORY_LIMIT: "512Mi"
  needs:
-    - job: compile-services
+    - job: compile-api-gateway
+      optional: true
+      artifacts: true
+    - job: compile-trading-service
+      optional: true
+      artifacts: true
+    - job: compile-ml-training-service
+      optional: true
+      artifacts: true
+    - job: compile-backtesting-service
+      optional: true
+      artifacts: true
+    - job: compile-trading-agent-service
+      optional: true
+      artifacts: true
+    - job: compile-broker-gateway
+      optional: true
+      artifacts: true
+    - job: compile-data-acquisition-service
+      optional: true
+      artifacts: true
+    - job: compile-web-gateway
      optional: true
      artifacts: true
    - job: compile-training
@@ -1622,18 +1770,24 @@ deploy:
    # ── Step 2: Apply service manifests (binaries uploaded to MinIO) ──
    # GPU overlays are in gpu-overlays/ — NOT auto-applied. Apply manually when needed.
    - kubectl apply -f infra/k8s/services/
-    # ── Step 3: Rolling restart + wait ──
+    # ── Step 3: Selective rolling restart — only services whose binary was compiled ──
    - |
-      for svc in trading-service api-gateway broker-gateway \
-                 ml-training-service backtesting-service \
-                 trading-agent-service data-acquisition-service web-gateway; do
-        kubectl -n foxhunt rollout restart deployment/$svc
-        echo "Restarted $svc"
+      DEPLOYED=""
+      for bin in build-out/services/*; do
+        [ -f "$bin" ] || continue
+        name=$(basename "$bin")
+        DEPLOYED="$DEPLOYED $name"
      done
-    - |
-      for svc in trading-service api-gateway broker-gateway \
-                 ml-training-service backtesting-service \
-                 trading-agent-service data-acquisition-service web-gateway; do
-        kubectl -n foxhunt rollout status deployment/$svc --timeout=300s
-      done
-    - echo "All services deployed"
+
+      if [ -n "$DEPLOYED" ]; then
+        for svc in $DEPLOYED; do
+          kubectl -n foxhunt rollout restart deployment/$svc
+          echo "Restarted $svc"
+        done
+        for svc in $DEPLOYED; do
+          kubectl -n foxhunt rollout status deployment/$svc --timeout=300s
+        done
+        echo "Deployed:$DEPLOYED"
+      else
+        echo "No service binaries changed — manifests-only deploy"
+      fi