From e58ce4f5854cbce6144f2fda9ce885d597d7dd3d Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Sat, 28 Feb 2026 19:12:34 +0100
Subject: [PATCH] fix(ci): halve CPU requests for parallel compilation on
 single node
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Scaleway quota limits POP2-32C-128G to 1 instance. Reduce each compile
job from 28→14 CPUs so compile-services and compile-training run in
parallel on the same 32-vCPU node instead of sequentially.

Also fix data-acquisition-service health probe: service doesn't implement
gRPC health protocol, switch to tcpSocket probe on port 50057.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .gitlab-ci.yml                                | 19 +++++++++++--------
 .../services/data-acquisition-service.yaml    | 12 ++++--------
 2 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9c5d1d378..5390a41bc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -254,7 +254,9 @@ build-foxhunt-training-runtime:
 
 # Base template for Rust jobs — pre-baked CI builder from Scaleway CR
 # Image pre-exists in SCR; rebuild via build-ci-builder when Dockerfile changes
-# Runs on ci-compile-cpu pool (GP1-L: 32 vCPU, 128GB RAM, 559GB SSD) via CPU runner
+# Runs on ci-compile-cpu pool (POP2-32C-128G: 32 vCPU, 128GB RAM) via CPU runner
+# CPU budget: 14 CPUs per job → compile-services + compile-training run in parallel on 1 node
+# (Scaleway quota: 1× POP2-32C-128G, so both jobs must share)
 # Base template for jobs needing CUDA builder (test stage only — compile uses .rust-base-cpu)
 .rust-base:
   image: ${CI_BUILDER_IMAGE}
@@ -263,9 +265,9 @@ build-foxhunt-training-runtime:
     - rust
   variables:
     CUDA_COMPUTE_CAP: "89"
-    CARGO_BUILD_JOBS: "30"
-    KUBERNETES_CPU_REQUEST: "28000m"
-    KUBERNETES_CPU_LIMIT: "31000m"
+    CARGO_BUILD_JOBS: "14"
+    KUBERNETES_CPU_REQUEST: "14000m"
+    KUBERNETES_CPU_LIMIT: "15500m"
     KUBERNETES_MEMORY_REQUEST: "24Gi"
     KUBERNETES_MEMORY_LIMIT: "56Gi"
   before_script:
@@ -281,6 +283,7 @@ build-foxhunt-training-runtime:
 
 # Base template for CPU-only Rust jobs — lightweight builder without CUDA
 # Uses ci-builder-cpu image (rust:1.89-slim, ~2-3GB, no CUDA toolkit)
+# CPU budget: 14 CPUs per job → runs in parallel with compile-training on same node
 .rust-base-cpu:
   image: ${CI_BUILDER_CPU_IMAGE}
   tags:
@@ -288,9 +291,9 @@ build-foxhunt-training-runtime:
     - rust
   variables:
     # No CUDA_COMPUTE_CAP — CPU-only builds don't compile CUDA kernels
-    CARGO_BUILD_JOBS: "30"
-    KUBERNETES_CPU_REQUEST: "28000m"
-    KUBERNETES_CPU_LIMIT: "31000m"
+    CARGO_BUILD_JOBS: "14"
+    KUBERNETES_CPU_REQUEST: "14000m"
+    KUBERNETES_CPU_LIMIT: "15500m"
     KUBERNETES_MEMORY_REQUEST: "24Gi"
     KUBERNETES_MEMORY_LIMIT: "56Gi"
   before_script:
@@ -404,7 +407,7 @@ compile-training:
   extends: .rust-base-cpu
   stage: compile
   image: ${CI_BUILDER_IMAGE}
-  needs: []  # parallel — CPU pool autoscales to 2 nodes (POP2-32C-128G each)
+  needs: []  # parallel with compile-services — both share 1 POP2-32C-128G node (14 CPUs each)
   variables:
     # CUDA stubs from ci-builder image — compiles CUDA kernels without GPU hardware.
     # L4 GPU = compute capability 8.9 → candle-kernels generates sm_89 PTX
diff --git a/infra/k8s/services/data-acquisition-service.yaml b/infra/k8s/services/data-acquisition-service.yaml
index 9cf8a6626..e54d53f36 100644
--- a/infra/k8s/services/data-acquisition-service.yaml
+++ b/infra/k8s/services/data-acquisition-service.yaml
@@ -106,17 +106,13 @@ spec:
               mountPath: /binaries
               readOnly: true
           readinessProbe:
-            exec:
-              command:
-                - grpc_health_probe
-                - -addr=localhost:50057
+            tcpSocket:
+              port: 50057
             initialDelaySeconds: 15
             periodSeconds: 10
           livenessProbe:
-            exec:
-              command:
-                - grpc_health_probe
-                - -addr=localhost:50057
+            tcpSocket:
+              port: 50057
             initialDelaySeconds: 30
             periodSeconds: 15
             failureThreshold: 5