fix(ci): halve CPU requests for parallel compilation on single node

Scaleway quota limits POP2-32C-128G to 1 instance. Reduce each compile
job from 28→14 CPUs so compile-services and compile-training run in
parallel on the same 32-vCPU node instead of sequentially.

Also fix data-acquisition-service health probe: service doesn't implement
gRPC health protocol, switch to tcpSocket probe on port 50057.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-28 19:12:34 +01:00
parent f5b836f98f
commit e58ce4f585
2 changed files with 15 additions and 16 deletions

View File

@@ -254,7 +254,9 @@ build-foxhunt-training-runtime:
# Base template for Rust jobs — pre-baked CI builder from Scaleway CR
# Image pre-exists in SCR; rebuild via build-ci-builder when Dockerfile changes
# Runs on ci-compile-cpu pool (GP1-L: 32 vCPU, 128GB RAM, 559GB SSD) via CPU runner
# Runs on ci-compile-cpu pool (POP2-32C-128G: 32 vCPU, 128GB RAM) via CPU runner
# CPU budget: 14 CPUs per job → compile-services + compile-training run in parallel on 1 node
# (Scaleway quota: 1× POP2-32C-128G, so both jobs must share)
# Base template for jobs needing CUDA builder (test stage only — compile uses .rust-base-cpu)
.rust-base:
image: ${CI_BUILDER_IMAGE}
@@ -263,9 +265,9 @@ build-foxhunt-training-runtime:
- rust
variables:
CUDA_COMPUTE_CAP: "89"
CARGO_BUILD_JOBS: "30"
KUBERNETES_CPU_REQUEST: "28000m"
KUBERNETES_CPU_LIMIT: "31000m"
CARGO_BUILD_JOBS: "14"
KUBERNETES_CPU_REQUEST: "14000m"
KUBERNETES_CPU_LIMIT: "15500m"
KUBERNETES_MEMORY_REQUEST: "24Gi"
KUBERNETES_MEMORY_LIMIT: "56Gi"
before_script:
@@ -281,6 +283,7 @@ build-foxhunt-training-runtime:
# Base template for CPU-only Rust jobs — lightweight builder without CUDA
# Uses ci-builder-cpu image (rust:1.89-slim, ~2-3GB, no CUDA toolkit)
# CPU budget: 14 CPUs per job → runs in parallel with compile-training on same node
.rust-base-cpu:
image: ${CI_BUILDER_CPU_IMAGE}
tags:
@@ -288,9 +291,9 @@ build-foxhunt-training-runtime:
- rust
variables:
# No CUDA_COMPUTE_CAP — CPU-only builds don't compile CUDA kernels
CARGO_BUILD_JOBS: "30"
KUBERNETES_CPU_REQUEST: "28000m"
KUBERNETES_CPU_LIMIT: "31000m"
CARGO_BUILD_JOBS: "14"
KUBERNETES_CPU_REQUEST: "14000m"
KUBERNETES_CPU_LIMIT: "15500m"
KUBERNETES_MEMORY_REQUEST: "24Gi"
KUBERNETES_MEMORY_LIMIT: "56Gi"
before_script:
@@ -404,7 +407,7 @@ compile-training:
extends: .rust-base-cpu
stage: compile
image: ${CI_BUILDER_IMAGE}
needs: [] # parallel — CPU pool autoscales to 2 nodes (POP2-32C-128G each)
needs: [] # parallel with compile-services — both share 1 POP2-32C-128G node (14 CPUs each)
variables:
# CUDA stubs from ci-builder image — compiles CUDA kernels without GPU hardware.
# L4 GPU = compute capability 8.9 → candle-kernels generates sm_89 PTX

View File

@@ -106,17 +106,13 @@ spec:
mountPath: /binaries
readOnly: true
readinessProbe:
exec:
command:
- grpc_health_probe
- -addr=localhost:50057
tcpSocket:
port: 50057
initialDelaySeconds: 15
periodSeconds: 10
livenessProbe:
exec:
command:
- grpc_health_probe
- -addr=localhost:50057
tcpSocket:
port: 50057
initialDelaySeconds: 30
periodSeconds: 15
failureThreshold: 5