fix(ci): halve CPU requests for parallel compilation on single node
Scaleway quota limits POP2-32C-128G to 1 instance. Reduce each compile job from 28→14 CPUs so compile-services and compile-training run in parallel on the same 32-vCPU node instead of sequentially. Also fix data-acquisition-service health probe: service doesn't implement gRPC health protocol, switch to tcpSocket probe on port 50057. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -254,7 +254,9 @@ build-foxhunt-training-runtime:
|
||||
|
||||
# Base template for Rust jobs — pre-baked CI builder from Scaleway CR
|
||||
# Image pre-exists in SCR; rebuild via build-ci-builder when Dockerfile changes
|
||||
# Runs on ci-compile-cpu pool (GP1-L: 32 vCPU, 128GB RAM, 559GB SSD) via CPU runner
|
||||
# Runs on ci-compile-cpu pool (POP2-32C-128G: 32 vCPU, 128GB RAM) via CPU runner
|
||||
# CPU budget: 14 CPUs per job → compile-services + compile-training run in parallel on 1 node
|
||||
# (Scaleway quota: 1× POP2-32C-128G, so both jobs must share)
|
||||
# Base template for jobs needing CUDA builder (test stage only — compile uses .rust-base-cpu)
|
||||
.rust-base:
|
||||
image: ${CI_BUILDER_IMAGE}
|
||||
@@ -263,9 +265,9 @@ build-foxhunt-training-runtime:
|
||||
- rust
|
||||
variables:
|
||||
CUDA_COMPUTE_CAP: "89"
|
||||
CARGO_BUILD_JOBS: "30"
|
||||
KUBERNETES_CPU_REQUEST: "28000m"
|
||||
KUBERNETES_CPU_LIMIT: "31000m"
|
||||
CARGO_BUILD_JOBS: "14"
|
||||
KUBERNETES_CPU_REQUEST: "14000m"
|
||||
KUBERNETES_CPU_LIMIT: "15500m"
|
||||
KUBERNETES_MEMORY_REQUEST: "24Gi"
|
||||
KUBERNETES_MEMORY_LIMIT: "56Gi"
|
||||
before_script:
|
||||
@@ -281,6 +283,7 @@ build-foxhunt-training-runtime:
|
||||
|
||||
# Base template for CPU-only Rust jobs — lightweight builder without CUDA
|
||||
# Uses ci-builder-cpu image (rust:1.89-slim, ~2-3GB, no CUDA toolkit)
|
||||
# CPU budget: 14 CPUs per job → runs in parallel with compile-training on same node
|
||||
.rust-base-cpu:
|
||||
image: ${CI_BUILDER_CPU_IMAGE}
|
||||
tags:
|
||||
@@ -288,9 +291,9 @@ build-foxhunt-training-runtime:
|
||||
- rust
|
||||
variables:
|
||||
# No CUDA_COMPUTE_CAP — CPU-only builds don't compile CUDA kernels
|
||||
CARGO_BUILD_JOBS: "30"
|
||||
KUBERNETES_CPU_REQUEST: "28000m"
|
||||
KUBERNETES_CPU_LIMIT: "31000m"
|
||||
CARGO_BUILD_JOBS: "14"
|
||||
KUBERNETES_CPU_REQUEST: "14000m"
|
||||
KUBERNETES_CPU_LIMIT: "15500m"
|
||||
KUBERNETES_MEMORY_REQUEST: "24Gi"
|
||||
KUBERNETES_MEMORY_LIMIT: "56Gi"
|
||||
before_script:
|
||||
@@ -404,7 +407,7 @@ compile-training:
|
||||
extends: .rust-base-cpu
|
||||
stage: compile
|
||||
image: ${CI_BUILDER_IMAGE}
|
||||
needs: [] # parallel — CPU pool autoscales to 2 nodes (POP2-32C-128G each)
|
||||
needs: [] # parallel with compile-services — both share 1 POP2-32C-128G node (14 CPUs each)
|
||||
variables:
|
||||
# CUDA stubs from ci-builder image — compiles CUDA kernels without GPU hardware.
|
||||
# L4 GPU = compute capability 8.9 → candle-kernels generates sm_89 PTX
|
||||
|
||||
@@ -106,17 +106,13 @@ spec:
|
||||
mountPath: /binaries
|
||||
readOnly: true
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- grpc_health_probe
|
||||
- -addr=localhost:50057
|
||||
tcpSocket:
|
||||
port: 50057
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- grpc_health_probe
|
||||
- -addr=localhost:50057
|
||||
tcpSocket:
|
||||
port: 50057
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 15
|
||||
failureThreshold: 5
|
||||
|
||||
Reference in New Issue
Block a user