diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e59e96a08..2f8f272b2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,12 +1,14 @@ # GitLab CI/CD — Foxhunt -# Compile: ci-compile pool (L4, CUDA 8.9) | Training: ci-training pool (L40S, 48GB VRAM) +# Two runners: CPU (compile/build/deploy) and GPU (training) # CI builder image: CUDA 12.4 + Rust 1.89 + protoc + sccache (hosted on Scaleway CR) # Service images: pushed to Scaleway Container Registry (Kaniko) # Note: .gitlab-ci.yml itself is NOT in changes: filters — CI-only edits skip build # -# Node pool routing: Runner default nodeSelector pool=ci-compile. -# Override via KUBERNETES_NODE_SELECTOR_pool: "pool=" (k=v format required) -# Pools: ci-compile (L4 GPU), ci-training (L40S GPU), services (DEV1-M), gitlab (GP1-XS) +# Runner routing (by tags): +# kapsule,rust,docker → CPU runner → ci-compile-cpu pool (POP2-HC-32C-64G, 32 vCPU) +# kapsule,gpu → GPU runner → ci-compile (L4) or ci-training (L40S) +# Override nodeSelector: KUBERNETES_NODE_SELECTOR_pool: "pool=" (k=v format) +# Pools: ci-compile-cpu (POP2-HC), ci-compile (L4), ci-training (L40S), services, gitlab # Ensure pipeline is always created — individual job rules handle filtering. # Without this, GitLab CE may silently drop push pipelines when changes: @@ -151,8 +153,7 @@ build-infra-runner: # Base template for Rust jobs — pre-baked CI builder from Scaleway CR # Image pre-exists in SCR; rebuild via build-ci-builder when Dockerfile changes -# Runs on pool=ci-compile (L4: 24GB VRAM, CUDA CC 89) by default (runner nodeSelector) -# Training jobs override via KUBERNETES_NODE_SELECTOR_pool: ci-training (L40S: 48GB VRAM) +# Runs on ci-compile-cpu pool (POP2-HC-32C-64G: 32 vCPU, 64GB RAM) via CPU runner .rust-base: image: ${CI_BUILDER_IMAGE} tags: @@ -162,11 +163,12 @@ build-infra-runner: # Compile uses CUDA stubs from ci-builder image — no nvidia runtime needed. # L4 GPU = compute capability 8.9 → candle-kernels generates sm_89 PTX CUDA_COMPUTE_CAP: "89" - # Give compile pods all available CPU for fast Rust compilation - KUBERNETES_CPU_REQUEST: "7000m" - KUBERNETES_CPU_LIMIT: "7800m" - KUBERNETES_MEMORY_REQUEST: "16Gi" - KUBERNETES_MEMORY_LIMIT: "80Gi" + # Use all 32 vCPUs on POP2-HC for maximum Rust compilation parallelism + CARGO_BUILD_JOBS: "30" + KUBERNETES_CPU_REQUEST: "28000m" + KUBERNETES_CPU_LIMIT: "31000m" + KUBERNETES_MEMORY_REQUEST: "24Gi" + KUBERNETES_MEMORY_LIMIT: "56Gi" before_script: # Mold linker fallback — .cargo/config.toml uses -fuse-ld=mold via clang. # clang's -fuse-ld=mold searches for "ld.mold" (not "mold") in its linker paths. @@ -1154,6 +1156,7 @@ infra-plan: image: ${INFRA_RUNNER_IMAGE} tags: - kapsule + - docker needs: [] rules: - if: $CI_PIPELINE_SOURCE == "merge_request_event" @@ -1186,6 +1189,7 @@ infra-apply: image: ${INFRA_RUNNER_IMAGE} tags: - kapsule + - docker needs: - job: infra-plan optional: true @@ -1214,6 +1218,7 @@ infra-drift-check: image: ${INFRA_RUNNER_IMAGE} tags: - kapsule + - docker needs: [] rules: - if: $CI_PIPELINE_SOURCE == "schedule" && $DRIFT_CHECK == "true" @@ -1253,9 +1258,9 @@ deploy: image: ${INFRA_RUNNER_IMAGE} tags: - kapsule + - docker variables: - # Deploy runs on ci-compile (nvidia runtime required by runner default). - # 200m CPU fits alongside compile since deploy runs in a later stage. + # Deploy runs on CPU compile pool (lightweight kubectl apply) KUBERNETES_CPU_REQUEST: "200m" KUBERNETES_CPU_LIMIT: "500m" KUBERNETES_MEMORY_REQUEST: "256Mi" diff --git a/infra/live/production/kapsule/terragrunt.hcl b/infra/live/production/kapsule/terragrunt.hcl index 003837fdd..1e3709b53 100644 --- a/infra/live/production/kapsule/terragrunt.hcl +++ b/infra/live/production/kapsule/terragrunt.hcl @@ -21,6 +21,11 @@ inputs = { ci_compile_type = "L4-1-24G" ci_compile_max_size = 1 + # CPU compile pool (POP2-HC for cargo check/build — 32 vCPU, no GPU) + enable_ci_compile_cpu_pool = true + ci_compile_cpu_type = "POP2-HC-32C-64G" + ci_compile_cpu_max_size = 1 + # CI training pool (L40S for hyperopt + training — 48GB VRAM, CUDA CC 89) enable_ci_training_pool = true ci_training_type = "L40S-1-48G" diff --git a/infra/modules/kapsule/main.tf b/infra/modules/kapsule/main.tf index 17f87e473..c7932b42a 100644 --- a/infra/modules/kapsule/main.tf +++ b/infra/modules/kapsule/main.tf @@ -109,6 +109,25 @@ resource "scaleway_k8s_pool" "ci_training" { } } +# CPU compile pool (POP2-HC — high CPU, no GPU, for cargo check/build) +resource "scaleway_k8s_pool" "ci_compile_cpu" { + count = var.enable_ci_compile_cpu_pool ? 1 : 0 + cluster_id = scaleway_k8s_cluster.foxhunt.id + name = "ci-compile-cpu" + node_type = var.ci_compile_cpu_type + size = 1 + min_size = 0 + max_size = var.ci_compile_cpu_max_size + autoscaling = true + autohealing = true + public_ip_disabled = var.public_ip_disabled + region = var.region + + lifecycle { + ignore_changes = [size] + } +} + # CPU pool for remote development (DevPod — autoscales to zero) resource "scaleway_k8s_pool" "dev" { count = var.enable_dev_pool ? 1 : 0 diff --git a/infra/modules/kapsule/outputs.tf b/infra/modules/kapsule/outputs.tf index b0a6582b6..1b1d265da 100644 --- a/infra/modules/kapsule/outputs.tf +++ b/infra/modules/kapsule/outputs.tf @@ -29,6 +29,11 @@ output "ci_compile_pool_id" { value = var.enable_ci_compile_pool ? scaleway_k8s_pool.ci_compile[0].id : "" } +output "ci_compile_cpu_pool_id" { + description = "ID of the CPU compile node pool (POP2-HC)" + value = var.enable_ci_compile_cpu_pool ? scaleway_k8s_pool.ci_compile_cpu[0].id : "" +} + output "ci_training_pool_id" { description = "ID of the CI training node pool (L40S)" value = var.enable_ci_training_pool ? scaleway_k8s_pool.ci_training[0].id : "" diff --git a/infra/modules/kapsule/variables.tf b/infra/modules/kapsule/variables.tf index a0d40da8d..d75b687d1 100644 --- a/infra/modules/kapsule/variables.tf +++ b/infra/modules/kapsule/variables.tf @@ -75,6 +75,24 @@ variable "ci_training_max_size" { default = 1 } +variable "enable_ci_compile_cpu_pool" { + description = "Create high-CPU node pool for CI compilation (no GPU needed)" + type = bool + default = false +} + +variable "ci_compile_cpu_type" { + description = "Instance type for the CPU compile pool (high CPU, no GPU)" + type = string + default = "POP2-HC-32C-64G" +} + +variable "ci_compile_cpu_max_size" { + description = "Maximum number of nodes in the CPU compile pool" + type = number + default = 1 +} + variable "enable_dev_pool" { description = "Create CPU node pool for remote development (DevPod)" type = bool