From e67f828f3c0ecd517a6622ef970f3c41bdca1cee Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 27 Feb 2026 23:37:43 +0100 Subject: [PATCH] fix(infra): use Kapsule built-in pool-name label for node selection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kapsule doesn't allow node-labels in kubelet_args. Instead, use the auto-assigned k8s.scaleway.com/pool-name label that every node gets. - Revert kubelet_args from Terraform (not supported) - Switch runner node_selector: pool→k8s.scaleway.com/pool-name - Update CI pipeline KUBERNETES_NODE_SELECTOR to match - Helm upgraded both runners (CPU + GPU) No more manual kubectl label after node scale-up. Co-Authored-By: Claude Opus 4.6 --- .gitlab-ci.yml | 8 ++++---- infra/modules/kapsule/main.tf | 35 +++++++---------------------------- 2 files changed, 11 insertions(+), 32 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bc251f09b..a4a5eb6bd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ # Runner routing (by tags): # kapsule,rust,docker → CPU runner → ci-compile-cpu pool (POP2-32C-128G, 32 vCPU) # kapsule,gpu → GPU runner → ci-compile (L4) or ci-training (L40S) -# Override nodeSelector: KUBERNETES_NODE_SELECTOR_pool: "pool=" (k=v format) +# Override nodeSelector: KUBERNETES_NODE_SELECTOR_k8s.scaleway.com/pool-name: "" (k=v format) # Pools: ci-compile-cpu (POP2-32C-128G), ci-compile (L4), ci-training (L40S), services, gitlab # Ensure pipeline is always created — individual job rules handle filtering. @@ -438,7 +438,7 @@ build-training: tags: - kapsule - gpu - # Runner default node_selector: pool=ci-compile (L4-1-24G) + # Runner default node_selector: k8s.scaleway.com/pool-name=ci-compile (L4-1-24G) variables: # nvidia runtime provided by runner default (runtime_class_name = "nvidia") # RL hyperopt is CPU-bound (parallel PSO trials). Use full L4 (8 vCPU). @@ -472,8 +472,8 @@ build-training: variables: # nvidia runtime provided by runner default (runtime_class_name = "nvidia") # Route to L40S pool (48GB VRAM for large supervised models) - # Overrides runner default pool=ci-compile - KUBERNETES_NODE_SELECTOR_pool: "pool=ci-training" + # Overrides runner default to L40S pool + KUBERNETES_NODE_SELECTOR_k8s.scaleway.com/pool-name: "ci-training" # Supervised is GPU-bound — minimal CPU. Low request allows 3-4 concurrent jobs. KUBERNETES_CPU_REQUEST: "1000m" KUBERNETES_CPU_LIMIT: "2000m" diff --git a/infra/modules/kapsule/main.tf b/infra/modules/kapsule/main.tf index 7cecd50bc..a153a76da 100644 --- a/infra/modules/kapsule/main.tf +++ b/infra/modules/kapsule/main.tf @@ -51,10 +51,6 @@ resource "scaleway_k8s_pool" "services" { public_ip_disabled = var.public_ip_disabled region = var.region - kubelet_args = { - "node-labels" = "pool=services" - } - lifecycle { ignore_changes = [size] } @@ -73,13 +69,10 @@ resource "scaleway_k8s_pool" "gitlab" { autohealing = true public_ip_disabled = var.public_ip_disabled region = var.region - - kubelet_args = { - "node-labels" = "pool=gitlab" - } } -# CI compile pool (L4 — 24GB VRAM, CUDA CC 89, for cargo check/test/build) +# CI compile pool (L4 — 24GB VRAM, CUDA CC 89, for RL training + hyperopt) +# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-compile resource "scaleway_k8s_pool" "ci_compile" { count = var.enable_ci_compile_pool ? 1 : 0 cluster_id = scaleway_k8s_cluster.foxhunt.id @@ -93,16 +86,13 @@ resource "scaleway_k8s_pool" "ci_compile" { public_ip_disabled = var.public_ip_disabled region = var.region - kubelet_args = { - "node-labels" = "pool=ci-compile" - } - lifecycle { ignore_changes = [size] } } -# CI training pool (L40S — 48GB VRAM, for hyperopt + training jobs) +# CI training pool (L40S — 48GB VRAM, for supervised hyperopt + training) +# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-training resource "scaleway_k8s_pool" "ci_training" { count = var.enable_ci_training_pool ? 1 : 0 cluster_id = scaleway_k8s_cluster.foxhunt.id @@ -116,16 +106,13 @@ resource "scaleway_k8s_pool" "ci_training" { public_ip_disabled = var.public_ip_disabled region = var.region - kubelet_args = { - "node-labels" = "pool=ci-training" - } - lifecycle { ignore_changes = [size] } } -# CPU compile pool (POP2-HC — high CPU, no GPU, for cargo check/build) +# CPU compile pool (POP2 — high CPU, no GPU, for cargo check/build) +# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-compile-cpu resource "scaleway_k8s_pool" "ci_compile_cpu" { count = var.enable_ci_compile_cpu_pool ? 1 : 0 cluster_id = scaleway_k8s_cluster.foxhunt.id @@ -139,16 +126,13 @@ resource "scaleway_k8s_pool" "ci_compile_cpu" { public_ip_disabled = var.public_ip_disabled region = var.region - kubelet_args = { - "node-labels" = "pool=ci-compile-cpu" - } - lifecycle { ignore_changes = [size] } } # CPU pool for remote development (DevPod — autoscales to zero) +# Nodes auto-labeled: k8s.scaleway.com/pool-name=gpu-dev resource "scaleway_k8s_pool" "dev" { count = var.enable_dev_pool ? 1 : 0 cluster_id = scaleway_k8s_cluster.foxhunt.id @@ -163,12 +147,7 @@ resource "scaleway_k8s_pool" "dev" { wait_for_pool_ready = false region = var.region - kubelet_args = { - "node-labels" = "pool=gpu-dev" - } - lifecycle { ignore_changes = [size] } } -