From c97c02677ebdaf95c94ef91a3fd6aaf6aeae100a Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 13:59:44 +0100 Subject: [PATCH] =?UTF-8?q?fix(infra):=20rename=20L4=20pool=20ci-compile?= =?UTF-8?q?=20=E2=86=92=20ci-rl,=20remove=20stale=20moved=20blocks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Applied terragrunt to recreate the L4 GPU pool with the correct name `ci-rl` (was `ci-compile` due to immutable Scaleway pool names). Updated all K8s manifests and comments to match. Removed the 3 stale `moved` blocks from main.tf since the state renames are now applied. Pool naming is now consistent across Terraform, Scaleway, and K8s configs: - ci-compile-cpu (POP2-32C-128G) — CPU compilation - ci-rl (L4-1-24G) — RL training / CUDA compile - ci-training (L40S-1-48G) — supervised training + hyperopt Co-Authored-By: Claude Opus 4.6 --- .gitlab-ci.yml | 2 +- infra/k8s/monitoring/APPLY-ORDER.md | 2 +- infra/k8s/monitoring/dcgm-exporter.yaml | 2 +- infra/k8s/training/image-prepuller.yaml | 5 ++--- infra/modules/kapsule/main.tf | 18 ------------------ 5 files changed, 5 insertions(+), 24 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9acb682a4..c28bd8bd6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,7 @@ # kapsule,rust → CPU runner → ci-compile-cpu pool (POP2-32C-128G, 32 vCPU) # kapsule,gpu → GPU runner → ci-training (L40S) # Override nodeSelector: KUBERNETES_NODE_SELECTOR_k8s.scaleway.com/pool-name: "" (k=v format) -# Pools: ci-compile-cpu (POP2-32C-128G), ci-compile (L4 compile), ci-training (L40S training), services, gitlab +# Pools: ci-compile-cpu (POP2-32C-128G), ci-rl (L4 GPU), ci-training (L40S training), services, gitlab # All compilation (services + training) runs on ci-compile-cpu. CUDA stubs in ci-builder image. # Ensure pipeline is always created — individual job rules handle filtering. diff --git a/infra/k8s/monitoring/APPLY-ORDER.md b/infra/k8s/monitoring/APPLY-ORDER.md index 01bdc2396..9f0299c4b 100644 --- a/infra/k8s/monitoring/APPLY-ORDER.md +++ b/infra/k8s/monitoring/APPLY-ORDER.md @@ -94,7 +94,7 @@ 4. Repeat for gitlab pool (if not autoscaling, same process). -5. Autoscaling pools (ci-training, ci-compile, gpu-dev): +5. Autoscaling pools (ci-training, ci-rl, gpu-dev): - No action needed — next scale-up automatically uses private IP. 6. Verify Tailscale still works: diff --git a/infra/k8s/monitoring/dcgm-exporter.yaml b/infra/k8s/monitoring/dcgm-exporter.yaml index 58cb54d3c..7812222a9 100644 --- a/infra/k8s/monitoring/dcgm-exporter.yaml +++ b/infra/k8s/monitoring/dcgm-exporter.yaml @@ -1,5 +1,5 @@ # NVIDIA DCGM Exporter — GPU metrics for Cockpit -# Runs on all GPU node pools (ci-training, ci-compile, gpu-dev) +# Runs on all GPU node pools (ci-training, ci-rl, gpu-dev) # Alloy auto-discovers and scrapes these pods via annotations --- apiVersion: v1 diff --git a/infra/k8s/training/image-prepuller.yaml b/infra/k8s/training/image-prepuller.yaml index 68d071dc9..481846e3a 100644 --- a/infra/k8s/training/image-prepuller.yaml +++ b/infra/k8s/training/image-prepuller.yaml @@ -1,5 +1,5 @@ # DaemonSet image pre-puller — keeps training images cached on GPU nodes -# Runs on ci-compile (L4) and ci-training (L40S) pools so training jobs skip the pull. +# Runs on ci-rl (L4) and ci-training (L40S) pools so training jobs skip the pull. # Init containers pull :latest tags, then the main container sleeps forever. --- apiVersion: apps/v1 @@ -27,8 +27,7 @@ spec: - key: k8s.scaleway.com/pool-name operator: In values: - - ci-compile - - ci-compile # remove after terragrunt apply renames pool + - ci-rl - ci-training tolerations: - key: nvidia.com/gpu diff --git a/infra/modules/kapsule/main.tf b/infra/modules/kapsule/main.tf index d38ff737e..24137b239 100644 --- a/infra/modules/kapsule/main.tf +++ b/infra/modules/kapsule/main.tf @@ -1,21 +1,3 @@ -# Terraform state renames — prevents destroy+recreate of state entries. -# Pool name attribute changes (always-on→services, ci-build→ci-training) -# will still force replacement since Scaleway pool names are immutable. -moved { - from = scaleway_k8s_pool.always_on - to = scaleway_k8s_pool.services -} - -moved { - from = scaleway_k8s_pool.ci_build - to = scaleway_k8s_pool.ci_training -} - -moved { - from = scaleway_k8s_pool.ci_compile - to = scaleway_k8s_pool.ci_rl -} - resource "scaleway_vpc_private_network" "foxhunt" { name = "${var.cluster_name}-pn" region = var.region