From ce65e4ca42dd390ec2bf44709bb50dec69878ed3 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 27 Feb 2026 10:11:54 +0100 Subject: [PATCH] feat(infra): add L4 compile pool alongside L40S training pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split GPU workloads across two pools: - ci-compile (L4-1-24G): for cargo check/test/build — cheaper, same CUDA CC 8.9 as L40S, sufficient 24GB VRAM for compilation - ci-build (L40S-1-48G): for hyperopt + training — 48GB VRAM needed for large model training batches Runner defaults to ci-compile; training jobs override to ci-build via KUBERNETES_NODE_SELECTOR. Both pools autoscale to zero when idle. Co-Authored-By: Claude Opus 4.6 --- .gitlab-ci.yml | 12 +++++---- infra/live/production/kapsule/terragrunt.hcl | 10 +++++--- infra/modules/kapsule/main.tf | 21 +++++++++++++++- infra/modules/kapsule/variables.tf | 26 +++++++++++++++++--- 4 files changed, 56 insertions(+), 13 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 98c291a0f..8bf5deb5a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,5 +1,5 @@ # GitLab CI/CD — Foxhunt -# Runs on ci-build pool (L40S, CUDA 8.9, autoscales 0-2), linker: clang+mold +# Compile: ci-compile pool (L4, CUDA 8.9) | Training: ci-build pool (L40S, 48GB VRAM) # CI builder image: CUDA 12.4 + Rust 1.89 + protoc + sccache (hosted on Scaleway CR) # Service images: pushed to Scaleway Container Registry (Kaniko) # Note: .gitlab-ci.yml itself is NOT in changes: filters — CI-only edits skip build @@ -148,16 +148,15 @@ build-infra-runner: # Base template for Rust jobs — pre-baked CI builder from Scaleway CR # Image pre-exists in SCR; rebuild via build-ci-builder when Dockerfile changes -# L40S node: 8 vCPU (7800m allocatable), 91GB RAM, 1 GPU (48GB VRAM) -# Compile jobs get all cores (CPU-bound); training jobs get fewer (GPU-bound) +# Runs on ci-compile pool (L4: 24GB VRAM, CUDA CC 89) by default (runner config) +# Training jobs override to ci-build pool (L40S: 48GB VRAM) via node_selector .rust-base: image: ${CI_BUILDER_IMAGE} tags: - kapsule - rust variables: - # Default: give build pods all available CPU for fast Rust compilation - # Training jobs override these to lower values (they're GPU-bound) + # Give compile pods all available CPU for fast Rust compilation KUBERNETES_CPU_REQUEST: "7000m" KUBERNETES_CPU_LIMIT: "7800m" KUBERNETES_MEMORY_REQUEST: "16Gi" @@ -478,6 +477,9 @@ build-training: - kapsule - gpu variables: + # Route training to L40S pool (48GB VRAM needed for large models) + # Runner default is ci-compile (L4); override here for training + "KUBERNETES_NODE_SELECTOR_k8s.scaleway.com/pool-name": "ci-build" # Training is GPU-bound — needs minimal CPU. Low request allows 2 concurrent # training jobs on same L40S node (7800m allocatable ÷ 2 = 3900m each) KUBERNETES_CPU_REQUEST: "2000m" diff --git a/infra/live/production/kapsule/terragrunt.hcl b/infra/live/production/kapsule/terragrunt.hcl index afd7205b0..7a23606bf 100644 --- a/infra/live/production/kapsule/terragrunt.hcl +++ b/infra/live/production/kapsule/terragrunt.hcl @@ -16,11 +16,15 @@ inputs = { enable_gitlab_pool = true gitlab_type = "GP1-XS" - # CI build pool (L40S for faster CI: 48GB VRAM, ~2x L4 throughput, CUDA CC 89) - # max_size=2 allows autoscaler to add a second node for concurrent jobs + # CI compile pool (L4 for cargo check/test/build — CUDA CC 89, 24GB VRAM) + enable_ci_compile_pool = true + ci_compile_type = "L4-1-24G" + ci_compile_max_size = 1 + + # CI training pool (L40S for hyperopt + training — 48GB VRAM, CUDA CC 89) enable_ci_build_pool = true ci_build_type = "L40S-1-48G" - ci_build_max_size = 2 + ci_build_max_size = 1 # Dev pool (GP1-L for DevPod remote development, autoscales to zero) enable_dev_pool = true diff --git a/infra/modules/kapsule/main.tf b/infra/modules/kapsule/main.tf index ea13ad4d3..1b980fcea 100644 --- a/infra/modules/kapsule/main.tf +++ b/infra/modules/kapsule/main.tf @@ -58,7 +58,26 @@ resource "scaleway_k8s_pool" "gitlab" { region = var.region } -# CI build pool (L40S — 48GB VRAM, ~2x L4 throughput, same Ada arch / CUDA CC 89) +# CI compile pool (L4 — 24GB VRAM, CUDA CC 89, for cargo check/test/build) +resource "scaleway_k8s_pool" "ci_compile" { + count = var.enable_ci_compile_pool ? 1 : 0 + cluster_id = scaleway_k8s_cluster.foxhunt.id + name = "ci-compile" + node_type = var.ci_compile_type + size = 1 + min_size = 0 + max_size = var.ci_compile_max_size + autoscaling = true + autohealing = true + public_ip_disabled = var.public_ip_disabled + region = var.region + + lifecycle { + ignore_changes = [size] + } +} + +# CI training pool (L40S — 48GB VRAM, for hyperopt + training jobs) resource "scaleway_k8s_pool" "ci_build" { count = var.enable_ci_build_pool ? 1 : 0 cluster_id = scaleway_k8s_cluster.foxhunt.id diff --git a/infra/modules/kapsule/variables.tf b/infra/modules/kapsule/variables.tf index 9262b2dce..e9bed2f88 100644 --- a/infra/modules/kapsule/variables.tf +++ b/infra/modules/kapsule/variables.tf @@ -39,22 +39,40 @@ variable "gitlab_type" { default = "DEV1-L" } +variable "enable_ci_compile_pool" { + description = "Create L4 GPU node pool for CI compilation + testing" + type = bool + default = false +} + +variable "ci_compile_type" { + description = "Instance type for the CI compile pool (L4 for CUDA compilation)" + type = string + default = "L4-1-24G" +} + +variable "ci_compile_max_size" { + description = "Maximum number of nodes in the CI compile pool" + type = number + default = 1 +} + variable "enable_ci_build_pool" { - description = "Create GPU node pool for CI builds (L4)" + description = "Create L40S GPU node pool for training + hyperopt" type = bool default = false } variable "ci_build_type" { - description = "Instance type for the CI build node pool (L40S for fast compilation + training)" + description = "Instance type for the CI training pool (L40S for 48GB VRAM training)" type = string default = "L40S-1-48G" } variable "ci_build_max_size" { - description = "Maximum number of nodes in the CI build pool (2 allows concurrent jobs)" + description = "Maximum number of nodes in the CI training pool" type = number - default = 2 + default = 1 } variable "enable_dev_pool" {