feat(infra): add L4 compile pool alongside L40S training pool

Split GPU workloads across two pools:
- ci-compile (L4-1-24G): for cargo check/test/build — cheaper, same
  CUDA CC 8.9 as L40S, sufficient 24GB VRAM for compilation
- ci-build (L40S-1-48G): for hyperopt + training — 48GB VRAM needed
  for large model training batches

Runner defaults to ci-compile; training jobs override to ci-build
via KUBERNETES_NODE_SELECTOR. Both pools autoscale to zero when idle.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

.gitlab-ci.yml

@@ -1,5 +1,5 @@
 # GitLab CI/CD — Foxhunt
-# Runs on ci-build pool (L40S, CUDA 8.9, autoscales 0-2), linker: clang+mold
+# Compile: ci-compile pool (L4, CUDA 8.9)  |  Training: ci-build pool (L40S, 48GB VRAM)
 # CI builder image: CUDA 12.4 + Rust 1.89 + protoc + sccache (hosted on Scaleway CR)
 # Service images: pushed to Scaleway Container Registry (Kaniko)
 # Note: .gitlab-ci.yml itself is NOT in changes: filters — CI-only edits skip build
@@ -148,16 +148,15 @@ build-infra-runner:
 
 # Base template for Rust jobs — pre-baked CI builder from Scaleway CR
 # Image pre-exists in SCR; rebuild via build-ci-builder when Dockerfile changes
-# L40S node: 8 vCPU (7800m allocatable), 91GB RAM, 1 GPU (48GB VRAM)
-# Compile jobs get all cores (CPU-bound); training jobs get fewer (GPU-bound)
+# Runs on ci-compile pool (L4: 24GB VRAM, CUDA CC 89) by default (runner config)
+# Training jobs override to ci-build pool (L40S: 48GB VRAM) via node_selector
 .rust-base:
   image: ${CI_BUILDER_IMAGE}
   tags:
     - kapsule
     - rust
   variables:
-    # Default: give build pods all available CPU for fast Rust compilation
-    # Training jobs override these to lower values (they're GPU-bound)
+    # Give compile pods all available CPU for fast Rust compilation
     KUBERNETES_CPU_REQUEST: "7000m"
     KUBERNETES_CPU_LIMIT: "7800m"
     KUBERNETES_MEMORY_REQUEST: "16Gi"
@@ -478,6 +477,9 @@ build-training:
     - kapsule
     - gpu
   variables:
+    # Route training to L40S pool (48GB VRAM needed for large models)
+    # Runner default is ci-compile (L4); override here for training
+    "KUBERNETES_NODE_SELECTOR_k8s.scaleway.com/pool-name": "ci-build"
     # Training is GPU-bound — needs minimal CPU. Low request allows 2 concurrent
     # training jobs on same L40S node (7800m allocatable ÷ 2 = 3900m each)
     KUBERNETES_CPU_REQUEST: "2000m"

infra/live/production/kapsule/terragrunt.hcl

@@ -16,11 +16,15 @@ inputs = {
   enable_gitlab_pool = true
   gitlab_type        = "GP1-XS"
 
-  # CI build pool (L40S for faster CI: 48GB VRAM, ~2x L4 throughput, CUDA CC 89)
-  # max_size=2 allows autoscaler to add a second node for concurrent jobs
+  # CI compile pool (L4 for cargo check/test/build — CUDA CC 89, 24GB VRAM)
+  enable_ci_compile_pool = true
+  ci_compile_type        = "L4-1-24G"
+  ci_compile_max_size    = 1
+
+  # CI training pool (L40S for hyperopt + training — 48GB VRAM, CUDA CC 89)
   enable_ci_build_pool = true
   ci_build_type        = "L40S-1-48G"
-  ci_build_max_size    = 2
+  ci_build_max_size    = 1
 
   # Dev pool (GP1-L for DevPod remote development, autoscales to zero)
   enable_dev_pool   = true

infra/modules/kapsule/main.tf

@@ -58,7 +58,26 @@ resource "scaleway_k8s_pool" "gitlab" {
   region             = var.region
 }
 
-# CI build pool (L40S — 48GB VRAM, ~2x L4 throughput, same Ada arch / CUDA CC 89)
+# CI compile pool (L4 — 24GB VRAM, CUDA CC 89, for cargo check/test/build)
+resource "scaleway_k8s_pool" "ci_compile" {
+  count       = var.enable_ci_compile_pool ? 1 : 0
+  cluster_id  = scaleway_k8s_cluster.foxhunt.id
+  name        = "ci-compile"
+  node_type   = var.ci_compile_type
+  size        = 1
+  min_size    = 0
+  max_size    = var.ci_compile_max_size
+  autoscaling        = true
+  autohealing        = true
+  public_ip_disabled = var.public_ip_disabled
+  region             = var.region
+
+  lifecycle {
+    ignore_changes = [size]
+  }
+}
+
+# CI training pool (L40S — 48GB VRAM, for hyperopt + training jobs)
 resource "scaleway_k8s_pool" "ci_build" {
   count       = var.enable_ci_build_pool ? 1 : 0
   cluster_id  = scaleway_k8s_cluster.foxhunt.id

infra/modules/kapsule/variables.tf

@@ -39,22 +39,40 @@ variable "gitlab_type" {
   default     = "DEV1-L"
 }
 
+variable "enable_ci_compile_pool" {
+  description = "Create L4 GPU node pool for CI compilation + testing"
+  type        = bool
+  default     = false
+}
+
+variable "ci_compile_type" {
+  description = "Instance type for the CI compile pool (L4 for CUDA compilation)"
+  type        = string
+  default     = "L4-1-24G"
+}
+
+variable "ci_compile_max_size" {
+  description = "Maximum number of nodes in the CI compile pool"
+  type        = number
+  default     = 1
+}
+
 variable "enable_ci_build_pool" {
-  description = "Create GPU node pool for CI builds (L4)"
+  description = "Create L40S GPU node pool for training + hyperopt"
   type        = bool
   default     = false
 }
 
 variable "ci_build_type" {
-  description = "Instance type for the CI build node pool (L40S for fast compilation + training)"
+  description = "Instance type for the CI training pool (L40S for 48GB VRAM training)"
   type        = string
   default     = "L40S-1-48G"
 }
 
 variable "ci_build_max_size" {
-  description = "Maximum number of nodes in the CI build pool (2 allows concurrent jobs)"
+  description = "Maximum number of nodes in the CI training pool"
   type        = number
-  default     = 2
+  default     = 1
 }
 
 variable "enable_dev_pool" {