From a252119fd408f5bd30e57ed602db2c1ba7979a50 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 16 May 2026 23:43:12 +0200 Subject: [PATCH] infra(kapsule): remove h100x2 + h100-sxm pools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reasons: - h100-sxm: Scaleway account quota is 0/0 for the SXM instance type (cp_servers_type_H100_SXM_2_80G). The pool was perpetually trying to maintain size=1 with a creation_error node, which JAMMED the cluster autoscaler. That blocked L40S scale-up entirely during today's alpha-perception cluster run. - h100x2: more expensive than current workloads justify. Every training path (alpha perception, future PPO) fits on either the single-GPU H100 or L40S. Removed: - Two `scaleway_k8s_pool` resources from infra/modules/kapsule/main.tf - Six variables (enable + type + max_size for each pool) - Two outputs (pool_id for each) - Corresponding inputs in infra/live/production/kapsule/terragrunt.hcl The live cluster has the SXM pool stuck node manually deleted via `scw k8s node delete` (this commit-session); the pool resource itself will be destroyed on next `terragrunt apply`. Post-cleanup pool inventory: - platform (DEV1-L × 3) - ci-training-h100 (H100-1-80G, max 1) <- regular single-GPU - ci-training-l40s (L40S-1-48G, max 1) <- primary training target - ci-compile-cpu (POP2-HC, max 4) - ci-compile-cpu-hm (POP2-HM, max 1) Co-Authored-By: Claude Opus 4.7 --- infra/live/production/kapsule/terragrunt.hcl | 14 +++---- infra/modules/kapsule/main.tf | 43 +++----------------- infra/modules/kapsule/outputs.tf | 10 ----- infra/modules/kapsule/variables.tf | 36 ---------------- 4 files changed, 11 insertions(+), 92 deletions(-) diff --git a/infra/live/production/kapsule/terragrunt.hcl b/infra/live/production/kapsule/terragrunt.hcl index ad6d9d2b2..b65f0ee7a 100644 --- a/infra/live/production/kapsule/terragrunt.hcl +++ b/infra/live/production/kapsule/terragrunt.hcl @@ -38,15 +38,11 @@ inputs = { ci_training_h100_type = "H100-1-80G" ci_training_h100_max_size = 1 - # H100x2 training pool (2x 80GB — multi-GPU parallel hyperopt) - enable_ci_training_h100x2_pool = true - ci_training_h100x2_type = "H100-2-80G" - ci_training_h100x2_max_size = 1 - - # H100-SXM training pool (2x 80GB SXM, NVLink) - enable_ci_training_h100_sxm_pool = true - ci_training_h100_sxm_type = "H100-SXM-2-80G" - ci_training_h100_sxm_max_size = 1 + # H100x2 and H100-SXM pools removed 2026-05-16: + # - SXM: Scaleway account quota is 0/0 (cp_servers_type_H100_SXM_2_80G). + # The pool was stuck trying to provision and jammed the autoscaler. + # - H100x2: more expensive than current workloads justify; the single-GPU + # H100 pool covers all training paths. # NAT gateway handles egress — no public IPs on nodes public_ip_disabled = true diff --git a/infra/modules/kapsule/main.tf b/infra/modules/kapsule/main.tf index bb311fc64..aa7342639 100644 --- a/infra/modules/kapsule/main.tf +++ b/infra/modules/kapsule/main.tf @@ -108,43 +108,12 @@ resource "scaleway_k8s_pool" "ci_training_h100" { } } -# H100x2 training pool (2x 80GB VRAM — multi-GPU parallel hyperopt) -resource "scaleway_k8s_pool" "ci_training_h100x2" { - count = var.enable_ci_training_h100x2_pool ? 1 : 0 - cluster_id = scaleway_k8s_cluster.foxhunt.id - name = "ci-training-h100x2" - node_type = var.ci_training_h100x2_type - size = 1 - min_size = 0 - max_size = var.ci_training_h100x2_max_size - autoscaling = true - autohealing = true - public_ip_disabled = var.public_ip_disabled - region = var.region - - lifecycle { - ignore_changes = [size] - } -} - -# H100-SXM training pool (2x 80GB SXM VRAM — NVLink multi-GPU) -resource "scaleway_k8s_pool" "ci_training_h100_sxm" { - count = var.enable_ci_training_h100_sxm_pool ? 1 : 0 - cluster_id = scaleway_k8s_cluster.foxhunt.id - name = "ci-training-h100-sxm" - node_type = var.ci_training_h100_sxm_type - size = 1 - min_size = 0 - max_size = var.ci_training_h100_sxm_max_size - autoscaling = true - autohealing = true - public_ip_disabled = var.public_ip_disabled - region = var.region - - lifecycle { - ignore_changes = [size] - } -} +# NOTE: ci-training-h100x2 and ci-training-h100-sxm pools removed +# 2026-05-16 — Scaleway account doesn't have quota for the SXM variant +# (cp_servers_type_H100_SXM_2_80G 0/0), and the H100x2 variant is +# more expensive than our actual usage justifies. Keeping the regular +# single-GPU H100 pool (ci_training_h100) + the L40S pool covers +# every current training workload. # CPU compile pool (POP2-HC — high CPU, no GPU, for cargo check/build) resource "scaleway_k8s_pool" "ci_compile_cpu" { diff --git a/infra/modules/kapsule/outputs.tf b/infra/modules/kapsule/outputs.tf index eb738af91..f4e760b14 100644 --- a/infra/modules/kapsule/outputs.tf +++ b/infra/modules/kapsule/outputs.tf @@ -34,16 +34,6 @@ output "ci_training_h100_pool_id" { value = var.enable_ci_training_h100_pool ? scaleway_k8s_pool.ci_training_h100[0].id : "" } -output "ci_training_h100x2_pool_id" { - description = "ID of the H100x2 training node pool" - value = var.enable_ci_training_h100x2_pool ? scaleway_k8s_pool.ci_training_h100x2[0].id : "" -} - -output "ci_training_h100_sxm_pool_id" { - description = "ID of the H100-SXM training node pool" - value = var.enable_ci_training_h100_sxm_pool ? scaleway_k8s_pool.ci_training_h100_sxm[0].id : "" -} - output "private_network_id" { description = "ID of the VPC private network the cluster is attached to" value = scaleway_vpc_private_network.foxhunt.id diff --git a/infra/modules/kapsule/variables.tf b/infra/modules/kapsule/variables.tf index cd4239e33..4f097476a 100644 --- a/infra/modules/kapsule/variables.tf +++ b/infra/modules/kapsule/variables.tf @@ -59,42 +59,6 @@ variable "ci_training_h100_max_size" { default = 1 } -variable "enable_ci_training_h100x2_pool" { - description = "Create H100x2 GPU node pool for multi-GPU training" - type = bool - default = false -} - -variable "ci_training_h100x2_type" { - description = "Instance type for the H100x2 training pool" - type = string - default = "H100-2-80G" -} - -variable "ci_training_h100x2_max_size" { - description = "Maximum number of nodes in the H100x2 training pool" - type = number - default = 1 -} - -variable "enable_ci_training_h100_sxm_pool" { - description = "Create H100-SXM GPU node pool for NVLink multi-GPU training" - type = bool - default = false -} - -variable "ci_training_h100_sxm_type" { - description = "Instance type for the H100-SXM training pool" - type = string - default = "H100-SXM-2-80G" -} - -variable "ci_training_h100_sxm_max_size" { - description = "Maximum number of nodes in the H100-SXM training pool" - type = number - default = 1 -} - variable "enable_ci_compile_cpu_pool" { description = "Create high-CPU node pool for CI compilation" type = bool