diff --git a/infra/live/production/kapsule/terragrunt.hcl b/infra/live/production/kapsule/terragrunt.hcl index ad6d9d2b2..b65f0ee7a 100644 --- a/infra/live/production/kapsule/terragrunt.hcl +++ b/infra/live/production/kapsule/terragrunt.hcl @@ -38,15 +38,11 @@ inputs = { ci_training_h100_type = "H100-1-80G" ci_training_h100_max_size = 1 - # H100x2 training pool (2x 80GB — multi-GPU parallel hyperopt) - enable_ci_training_h100x2_pool = true - ci_training_h100x2_type = "H100-2-80G" - ci_training_h100x2_max_size = 1 - - # H100-SXM training pool (2x 80GB SXM, NVLink) - enable_ci_training_h100_sxm_pool = true - ci_training_h100_sxm_type = "H100-SXM-2-80G" - ci_training_h100_sxm_max_size = 1 + # H100x2 and H100-SXM pools removed 2026-05-16: + # - SXM: Scaleway account quota is 0/0 (cp_servers_type_H100_SXM_2_80G). + # The pool was stuck trying to provision and jammed the autoscaler. + # - H100x2: more expensive than current workloads justify; the single-GPU + # H100 pool covers all training paths. # NAT gateway handles egress — no public IPs on nodes public_ip_disabled = true diff --git a/infra/modules/kapsule/main.tf b/infra/modules/kapsule/main.tf index bb311fc64..aa7342639 100644 --- a/infra/modules/kapsule/main.tf +++ b/infra/modules/kapsule/main.tf @@ -108,43 +108,12 @@ resource "scaleway_k8s_pool" "ci_training_h100" { } } -# H100x2 training pool (2x 80GB VRAM — multi-GPU parallel hyperopt) -resource "scaleway_k8s_pool" "ci_training_h100x2" { - count = var.enable_ci_training_h100x2_pool ? 1 : 0 - cluster_id = scaleway_k8s_cluster.foxhunt.id - name = "ci-training-h100x2" - node_type = var.ci_training_h100x2_type - size = 1 - min_size = 0 - max_size = var.ci_training_h100x2_max_size - autoscaling = true - autohealing = true - public_ip_disabled = var.public_ip_disabled - region = var.region - - lifecycle { - ignore_changes = [size] - } -} - -# H100-SXM training pool (2x 80GB SXM VRAM — NVLink multi-GPU) -resource "scaleway_k8s_pool" "ci_training_h100_sxm" { - count = var.enable_ci_training_h100_sxm_pool ? 1 : 0 - cluster_id = scaleway_k8s_cluster.foxhunt.id - name = "ci-training-h100-sxm" - node_type = var.ci_training_h100_sxm_type - size = 1 - min_size = 0 - max_size = var.ci_training_h100_sxm_max_size - autoscaling = true - autohealing = true - public_ip_disabled = var.public_ip_disabled - region = var.region - - lifecycle { - ignore_changes = [size] - } -} +# NOTE: ci-training-h100x2 and ci-training-h100-sxm pools removed +# 2026-05-16 — Scaleway account doesn't have quota for the SXM variant +# (cp_servers_type_H100_SXM_2_80G 0/0), and the H100x2 variant is +# more expensive than our actual usage justifies. Keeping the regular +# single-GPU H100 pool (ci_training_h100) + the L40S pool covers +# every current training workload. # CPU compile pool (POP2-HC — high CPU, no GPU, for cargo check/build) resource "scaleway_k8s_pool" "ci_compile_cpu" { diff --git a/infra/modules/kapsule/outputs.tf b/infra/modules/kapsule/outputs.tf index eb738af91..f4e760b14 100644 --- a/infra/modules/kapsule/outputs.tf +++ b/infra/modules/kapsule/outputs.tf @@ -34,16 +34,6 @@ output "ci_training_h100_pool_id" { value = var.enable_ci_training_h100_pool ? scaleway_k8s_pool.ci_training_h100[0].id : "" } -output "ci_training_h100x2_pool_id" { - description = "ID of the H100x2 training node pool" - value = var.enable_ci_training_h100x2_pool ? scaleway_k8s_pool.ci_training_h100x2[0].id : "" -} - -output "ci_training_h100_sxm_pool_id" { - description = "ID of the H100-SXM training node pool" - value = var.enable_ci_training_h100_sxm_pool ? scaleway_k8s_pool.ci_training_h100_sxm[0].id : "" -} - output "private_network_id" { description = "ID of the VPC private network the cluster is attached to" value = scaleway_vpc_private_network.foxhunt.id diff --git a/infra/modules/kapsule/variables.tf b/infra/modules/kapsule/variables.tf index cd4239e33..4f097476a 100644 --- a/infra/modules/kapsule/variables.tf +++ b/infra/modules/kapsule/variables.tf @@ -59,42 +59,6 @@ variable "ci_training_h100_max_size" { default = 1 } -variable "enable_ci_training_h100x2_pool" { - description = "Create H100x2 GPU node pool for multi-GPU training" - type = bool - default = false -} - -variable "ci_training_h100x2_type" { - description = "Instance type for the H100x2 training pool" - type = string - default = "H100-2-80G" -} - -variable "ci_training_h100x2_max_size" { - description = "Maximum number of nodes in the H100x2 training pool" - type = number - default = 1 -} - -variable "enable_ci_training_h100_sxm_pool" { - description = "Create H100-SXM GPU node pool for NVLink multi-GPU training" - type = bool - default = false -} - -variable "ci_training_h100_sxm_type" { - description = "Instance type for the H100-SXM training pool" - type = string - default = "H100-SXM-2-80G" -} - -variable "ci_training_h100_sxm_max_size" { - description = "Maximum number of nodes in the H100-SXM training pool" - type = number - default = 1 -} - variable "enable_ci_compile_cpu_pool" { description = "Create high-CPU node pool for CI compilation" type = bool