infra(kapsule): remove h100x2 + h100-sxm pools

Reasons:
- h100-sxm: Scaleway account quota is 0/0 for the SXM instance type
  (cp_servers_type_H100_SXM_2_80G). The pool was perpetually trying
  to maintain size=1 with a creation_error node, which JAMMED the
  cluster autoscaler. That blocked L40S scale-up entirely during
  today's alpha-perception cluster run.
- h100x2: more expensive than current workloads justify. Every
  training path (alpha perception, future PPO) fits on either the
  single-GPU H100 or L40S.

Removed:
- Two `scaleway_k8s_pool` resources from infra/modules/kapsule/main.tf
- Six variables (enable + type + max_size for each pool)
- Two outputs (pool_id for each)
- Corresponding inputs in infra/live/production/kapsule/terragrunt.hcl

The live cluster has the SXM pool stuck node manually deleted via
`scw k8s node delete` (this commit-session); the pool resource itself
will be destroyed on next `terragrunt apply`.

Post-cleanup pool inventory:
- platform (DEV1-L × 3)
- ci-training-h100  (H100-1-80G, max 1)  <- regular single-GPU
- ci-training-l40s  (L40S-1-48G, max 1)  <- primary training target
- ci-compile-cpu    (POP2-HC, max 4)
- ci-compile-cpu-hm (POP2-HM, max 1)

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-05-16 23:43:12 +02:00
parent 3dc42a6827
commit a252119fd4
4 changed files with 11 additions and 92 deletions

View File

@@ -38,15 +38,11 @@ inputs = {
ci_training_h100_type = "H100-1-80G"
ci_training_h100_max_size = 1
# H100x2 training pool (2x 80GB — multi-GPU parallel hyperopt)
enable_ci_training_h100x2_pool = true
ci_training_h100x2_type = "H100-2-80G"
ci_training_h100x2_max_size = 1
# H100-SXM training pool (2x 80GB SXM, NVLink)
enable_ci_training_h100_sxm_pool = true
ci_training_h100_sxm_type = "H100-SXM-2-80G"
ci_training_h100_sxm_max_size = 1
# H100x2 and H100-SXM pools removed 2026-05-16:
# - SXM: Scaleway account quota is 0/0 (cp_servers_type_H100_SXM_2_80G).
# The pool was stuck trying to provision and jammed the autoscaler.
# - H100x2: more expensive than current workloads justify; the single-GPU
# H100 pool covers all training paths.
# NAT gateway handles egress — no public IPs on nodes
public_ip_disabled = true

View File

@@ -108,43 +108,12 @@ resource "scaleway_k8s_pool" "ci_training_h100" {
}
}
# H100x2 training pool (2x 80GB VRAM — multi-GPU parallel hyperopt)
resource "scaleway_k8s_pool" "ci_training_h100x2" {
count = var.enable_ci_training_h100x2_pool ? 1 : 0
cluster_id = scaleway_k8s_cluster.foxhunt.id
name = "ci-training-h100x2"
node_type = var.ci_training_h100x2_type
size = 1
min_size = 0
max_size = var.ci_training_h100x2_max_size
autoscaling = true
autohealing = true
public_ip_disabled = var.public_ip_disabled
region = var.region
lifecycle {
ignore_changes = [size]
}
}
# H100-SXM training pool (2x 80GB SXM VRAM — NVLink multi-GPU)
resource "scaleway_k8s_pool" "ci_training_h100_sxm" {
count = var.enable_ci_training_h100_sxm_pool ? 1 : 0
cluster_id = scaleway_k8s_cluster.foxhunt.id
name = "ci-training-h100-sxm"
node_type = var.ci_training_h100_sxm_type
size = 1
min_size = 0
max_size = var.ci_training_h100_sxm_max_size
autoscaling = true
autohealing = true
public_ip_disabled = var.public_ip_disabled
region = var.region
lifecycle {
ignore_changes = [size]
}
}
# NOTE: ci-training-h100x2 and ci-training-h100-sxm pools removed
# 2026-05-16 — Scaleway account doesn't have quota for the SXM variant
# (cp_servers_type_H100_SXM_2_80G 0/0), and the H100x2 variant is
# more expensive than our actual usage justifies. Keeping the regular
# single-GPU H100 pool (ci_training_h100) + the L40S pool covers
# every current training workload.
# CPU compile pool (POP2-HC — high CPU, no GPU, for cargo check/build)
resource "scaleway_k8s_pool" "ci_compile_cpu" {

View File

@@ -34,16 +34,6 @@ output "ci_training_h100_pool_id" {
value = var.enable_ci_training_h100_pool ? scaleway_k8s_pool.ci_training_h100[0].id : ""
}
output "ci_training_h100x2_pool_id" {
description = "ID of the H100x2 training node pool"
value = var.enable_ci_training_h100x2_pool ? scaleway_k8s_pool.ci_training_h100x2[0].id : ""
}
output "ci_training_h100_sxm_pool_id" {
description = "ID of the H100-SXM training node pool"
value = var.enable_ci_training_h100_sxm_pool ? scaleway_k8s_pool.ci_training_h100_sxm[0].id : ""
}
output "private_network_id" {
description = "ID of the VPC private network the cluster is attached to"
value = scaleway_vpc_private_network.foxhunt.id

View File

@@ -59,42 +59,6 @@ variable "ci_training_h100_max_size" {
default = 1
}
variable "enable_ci_training_h100x2_pool" {
description = "Create H100x2 GPU node pool for multi-GPU training"
type = bool
default = false
}
variable "ci_training_h100x2_type" {
description = "Instance type for the H100x2 training pool"
type = string
default = "H100-2-80G"
}
variable "ci_training_h100x2_max_size" {
description = "Maximum number of nodes in the H100x2 training pool"
type = number
default = 1
}
variable "enable_ci_training_h100_sxm_pool" {
description = "Create H100-SXM GPU node pool for NVLink multi-GPU training"
type = bool
default = false
}
variable "ci_training_h100_sxm_type" {
description = "Instance type for the H100-SXM training pool"
type = string
default = "H100-SXM-2-80G"
}
variable "ci_training_h100_sxm_max_size" {
description = "Maximum number of nodes in the H100-SXM training pool"
type = number
default = 1
}
variable "enable_ci_compile_cpu_pool" {
description = "Create high-CPU node pool for CI compilation"
type = bool