infra(kapsule): remove h100x2 + h100-sxm pools
Reasons: - h100-sxm: Scaleway account quota is 0/0 for the SXM instance type (cp_servers_type_H100_SXM_2_80G). The pool was perpetually trying to maintain size=1 with a creation_error node, which JAMMED the cluster autoscaler. That blocked L40S scale-up entirely during today's alpha-perception cluster run. - h100x2: more expensive than current workloads justify. Every training path (alpha perception, future PPO) fits on either the single-GPU H100 or L40S. Removed: - Two `scaleway_k8s_pool` resources from infra/modules/kapsule/main.tf - Six variables (enable + type + max_size for each pool) - Two outputs (pool_id for each) - Corresponding inputs in infra/live/production/kapsule/terragrunt.hcl The live cluster has the SXM pool stuck node manually deleted via `scw k8s node delete` (this commit-session); the pool resource itself will be destroyed on next `terragrunt apply`. Post-cleanup pool inventory: - platform (DEV1-L × 3) - ci-training-h100 (H100-1-80G, max 1) <- regular single-GPU - ci-training-l40s (L40S-1-48G, max 1) <- primary training target - ci-compile-cpu (POP2-HC, max 4) - ci-compile-cpu-hm (POP2-HM, max 1) Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -38,15 +38,11 @@ inputs = {
|
|||||||
ci_training_h100_type = "H100-1-80G"
|
ci_training_h100_type = "H100-1-80G"
|
||||||
ci_training_h100_max_size = 1
|
ci_training_h100_max_size = 1
|
||||||
|
|
||||||
# H100x2 training pool (2x 80GB — multi-GPU parallel hyperopt)
|
# H100x2 and H100-SXM pools removed 2026-05-16:
|
||||||
enable_ci_training_h100x2_pool = true
|
# - SXM: Scaleway account quota is 0/0 (cp_servers_type_H100_SXM_2_80G).
|
||||||
ci_training_h100x2_type = "H100-2-80G"
|
# The pool was stuck trying to provision and jammed the autoscaler.
|
||||||
ci_training_h100x2_max_size = 1
|
# - H100x2: more expensive than current workloads justify; the single-GPU
|
||||||
|
# H100 pool covers all training paths.
|
||||||
# H100-SXM training pool (2x 80GB SXM, NVLink)
|
|
||||||
enable_ci_training_h100_sxm_pool = true
|
|
||||||
ci_training_h100_sxm_type = "H100-SXM-2-80G"
|
|
||||||
ci_training_h100_sxm_max_size = 1
|
|
||||||
|
|
||||||
# NAT gateway handles egress — no public IPs on nodes
|
# NAT gateway handles egress — no public IPs on nodes
|
||||||
public_ip_disabled = true
|
public_ip_disabled = true
|
||||||
|
|||||||
@@ -108,43 +108,12 @@ resource "scaleway_k8s_pool" "ci_training_h100" {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
# H100x2 training pool (2x 80GB VRAM — multi-GPU parallel hyperopt)
|
# NOTE: ci-training-h100x2 and ci-training-h100-sxm pools removed
|
||||||
resource "scaleway_k8s_pool" "ci_training_h100x2" {
|
# 2026-05-16 — Scaleway account doesn't have quota for the SXM variant
|
||||||
count = var.enable_ci_training_h100x2_pool ? 1 : 0
|
# (cp_servers_type_H100_SXM_2_80G 0/0), and the H100x2 variant is
|
||||||
cluster_id = scaleway_k8s_cluster.foxhunt.id
|
# more expensive than our actual usage justifies. Keeping the regular
|
||||||
name = "ci-training-h100x2"
|
# single-GPU H100 pool (ci_training_h100) + the L40S pool covers
|
||||||
node_type = var.ci_training_h100x2_type
|
# every current training workload.
|
||||||
size = 1
|
|
||||||
min_size = 0
|
|
||||||
max_size = var.ci_training_h100x2_max_size
|
|
||||||
autoscaling = true
|
|
||||||
autohealing = true
|
|
||||||
public_ip_disabled = var.public_ip_disabled
|
|
||||||
region = var.region
|
|
||||||
|
|
||||||
lifecycle {
|
|
||||||
ignore_changes = [size]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# H100-SXM training pool (2x 80GB SXM VRAM — NVLink multi-GPU)
|
|
||||||
resource "scaleway_k8s_pool" "ci_training_h100_sxm" {
|
|
||||||
count = var.enable_ci_training_h100_sxm_pool ? 1 : 0
|
|
||||||
cluster_id = scaleway_k8s_cluster.foxhunt.id
|
|
||||||
name = "ci-training-h100-sxm"
|
|
||||||
node_type = var.ci_training_h100_sxm_type
|
|
||||||
size = 1
|
|
||||||
min_size = 0
|
|
||||||
max_size = var.ci_training_h100_sxm_max_size
|
|
||||||
autoscaling = true
|
|
||||||
autohealing = true
|
|
||||||
public_ip_disabled = var.public_ip_disabled
|
|
||||||
region = var.region
|
|
||||||
|
|
||||||
lifecycle {
|
|
||||||
ignore_changes = [size]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# CPU compile pool (POP2-HC — high CPU, no GPU, for cargo check/build)
|
# CPU compile pool (POP2-HC — high CPU, no GPU, for cargo check/build)
|
||||||
resource "scaleway_k8s_pool" "ci_compile_cpu" {
|
resource "scaleway_k8s_pool" "ci_compile_cpu" {
|
||||||
|
|||||||
@@ -34,16 +34,6 @@ output "ci_training_h100_pool_id" {
|
|||||||
value = var.enable_ci_training_h100_pool ? scaleway_k8s_pool.ci_training_h100[0].id : ""
|
value = var.enable_ci_training_h100_pool ? scaleway_k8s_pool.ci_training_h100[0].id : ""
|
||||||
}
|
}
|
||||||
|
|
||||||
output "ci_training_h100x2_pool_id" {
|
|
||||||
description = "ID of the H100x2 training node pool"
|
|
||||||
value = var.enable_ci_training_h100x2_pool ? scaleway_k8s_pool.ci_training_h100x2[0].id : ""
|
|
||||||
}
|
|
||||||
|
|
||||||
output "ci_training_h100_sxm_pool_id" {
|
|
||||||
description = "ID of the H100-SXM training node pool"
|
|
||||||
value = var.enable_ci_training_h100_sxm_pool ? scaleway_k8s_pool.ci_training_h100_sxm[0].id : ""
|
|
||||||
}
|
|
||||||
|
|
||||||
output "private_network_id" {
|
output "private_network_id" {
|
||||||
description = "ID of the VPC private network the cluster is attached to"
|
description = "ID of the VPC private network the cluster is attached to"
|
||||||
value = scaleway_vpc_private_network.foxhunt.id
|
value = scaleway_vpc_private_network.foxhunt.id
|
||||||
|
|||||||
@@ -59,42 +59,6 @@ variable "ci_training_h100_max_size" {
|
|||||||
default = 1
|
default = 1
|
||||||
}
|
}
|
||||||
|
|
||||||
variable "enable_ci_training_h100x2_pool" {
|
|
||||||
description = "Create H100x2 GPU node pool for multi-GPU training"
|
|
||||||
type = bool
|
|
||||||
default = false
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "ci_training_h100x2_type" {
|
|
||||||
description = "Instance type for the H100x2 training pool"
|
|
||||||
type = string
|
|
||||||
default = "H100-2-80G"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "ci_training_h100x2_max_size" {
|
|
||||||
description = "Maximum number of nodes in the H100x2 training pool"
|
|
||||||
type = number
|
|
||||||
default = 1
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "enable_ci_training_h100_sxm_pool" {
|
|
||||||
description = "Create H100-SXM GPU node pool for NVLink multi-GPU training"
|
|
||||||
type = bool
|
|
||||||
default = false
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "ci_training_h100_sxm_type" {
|
|
||||||
description = "Instance type for the H100-SXM training pool"
|
|
||||||
type = string
|
|
||||||
default = "H100-SXM-2-80G"
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "ci_training_h100_sxm_max_size" {
|
|
||||||
description = "Maximum number of nodes in the H100-SXM training pool"
|
|
||||||
type = number
|
|
||||||
default = 1
|
|
||||||
}
|
|
||||||
|
|
||||||
variable "enable_ci_compile_cpu_pool" {
|
variable "enable_ci_compile_cpu_pool" {
|
||||||
description = "Create high-CPU node pool for CI compilation"
|
description = "Create high-CPU node pool for CI compilation"
|
||||||
type = bool
|
type = bool
|
||||||
|
|||||||
Reference in New Issue
Block a user