a252119fd4
Reasons: - h100-sxm: Scaleway account quota is 0/0 for the SXM instance type (cp_servers_type_H100_SXM_2_80G). The pool was perpetually trying to maintain size=1 with a creation_error node, which JAMMED the cluster autoscaler. That blocked L40S scale-up entirely during today's alpha-perception cluster run. - h100x2: more expensive than current workloads justify. Every training path (alpha perception, future PPO) fits on either the single-GPU H100 or L40S. Removed: - Two `scaleway_k8s_pool` resources from infra/modules/kapsule/main.tf - Six variables (enable + type + max_size for each pool) - Two outputs (pool_id for each) - Corresponding inputs in infra/live/production/kapsule/terragrunt.hcl The live cluster has the SXM pool stuck node manually deleted via `scw k8s node delete` (this commit-session); the pool resource itself will be destroyed on next `terragrunt apply`. Post-cleanup pool inventory: - platform (DEV1-L × 3) - ci-training-h100 (H100-1-80G, max 1) <- regular single-GPU - ci-training-l40s (L40S-1-48G, max 1) <- primary training target - ci-compile-cpu (POP2-HC, max 4) - ci-compile-cpu-hm (POP2-HM, max 1) Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
108 lines
2.9 KiB
HCL
108 lines
2.9 KiB
HCL
variable "region" {
|
|
description = "Scaleway region for the Kapsule cluster"
|
|
type = string
|
|
}
|
|
|
|
variable "cluster_name" {
|
|
description = "Name of the Kubernetes cluster"
|
|
type = string
|
|
}
|
|
|
|
variable "k8s_version" {
|
|
description = "Kubernetes version for the Kapsule cluster"
|
|
type = string
|
|
}
|
|
|
|
variable "platform_type" {
|
|
description = "Instance type for the platform node pool"
|
|
type = string
|
|
}
|
|
|
|
variable "platform_max_size" {
|
|
description = "Maximum number of nodes in the platform pool"
|
|
type = number
|
|
}
|
|
|
|
variable "enable_ci_training_l40s_pool" {
|
|
description = "Create L40S GPU node pool for training + hyperopt"
|
|
type = bool
|
|
default = false
|
|
}
|
|
|
|
variable "ci_training_l40s_type" {
|
|
description = "Instance type for the L40S training pool"
|
|
type = string
|
|
default = "L40S-1-48G"
|
|
}
|
|
|
|
variable "ci_training_l40s_max_size" {
|
|
description = "Maximum number of nodes in the L40S training pool"
|
|
type = number
|
|
default = 1
|
|
}
|
|
|
|
variable "enable_ci_training_h100_pool" {
|
|
description = "Create H100 GPU node pool for training + hyperopt"
|
|
type = bool
|
|
default = false
|
|
}
|
|
|
|
variable "ci_training_h100_type" {
|
|
description = "Instance type for the H100 training pool"
|
|
type = string
|
|
default = "H100-1-80G"
|
|
}
|
|
|
|
variable "ci_training_h100_max_size" {
|
|
description = "Maximum number of nodes in the H100 training pool"
|
|
type = number
|
|
default = 1
|
|
}
|
|
|
|
variable "enable_ci_compile_cpu_pool" {
|
|
description = "Create high-CPU node pool for CI compilation"
|
|
type = bool
|
|
default = false
|
|
}
|
|
|
|
variable "ci_compile_cpu_type" {
|
|
description = "Instance type for the CPU compile pool"
|
|
type = string
|
|
}
|
|
|
|
variable "ci_compile_cpu_max_size" {
|
|
description = "Maximum number of nodes in the CPU compile pool"
|
|
type = number
|
|
default = 1
|
|
}
|
|
|
|
# ─── High-memory CPU compile pool (rare full-rebuild path) ──────────────
|
|
# Separate pool with min_size=0 autoscaling — pays nothing when idle, only
|
|
# provisions a large-RAM node when `ensure-fxcache` requests this pool
|
|
# (9-quarter precompute_features needs ~50-60GB peak which won't fit on
|
|
# the standard 64GB ci-compile-cpu pool with kernel + page-cache overhead).
|
|
|
|
variable "enable_ci_compile_cpu_hm_pool" {
|
|
description = "Create a high-memory CPU pool for rare full-rebuild fxcache runs"
|
|
type = bool
|
|
default = false
|
|
}
|
|
|
|
variable "ci_compile_cpu_hm_type" {
|
|
description = "Instance type for the high-memory CPU pool (e.g. POP2-HM-32C-256G)"
|
|
type = string
|
|
default = "POP2-HM-32C-256G"
|
|
}
|
|
|
|
variable "ci_compile_cpu_hm_max_size" {
|
|
description = "Maximum number of nodes in the high-memory CPU pool"
|
|
type = number
|
|
default = 1
|
|
}
|
|
|
|
variable "public_ip_disabled" {
|
|
description = "Disable public IPs on all node pools (requires Public Gateway for egress)"
|
|
type = bool
|
|
default = false
|
|
}
|