a252119fd4
Reasons: - h100-sxm: Scaleway account quota is 0/0 for the SXM instance type (cp_servers_type_H100_SXM_2_80G). The pool was perpetually trying to maintain size=1 with a creation_error node, which JAMMED the cluster autoscaler. That blocked L40S scale-up entirely during today's alpha-perception cluster run. - h100x2: more expensive than current workloads justify. Every training path (alpha perception, future PPO) fits on either the single-GPU H100 or L40S. Removed: - Two `scaleway_k8s_pool` resources from infra/modules/kapsule/main.tf - Six variables (enable + type + max_size for each pool) - Two outputs (pool_id for each) - Corresponding inputs in infra/live/production/kapsule/terragrunt.hcl The live cluster has the SXM pool stuck node manually deleted via `scw k8s node delete` (this commit-session); the pool resource itself will be destroyed on next `terragrunt apply`. Post-cleanup pool inventory: - platform (DEV1-L × 3) - ci-training-h100 (H100-1-80G, max 1) <- regular single-GPU - ci-training-l40s (L40S-1-48G, max 1) <- primary training target - ci-compile-cpu (POP2-HC, max 4) - ci-compile-cpu-hm (POP2-HM, max 1) Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
50 lines
1.7 KiB
HCL
50 lines
1.7 KiB
HCL
include "root" {
|
|
path = find_in_parent_folders("root.hcl")
|
|
}
|
|
|
|
terraform {
|
|
source = "../../../modules/kapsule"
|
|
}
|
|
|
|
inputs = {
|
|
cluster_name = "foxhunt"
|
|
k8s_version = "1.34"
|
|
|
|
# Platform pool — runs everything: app services, databases, GitLab, monitoring
|
|
platform_type = "DEV1-L"
|
|
platform_max_size = 3
|
|
|
|
# CPU compile pool (POP2-HC-32C-64G — 32 vCPU, 64GB RAM, high clock)
|
|
enable_ci_compile_cpu_pool = true
|
|
ci_compile_cpu_type = "POP2-HC-32C-64G"
|
|
ci_compile_cpu_max_size = 4
|
|
|
|
# High-memory CPU pool (POP2-HM-32C-256G — 32 vCPU, 256GB RAM).
|
|
# Reserved for the rare full-rebuild fxcache path that can't fit on
|
|
# the 64GB ci-compile-cpu pool (9-quarter precompute_features peaks
|
|
# ~50-60GB during the post-OFI alpha_trades conversion).
|
|
# min_size=0 + size=0 initial so this pool costs nothing when idle.
|
|
enable_ci_compile_cpu_hm_pool = true
|
|
ci_compile_cpu_hm_type = "POP2-HM-32C-256G"
|
|
ci_compile_cpu_hm_max_size = 1
|
|
|
|
# L40S training pool (48GB VRAM, CUDA CC 89)
|
|
enable_ci_training_l40s_pool = true
|
|
ci_training_l40s_type = "L40S-1-48G"
|
|
ci_training_l40s_max_size = 1
|
|
|
|
# H100 training pool (80GB VRAM, CUDA CC 90)
|
|
enable_ci_training_h100_pool = true
|
|
ci_training_h100_type = "H100-1-80G"
|
|
ci_training_h100_max_size = 1
|
|
|
|
# H100x2 and H100-SXM pools removed 2026-05-16:
|
|
# - SXM: Scaleway account quota is 0/0 (cp_servers_type_H100_SXM_2_80G).
|
|
# The pool was stuck trying to provision and jammed the autoscaler.
|
|
# - H100x2: more expensive than current workloads justify; the single-GPU
|
|
# H100 pool covers all training paths.
|
|
|
|
# NAT gateway handles egress — no public IPs on nodes
|
|
public_ip_disabled = true
|
|
}
|