diff --git a/infra/k8s/argo/train-template.yaml b/infra/k8s/argo/train-template.yaml index 9af442590..3cadea340 100644 --- a/infra/k8s/argo/train-template.yaml +++ b/infra/k8s/argo/train-template.yaml @@ -352,12 +352,21 @@ spec: echo "$SHORT_SHA" > /tmp/sha # ── ensure-fxcache: precompute feature cache if needed ── + # + # Pinned to the high-memory ci-compile-cpu-hm pool (POP2-HM-32C-256G, + # 32 vCPU + 256GB RAM, min_size=0). The 9-quarter precompute_features + # peaks ~50-60GB during the post-OFI alpha_trades conversion which + # OOM-killed the standard 64GB ci-compile-cpu pool twice on + # 2026-05-16 (workflows train-wq8b8 + train-2l6p4 both died at + # exitCode 137 right after "OFI computed"). The HM pool autoscales + # to zero when idle so this pin only costs anything during an + # actual fxcache rebuild. - name: ensure-fxcache inputs: parameters: - name: sha nodeSelector: - k8s.scaleway.com/pool-name: ci-compile-cpu + k8s.scaleway.com/pool-name: ci-compile-cpu-hm tolerations: - key: node.cilium.io/agent-not-ready operator: Exists @@ -374,7 +383,7 @@ spec: memory: 16Gi limits: cpu: "28" - memory: 56Gi + memory: 200Gi volumeMounts: - name: training-data mountPath: /data diff --git a/infra/live/production/kapsule/terragrunt.hcl b/infra/live/production/kapsule/terragrunt.hcl index 5e6b7197e..ad6d9d2b2 100644 --- a/infra/live/production/kapsule/terragrunt.hcl +++ b/infra/live/production/kapsule/terragrunt.hcl @@ -19,6 +19,15 @@ inputs = { ci_compile_cpu_type = "POP2-HC-32C-64G" ci_compile_cpu_max_size = 4 + # High-memory CPU pool (POP2-HM-32C-256G — 32 vCPU, 256GB RAM). + # Reserved for the rare full-rebuild fxcache path that can't fit on + # the 64GB ci-compile-cpu pool (9-quarter precompute_features peaks + # ~50-60GB during the post-OFI alpha_trades conversion). + # min_size=0 + size=0 initial so this pool costs nothing when idle. + enable_ci_compile_cpu_hm_pool = true + ci_compile_cpu_hm_type = "POP2-HM-32C-256G" + ci_compile_cpu_hm_max_size = 1 + # L40S training pool (48GB VRAM, CUDA CC 89) enable_ci_training_l40s_pool = true ci_training_l40s_type = "L40S-1-48G" diff --git a/infra/modules/kapsule/main.tf b/infra/modules/kapsule/main.tf index 40f95b8e6..bb311fc64 100644 --- a/infra/modules/kapsule/main.tf +++ b/infra/modules/kapsule/main.tf @@ -164,3 +164,26 @@ resource "scaleway_k8s_pool" "ci_compile_cpu" { ignore_changes = [size] } } + +# High-memory CPU compile pool (POP2-HM — same CPU count as ci-compile-cpu +# but 4× the RAM; reserved for rare full-rebuild precompute_features runs +# that can't fit on the standard 64GB nodes). min_size=0 + size=0 initial +# so the pool costs nothing when idle — autoscaler provisions one node +# only when a pod targets `nodeSelector: ci-compile-cpu-hm`. +resource "scaleway_k8s_pool" "ci_compile_cpu_hm" { + count = var.enable_ci_compile_cpu_hm_pool ? 1 : 0 + cluster_id = scaleway_k8s_cluster.foxhunt.id + name = "ci-compile-cpu-hm" + node_type = var.ci_compile_cpu_hm_type + size = 0 + min_size = 0 + max_size = var.ci_compile_cpu_hm_max_size + autoscaling = true + autohealing = true + public_ip_disabled = var.public_ip_disabled + region = var.region + + lifecycle { + ignore_changes = [size] + } +} diff --git a/infra/modules/kapsule/variables.tf b/infra/modules/kapsule/variables.tf index c0a9c687f..cd4239e33 100644 --- a/infra/modules/kapsule/variables.tf +++ b/infra/modules/kapsule/variables.tf @@ -112,6 +112,30 @@ variable "ci_compile_cpu_max_size" { default = 1 } +# ─── High-memory CPU compile pool (rare full-rebuild path) ────────────── +# Separate pool with min_size=0 autoscaling — pays nothing when idle, only +# provisions a large-RAM node when `ensure-fxcache` requests this pool +# (9-quarter precompute_features needs ~50-60GB peak which won't fit on +# the standard 64GB ci-compile-cpu pool with kernel + page-cache overhead). + +variable "enable_ci_compile_cpu_hm_pool" { + description = "Create a high-memory CPU pool for rare full-rebuild fxcache runs" + type = bool + default = false +} + +variable "ci_compile_cpu_hm_type" { + description = "Instance type for the high-memory CPU pool (e.g. POP2-HM-32C-256G)" + type = string + default = "POP2-HM-32C-256G" +} + +variable "ci_compile_cpu_hm_max_size" { + description = "Maximum number of nodes in the high-memory CPU pool" + type = number + default = 1 +} + variable "public_ip_disabled" { description = "Disable public IPs on all node pools (requires Public Gateway for egress)" type = bool