From 86de265fc8bacfceb9aab2e80c551ce1fe121ec3 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 16 May 2026 11:13:03 +0200 Subject: [PATCH] infra(kapsule): add ci-compile-cpu-hm pool for 9-quarter fxcache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two 9-quarter precompute_features runs OOM-killed on the existing ci-compile-cpu pool (POP2-HC-32C-64G, 56Gi cgroup limit): - train-wq8b8: exit 137 ~12s after "OFI computed" at ~57Gi - train-2l6p4: exit 137 ~12s after "OFI computed" at ~52Gi peak (despite the into_iter + drop(feature_vectors) + normalize_in_place refactors landed in a27cb40a9 + 623ebfcf7, which trimmed ~12GB of avoidable retention) The remaining ~52-60GB peak is the irreducible working set at the post-OFI / pre-alpha-pipeline step: - 9-quarter MBP-10 snapshots (~10GB) - 199M front-month trades as Mbp10Trade (~13GB) - 17.8M bars × features + targets + OFI buffers (~14GB) - alpha_snapshots (move-handoff from all_snapshots, ~10GB) - feature/target Vecs (~7GB) + Rust allocator overhead Adds a dedicated high-memory pool sized for this rare path: - New scaleway_k8s_pool.ci_compile_cpu_hm (POP2-HM-32C-256G, 32 vCPU + 256GB RAM) with size=0 + min_size=0 autoscaling. Costs zero when idle; autoscaler provisions one node when a pod targets `nodeSelector: ci-compile-cpu-hm`. - New variables: enable_ci_compile_cpu_hm_pool, ci_compile_cpu_hm_type (default POP2-HM-32C-256G), ci_compile_cpu_hm_max_size (default 1). - terragrunt.hcl: enable the pool, max_size=1. - train-template.yaml: ensure-fxcache nodeSelector pinned to ci-compile-cpu-hm; memory limit raised 56Gi → 200Gi. The ci-compile-cpu pool stays as the standard CI compile target for ensure-binary + every other CPU-heavy task. Apply with: cd infra/live/production/kapsule terragrunt apply -target=module.kapsule.scaleway_k8s_pool.ci_compile_cpu_hm kubectl apply -n foxhunt -f ../../../../infra/k8s/argo/train-template.yaml Co-Authored-By: Claude Opus 4.7 --- infra/k8s/argo/train-template.yaml | 13 +++++++++-- infra/live/production/kapsule/terragrunt.hcl | 9 ++++++++ infra/modules/kapsule/main.tf | 23 +++++++++++++++++++ infra/modules/kapsule/variables.tf | 24 ++++++++++++++++++++ 4 files changed, 67 insertions(+), 2 deletions(-) diff --git a/infra/k8s/argo/train-template.yaml b/infra/k8s/argo/train-template.yaml index 9af442590..3cadea340 100644 --- a/infra/k8s/argo/train-template.yaml +++ b/infra/k8s/argo/train-template.yaml @@ -352,12 +352,21 @@ spec: echo "$SHORT_SHA" > /tmp/sha # ── ensure-fxcache: precompute feature cache if needed ── + # + # Pinned to the high-memory ci-compile-cpu-hm pool (POP2-HM-32C-256G, + # 32 vCPU + 256GB RAM, min_size=0). The 9-quarter precompute_features + # peaks ~50-60GB during the post-OFI alpha_trades conversion which + # OOM-killed the standard 64GB ci-compile-cpu pool twice on + # 2026-05-16 (workflows train-wq8b8 + train-2l6p4 both died at + # exitCode 137 right after "OFI computed"). The HM pool autoscales + # to zero when idle so this pin only costs anything during an + # actual fxcache rebuild. - name: ensure-fxcache inputs: parameters: - name: sha nodeSelector: - k8s.scaleway.com/pool-name: ci-compile-cpu + k8s.scaleway.com/pool-name: ci-compile-cpu-hm tolerations: - key: node.cilium.io/agent-not-ready operator: Exists @@ -374,7 +383,7 @@ spec: memory: 16Gi limits: cpu: "28" - memory: 56Gi + memory: 200Gi volumeMounts: - name: training-data mountPath: /data diff --git a/infra/live/production/kapsule/terragrunt.hcl b/infra/live/production/kapsule/terragrunt.hcl index 5e6b7197e..ad6d9d2b2 100644 --- a/infra/live/production/kapsule/terragrunt.hcl +++ b/infra/live/production/kapsule/terragrunt.hcl @@ -19,6 +19,15 @@ inputs = { ci_compile_cpu_type = "POP2-HC-32C-64G" ci_compile_cpu_max_size = 4 + # High-memory CPU pool (POP2-HM-32C-256G — 32 vCPU, 256GB RAM). + # Reserved for the rare full-rebuild fxcache path that can't fit on + # the 64GB ci-compile-cpu pool (9-quarter precompute_features peaks + # ~50-60GB during the post-OFI alpha_trades conversion). + # min_size=0 + size=0 initial so this pool costs nothing when idle. + enable_ci_compile_cpu_hm_pool = true + ci_compile_cpu_hm_type = "POP2-HM-32C-256G" + ci_compile_cpu_hm_max_size = 1 + # L40S training pool (48GB VRAM, CUDA CC 89) enable_ci_training_l40s_pool = true ci_training_l40s_type = "L40S-1-48G" diff --git a/infra/modules/kapsule/main.tf b/infra/modules/kapsule/main.tf index 40f95b8e6..bb311fc64 100644 --- a/infra/modules/kapsule/main.tf +++ b/infra/modules/kapsule/main.tf @@ -164,3 +164,26 @@ resource "scaleway_k8s_pool" "ci_compile_cpu" { ignore_changes = [size] } } + +# High-memory CPU compile pool (POP2-HM — same CPU count as ci-compile-cpu +# but 4× the RAM; reserved for rare full-rebuild precompute_features runs +# that can't fit on the standard 64GB nodes). min_size=0 + size=0 initial +# so the pool costs nothing when idle — autoscaler provisions one node +# only when a pod targets `nodeSelector: ci-compile-cpu-hm`. +resource "scaleway_k8s_pool" "ci_compile_cpu_hm" { + count = var.enable_ci_compile_cpu_hm_pool ? 1 : 0 + cluster_id = scaleway_k8s_cluster.foxhunt.id + name = "ci-compile-cpu-hm" + node_type = var.ci_compile_cpu_hm_type + size = 0 + min_size = 0 + max_size = var.ci_compile_cpu_hm_max_size + autoscaling = true + autohealing = true + public_ip_disabled = var.public_ip_disabled + region = var.region + + lifecycle { + ignore_changes = [size] + } +} diff --git a/infra/modules/kapsule/variables.tf b/infra/modules/kapsule/variables.tf index c0a9c687f..cd4239e33 100644 --- a/infra/modules/kapsule/variables.tf +++ b/infra/modules/kapsule/variables.tf @@ -112,6 +112,30 @@ variable "ci_compile_cpu_max_size" { default = 1 } +# ─── High-memory CPU compile pool (rare full-rebuild path) ────────────── +# Separate pool with min_size=0 autoscaling — pays nothing when idle, only +# provisions a large-RAM node when `ensure-fxcache` requests this pool +# (9-quarter precompute_features needs ~50-60GB peak which won't fit on +# the standard 64GB ci-compile-cpu pool with kernel + page-cache overhead). + +variable "enable_ci_compile_cpu_hm_pool" { + description = "Create a high-memory CPU pool for rare full-rebuild fxcache runs" + type = bool + default = false +} + +variable "ci_compile_cpu_hm_type" { + description = "Instance type for the high-memory CPU pool (e.g. POP2-HM-32C-256G)" + type = string + default = "POP2-HM-32C-256G" +} + +variable "ci_compile_cpu_hm_max_size" { + description = "Maximum number of nodes in the high-memory CPU pool" + type = number + default = 1 +} + variable "public_ip_disabled" { description = "Disable public IPs on all node pools (requires Public Gateway for egress)" type = bool