Files
foxhunt/infra/modules/kapsule/variables.tf
jgrusewski 86de265fc8 infra(kapsule): add ci-compile-cpu-hm pool for 9-quarter fxcache
Two 9-quarter precompute_features runs OOM-killed on the existing
ci-compile-cpu pool (POP2-HC-32C-64G, 56Gi cgroup limit):
  - train-wq8b8: exit 137 ~12s after "OFI computed" at ~57Gi
  - train-2l6p4: exit 137 ~12s after "OFI computed" at ~52Gi peak
    (despite the into_iter + drop(feature_vectors) +
    normalize_in_place refactors landed in a27cb40a9 + 623ebfcf7,
    which trimmed ~12GB of avoidable retention)

The remaining ~52-60GB peak is the irreducible working set at the
post-OFI / pre-alpha-pipeline step:
  - 9-quarter MBP-10 snapshots (~10GB)
  - 199M front-month trades as Mbp10Trade (~13GB)
  - 17.8M bars × features + targets + OFI buffers (~14GB)
  - alpha_snapshots (move-handoff from all_snapshots, ~10GB)
  - feature/target Vecs (~7GB) + Rust allocator overhead

Adds a dedicated high-memory pool sized for this rare path:

- New scaleway_k8s_pool.ci_compile_cpu_hm (POP2-HM-32C-256G,
  32 vCPU + 256GB RAM) with size=0 + min_size=0 autoscaling. Costs
  zero when idle; autoscaler provisions one node when a pod targets
  `nodeSelector: ci-compile-cpu-hm`.
- New variables: enable_ci_compile_cpu_hm_pool,
  ci_compile_cpu_hm_type (default POP2-HM-32C-256G),
  ci_compile_cpu_hm_max_size (default 1).
- terragrunt.hcl: enable the pool, max_size=1.
- train-template.yaml: ensure-fxcache nodeSelector pinned to
  ci-compile-cpu-hm; memory limit raised 56Gi → 200Gi. The
  ci-compile-cpu pool stays as the standard CI compile target for
  ensure-binary + every other CPU-heavy task.

Apply with:
  cd infra/live/production/kapsule
  terragrunt apply -target=module.kapsule.scaleway_k8s_pool.ci_compile_cpu_hm
  kubectl apply -n foxhunt -f ../../../../infra/k8s/argo/train-template.yaml

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
2026-05-16 11:13:03 +02:00

144 lines
3.8 KiB
HCL

variable "region" {
description = "Scaleway region for the Kapsule cluster"
type = string
}
variable "cluster_name" {
description = "Name of the Kubernetes cluster"
type = string
}
variable "k8s_version" {
description = "Kubernetes version for the Kapsule cluster"
type = string
}
variable "platform_type" {
description = "Instance type for the platform node pool"
type = string
}
variable "platform_max_size" {
description = "Maximum number of nodes in the platform pool"
type = number
}
variable "enable_ci_training_l40s_pool" {
description = "Create L40S GPU node pool for training + hyperopt"
type = bool
default = false
}
variable "ci_training_l40s_type" {
description = "Instance type for the L40S training pool"
type = string
default = "L40S-1-48G"
}
variable "ci_training_l40s_max_size" {
description = "Maximum number of nodes in the L40S training pool"
type = number
default = 1
}
variable "enable_ci_training_h100_pool" {
description = "Create H100 GPU node pool for training + hyperopt"
type = bool
default = false
}
variable "ci_training_h100_type" {
description = "Instance type for the H100 training pool"
type = string
default = "H100-1-80G"
}
variable "ci_training_h100_max_size" {
description = "Maximum number of nodes in the H100 training pool"
type = number
default = 1
}
variable "enable_ci_training_h100x2_pool" {
description = "Create H100x2 GPU node pool for multi-GPU training"
type = bool
default = false
}
variable "ci_training_h100x2_type" {
description = "Instance type for the H100x2 training pool"
type = string
default = "H100-2-80G"
}
variable "ci_training_h100x2_max_size" {
description = "Maximum number of nodes in the H100x2 training pool"
type = number
default = 1
}
variable "enable_ci_training_h100_sxm_pool" {
description = "Create H100-SXM GPU node pool for NVLink multi-GPU training"
type = bool
default = false
}
variable "ci_training_h100_sxm_type" {
description = "Instance type for the H100-SXM training pool"
type = string
default = "H100-SXM-2-80G"
}
variable "ci_training_h100_sxm_max_size" {
description = "Maximum number of nodes in the H100-SXM training pool"
type = number
default = 1
}
variable "enable_ci_compile_cpu_pool" {
description = "Create high-CPU node pool for CI compilation"
type = bool
default = false
}
variable "ci_compile_cpu_type" {
description = "Instance type for the CPU compile pool"
type = string
}
variable "ci_compile_cpu_max_size" {
description = "Maximum number of nodes in the CPU compile pool"
type = number
default = 1
}
# ─── High-memory CPU compile pool (rare full-rebuild path) ──────────────
# Separate pool with min_size=0 autoscaling — pays nothing when idle, only
# provisions a large-RAM node when `ensure-fxcache` requests this pool
# (9-quarter precompute_features needs ~50-60GB peak which won't fit on
# the standard 64GB ci-compile-cpu pool with kernel + page-cache overhead).
variable "enable_ci_compile_cpu_hm_pool" {
description = "Create a high-memory CPU pool for rare full-rebuild fxcache runs"
type = bool
default = false
}
variable "ci_compile_cpu_hm_type" {
description = "Instance type for the high-memory CPU pool (e.g. POP2-HM-32C-256G)"
type = string
default = "POP2-HM-32C-256G"
}
variable "ci_compile_cpu_hm_max_size" {
description = "Maximum number of nodes in the high-memory CPU pool"
type = number
default = 1
}
variable "public_ip_disabled" {
description = "Disable public IPs on all node pools (requires Public Gateway for egress)"
type = bool
default = false
}