Two 9-quarter precompute_features runs OOM-killed on the existing
ci-compile-cpu pool (POP2-HC-32C-64G, 56Gi cgroup limit):
- train-wq8b8: exit 137 ~12s after "OFI computed" at ~57Gi
- train-2l6p4: exit 137 ~12s after "OFI computed" at ~52Gi peak
(despite the into_iter + drop(feature_vectors) +
normalize_in_place refactors landed in a27cb40a9 + 623ebfcf7,
which trimmed ~12GB of avoidable retention)
The remaining ~52-60GB peak is the irreducible working set at the
post-OFI / pre-alpha-pipeline step:
- 9-quarter MBP-10 snapshots (~10GB)
- 199M front-month trades as Mbp10Trade (~13GB)
- 17.8M bars × features + targets + OFI buffers (~14GB)
- alpha_snapshots (move-handoff from all_snapshots, ~10GB)
- feature/target Vecs (~7GB) + Rust allocator overhead
Adds a dedicated high-memory pool sized for this rare path:
- New scaleway_k8s_pool.ci_compile_cpu_hm (POP2-HM-32C-256G,
32 vCPU + 256GB RAM) with size=0 + min_size=0 autoscaling. Costs
zero when idle; autoscaler provisions one node when a pod targets
`nodeSelector: ci-compile-cpu-hm`.
- New variables: enable_ci_compile_cpu_hm_pool,
ci_compile_cpu_hm_type (default POP2-HM-32C-256G),
ci_compile_cpu_hm_max_size (default 1).
- terragrunt.hcl: enable the pool, max_size=1.
- train-template.yaml: ensure-fxcache nodeSelector pinned to
ci-compile-cpu-hm; memory limit raised 56Gi → 200Gi. The
ci-compile-cpu pool stays as the standard CI compile target for
ensure-binary + every other CPU-heavy task.
Apply with:
cd infra/live/production/kapsule
terragrunt apply -target=module.kapsule.scaleway_k8s_pool.ci_compile_cpu_hm
kubectl apply -n foxhunt -f ../../../../infra/k8s/argo/train-template.yaml
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
144 lines
3.8 KiB
HCL
144 lines
3.8 KiB
HCL
variable "region" {
|
|
description = "Scaleway region for the Kapsule cluster"
|
|
type = string
|
|
}
|
|
|
|
variable "cluster_name" {
|
|
description = "Name of the Kubernetes cluster"
|
|
type = string
|
|
}
|
|
|
|
variable "k8s_version" {
|
|
description = "Kubernetes version for the Kapsule cluster"
|
|
type = string
|
|
}
|
|
|
|
variable "platform_type" {
|
|
description = "Instance type for the platform node pool"
|
|
type = string
|
|
}
|
|
|
|
variable "platform_max_size" {
|
|
description = "Maximum number of nodes in the platform pool"
|
|
type = number
|
|
}
|
|
|
|
variable "enable_ci_training_l40s_pool" {
|
|
description = "Create L40S GPU node pool for training + hyperopt"
|
|
type = bool
|
|
default = false
|
|
}
|
|
|
|
variable "ci_training_l40s_type" {
|
|
description = "Instance type for the L40S training pool"
|
|
type = string
|
|
default = "L40S-1-48G"
|
|
}
|
|
|
|
variable "ci_training_l40s_max_size" {
|
|
description = "Maximum number of nodes in the L40S training pool"
|
|
type = number
|
|
default = 1
|
|
}
|
|
|
|
variable "enable_ci_training_h100_pool" {
|
|
description = "Create H100 GPU node pool for training + hyperopt"
|
|
type = bool
|
|
default = false
|
|
}
|
|
|
|
variable "ci_training_h100_type" {
|
|
description = "Instance type for the H100 training pool"
|
|
type = string
|
|
default = "H100-1-80G"
|
|
}
|
|
|
|
variable "ci_training_h100_max_size" {
|
|
description = "Maximum number of nodes in the H100 training pool"
|
|
type = number
|
|
default = 1
|
|
}
|
|
|
|
variable "enable_ci_training_h100x2_pool" {
|
|
description = "Create H100x2 GPU node pool for multi-GPU training"
|
|
type = bool
|
|
default = false
|
|
}
|
|
|
|
variable "ci_training_h100x2_type" {
|
|
description = "Instance type for the H100x2 training pool"
|
|
type = string
|
|
default = "H100-2-80G"
|
|
}
|
|
|
|
variable "ci_training_h100x2_max_size" {
|
|
description = "Maximum number of nodes in the H100x2 training pool"
|
|
type = number
|
|
default = 1
|
|
}
|
|
|
|
variable "enable_ci_training_h100_sxm_pool" {
|
|
description = "Create H100-SXM GPU node pool for NVLink multi-GPU training"
|
|
type = bool
|
|
default = false
|
|
}
|
|
|
|
variable "ci_training_h100_sxm_type" {
|
|
description = "Instance type for the H100-SXM training pool"
|
|
type = string
|
|
default = "H100-SXM-2-80G"
|
|
}
|
|
|
|
variable "ci_training_h100_sxm_max_size" {
|
|
description = "Maximum number of nodes in the H100-SXM training pool"
|
|
type = number
|
|
default = 1
|
|
}
|
|
|
|
variable "enable_ci_compile_cpu_pool" {
|
|
description = "Create high-CPU node pool for CI compilation"
|
|
type = bool
|
|
default = false
|
|
}
|
|
|
|
variable "ci_compile_cpu_type" {
|
|
description = "Instance type for the CPU compile pool"
|
|
type = string
|
|
}
|
|
|
|
variable "ci_compile_cpu_max_size" {
|
|
description = "Maximum number of nodes in the CPU compile pool"
|
|
type = number
|
|
default = 1
|
|
}
|
|
|
|
# ─── High-memory CPU compile pool (rare full-rebuild path) ──────────────
|
|
# Separate pool with min_size=0 autoscaling — pays nothing when idle, only
|
|
# provisions a large-RAM node when `ensure-fxcache` requests this pool
|
|
# (9-quarter precompute_features needs ~50-60GB peak which won't fit on
|
|
# the standard 64GB ci-compile-cpu pool with kernel + page-cache overhead).
|
|
|
|
variable "enable_ci_compile_cpu_hm_pool" {
|
|
description = "Create a high-memory CPU pool for rare full-rebuild fxcache runs"
|
|
type = bool
|
|
default = false
|
|
}
|
|
|
|
variable "ci_compile_cpu_hm_type" {
|
|
description = "Instance type for the high-memory CPU pool (e.g. POP2-HM-32C-256G)"
|
|
type = string
|
|
default = "POP2-HM-32C-256G"
|
|
}
|
|
|
|
variable "ci_compile_cpu_hm_max_size" {
|
|
description = "Maximum number of nodes in the high-memory CPU pool"
|
|
type = number
|
|
default = 1
|
|
}
|
|
|
|
variable "public_ip_disabled" {
|
|
description = "Disable public IPs on all node pools (requires Public Gateway for egress)"
|
|
type = bool
|
|
default = false
|
|
}
|