From a36fce0b87c3dd8a4b048681b6e69b378a109df3 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 1 Mar 2026 13:04:26 +0100 Subject: [PATCH] fix(infra): increase Tempo memory to 1Gi, RL runner CPU limits to 8000m Tempo was OOMKilled every ~30 min at 512Mi, causing web-gateway OTLP export errors. Increased to 1Gi. RL runner had cpu_request_overwrite_max_allowed=4000m but hyperopt RL jobs need 6000m for parallel PSO trials. Increased to 8000m. Co-Authored-By: Claude Opus 4.6 --- infra/k8s/gitlab/runner-rl-values.yaml | 9 +++++---- infra/k8s/monitoring/tempo.yaml | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/infra/k8s/gitlab/runner-rl-values.yaml b/infra/k8s/gitlab/runner-rl-values.yaml index 5fafb3c8e..8a81ed7da 100644 --- a/infra/k8s/gitlab/runner-rl-values.yaml +++ b/infra/k8s/gitlab/runner-rl-values.yaml @@ -34,10 +34,11 @@ runners: image = "rust:1.89-slim" privileged = false node_selector_overwrite_allowed = ".*" - cpu_request_overwrite_max_allowed = "4000m" - cpu_limit_overwrite_max_allowed = "4000m" - memory_request_overwrite_max_allowed = "16Gi" - memory_limit_overwrite_max_allowed = "16Gi" + # Hyperopt RL jobs need 6000m CPU + 20Gi mem (PSO parallel trials on L4 8 vCPU) + cpu_request_overwrite_max_allowed = "8000m" + cpu_limit_overwrite_max_allowed = "8000m" + memory_request_overwrite_max_allowed = "24Gi" + memory_limit_overwrite_max_allowed = "24Gi" poll_timeout = 600 # Allow CI jobs to override runtimeClassName via KUBERNETES_RUNTIME_CLASS_NAME runtime_class_name_overwrite_allowed = ".*" diff --git a/infra/k8s/monitoring/tempo.yaml b/infra/k8s/monitoring/tempo.yaml index d4f2dc764..8f4948f61 100644 --- a/infra/k8s/monitoring/tempo.yaml +++ b/infra/k8s/monitoring/tempo.yaml @@ -77,10 +77,10 @@ spec: resources: requests: cpu: 100m - memory: 128Mi + memory: 256Mi limits: cpu: 500m - memory: 512Mi + memory: 1Gi readinessProbe: httpGet: path: /ready