From a7c22ca5aaaccfb57917e78ebd30c448569599cc Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Sun, 22 Mar 2026 10:45:43 +0100
Subject: [PATCH] fix: CI OOMKilled on submit-gpu-test + YAML indentation fixes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

submit-gpu-test wait sidecar had 64Mi limit — OOMKilled when tracking
large child workflow status JSON. Bumped to 256Mi.

Fixed YAML indentation errors in compile-and-train and training-pipeline
templates (misaligned labels, duplicate component keys).

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 infra/k8s/argo/ci-pipeline-template.yaml       | 3 +++
 infra/k8s/argo/compile-and-train-template.yaml | 3 +--
 infra/k8s/argo/training-workflow-template.yaml | 3 +--
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/infra/k8s/argo/ci-pipeline-template.yaml b/infra/k8s/argo/ci-pipeline-template.yaml
index 5efa19b37..a2c59352b 100644
--- a/infra/k8s/argo/ci-pipeline-template.yaml
+++ b/infra/k8s/argo/ci-pipeline-template.yaml
@@ -595,6 +595,9 @@ spec:
 
     # ── submit-gpu-test: launch GPU test workflow as a child workflow ──
     - name: submit-gpu-test
+      # Argo wait sidecar needs 256Mi to track large child workflow status JSON.
+      # Default 64Mi causes OOMKilled when child has many parameters.
+      podSpecPatch: '{"containers":[{"name":"wait","resources":{"requests":{"memory":"128Mi"},"limits":{"memory":"256Mi"}}}]}'
       resource:
         action: create
         setOwnerReference: true
diff --git a/infra/k8s/argo/compile-and-train-template.yaml b/infra/k8s/argo/compile-and-train-template.yaml
index 1e5be9635..07d4e10c7 100644
--- a/infra/k8s/argo/compile-and-train-template.yaml
+++ b/infra/k8s/argo/compile-and-train-template.yaml
@@ -17,7 +17,7 @@ metadata:
   labels:
     app.kubernetes.io/name: compile-and-train
     app.kubernetes.io/part-of: foxhunt
-      app.kubernetes.io/component: gpu-test
+    app.kubernetes.io/component: gpu-test
 spec:
   entrypoint: pipeline
   onExit: notify-result
@@ -25,7 +25,6 @@ spec:
   podMetadata:
     labels:
       app.kubernetes.io/part-of: foxhunt
-      app.kubernetes.io/component: gpu-test
       app.kubernetes.io/component: compile-and-train
   # No runAsUser — compile (ci-builder) needs root, training runtime defaults
   # to its image user. fsGroup ensures workspace PVC is writable by all pods.
diff --git a/infra/k8s/argo/training-workflow-template.yaml b/infra/k8s/argo/training-workflow-template.yaml
index 35a0477da..16918e1bb 100644
--- a/infra/k8s/argo/training-workflow-template.yaml
+++ b/infra/k8s/argo/training-workflow-template.yaml
@@ -6,7 +6,7 @@ metadata:
   labels:
     app.kubernetes.io/name: training-pipeline
     app.kubernetes.io/part-of: foxhunt
-      app.kubernetes.io/component: gpu-test
+    app.kubernetes.io/component: gpu-test
 spec:
   entrypoint: train-model
   serviceAccountName: argo-workflow
@@ -14,7 +14,6 @@ spec:
   podMetadata:
     labels:
       app.kubernetes.io/part-of: foxhunt
-      app.kubernetes.io/component: gpu-test
       app.kubernetes.io/component: training-workflow
   securityContext:
     runAsUser: 1000