diff --git a/infra/k8s/argo/ci-pipeline-template.yaml b/infra/k8s/argo/ci-pipeline-template.yaml index 5efa19b37..a2c59352b 100644 --- a/infra/k8s/argo/ci-pipeline-template.yaml +++ b/infra/k8s/argo/ci-pipeline-template.yaml @@ -595,6 +595,9 @@ spec: # ── submit-gpu-test: launch GPU test workflow as a child workflow ── - name: submit-gpu-test + # Argo wait sidecar needs 256Mi to track large child workflow status JSON. + # Default 64Mi causes OOMKilled when child has many parameters. + podSpecPatch: '{"containers":[{"name":"wait","resources":{"requests":{"memory":"128Mi"},"limits":{"memory":"256Mi"}}}]}' resource: action: create setOwnerReference: true diff --git a/infra/k8s/argo/compile-and-train-template.yaml b/infra/k8s/argo/compile-and-train-template.yaml index 1e5be9635..07d4e10c7 100644 --- a/infra/k8s/argo/compile-and-train-template.yaml +++ b/infra/k8s/argo/compile-and-train-template.yaml @@ -17,7 +17,7 @@ metadata: labels: app.kubernetes.io/name: compile-and-train app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/component: gpu-test + app.kubernetes.io/component: gpu-test spec: entrypoint: pipeline onExit: notify-result @@ -25,7 +25,6 @@ spec: podMetadata: labels: app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/component: gpu-test app.kubernetes.io/component: compile-and-train # No runAsUser — compile (ci-builder) needs root, training runtime defaults # to its image user. fsGroup ensures workspace PVC is writable by all pods. diff --git a/infra/k8s/argo/training-workflow-template.yaml b/infra/k8s/argo/training-workflow-template.yaml index 35a0477da..16918e1bb 100644 --- a/infra/k8s/argo/training-workflow-template.yaml +++ b/infra/k8s/argo/training-workflow-template.yaml @@ -6,7 +6,7 @@ metadata: labels: app.kubernetes.io/name: training-pipeline app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/component: gpu-test + app.kubernetes.io/component: gpu-test spec: entrypoint: train-model serviceAccountName: argo-workflow @@ -14,7 +14,6 @@ spec: podMetadata: labels: app.kubernetes.io/part-of: foxhunt - app.kubernetes.io/component: gpu-test app.kubernetes.io/component: training-workflow securityContext: runAsUser: 1000