From 8e6effb7f67775ee5a567e289e400c98b49dee0e Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 14 Mar 2026 01:10:06 +0100 Subject: [PATCH] fix(ci): retain completed pods 10 min for log access (CPU only) Add deleteDelayDuration: 600s to podGC on CPU workflow templates and sensors (ci-pipeline, compile-deploy, build-image). GPU training workflows keep immediate cleanup to avoid wasting expensive GPU time. Co-Authored-By: Claude Opus 4.6 --- infra/k8s/argo/ci-pipeline-template.yaml | 1 + infra/k8s/argo/compile-and-deploy-template.yaml | 1 + infra/k8s/argo/events/ci-pipeline-sensor.yaml | 1 + infra/k8s/argo/events/workflow-trigger-sensor.yaml | 2 ++ 4 files changed, 5 insertions(+) diff --git a/infra/k8s/argo/ci-pipeline-template.yaml b/infra/k8s/argo/ci-pipeline-template.yaml index 0cd274878..80a4797c2 100644 --- a/infra/k8s/argo/ci-pipeline-template.yaml +++ b/infra/k8s/argo/ci-pipeline-template.yaml @@ -17,6 +17,7 @@ spec: app.kubernetes.io/part-of: foxhunt podGC: strategy: OnPodCompletion + deleteDelayDuration: 600s ttlStrategy: secondsAfterCompletion: 3600 volumes: diff --git a/infra/k8s/argo/compile-and-deploy-template.yaml b/infra/k8s/argo/compile-and-deploy-template.yaml index 3dfd4682d..746440191 100644 --- a/infra/k8s/argo/compile-and-deploy-template.yaml +++ b/infra/k8s/argo/compile-and-deploy-template.yaml @@ -24,6 +24,7 @@ spec: app.kubernetes.io/component: compile-and-deploy podGC: strategy: OnPodCompletion + deleteDelayDuration: 600s ttlStrategy: secondsAfterCompletion: 3600 activeDeadlineSeconds: 3600 # 1 hour (compile + deploy) diff --git a/infra/k8s/argo/events/ci-pipeline-sensor.yaml b/infra/k8s/argo/events/ci-pipeline-sensor.yaml index c01bcf8ea..8cfb2adf2 100644 --- a/infra/k8s/argo/events/ci-pipeline-sensor.yaml +++ b/infra/k8s/argo/events/ci-pipeline-sensor.yaml @@ -46,6 +46,7 @@ spec: serviceAccountName: argo-workflow podGC: strategy: OnPodCompletion + deleteDelayDuration: 600s ttlStrategy: secondsAfterCompletion: 3600 workflowTemplateRef: diff --git a/infra/k8s/argo/events/workflow-trigger-sensor.yaml b/infra/k8s/argo/events/workflow-trigger-sensor.yaml index a4405d817..350fb554c 100644 --- a/infra/k8s/argo/events/workflow-trigger-sensor.yaml +++ b/infra/k8s/argo/events/workflow-trigger-sensor.yaml @@ -63,6 +63,7 @@ spec: serviceAccountName: argo-workflow podGC: strategy: OnPodCompletion + deleteDelayDuration: 600s ttlStrategy: secondsAfterCompletion: 3600 workflowTemplateRef: @@ -193,6 +194,7 @@ spec: serviceAccountName: argo-workflow podGC: strategy: OnPodCompletion + deleteDelayDuration: 600s ttlStrategy: secondsAfterCompletion: 3600 workflowTemplateRef: