From 4e0d1fcbe618706c82d3f2df1387c27792777b39 Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Fri, 6 Mar 2026 12:49:34 +0100
Subject: [PATCH] fix(ci): add imagePullPolicy: IfNotPresent to training
 workflow

Kubernetes defaults to Always for :latest tags, forcing registry
round-trips that fail on fresh GPU nodes where containerd HTTP-only
registry config has a race condition with HTTPS fallback.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 infra/k8s/argo/training-workflow-template.yaml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/infra/k8s/argo/training-workflow-template.yaml b/infra/k8s/argo/training-workflow-template.yaml
index c86de2b48..1c00bc220 100644
--- a/infra/k8s/argo/training-workflow-template.yaml
+++ b/infra/k8s/argo/training-workflow-template.yaml
@@ -26,7 +26,7 @@ spec:
     parameters:
       - name: model
       - name: gpu-pool
-        value: ci-training-h100
+        value: ci-training-h100  # Pools: ci-training-h100, ci-training-l40s, ci-training-h100x2
       - name: hyperopt-trials
         value: "20"
       - name: hyperopt-epochs
@@ -97,6 +97,7 @@ spec:
           effect: NoSchedule
       container:
         image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
+        imagePullPolicy: IfNotPresent
         command: ["/bin/sh", "-c"]
         args:
           - |
@@ -118,6 +119,7 @@ spec:
         k8s.scaleway.com/pool-name: platform
       container:
         image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
+        imagePullPolicy: IfNotPresent
         command: ["/bin/sh", "-c"]
         args:
           - |
@@ -168,6 +170,7 @@ spec:
           effect: NoSchedule
       container:
         image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
+        imagePullPolicy: IfNotPresent
         command: ["/bin/sh", "-c"]
         args:
           - |
@@ -264,6 +267,7 @@ spec:
           effect: NoSchedule
       container:
         image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
+        imagePullPolicy: IfNotPresent
         command: ["/bin/sh", "-c"]
         args:
           - |
@@ -355,6 +359,7 @@ spec:
           effect: NoSchedule
       container:
         image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-training-runtime:latest
+        imagePullPolicy: IfNotPresent
         command: ["/bin/sh", "-c"]
         args:
           - |
@@ -429,6 +434,7 @@ spec:
         k8s.scaleway.com/pool-name: platform
       container:
         image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest
+        imagePullPolicy: IfNotPresent
         command: ["/bin/sh", "-c"]
         args:
           - |