From 29cf09255106e83c058ab2bf7ff3560f3ae78fab Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Sat, 16 May 2026 23:31:40 +0200
Subject: [PATCH] =?UTF-8?q?infra(argo):=20warmup-gpu=20exits=20immediately?=
 =?UTF-8?q?=20=E2=80=94=20relies=20on=20Scaleway=2015min=20scaledown=20gra?=
 =?UTF-8?q?ce?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Removed the 30s sleep. The warmup pod's purpose is to make the L40S
pool autoscaler scale 0 → 1; once the pod lands on the new node
(Scheduled → Running → Succeeded), the node enters Scaleway Kapsule's
scaledown-grace window (~15 min). That single window covers the
entire range of ensure-binary durations (sccache-hit ~10s through
cold ~15 min), so train always lands on a hot node without holding
the warmup pod open.

Trimmed cpu request 100m → 50m and mem 64Mi → 32Mi: the pod runs
~one shell command then exits; tiny resource footprint = faster
scheduling and no kubelet noise.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
---
 infra/k8s/argo/alpha-perception-template.yaml | 20 +++++++++----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/infra/k8s/argo/alpha-perception-template.yaml b/infra/k8s/argo/alpha-perception-template.yaml
index 09579b805..5c61bd52c 100644
--- a/infra/k8s/argo/alpha-perception-template.yaml
+++ b/infra/k8s/argo/alpha-perception-template.yaml
@@ -218,10 +218,11 @@ spec:
     #
     # Schedules a tiny CPU-only pod on the gpu-pool's nodeSelector.
     # Kubernetes sees an unschedulable pod (no node currently in the
-    # pool), autoscaler scales the pool from 0 → 1. The pod sleeps
-    # briefly, exits; the node enters "empty" state with the
-    # autoscaler's scaledown grace window (~10 min on Scaleway), so
-    # train can land on it without provisioning latency.
+    # pool), autoscaler scales the pool 0 → 1. The pod exits the
+    # instant it lands; the node enters scaledown-grace state for
+    # ~15 min (Scaleway Kapsule default), which spans the entire
+    # ensure-binary compile window so train lands on a hot node with
+    # zero provisioning latency.
     #
     # No GPU resource request — that would compete with `train` for
     # the single GPU and serialise the steps. The nodeSelector +
@@ -243,17 +244,14 @@ spec:
         command: ["/bin/sh", "-c"]
         resources:
           requests:
+            cpu: "50m"
+            memory: 32Mi
+          limits:
             cpu: "100m"
             memory: 64Mi
-          limits:
-            cpu: "200m"
-            memory: 128Mi
         args:
           - |
-            echo "GPU warmup pod scheduled on $(hostname) — autoscaler scale-up triggered."
-            echo "Sleeping 30s to keep the node alive past provisioning; train will land here."
-            sleep 30
-            echo "Warmup complete; node remains in autoscaler scaledown grace window."
+            echo "GPU warmup pod scheduled on $(hostname) — autoscaler scale-up triggered, node now in scaledown-grace window."
 
     # ── train: run alpha_train on L40S ──
     - name: train