From 260b07c4927a8fb86527dd035c52df37976fee48 Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Sat, 16 May 2026 23:29:16 +0200
Subject: [PATCH] =?UTF-8?q?infra(argo):=20warmup-gpu=20DAG=20branch=20?=
 =?UTF-8?q?=E2=80=94=20pipeline=20compile=20+=20L40S=20provisioning?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds a parallel warmup-gpu task that runs concurrently with
ensure-binary. The warmup pod is a tiny CPU-only alpine container
scheduled on the gpu-pool's nodeSelector — its presence triggers
cluster-autoscaler scale-up of the L40S pool. After a 30s sleep, the
warmup pod exits; the node enters Scaleway's scaledown grace window
(~10 min), so the train pod lands on a hot node without waiting for
autoscaler provisioning.

No GPU resource request on the warmup pod — that would serialise
warmup and train on the same GPU. nodeSelector + nvidia.com/gpu
toleration are sufficient to force placement on the L40S pool.

Expected savings: ~3-5 min per cold cluster submission. First run
(this session, alpha-perception-4vl7c) showed compile took 113s
(sccache warm) with serial GPU provisioning following; subsequent
submissions should overlap the two stages.

DAG topology:
  ensure-binary ──┐
                  ├──> train
  warmup-gpu  ────┘   (only depends on ensure-binary; warmup is
                       fire-and-forget infrastructure)

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
---
 infra/k8s/argo/alpha-perception-template.yaml | 50 +++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/infra/k8s/argo/alpha-perception-template.yaml b/infra/k8s/argo/alpha-perception-template.yaml
index 4ee38c9f9..09579b805 100644
--- a/infra/k8s/argo/alpha-perception-template.yaml
+++ b/infra/k8s/argo/alpha-perception-template.yaml
@@ -80,11 +80,20 @@ spec:
 
   templates:
     # ── DAG ──
+    #
+    # ensure-binary (CPU compile) and warmup-gpu (autoscaler trigger
+    # for the L40S pool) run in parallel. train only depends on
+    # ensure-binary — warmup-gpu is fire-and-forget; the autoscaler
+    # keeps the node warm for its scaledown grace window after the
+    # warmup pod exits, so train lands on a hot node without waiting
+    # for autoscaler provisioning.
     - name: pipeline
       dag:
         tasks:
           - name: ensure-binary
             template: ensure-binary
+          - name: warmup-gpu
+            template: warmup-gpu
           - name: train
             template: train
             dependencies: [ensure-binary]
@@ -205,6 +214,47 @@ spec:
             done
             echo "$SHORT_SHA" > /tmp/sha
 
+    # ── warmup-gpu: trigger L40S autoscaler scale-up in parallel ──
+    #
+    # Schedules a tiny CPU-only pod on the gpu-pool's nodeSelector.
+    # Kubernetes sees an unschedulable pod (no node currently in the
+    # pool), autoscaler scales the pool from 0 → 1. The pod sleeps
+    # briefly, exits; the node enters "empty" state with the
+    # autoscaler's scaledown grace window (~10 min on Scaleway), so
+    # train can land on it without provisioning latency.
+    #
+    # No GPU resource request — that would compete with `train` for
+    # the single GPU and serialise the steps. The nodeSelector +
+    # nvidia.com/gpu toleration are enough to force placement on the
+    # right pool.
+    - name: warmup-gpu
+      nodeSelector:
+        k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
+        topology.kubernetes.io/zone: fr-par-2
+      tolerations:
+        - key: nvidia.com/gpu
+          operator: Exists
+          effect: NoSchedule
+        - key: node.cilium.io/agent-not-ready
+          operator: Exists
+          effect: NoSchedule
+      container:
+        image: alpine:3.20
+        command: ["/bin/sh", "-c"]
+        resources:
+          requests:
+            cpu: "100m"
+            memory: 64Mi
+          limits:
+            cpu: "200m"
+            memory: 128Mi
+        args:
+          - |
+            echo "GPU warmup pod scheduled on $(hostname) — autoscaler scale-up triggered."
+            echo "Sleeping 30s to keep the node alive past provisioning; train will land here."
+            sleep 30
+            echo "Warmup complete; node remains in autoscaler scaledown grace window."
+
     # ── train: run alpha_train on L40S ──
     - name: train
       inputs: