From d5896ef8094e2e38cfd28f8826c496d4fd681a38 Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Sat, 16 May 2026 09:35:20 +0200
Subject: [PATCH] infra(argo): skip gpu-warmup on the fxcache-only path

gpu-warmup pre-provisions an L40S node so hyperopt / train-best don't
pay autoscaler latency at workflow start. When hyperopt-trials=0
(the precompute-only path used by scripts/argo-precompute.sh), both
downstream consumers are skipped via their `when:` clauses and the
warmup-provisioned GPU node would sit idle until workflow end.

Add the same `when:` clause to gpu-warmup so it skips alongside its
consumers, saving an L40S node's provisioning + idle time for every
fxcache-rebuild submission.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
---
 infra/k8s/argo/train-template.yaml | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/infra/k8s/argo/train-template.yaml b/infra/k8s/argo/train-template.yaml
index e424ef886..9af442590 100644
--- a/infra/k8s/argo/train-template.yaml
+++ b/infra/k8s/argo/train-template.yaml
@@ -7,10 +7,15 @@
 #   ensure-binary ──┐
 #   gpu-warmup ─────┼──> ensure-fxcache ──> hyperopt ──> train-best ──> evaluate ──> upload-results
 #
+# When hyperopt-trials=0 (the `scripts/argo-precompute.sh` path),
+# gpu-warmup / hyperopt / train-best / evaluate all skip and the
+# workflow runs ensure-binary → ensure-fxcache only — no GPU node
+# gets provisioned for nothing.
+#
 # Usage:
 #   argo submit -n foxhunt --from=wftmpl/train
 #   argo submit -n foxhunt --from=wftmpl/train -p model=ppo -p train-epochs=100
-#   argo submit -n foxhunt --from=wftmpl/train -p hyperopt-trials=0   # skip hyperopt
+#   argo submit -n foxhunt --from=wftmpl/train -p hyperopt-trials=0   # fxcache-only path
 apiVersion: argoproj.io/v1alpha1
 kind: WorkflowTemplate
 metadata:
@@ -125,6 +130,14 @@ spec:
             template: ensure-binary
           - name: gpu-warmup
             template: gpu-warmup
+            # gpu-warmup pre-provisions an L40S node so hyperopt /
+            # train-best don't pay autoscaler latency at start. When
+            # hyperopt-trials==0 (precompute-only path used by
+            # `scripts/argo-precompute.sh`), both downstream consumers
+            # are skipped and the GPU node would sit idle until the
+            # workflow ends. Skip the warmup in that case to avoid
+            # provisioning expensive L40S capacity for nothing.
+            when: "{{workflow.parameters.hyperopt-trials}} != 0"
           - name: ensure-fxcache
             template: ensure-fxcache
             dependencies: [ensure-binary]