diff --git a/infra/k8s/argo/train-template.yaml b/infra/k8s/argo/train-template.yaml index e424ef886..9af442590 100644 --- a/infra/k8s/argo/train-template.yaml +++ b/infra/k8s/argo/train-template.yaml @@ -7,10 +7,15 @@ # ensure-binary ──┐ # gpu-warmup ─────┼──> ensure-fxcache ──> hyperopt ──> train-best ──> evaluate ──> upload-results # +# When hyperopt-trials=0 (the `scripts/argo-precompute.sh` path), +# gpu-warmup / hyperopt / train-best / evaluate all skip and the +# workflow runs ensure-binary → ensure-fxcache only — no GPU node +# gets provisioned for nothing. +# # Usage: # argo submit -n foxhunt --from=wftmpl/train # argo submit -n foxhunt --from=wftmpl/train -p model=ppo -p train-epochs=100 -# argo submit -n foxhunt --from=wftmpl/train -p hyperopt-trials=0 # skip hyperopt +# argo submit -n foxhunt --from=wftmpl/train -p hyperopt-trials=0 # fxcache-only path apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: @@ -125,6 +130,14 @@ spec: template: ensure-binary - name: gpu-warmup template: gpu-warmup + # gpu-warmup pre-provisions an L40S node so hyperopt / + # train-best don't pay autoscaler latency at start. When + # hyperopt-trials==0 (precompute-only path used by + # `scripts/argo-precompute.sh`), both downstream consumers + # are skipped and the GPU node would sit idle until the + # workflow ends. Skip the warmup in that case to avoid + # provisioning expensive L40S capacity for nothing. + when: "{{workflow.parameters.hyperopt-trials}} != 0" - name: ensure-fxcache template: ensure-fxcache dependencies: [ensure-binary]