diff --git a/scripts/argo-train.sh b/scripts/argo-train.sh index e257bb456..7ab881dd4 100755 --- a/scripts/argo-train.sh +++ b/scripts/argo-train.sh @@ -152,6 +152,14 @@ if [[ "$MULTI_SEED" -gt 1 || "$FOLDS" -gt 1 || "$PROFILE" == "true" ]]; then fi if [[ "$USE_MULTI_SEED" == "false" ]]; then + # Apply the local train-template.yaml to the cluster BEFORE submission. + # Without this, `argo submit --from=wftmpl/train` runs against whatever + # was last applied — defaults can drift from source. Caused workflow + # train-jpxvn (2026-05-10) to dispatch with a stale threshold=0.5 + # default and trigger near-OOM. Multi-seed path already does this; the + # single-job path was missing it. + kubectl apply -n foxhunt -f infra/k8s/argo/train-template.yaml >/dev/null + CMD="argo submit -n foxhunt --from=wftmpl/train" CMD="$CMD -p commit-sha=$SHA" CMD="$CMD -p git-branch=$BRANCH"