From b8343183c244664d528811aef104bc96254cf972 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 10 Apr 2026 20:38:47 +0200 Subject: [PATCH] fix: pass SHA via inputs.parameters in Argo DAG task arguments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Argo requires task output parameters to flow through DAG arguments → template inputs, not direct {{tasks.X.outputs}} in container args. Co-Authored-By: Claude Opus 4.6 (1M context) --- infra/k8s/argo/train-template.yaml | 36 ++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/infra/k8s/argo/train-template.yaml b/infra/k8s/argo/train-template.yaml index 5628d28df..ba624d671 100644 --- a/infra/k8s/argo/train-template.yaml +++ b/infra/k8s/argo/train-template.yaml @@ -103,16 +103,32 @@ spec: - name: ensure-fxcache template: ensure-fxcache dependencies: [ensure-binary] + arguments: + parameters: + - name: sha + value: "{{tasks.ensure-binary.outputs.parameters.sha}}" - name: hyperopt template: hyperopt dependencies: [ensure-binary, gpu-warmup, ensure-fxcache] when: "{{workflow.parameters.hyperopt-trials}} != 0" + arguments: + parameters: + - name: sha + value: "{{tasks.ensure-binary.outputs.parameters.sha}}" - name: train-best template: train-best dependencies: [ensure-binary, gpu-warmup, ensure-fxcache, hyperopt] + arguments: + parameters: + - name: sha + value: "{{tasks.ensure-binary.outputs.parameters.sha}}" - name: evaluate template: evaluate dependencies: [train-best] + arguments: + parameters: + - name: sha + value: "{{tasks.ensure-binary.outputs.parameters.sha}}" - name: upload-results template: upload-results dependencies: [evaluate] @@ -279,6 +295,9 @@ spec: # ── ensure-fxcache: precompute feature cache if needed ── - name: ensure-fxcache + inputs: + parameters: + - name: sha nodeSelector: k8s.scaleway.com/pool-name: ci-compile-cpu tolerations: @@ -307,7 +326,7 @@ spec: args: - | set -e - SHA="{{tasks.ensure-binary.outputs.parameters.sha}}" + SHA="{{inputs.parameters.sha}}" BINARY="/data/bin/$SHA/precompute_features" if [ ! -x "$BINARY" ]; then @@ -356,6 +375,9 @@ spec: # ── hyperopt: PSO/TPE hyperparameter optimization on GPU ── - name: hyperopt + inputs: + parameters: + - name: sha nodeSelector: k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" tolerations: @@ -399,7 +421,7 @@ spec: args: - | set -e - SHA="{{tasks.ensure-binary.outputs.parameters.sha}}" + SHA="{{inputs.parameters.sha}}" export PATH="/data/bin/$SHA:$PATH" export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') nvidia-smi @@ -437,6 +459,9 @@ spec: # ── train-best: full training with best hyperparams ── - name: train-best + inputs: + parameters: + - name: sha nodeSelector: k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" tolerations: @@ -480,7 +505,7 @@ spec: args: - | set -e - SHA="{{tasks.ensure-binary.outputs.parameters.sha}}" + SHA="{{inputs.parameters.sha}}" export PATH="/data/bin/$SHA:$PATH" export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') nvidia-smi @@ -520,6 +545,9 @@ spec: # ── evaluate: run evaluation on trained model ── - name: evaluate + inputs: + parameters: + - name: sha nodeSelector: k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" tolerations: @@ -559,7 +587,7 @@ spec: args: - | set -e - SHA="{{tasks.ensure-binary.outputs.parameters.sha}}" + SHA="{{inputs.parameters.sha}}" export PATH="/data/bin/$SHA:$PATH" export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')