diff --git a/infra/k8s/argo/train-template.yaml b/infra/k8s/argo/train-template.yaml index 5628d28df..ba624d671 100644 --- a/infra/k8s/argo/train-template.yaml +++ b/infra/k8s/argo/train-template.yaml @@ -103,16 +103,32 @@ spec: - name: ensure-fxcache template: ensure-fxcache dependencies: [ensure-binary] + arguments: + parameters: + - name: sha + value: "{{tasks.ensure-binary.outputs.parameters.sha}}" - name: hyperopt template: hyperopt dependencies: [ensure-binary, gpu-warmup, ensure-fxcache] when: "{{workflow.parameters.hyperopt-trials}} != 0" + arguments: + parameters: + - name: sha + value: "{{tasks.ensure-binary.outputs.parameters.sha}}" - name: train-best template: train-best dependencies: [ensure-binary, gpu-warmup, ensure-fxcache, hyperopt] + arguments: + parameters: + - name: sha + value: "{{tasks.ensure-binary.outputs.parameters.sha}}" - name: evaluate template: evaluate dependencies: [train-best] + arguments: + parameters: + - name: sha + value: "{{tasks.ensure-binary.outputs.parameters.sha}}" - name: upload-results template: upload-results dependencies: [evaluate] @@ -279,6 +295,9 @@ spec: # ── ensure-fxcache: precompute feature cache if needed ── - name: ensure-fxcache + inputs: + parameters: + - name: sha nodeSelector: k8s.scaleway.com/pool-name: ci-compile-cpu tolerations: @@ -307,7 +326,7 @@ spec: args: - | set -e - SHA="{{tasks.ensure-binary.outputs.parameters.sha}}" + SHA="{{inputs.parameters.sha}}" BINARY="/data/bin/$SHA/precompute_features" if [ ! -x "$BINARY" ]; then @@ -356,6 +375,9 @@ spec: # ── hyperopt: PSO/TPE hyperparameter optimization on GPU ── - name: hyperopt + inputs: + parameters: + - name: sha nodeSelector: k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" tolerations: @@ -399,7 +421,7 @@ spec: args: - | set -e - SHA="{{tasks.ensure-binary.outputs.parameters.sha}}" + SHA="{{inputs.parameters.sha}}" export PATH="/data/bin/$SHA:$PATH" export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') nvidia-smi @@ -437,6 +459,9 @@ spec: # ── train-best: full training with best hyperparams ── - name: train-best + inputs: + parameters: + - name: sha nodeSelector: k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" tolerations: @@ -480,7 +505,7 @@ spec: args: - | set -e - SHA="{{tasks.ensure-binary.outputs.parameters.sha}}" + SHA="{{inputs.parameters.sha}}" export PATH="/data/bin/$SHA:$PATH" export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//') nvidia-smi @@ -520,6 +545,9 @@ spec: # ── evaluate: run evaluation on trained model ── - name: evaluate + inputs: + parameters: + - name: sha nodeSelector: k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" tolerations: @@ -559,7 +587,7 @@ spec: args: - | set -e - SHA="{{tasks.ensure-binary.outputs.parameters.sha}}" + SHA="{{inputs.parameters.sha}}" export PATH="/data/bin/$SHA:$PATH" export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')