From eb78f078a22afc8265a180f2ce1bb9054cb29bf5 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 13 Mar 2026 11:27:16 +0100 Subject: [PATCH] docs: fix 8 reviewer issues in GPU test workflow plan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add bayesian_changepoint_test to core tests (spec 4.5) - Replace fragmented YAML with complete WorkflowTemplate (all 4 templates) - Add complete notify-result template with notify parameter gating - Remove dead DQN_SMOKE_EPOCHS env var (no consumer in codebase) - Replace templateRef with resource template (workflow-of-workflows) to fix PVC/volume context issue in ci-pipeline integration - Fix supervised integration test || true → compile-check guard - Remove dead epochs parameter from all consumers - Inline shell script into YAML (no separate code block) Co-Authored-By: Claude Opus 4.6 --- .../plans/2026-03-13-gpu-test-workflow.md | 580 ++++++++++-------- 1 file changed, 323 insertions(+), 257 deletions(-) diff --git a/docs/superpowers/plans/2026-03-13-gpu-test-workflow.md b/docs/superpowers/plans/2026-03-13-gpu-test-workflow.md index b0ce53420..9ccf380e1 100644 --- a/docs/superpowers/plans/2026-03-13-gpu-test-workflow.md +++ b/docs/superpowers/plans/2026-03-13-gpu-test-workflow.md @@ -381,10 +381,14 @@ This is the core workflow. Reference `compile-and-train-template.yaml` for patte - [ ] **Step 1: Create the WorkflowTemplate** -Create `infra/k8s/argo/gpu-test-pipeline-template.yaml` with: +Create `infra/k8s/argo/gpu-test-pipeline-template.yaml`. The complete YAML is provided in a single block below. + +The file has 4 templates: `pipeline` (DAG entrypoint), `gpu-warmup`, `compile-and-test`, and `notify-result`. + +The shell script for `compile-and-test` is shown separately after the YAML structure for readability, but must be inlined in the `args: [|]` block. -**Header:** ```yaml +# infra/k8s/argo/gpu-test-pipeline-template.yaml apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: @@ -408,10 +412,6 @@ spec: ttlStrategy: secondsAfterCompletion: 3600 activeDeadlineSeconds: 5400 -``` - -**Parameters:** -```yaml arguments: parameters: - name: commit-ref @@ -420,18 +420,12 @@ spec: value: "dqn,ppo,tft" - name: test-scope value: all - - name: epochs - value: "3" - name: gpu-pool value: ci-training-h100 - name: cuda-compute-cap value: "90" - name: notify value: "true" -``` - -**Volumes:** -```yaml volumes: - name: git-ssh-key secret: @@ -444,11 +438,9 @@ spec: persistentVolumeClaim: claimName: test-data-pvc readOnly: true -``` -**DAG:** -```yaml templates: + # ── pipeline: DAG entrypoint ── - name: pipeline dag: tasks: @@ -457,230 +449,291 @@ spec: - name: compile-and-test template: compile-and-test dependencies: [gpu-warmup] + + # ── gpu-warmup: trigger H100 autoscale ── + - name: gpu-warmup + nodeSelector: + k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + container: + image: busybox:1.37 + command: ["/bin/sh", "-c"] + args: + - | + echo "GPU warmup: triggering node autoscale..." + echo "GPU node scheduled, exiting to free resources" + resources: + requests: + nvidia.com/gpu: "1" + cpu: 100m + memory: 64Mi + limits: + nvidia.com/gpu: "1" + cpu: 200m + memory: 128Mi + + # ── compile-and-test: compile + run GPU tests in single H100 pod ── + - name: compile-and-test + nodeSelector: + k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + outputs: + parameters: + - name: results + valueFrom: + path: /tmp/outputs/results + default: "unknown:FAIL" + - name: failures + valueFrom: + path: /tmp/outputs/failures + default: "1" + container: + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest + command: ["/bin/sh", "-c"] + env: + - name: SQLX_OFFLINE + value: "true" + - name: CARGO_TERM_COLOR + value: always + - name: CARGO_TARGET_DIR + value: /cargo-target + - name: CARGO_HOME + value: /cargo-target/cargo-home + - name: CUDA_VISIBLE_DEVICES + value: "0" + - name: CUBLAS_WORKSPACE_CONFIG + value: ":4096:8" + - name: TEST_DATA_DIR + value: /data/test-data + - name: RUST_LOG + value: info + volumeMounts: + - name: git-ssh-key + mountPath: /etc/git-ssh + readOnly: true + - name: cargo-target + mountPath: /cargo-target + - name: test-data + mountPath: /data/test-data + readOnly: true + resources: + requests: + nvidia.com/gpu: "1" + cpu: "4" + memory: 32Gi + limits: + nvidia.com/gpu: "1" + cpu: "8" + memory: 64Gi + args: + - | + set -e + + REF="{{workflow.parameters.commit-ref}}" + MODELS="{{workflow.parameters.models}}" + SCOPE="{{workflow.parameters.test-scope}}" + + # --- SSH setup (same as compile-and-train) --- + mkdir -p ~/.ssh + cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config + chmod 600 ~/.ssh/config + + REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git" + BUILD="/cargo-target/src" + git config --global --add safe.directory "$BUILD" + + # --- Persistent checkout on PVC (same as compile-and-train) --- + if [ -d "$BUILD/.git" ]; then + cd "$BUILD" + CURRENT=$(git rev-parse HEAD 2>/dev/null || echo "none") + TARGET=$(git ls-remote origin "$REF" 2>/dev/null | cut -f1 || echo "$REF") + if [ "$CURRENT" = "$TARGET" ] || [ "$CURRENT" = "$REF" ]; then + echo "=== Already at $REF ===" + else + echo "=== Updating checkout ===" + git fetch origin + git checkout --force "$REF" + git clean -fd + fi + else + echo "=== Initial clone ===" + git clone --filter=blob:none "$REPO" "$BUILD" + cd "$BUILD" + git checkout "$REF" + fi + echo "Checked out $(git rev-parse --short HEAD)" + + export PATH="${CARGO_HOME}/bin:${PATH}" + export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}} + + # --- PVC size guard --- + TARGET_SIZE_MB=$(du -sm "$CARGO_TARGET_DIR" 2>/dev/null | cut -f1 || echo 0) + echo "PVC usage: ${TARGET_SIZE_MB}MB" + if [ "$TARGET_SIZE_MB" -gt 25000 ]; then + echo "PVC exceeds 25GB, pruning..." + rm -rf "$CARGO_TARGET_DIR/release" "$CARGO_TARGET_DIR/debug" + fi + + # --- Compile --- + echo "=== Compiling test binaries (--features cuda) ===" + cargo test -p ml -p ml-dqn -p ml-core --features cuda --no-run 2>&1 + echo "=== Compilation done ===" + + # --- Expand "all" --- + if [ "$MODELS" = "all" ]; then + MODELS="dqn,ppo,tft,mamba2,tggn,tlob,liquid,kan,xlstm,diffusion" + fi + + # --- Test runner --- + RESULTS="" + FAILURES=0 + + run_tests() { + local NAME="$1"; shift + echo "" + echo "========================================" + echo " TEST: $NAME" + echo "========================================" + set +e + "$@" --nocapture 2>&1 + EXIT=$? + set -e + if [ $EXIT -eq 0 ]; then + RESULTS="${RESULTS}${NAME}:PASS\n" + else + RESULTS="${RESULTS}${NAME}:FAIL\n" + FAILURES=$((FAILURES + 1)) + fi + } + + # --- Core tests (always run) --- + run_tests "core-lib" cargo test -p ml-core --features cuda --lib + run_tests "bayesian" cargo test -p ml --features cuda --test bayesian_changepoint_test + + # --- Per-model tests --- + IFS=',' read -ra MODEL_LIST <<< "$MODELS" + for MODEL in "${MODEL_LIST[@]}"; do + case "$MODEL" in + dqn) + if [ "$SCOPE" = "lib" ] || [ "$SCOPE" = "all" ]; then + run_tests "dqn-lib" cargo test -p ml-dqn --features cuda --lib + run_tests "dqn-ml-lib" cargo test -p ml --features cuda --lib -- dqn + fi + if [ "$SCOPE" = "integration" ] || [ "$SCOPE" = "all" ]; then + run_tests "dqn-smoke" cargo test -p ml --features cuda --test smoke_test_real_data + run_tests "dqn-pipeline" cargo test -p ml --features cuda --test dqn_training_pipeline_test + run_tests "dqn-smoke-train" cargo test -p ml --features cuda --test dqn_training_smoke_test + run_tests "dqn-early-stop" cargo test -p ml --features cuda --test dqn_early_stopping_termination_test + run_tests "dqn-collapse" cargo test -p ml --features cuda --test dqn_action_collapse_fix_test + fi + ;; + ppo) + if [ "$SCOPE" = "lib" ] || [ "$SCOPE" = "all" ]; then + run_tests "ppo-lib" cargo test -p ml --features cuda --lib -- ppo + fi + if [ "$SCOPE" = "integration" ] || [ "$SCOPE" = "all" ]; then + run_tests "ppo-barrier" cargo test -p ml --features cuda --test barrier_optimization_test + fi + ;; + *) + # Supervised models + if [ "$SCOPE" = "lib" ] || [ "$SCOPE" = "all" ]; then + run_tests "${MODEL}-lib" cargo test -p ml --features cuda --lib -- "$MODEL" + fi + if [ "$SCOPE" = "integration" ] || [ "$SCOPE" = "all" ]; then + # Only run if a matching test binary exists (avoid false FAIL for models without integration tests) + if cargo test -p ml --features cuda --test "${MODEL}_*" --no-run 2>/dev/null; then + run_tests "${MODEL}-integ" cargo test -p ml --features cuda --test "${MODEL}_*" + fi + fi + ;; + esac + done + + # --- Summary --- + echo "" + echo "========================================" + echo " GPU TEST RESULTS" + echo "========================================" + printf "$RESULTS" | while IFS=: read -r name status; do + [ -z "$name" ] && continue + printf " %-25s %s\n" "$name" "$status" + done + echo "========================================" + echo " Total failures: $FAILURES" + echo "========================================" + + # Write results for notification step + mkdir -p /tmp/outputs + printf "$RESULTS" > /tmp/outputs/results + echo "$FAILURES" > /tmp/outputs/failures + + exit $FAILURES + + # ── notify-result: post test outcome to Mattermost (onExit) ── + - name: notify-result + nodeSelector: + k8s.scaleway.com/pool-name: platform + container: + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/foxhunt-runtime:latest + command: ["/bin/sh", "-c"] + env: + - name: WEBHOOK_URL + valueFrom: + secretKeyRef: + name: notification-webhook + key: webhook-url + optional: true + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 100m + memory: 64Mi + args: + - | + NOTIFY="{{workflow.parameters.notify}}" + if [ "$NOTIFY" != "true" ]; then + echo "Notifications disabled, skipping" + exit 0 + fi + + STATUS="{{workflow.status}}" + NAME="{{workflow.name}}" + + if [ -z "$WEBHOOK_URL" ] || echo "$WEBHOOK_URL" | grep -q "PLACEHOLDER"; then + echo "No webhook configured, skipping notification" + exit 0 + fi + + if [ "$STATUS" = "Succeeded" ]; then + EMOJI=":white_check_mark:" + else + EMOJI=":x:" + fi + + PAYLOAD="{\"username\":\"Argo CI\",\"text\":\"${EMOJI} **GPU Tests** ${NAME} — ${STATUS} ({{workflow.duration}}s)\"}" + + curl -sf -X POST -H 'Content-Type: application/json' \ + -d "$PAYLOAD" "$WEBHOOK_URL" || echo "WARN: webhook post failed" ``` -**gpu-warmup template:** Copy from `compile-and-train-template.yaml:266-291`, replace hardcoded pool with `{{workflow.parameters.gpu-pool}}`. - -**compile-and-test template:** Single container on H100. The shell script: - -1. SSH + git checkout (same pattern as compile-and-train lines 170-206) -2. Set `CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}}` -3. PVC size guard (same as compile-and-train lines 214-220) -4. `cargo test -p ml -p ml-dqn -p ml-core --features cuda --no-run` (compile) -5. Parse `models` parameter, loop over each model -6. Per model: run `cargo test` commands, capture exit code, continue on failure -7. Print summary table, exit with aggregate status - -**Key env vars on the compile-and-test container:** -```yaml -env: - - name: SQLX_OFFLINE - value: "true" - - name: CARGO_TERM_COLOR - value: always - - name: CARGO_TARGET_DIR - value: /cargo-target - - name: CARGO_HOME - value: /cargo-target/cargo-home - - name: CUDA_VISIBLE_DEVICES - value: "0" - - name: CUBLAS_WORKSPACE_CONFIG - value: ":4096:8" - - name: TEST_DATA_DIR - value: /data/test-data - - name: RUST_LOG - value: info - - name: DQN_SMOKE_EPOCHS - value: "{{workflow.parameters.epochs}}" -``` - -**Key volume mounts:** -```yaml -volumeMounts: - - name: git-ssh-key - mountPath: /etc/git-ssh - readOnly: true - - name: cargo-target - mountPath: /cargo-target - - name: test-data - mountPath: /data/test-data - readOnly: true -``` - -**Resources:** -```yaml -resources: - requests: - nvidia.com/gpu: "1" - cpu: "4" - memory: 32Gi - limits: - nvidia.com/gpu: "1" - cpu: "8" - memory: 64Gi -``` - -**Shell script for compile-and-test (inline in args):** - -```bash -set -e - -REF="{{workflow.parameters.commit-ref}}" -MODELS="{{workflow.parameters.models}}" -SCOPE="{{workflow.parameters.test-scope}}" - -# --- SSH setup (same as compile-and-train) --- -mkdir -p ~/.ssh -cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519 -chmod 600 ~/.ssh/id_ed25519 -printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config -chmod 600 ~/.ssh/config - -REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git" -BUILD="/cargo-target/src" -git config --global --add safe.directory "$BUILD" - -# --- Persistent checkout on PVC (same as compile-and-train) --- -if [ -d "$BUILD/.git" ]; then - cd "$BUILD" - CURRENT=$(git rev-parse HEAD 2>/dev/null || echo "none") - TARGET=$(git ls-remote origin "$REF" 2>/dev/null | cut -f1 || echo "$REF") - if [ "$CURRENT" = "$TARGET" ] || [ "$CURRENT" = "$REF" ]; then - echo "=== Already at $REF ===" - else - echo "=== Updating checkout ===" - git fetch origin - git checkout --force "$REF" - git clean -fd - fi -else - echo "=== Initial clone ===" - git clone --filter=blob:none "$REPO" "$BUILD" - cd "$BUILD" - git checkout "$REF" -fi -echo "Checked out $(git rev-parse --short HEAD)" - -export PATH="${CARGO_HOME}/bin:${PATH}" -export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}} - -# --- PVC size guard --- -TARGET_SIZE_MB=$(du -sm "$CARGO_TARGET_DIR" 2>/dev/null | cut -f1 || echo 0) -echo "PVC usage: ${TARGET_SIZE_MB}MB" -if [ "$TARGET_SIZE_MB" -gt 25000 ]; then - echo "PVC exceeds 25GB, pruning..." - rm -rf "$CARGO_TARGET_DIR/release" "$CARGO_TARGET_DIR/debug" -fi - -# --- Compile --- -echo "=== Compiling test binaries (--features cuda) ===" -cargo test -p ml -p ml-dqn -p ml-core --features cuda --no-run 2>&1 -echo "=== Compilation done ===" - -# --- Expand "all" --- -if [ "$MODELS" = "all" ]; then - MODELS="dqn,ppo,tft,mamba2,tggn,tlob,liquid,kan,xlstm,diffusion" -fi - -# --- Test runner --- -RESULTS="" -FAILURES=0 - -run_tests() { - local NAME="$1"; shift - echo "" - echo "========================================" - echo " TEST: $NAME" - echo "========================================" - set +e - "$@" --nocapture 2>&1 - EXIT=$? - set -e - if [ $EXIT -eq 0 ]; then - RESULTS="${RESULTS}${NAME}:PASS\n" - else - RESULTS="${RESULTS}${NAME}:FAIL\n" - FAILURES=$((FAILURES + 1)) - fi -} - -# --- Core tests (always run) --- -run_tests "core-lib" cargo test -p ml-core --features cuda --lib - -# --- Per-model tests --- -IFS=',' read -ra MODEL_LIST <<< "$MODELS" -for MODEL in "${MODEL_LIST[@]}"; do - case "$MODEL" in - dqn) - if [ "$SCOPE" = "lib" ] || [ "$SCOPE" = "all" ]; then - run_tests "dqn-lib" cargo test -p ml-dqn --features cuda --lib - run_tests "dqn-ml-lib" cargo test -p ml --features cuda --lib -- dqn - fi - if [ "$SCOPE" = "integration" ] || [ "$SCOPE" = "all" ]; then - run_tests "dqn-smoke" cargo test -p ml --features cuda --test smoke_test_real_data - run_tests "dqn-pipeline" cargo test -p ml --features cuda --test dqn_training_pipeline_test - run_tests "dqn-smoke-train" cargo test -p ml --features cuda --test dqn_training_smoke_test - run_tests "dqn-early-stop" cargo test -p ml --features cuda --test dqn_early_stopping_termination_test - run_tests "dqn-collapse" cargo test -p ml --features cuda --test dqn_action_collapse_fix_test - fi - ;; - ppo) - if [ "$SCOPE" = "lib" ] || [ "$SCOPE" = "all" ]; then - run_tests "ppo-lib" cargo test -p ml --features cuda --lib -- ppo - fi - if [ "$SCOPE" = "integration" ] || [ "$SCOPE" = "all" ]; then - run_tests "ppo-barrier" cargo test -p ml --features cuda --test barrier_optimization_test - fi - ;; - *) - # Supervised models - if [ "$SCOPE" = "lib" ] || [ "$SCOPE" = "all" ]; then - run_tests "${MODEL}-lib" cargo test -p ml --features cuda --lib -- "$MODEL" - fi - if [ "$SCOPE" = "integration" ] || [ "$SCOPE" = "all" ]; then - # Run model-specific integration tests if they exist (non-fatal if none) - run_tests "${MODEL}-integ" cargo test -p ml --features cuda --test "${MODEL}_*" 2>/dev/null || true - fi - ;; - esac -done - -# --- Summary --- -echo "" -echo "========================================" -echo " GPU TEST RESULTS" -echo "========================================" -printf "$RESULTS" | while IFS=: read -r name status; do - [ -z "$name" ] && continue - printf " %-25s %s\n" "$name" "$status" -done -echo "========================================" -echo " Total failures: $FAILURES" -echo "========================================" - -# Write results for notification step -mkdir -p /tmp/outputs -printf "$RESULTS" > /tmp/outputs/results -echo "$FAILURES" > /tmp/outputs/failures - -exit $FAILURES -``` - -**Outputs on compile-and-test:** -```yaml -outputs: - parameters: - - name: results - valueFrom: - path: /tmp/outputs/results - default: "unknown:FAIL" - - name: failures - valueFrom: - path: /tmp/outputs/failures - default: "1" -``` - -**notify-result template:** Same pattern as `compile-and-train-template.yaml:586-627` and `ci-pipeline-template.yaml:553-594`. Use `{{workflow.status}}` and `{{workflow.name}}`. Gated on `notify` parameter. +The complete shell script is inlined in the YAML above within the `compile-and-test` template's `args` block. - [ ] **Step 2: Validate YAML syntax** @@ -728,8 +781,6 @@ spec: parameters: - name: models value: "dqn,ppo,tft,mamba2,tggn,tlob,liquid,kan,xlstm,diffusion" - - name: epochs - value: "5" - name: commit-ref value: "main" ``` @@ -759,7 +810,7 @@ Pattern matches `scripts/argo-train.sh` (same argument parsing style, same `argo # Usage: # ./scripts/argo-test.sh # defaults: dqn,ppo,tft # ./scripts/argo-test.sh --models dqn --scope lib # DQN lib tests only -# ./scripts/argo-test.sh --models all --epochs 5 # all 10 models, 5 epochs +# ./scripts/argo-test.sh --models all # all 10 models # ./scripts/argo-test.sh --watch # follow logs # # Requires: argo CLI @@ -767,7 +818,6 @@ set -euo pipefail MODELS="dqn,ppo,tft" SCOPE="all" -EPOCHS="3" COMMIT_REF="HEAD" GPU_POOL="" WATCH=false @@ -780,7 +830,6 @@ Options: --models Comma-separated model list (default: dqn,ppo,tft) Use "all" for all 10 models --scope lib, integration, or all (default: all) - --epochs Training epochs for e2e tests (default: 3) --ref Git ref to test (default: HEAD) --gpu-pool GPU node pool (default: ci-training-h100) --watch Follow workflow logs @@ -793,7 +842,6 @@ while [[ $# -gt 0 ]]; do case $1 in --models) MODELS="$2"; shift 2 ;; --scope) SCOPE="$2"; shift 2 ;; - --epochs) EPOCHS="$2"; shift 2 ;; --ref) COMMIT_REF="$2"; shift 2 ;; --gpu-pool) GPU_POOL="$2"; shift 2 ;; --watch) WATCH=true; shift ;; @@ -811,7 +859,6 @@ CMD="argo submit -n foxhunt --from=wftmpl/gpu-test-pipeline" CMD="$CMD -p commit-ref=$COMMIT_REF" CMD="$CMD -p models=$MODELS" CMD="$CMD -p test-scope=$SCOPE" -CMD="$CMD -p epochs=$EPOCHS" [[ -n "$GPU_POOL" ]] && CMD="$CMD -p gpu-pool=$GPU_POOL" @@ -823,7 +870,6 @@ echo "Submitting GPU test workflow..." echo " commit: $(echo $COMMIT_REF | cut -c1-8)" echo " models: $MODELS" echo " scope: $SCOPE" -echo " epochs: $EPOCHS" echo "" eval "$CMD" ``` @@ -870,26 +916,46 @@ In the `outputs.parameters` section of `detect-changes` (around line 238), add: path: /tmp/outputs/ml-changed ``` -- [ ] **Step 3: Add gpu-test DAG task** +- [ ] **Step 3: Add gpu-test DAG task + submit-gpu-test template** + +**Important**: Cannot use `templateRef` to invoke `gpu-test-pipeline/pipeline` directly from the CI pipeline DAG because `templateRef` runs the referenced template within the **calling** workflow's pod context. The CI pipeline lacks the required PVC volumes (`cargo-target-cuda-test`, `test-data-pvc`) and GPU nodeSelector/tolerations. Instead, use a `resource` template that submits a child Workflow from the WorkflowTemplate (workflow-of-workflows pattern). In the `pipeline` DAG `tasks` section (around line 143), add: ```yaml - name: gpu-test depends: "detect-changes" - templateRef: - name: gpu-test-pipeline - template: pipeline - arguments: - parameters: - - name: commit-ref - value: "{{workflow.parameters.commit-sha}}" - - name: models - value: "dqn,ppo,tft" - - name: epochs - value: "3" + template: submit-gpu-test when: "{{tasks.detect-changes.outputs.parameters.ml-changed}} == true" ``` +Add the `submit-gpu-test` template to the `templates` list: +```yaml + # ── submit-gpu-test: launch GPU test workflow as a child workflow ── + - name: submit-gpu-test + resource: + action: create + setOwnerReference: true + successCondition: status.phase == Succeeded + failureCondition: status.phase in (Failed, Error) + manifest: | + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + generateName: gpu-test-on-push- + namespace: foxhunt + spec: + workflowTemplateRef: + name: gpu-test-pipeline + arguments: + parameters: + - name: commit-ref + value: "{{workflow.parameters.commit-sha}}" + - name: models + value: "dqn,ppo,tft" +``` + +This creates a child Workflow from the `gpu-test-pipeline` WorkflowTemplate. The child workflow has its own volumes, nodeSelector, and tolerations defined in the WorkflowTemplate spec. `setOwnerReference: true` ensures the child workflow is cleaned up when the parent is deleted. + - [ ] **Step 4: Validate YAML syntax** ```bash