From 5dd5a0896363f2352c57f7faa37e21d2aa87f67c Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 26 May 2026 01:00:42 +0200 Subject: [PATCH] feat(infra): optional nsys profiling in Argo template (-p nsys-profile=true) Co-Authored-By: Claude Opus 4.7 --- infra/k8s/argo/alpha-rl-template.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/infra/k8s/argo/alpha-rl-template.yaml b/infra/k8s/argo/alpha-rl-template.yaml index bb2db7c9a..1d4f11919 100644 --- a/infra/k8s/argo/alpha-rl-template.yaml +++ b/infra/k8s/argo/alpha-rl-template.yaml @@ -59,6 +59,8 @@ spec: value: "1" - name: n-eval-steps value: "0" + - name: nsys-profile + value: "false" volumes: - name: git-ssh-key @@ -164,7 +166,13 @@ spec: OUT="/feature-cache/alpha-rl-runs/$SHA/fold{{workflow.parameters.fold-idx}}" mkdir -p "$OUT" echo "=== Train on $(nvidia-smi --query-gpu=name --format=csv,noheader) ===" - stdbuf -oL ${CARGO_TARGET_DIR}/release/examples/alpha_rl_train \ + TRAIN_BIN="${CARGO_TARGET_DIR}/release/examples/alpha_rl_train" + if [ "{{workflow.parameters.nsys-profile}}" = "true" ]; then + TRAIN_CMD="nsys profile -o $OUT/nsys_trace --stats=true --force-overwrite=true $TRAIN_BIN" + else + TRAIN_CMD="stdbuf -oL $TRAIN_BIN" + fi + $TRAIN_CMD \ --mbp10-data-dir /data/futures-baseline-mbp10/ES.FUT \ --predecoded-dir /feature-cache/predecoded \ --out "$OUT" \