diff --git a/infra/k8s/argo/alpha-rl-template.yaml b/infra/k8s/argo/alpha-rl-template.yaml index bb2db7c9a..1d4f11919 100644 --- a/infra/k8s/argo/alpha-rl-template.yaml +++ b/infra/k8s/argo/alpha-rl-template.yaml @@ -59,6 +59,8 @@ spec: value: "1" - name: n-eval-steps value: "0" + - name: nsys-profile + value: "false" volumes: - name: git-ssh-key @@ -164,7 +166,13 @@ spec: OUT="/feature-cache/alpha-rl-runs/$SHA/fold{{workflow.parameters.fold-idx}}" mkdir -p "$OUT" echo "=== Train on $(nvidia-smi --query-gpu=name --format=csv,noheader) ===" - stdbuf -oL ${CARGO_TARGET_DIR}/release/examples/alpha_rl_train \ + TRAIN_BIN="${CARGO_TARGET_DIR}/release/examples/alpha_rl_train" + if [ "{{workflow.parameters.nsys-profile}}" = "true" ]; then + TRAIN_CMD="nsys profile -o $OUT/nsys_trace --stats=true --force-overwrite=true $TRAIN_BIN" + else + TRAIN_CMD="stdbuf -oL $TRAIN_BIN" + fi + $TRAIN_CMD \ --mbp10-data-dir /data/futures-baseline-mbp10/ES.FUT \ --predecoded-dir /feature-cache/predecoded \ --out "$OUT" \