diff --git a/infra/k8s/argo/train-template.yaml b/infra/k8s/argo/train-template.yaml index 1e4438650..69a7c5588 100644 --- a/infra/k8s/argo/train-template.yaml +++ b/infra/k8s/argo/train-template.yaml @@ -548,8 +548,8 @@ spec: if [ "$SANITIZER" = "nsys" ]; then NSYS_BIN=$(which nsys 2>/dev/null || echo "/usr/local/cuda/bin/nsys") if [ -x "$NSYS_BIN" ]; then - NSYS_OUT="/data/nsys/nsys_$(date +%Y%m%d_%H%M%S)" - mkdir -p /data/nsys + NSYS_OUT="/workspace/output/nsys_$(date +%Y%m%d_%H%M%S)" + mkdir -p /workspace/output SANITIZER_PREFIX="$NSYS_BIN profile -o $NSYS_OUT --gpu-metrics-device=all --cuda-graph-trace=node --cudabacktrace=kernel --stats=true --show-output=true -f true --capture-range=cudaProfilerApi --capture-range-end=stop-shutdown" echo " nsys profiling enabled — output: ${NSYS_OUT}.nsys-rep (persistent PVC)" echo " --cuda-graph-trace=node: per-kernel timing inside parent graph"