From 43c6a3ad6184f28696a0dacecfe340472abce288 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 19 Apr 2026 10:53:15 +0200 Subject: [PATCH] fix: nsys output to feature-cache PVC (RW, persistent) --- infra/k8s/argo/train-template.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/k8s/argo/train-template.yaml b/infra/k8s/argo/train-template.yaml index 69a7c5588..d8e41c2d9 100644 --- a/infra/k8s/argo/train-template.yaml +++ b/infra/k8s/argo/train-template.yaml @@ -548,8 +548,8 @@ spec: if [ "$SANITIZER" = "nsys" ]; then NSYS_BIN=$(which nsys 2>/dev/null || echo "/usr/local/cuda/bin/nsys") if [ -x "$NSYS_BIN" ]; then - NSYS_OUT="/workspace/output/nsys_$(date +%Y%m%d_%H%M%S)" - mkdir -p /workspace/output + NSYS_OUT="/feature-cache/nsys_$(date +%Y%m%d_%H%M%S)" + mkdir -p /feature-cache SANITIZER_PREFIX="$NSYS_BIN profile -o $NSYS_OUT --gpu-metrics-device=all --cuda-graph-trace=node --cudabacktrace=kernel --stats=true --show-output=true -f true --capture-range=cudaProfilerApi --capture-range-end=stop-shutdown" echo " nsys profiling enabled — output: ${NSYS_OUT}.nsys-rep (persistent PVC)" echo " --cuda-graph-trace=node: per-kernel timing inside parent graph"