From d6ada5266be1dfd2102ba959dcd1ec0622cf49fb Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Mon, 13 Apr 2026 23:44:10 +0200 Subject: [PATCH] =?UTF-8?q?diag:=20enable=20FOXHUNT=5FNO=5FGRAPH=20in=20Ar?= =?UTF-8?q?go=20train=20template=20=E2=80=94=20isolate=20H100=20graph=20is?= =?UTF-8?q?sue?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Temporarily sets FOXHUNT_NO_GRAPH=1 in train-best pods to run forward+backward+Adam ungraphed. If grad_buf is non-zero ungraphed, the root cause is CUDA Graph capture of cuBLASLt on H100 SM90. Co-Authored-By: Claude Opus 4.6 (1M context) --- infra/k8s/argo/train-template.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infra/k8s/argo/train-template.yaml b/infra/k8s/argo/train-template.yaml index 818fd64b3..bb02f1375 100644 --- a/infra/k8s/argo/train-template.yaml +++ b/infra/k8s/argo/train-template.yaml @@ -400,6 +400,8 @@ spec: value: ":4096:8" - name: FOXHUNT_FEATURE_CACHE_DIR value: /feature-cache + - name: FOXHUNT_NO_GRAPH + value: "1" resources: requests: nvidia.com/gpu: "1" @@ -484,6 +486,8 @@ spec: value: ":4096:8" - name: FOXHUNT_FEATURE_CACHE_DIR value: /feature-cache + - name: FOXHUNT_NO_GRAPH + value: "1" resources: requests: nvidia.com/gpu: "1"