feat: compute-sanitizer support in Argo training workflow

Usage: ./scripts/argo-train.sh dqn --baseline --epochs 2 --sanitizer memcheck
       ./scripts/argo-train.sh dqn --baseline --epochs 1 --sanitizer synccheck

Tools: memcheck (OOB, uninitialized), racecheck (data races),
synccheck (__syncthreads divergence/deadlocks).
10-100x slower — use with 1-2 epochs for debugging.
Detects exact kernel + line causing GPU hang.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

infra/k8s/argo/train-template.yaml

@@ -65,6 +65,8 @@ spec:
         value: "1.0"
       - name: min-hold-bars
         value: "5"
+      - name: sanitizer
+        value: "none"  # "none", "memcheck", "racecheck", "synccheck"
 
   volumes:
     - name: git-ssh-key
@@ -541,7 +543,21 @@ spec:
 
             echo "=== Training: $MODEL ({{workflow.parameters.train-epochs}} epochs) ==="
 
-            stdbuf -oL ${BINARY} \
+            # compute-sanitizer: optional GPU debugging (10-100x slower)
+            SANITIZER="{{workflow.parameters.sanitizer}}"
+            SANITIZER_PREFIX=""
+            if [ "$SANITIZER" != "none" ] && [ -n "$SANITIZER" ]; then
+              SANITIZER_BIN=$(which compute-sanitizer 2>/dev/null || echo "/usr/local/cuda/bin/compute-sanitizer")
+              if [ -x "$SANITIZER_BIN" ]; then
+                SANITIZER_PREFIX="$SANITIZER_BIN --tool $SANITIZER --print-limit 20 --error-exitcode 1"
+                echo "  compute-sanitizer enabled: --tool $SANITIZER"
+                echo "  WARNING: 10-100x slower — use with 1-2 epochs only"
+              else
+                echo "  WARNING: compute-sanitizer not found at $SANITIZER_BIN — running without"
+              fi
+            fi
+
+            stdbuf -oL $SANITIZER_PREFIX ${BINARY} \
               --model "$MODEL" \
               --symbol {{workflow.parameters.symbol}} \
               --tx-cost-bps {{workflow.parameters.tx-cost-bps}} \

scripts/argo-train.sh

@@ -25,6 +25,7 @@ SYMBOL=""
 WATCH=false
 BASELINE=false
 CAPITAL=""
+SANITIZER="none"
 
 usage() {
   cat <<EOF
@@ -43,6 +44,7 @@ Options:
   --symbol <sym>       Trading symbol (default: ES.FUT)
   --capital <n>        Initial capital (default: 35000)
   --baseline           Skip hyperopt (trials=0)
+  --sanitizer <tool>   Run under compute-sanitizer (memcheck|racecheck|synccheck)
   --watch              Follow workflow logs
   -h, --help           Show this help
 EOF
@@ -68,6 +70,7 @@ while [[ $# -gt 0 ]]; do
     --symbol)    SYMBOL="$2"; shift 2 ;;
     --capital)   CAPITAL="$2"; shift 2 ;;
     --baseline)  BASELINE=true; shift ;;
+    --sanitizer) SANITIZER="$2"; shift 2 ;;
     --watch)     WATCH=true; shift ;;
     -h|--help)   usage ;;
     *)           echo "Unknown option: $1"; usage ;;
@@ -84,6 +87,7 @@ CMD="$CMD -p model=$MODEL"
 [[ -n "$GPU_POOL" ]]  && CMD="$CMD -p gpu-pool=$GPU_POOL"
 [[ -n "$SYMBOL" ]]    && CMD="$CMD -p symbol=$SYMBOL"
 [[ -n "$CAPITAL" ]]   && CMD="$CMD -p initial-capital=$CAPITAL"
+[[ "$SANITIZER" != "none" ]] && CMD="$CMD -p sanitizer=$SANITIZER"
 
 $BASELINE && CMD="$CMD -p hyperopt-trials=0"
 $WATCH    && CMD="$CMD --watch"