diff --git a/infra/k8s/argo/train-template.yaml b/infra/k8s/argo/train-template.yaml index c9011d5dd..a3b451673 100644 --- a/infra/k8s/argo/train-template.yaml +++ b/infra/k8s/argo/train-template.yaml @@ -65,6 +65,8 @@ spec: value: "1.0" - name: min-hold-bars value: "5" + - name: sanitizer + value: "none" # "none", "memcheck", "racecheck", "synccheck" volumes: - name: git-ssh-key @@ -541,7 +543,21 @@ spec: echo "=== Training: $MODEL ({{workflow.parameters.train-epochs}} epochs) ===" - stdbuf -oL ${BINARY} \ + # compute-sanitizer: optional GPU debugging (10-100x slower) + SANITIZER="{{workflow.parameters.sanitizer}}" + SANITIZER_PREFIX="" + if [ "$SANITIZER" != "none" ] && [ -n "$SANITIZER" ]; then + SANITIZER_BIN=$(which compute-sanitizer 2>/dev/null || echo "/usr/local/cuda/bin/compute-sanitizer") + if [ -x "$SANITIZER_BIN" ]; then + SANITIZER_PREFIX="$SANITIZER_BIN --tool $SANITIZER --print-limit 20 --error-exitcode 1" + echo " compute-sanitizer enabled: --tool $SANITIZER" + echo " WARNING: 10-100x slower — use with 1-2 epochs only" + else + echo " WARNING: compute-sanitizer not found at $SANITIZER_BIN — running without" + fi + fi + + stdbuf -oL $SANITIZER_PREFIX ${BINARY} \ --model "$MODEL" \ --symbol {{workflow.parameters.symbol}} \ --tx-cost-bps {{workflow.parameters.tx-cost-bps}} \ diff --git a/scripts/argo-train.sh b/scripts/argo-train.sh index 22b5668dc..55cf04882 100755 --- a/scripts/argo-train.sh +++ b/scripts/argo-train.sh @@ -25,6 +25,7 @@ SYMBOL="" WATCH=false BASELINE=false CAPITAL="" +SANITIZER="none" usage() { cat < Trading symbol (default: ES.FUT) --capital Initial capital (default: 35000) --baseline Skip hyperopt (trials=0) + --sanitizer Run under compute-sanitizer (memcheck|racecheck|synccheck) --watch Follow workflow logs -h, --help Show this help EOF @@ -68,6 +70,7 @@ while [[ $# -gt 0 ]]; do --symbol) SYMBOL="$2"; shift 2 ;; --capital) CAPITAL="$2"; shift 2 ;; --baseline) BASELINE=true; shift ;; + --sanitizer) SANITIZER="$2"; shift 2 ;; --watch) WATCH=true; shift ;; -h|--help) usage ;; *) echo "Unknown option: $1"; usage ;; @@ -84,6 +87,7 @@ CMD="$CMD -p model=$MODEL" [[ -n "$GPU_POOL" ]] && CMD="$CMD -p gpu-pool=$GPU_POOL" [[ -n "$SYMBOL" ]] && CMD="$CMD -p symbol=$SYMBOL" [[ -n "$CAPITAL" ]] && CMD="$CMD -p initial-capital=$CAPITAL" +[[ "$SANITIZER" != "none" ]] && CMD="$CMD -p sanitizer=$SANITIZER" $BASELINE && CMD="$CMD -p hyperopt-trials=0" $WATCH && CMD="$CMD --watch"