From 4d1d8ffa25a887be8b63017fd4bd5686716938ec Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 28 Apr 2026 21:57:32 +0200 Subject: [PATCH] infra(argo): add sanitizer-test + nsys-test workflow templates for L40S smoke validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two new one-shot WorkflowTemplates for validating smoke tests under GPU-instrumentation tools that don't fit in the laptop's 4 GB VRAM: - `sanitizer-test`: wraps cargo test smoke under `compute-sanitizer --tool=memcheck|racecheck|synccheck|initcheck` with --target-processes all so multi_fold_convergence's spawned `train_baseline_rl` subprocess is also instrumented. Pre-builds train_baseline_rl example to avoid sanitizer instrumenting cargo/rustc on the inner spawn. Triages internal-vs-real errors and exits non-zero only on real bugs. - `nsys-test`: wraps cargo test smoke under `nsys profile`. Captures CUDA + NVTX + osrt traces with GPU metrics (ga10x set). Uploads .nsys-rep to MinIO at foxhunt-training-artifacts/profiles/smoke//, mirroring the existing Plan 5 Task 3 production-training profile pattern in train-multi-seed-template. Wrapper scripts: - scripts/argo-sanitizer.sh — `argo submit` wrapper, supports --multi-fold shortcut for fold-boundary code paths (IQN sync, aux Adam reset, iqn_readiness reset, MSE clamp) that single-fold tests cannot exercise. - scripts/argo-nsys.sh — same shape as argo-sanitizer.sh, default test is performance::test_real_data_single_epoch for broad coverage. Why L40S: laptop RTX 3050 Ti's 4 GB cannot fit compute-sanitizer's instrumentation metadata (~2-3x app VRAM) — sanitizer falls back to "didn't track the launch" with 60k+ internal-allocation errors. L40S 48 GB has ample headroom for both memcheck and nsys overhead. Both templates compile cargo test --release --lib --no-run plus cargo build --release --example train_baseline_rl on the cargo-target PVC. Compile time dominated by sccache hit rate (production training image: 100% C/C++ cache, ~75% Rust cache after warmup). Templates registered in kustomization.yaml — apply with `kubectl apply -k infra/k8s/argo` before first submit. --- infra/k8s/argo/kustomization.yaml | 2 + infra/k8s/argo/nsys-test-template.yaml | 299 ++++++++++++++++++++ infra/k8s/argo/sanitizer-test-template.yaml | 294 +++++++++++++++++++ scripts/argo-nsys.sh | 91 ++++++ scripts/argo-sanitizer.sh | 104 +++++++ 5 files changed, 790 insertions(+) create mode 100644 infra/k8s/argo/nsys-test-template.yaml create mode 100644 infra/k8s/argo/sanitizer-test-template.yaml create mode 100755 scripts/argo-nsys.sh create mode 100755 scripts/argo-sanitizer.sh diff --git a/infra/k8s/argo/kustomization.yaml b/infra/k8s/argo/kustomization.yaml index 57eb8269c..b289f4e2a 100644 --- a/infra/k8s/argo/kustomization.yaml +++ b/infra/k8s/argo/kustomization.yaml @@ -6,6 +6,8 @@ resources: - compile-and-deploy-template.yaml - train-template.yaml - build-ci-image-template.yaml + - sanitizer-test-template.yaml + - nsys-test-template.yaml - argo-workflow-netpol.yaml - ci-deploy-rbac.yaml - archive-rbac.yaml diff --git a/infra/k8s/argo/nsys-test-template.yaml b/infra/k8s/argo/nsys-test-template.yaml new file mode 100644 index 000000000..4689d1a59 --- /dev/null +++ b/infra/k8s/argo/nsys-test-template.yaml @@ -0,0 +1,299 @@ +# infra/k8s/argo/nsys-test-template.yaml +# +# One-shot Nsight Systems profiling run on L40S. Wraps a smoke test under +# `nsys profile` and uploads the .nsys-rep to MinIO at +# foxhunt-training-artifacts/profiles/smoke//. +# +# Mirrors the nsys integration in train-multi-seed-template (Plan 5 Task 3, +# A.4.1) but for smoke tests. Use this for rapid kernel-timing iteration +# (per-step bottleneck hunts, SOL %, occupancy diffs) without the full +# multi-seed Argo cycle. +# +# Why L40S not local: laptop RTX 3050 Ti has 4 GB VRAM; nsys on L40S (48 GB) +# avoids any contention and runs at near-native speed (typical ~5-10% +# slowdown vs uninstrumented). +# +# Usage: +# argo submit --watch -n foxhunt nsys-test-template.yaml \ +# -p commit-ref=main \ +# -p test-name=performance::test_real_data_single_epoch +# +# Or via the wrapper script: scripts/argo-nsys.sh + +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: nsys-test + namespace: foxhunt + labels: + app.kubernetes.io/name: nsys-test + app.kubernetes.io/part-of: foxhunt +spec: + entrypoint: nsys-run + serviceAccountName: argo-workflow + podMetadata: + labels: + app.kubernetes.io/part-of: foxhunt + app.kubernetes.io/component: nsys-test + archiveLogs: true + podGC: + strategy: OnPodCompletion + ttlStrategy: + secondsAfterCompletion: 7200 + activeDeadlineSeconds: 3600 # nsys overhead is small — 1h cap is generous + arguments: + parameters: + - name: commit-ref + value: HEAD + - name: gpu-pool + value: ci-training-l40s + - name: cuda-compute-cap + value: "89" + - name: test-name + value: performance::test_real_data_single_epoch + # nsys capture options. Defaults capture the full kernel timeline. + # For range-based capture, use `--capture-range=cudaProfilerApi` and + # call cudaProfilerStart/Stop in the test (advanced). + - name: nsys-extra-args + value: "--trace=cuda,nvtx,osrt --gpu-metrics-devices=0 --gpu-metrics-set=ga10x" + + volumes: + - name: git-ssh-key + secret: + secretName: argo-git-ssh-key + defaultMode: 256 + - name: cargo-target + persistentVolumeClaim: + claimName: cargo-target-cuda-test + - name: test-data + persistentVolumeClaim: + claimName: test-data-pvc + readOnly: true + + templates: + - name: nsys-run + nodeSelector: + k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + container: + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest + command: ["/bin/bash", "-c"] + env: + - name: SQLX_OFFLINE + value: "true" + - name: CARGO_TERM_COLOR + value: always + - name: CARGO_TARGET_DIR + value: /cargo-target + - name: CARGO_HOME + value: /cargo-target/cargo-home + - name: CUDA_VISIBLE_DEVICES + value: "0" + - name: CUBLAS_WORKSPACE_CONFIG + value: ":4096:8" + - name: FOXHUNT_TEST_DATA + value: /data/test-data + - name: TEST_DATA_DIR + value: /data/test-data + - name: RUST_LOG + value: info + - name: LD_LIBRARY_PATH + value: /usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64 + # MinIO upload — same secret + optional pattern as train-multi-seed. + # Both refs are `optional: true` so env mount succeeds on clusters + # that do not pre-create `minio-credentials` (upload then warn-fails). + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: access-key + optional: true + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-credentials + key: secret-key + optional: true + volumeMounts: + - name: git-ssh-key + mountPath: /etc/git-ssh + readOnly: true + - name: cargo-target + mountPath: /cargo-target + - name: test-data + mountPath: /data/test-data + readOnly: true + resources: + requests: + nvidia.com/gpu: "1" + cpu: "4" + memory: 32Gi + limits: + nvidia.com/gpu: "1" + cpu: "8" + memory: 64Gi + args: + - | + set -euo pipefail + + REF="{{workflow.parameters.commit-ref}}" + TEST_NAME="{{workflow.parameters.test-name}}" + EXTRA_ARGS="{{workflow.parameters.nsys-extra-args}}" + + echo "===================================" + echo " nsys profile L40S run" + echo "===================================" + echo " Ref: $REF" + echo " Test: $TEST_NAME" + echo " Extra: $EXTRA_ARGS" + echo " Pool: {{workflow.parameters.gpu-pool}}" + echo "===================================" + + # --- SSH setup --- + mkdir -p ~/.ssh + cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config + chmod 600 ~/.ssh/config + + REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git" + BUILD="/cargo-target/src" + git config --global --add safe.directory "$BUILD" + + # --- Persistent checkout on PVC --- + if [ -d "$BUILD/.git" ]; then + cd "$BUILD" + git fetch origin + if git rev-parse --verify "origin/$REF" >/dev/null 2>&1; then + TARGET=$(git rev-parse "origin/$REF") + else + TARGET=$(git rev-parse --verify "$REF" 2>/dev/null || echo "$REF") + fi + CURRENT=$(git rev-parse HEAD 2>/dev/null || echo "none") + if [ "$CURRENT" != "$TARGET" ]; then + echo "=== Updating checkout to $REF ($TARGET) ===" + git checkout --force --detach "$TARGET" + git clean -fd + fi + else + git clone --filter=blob:none "$REPO" "$BUILD" + cd "$BUILD" + git fetch origin + if git rev-parse --verify "origin/$REF" >/dev/null 2>&1; then + git checkout --force --detach "origin/$REF" + else + git checkout "$REF" + fi + fi + SHA=$(git rev-parse --short HEAD) + echo "Checked out $SHA" + + export PATH="${CARGO_HOME}/bin:/usr/local/cuda/bin:${PATH}" + export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}} + + bash scripts/ptx-cache-invalidate.sh "${CARGO_TARGET_DIR}/.ptx_cache" + + # --- Verify nsys --- + NSYS_BIN=$(which nsys 2>/dev/null || echo /usr/local/cuda/bin/nsys) + if [ ! -x "$NSYS_BIN" ]; then + echo "ERROR: nsys not found at $NSYS_BIN" + exit 2 + fi + echo "=== nsys version ===" + "$NSYS_BIN" --version | head -3 + + # --- Compile test binary + train_baseline_rl --- + echo "=== Compiling (--release --no-run + train_baseline_rl) ===" + cargo test -p ml --release --lib --no-run 2>&1 | tee /cargo-target/nsys-compile.log + cargo build -p ml --release --example train_baseline_rl 2>&1 | tee -a /cargo-target/nsys-compile.log + + TEST_BIN=$(ls -t target/release/deps/ml-* 2>/dev/null \ + | grep -vE '\.(d|rcgu|rmeta|rlib|so|json)$' \ + | head -1) + if [ -z "$TEST_BIN" ] || [ ! -x "$TEST_BIN" ]; then + echo "ERROR: could not locate compiled test binary" + ls -la target/release/deps/ | head -20 + exit 3 + fi + echo "=== Test binary: $TEST_BIN ===" + + if ! "$TEST_BIN" --list 2>&1 | grep -q "$TEST_NAME"; then + echo "ERROR: test '$TEST_NAME' not found" + "$TEST_BIN" --list 2>&1 | grep smoke_tests | head -30 + exit 4 + fi + + # --- Run under nsys profile --- + mkdir -p /tmp/nsys-output + POD_NAME="${HOSTNAME:-pod}" + NSYS_OUT="/tmp/nsys-output/profile-${POD_NAME}.nsys-rep" + + echo "" + echo "===================================" + echo " Launching nsys profile..." + echo "===================================" + START_TS=$(date +%s) + set +e + "$NSYS_BIN" profile \ + --output="$NSYS_OUT" \ + --force-overwrite=true \ + --stats=true \ + ${EXTRA_ARGS} \ + -- "$TEST_BIN" "$TEST_NAME" --ignored --nocapture + EXIT_CODE=$? + set -e + END_TS=$(date +%s) + ELAPSED=$((END_TS - START_TS)) + + echo "" + echo "===================================" + echo " nsys run complete" + echo "===================================" + echo " Exit code: $EXIT_CODE" + echo " Elapsed: ${ELAPSED}s" + ls -la "$NSYS_OUT" 2>/dev/null && \ + echo " Profile: $(du -h "$NSYS_OUT" | cut -f1)" + echo "" + + # --- Upload to MinIO (same pattern as train-multi-seed) --- + if [ -f "$NSYS_OUT" ]; then + echo "=== Uploading nsys profile to MinIO ===" + MC_BIN=$(which mc 2>/dev/null || echo "") + if [ -z "$MC_BIN" ]; then + MC_BIN=/tmp/mc + curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o "$MC_BIN" || { + echo "WARN: failed to download mc — skipping upload" + echo "Profile available locally at: $NSYS_OUT (PVC: cargo-target)" + exit 0 + } + chmod +x "$MC_BIN" + fi + "$MC_BIN" alias set foxhunt http://minio.foxhunt.svc.cluster.local:9000 \ + "${MINIO_ACCESS_KEY:-}" "${MINIO_SECRET_KEY:-}" 2>/dev/null || true + # Smoke profiles go to a separate prefix to keep the + # production training profile bucket clean. + UPLOAD_PATH="foxhunt/foxhunt-training-artifacts/profiles/smoke/${SHA}/profile-${POD_NAME}.nsys-rep" + if "$MC_BIN" cp "$NSYS_OUT" "$UPLOAD_PATH"; then + echo "=== nsys profile uploaded to ${UPLOAD_PATH} ===" + echo "" + echo "Download with:" + echo " mc cp ${UPLOAD_PATH} ./profile.nsys-rep" + echo "Open with: nsys-ui profile.nsys-rep" + else + echo "WARN: nsys upload failed — profile remains on PVC at $NSYS_OUT" + fi + fi + + if [ "$EXIT_CODE" -ne 0 ]; then + echo "" + echo "FAIL: test exited with code $EXIT_CODE" + exit "$EXIT_CODE" + fi + echo "" + echo "PASS: nsys profile captured, test passed." diff --git a/infra/k8s/argo/sanitizer-test-template.yaml b/infra/k8s/argo/sanitizer-test-template.yaml new file mode 100644 index 000000000..85aae1537 --- /dev/null +++ b/infra/k8s/argo/sanitizer-test-template.yaml @@ -0,0 +1,294 @@ +# infra/k8s/argo/sanitizer-test-template.yaml +# +# One-shot compute-sanitizer run on L40S. Wraps a smoke test under +# `compute-sanitizer --tool memcheck` to validate no CUDA memory errors +# (out-of-bounds, leaks, sync violations, race conditions). +# +# Why L40S not local: the laptop RTX 3050 Ti (4 GB VRAM) cannot fit +# compute-sanitizer's instrumentation metadata alongside the production +# training workload — the sanitizer falls back to "didn't track the +# launch" with 60k+ internal-allocation errors. L40S (48 GB) has ample +# headroom for memcheck's 2-3× shadow-memory overhead. +# +# Usage: +# argo submit --watch -n foxhunt sanitizer-test-template.yaml \ +# -p commit-ref=main \ +# -p test-name=iqn_quantile_monotonicity::iqn_multi_quantile_heads_produce_monotonic_estimates \ +# -p sanitizer-tool=memcheck +# +# Or via the wrapper script: scripts/argo-sanitizer.sh + +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: sanitizer-test + namespace: foxhunt + labels: + app.kubernetes.io/name: sanitizer-test + app.kubernetes.io/part-of: foxhunt +spec: + entrypoint: sanitizer-run + serviceAccountName: argo-workflow + podMetadata: + labels: + app.kubernetes.io/part-of: foxhunt + app.kubernetes.io/component: sanitizer-test + archiveLogs: true + podGC: + strategy: OnPodCompletion + ttlStrategy: + secondsAfterCompletion: 7200 + activeDeadlineSeconds: 7200 # 2h cap — sanitizer is slow but bounded + arguments: + parameters: + - name: commit-ref + value: HEAD + - name: gpu-pool + value: ci-training-l40s + - name: cuda-compute-cap + value: "89" # L40S Ada Lovelace + - name: test-name + value: iqn_quantile_monotonicity::iqn_multi_quantile_heads_produce_monotonic_estimates + - name: sanitizer-tool + value: memcheck # memcheck | racecheck | synccheck | initcheck + - name: sanitizer-extra-args + value: "" + + volumes: + - name: git-ssh-key + secret: + secretName: argo-git-ssh-key + defaultMode: 256 + - name: cargo-target + persistentVolumeClaim: + claimName: cargo-target-cuda-test + - name: test-data + persistentVolumeClaim: + claimName: test-data-pvc + readOnly: true + + templates: + - name: sanitizer-run + nodeSelector: + k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + container: + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest + command: ["/bin/bash", "-c"] + env: + - name: SQLX_OFFLINE + value: "true" + - name: CARGO_TERM_COLOR + value: always + - name: CARGO_TARGET_DIR + value: /cargo-target + - name: CARGO_HOME + value: /cargo-target/cargo-home + - name: CUDA_VISIBLE_DEVICES + value: "0" + - name: CUBLAS_WORKSPACE_CONFIG + value: ":4096:8" + - name: FOXHUNT_TEST_DATA + value: /data/test-data + - name: TEST_DATA_DIR + value: /data/test-data + - name: RUST_LOG + value: info + - name: LD_LIBRARY_PATH + value: /usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64 + volumeMounts: + - name: git-ssh-key + mountPath: /etc/git-ssh + readOnly: true + - name: cargo-target + mountPath: /cargo-target + - name: test-data + mountPath: /data/test-data + readOnly: true + resources: + requests: + nvidia.com/gpu: "1" + cpu: "4" + memory: 32Gi + limits: + nvidia.com/gpu: "1" + cpu: "8" + memory: 64Gi + args: + - | + set -euo pipefail + + REF="{{workflow.parameters.commit-ref}}" + TEST_NAME="{{workflow.parameters.test-name}}" + SANITIZER_TOOL="{{workflow.parameters.sanitizer-tool}}" + EXTRA_ARGS="{{workflow.parameters.sanitizer-extra-args}}" + + echo "===================================" + echo " compute-sanitizer L40S run" + echo "===================================" + echo " Ref: $REF" + echo " Test: $TEST_NAME" + echo " Tool: $SANITIZER_TOOL" + echo " Pool: {{workflow.parameters.gpu-pool}}" + echo " Compute: {{workflow.parameters.cuda-compute-cap}}" + echo "===================================" + + # --- SSH setup (same as gpu-test-pipeline) --- + mkdir -p ~/.ssh + cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config + chmod 600 ~/.ssh/config + + REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git" + BUILD="/cargo-target/src" + git config --global --add safe.directory "$BUILD" + + # --- Persistent checkout on PVC --- + if [ -d "$BUILD/.git" ]; then + cd "$BUILD" + git fetch origin + if git rev-parse --verify "origin/$REF" >/dev/null 2>&1; then + TARGET=$(git rev-parse "origin/$REF") + else + TARGET=$(git rev-parse --verify "$REF" 2>/dev/null || echo "$REF") + fi + CURRENT=$(git rev-parse HEAD 2>/dev/null || echo "none") + if [ "$CURRENT" != "$TARGET" ]; then + echo "=== Updating checkout to $REF ($TARGET) ===" + git checkout --force --detach "$TARGET" + git clean -fd + fi + else + echo "=== Initial clone ===" + git clone --filter=blob:none "$REPO" "$BUILD" + cd "$BUILD" + git fetch origin + if git rev-parse --verify "origin/$REF" >/dev/null 2>&1; then + git checkout --force --detach "origin/$REF" + else + git checkout "$REF" + fi + fi + echo "Checked out $(git rev-parse --short HEAD)" + + export PATH="${CARGO_HOME}/bin:/usr/local/cuda/bin:${PATH}" + export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}} + + # --- PTX cache invalidation --- + bash scripts/ptx-cache-invalidate.sh "${CARGO_TARGET_DIR}/.ptx_cache" + + # --- Verify compute-sanitizer is available --- + if ! command -v compute-sanitizer >/dev/null 2>&1; then + echo "ERROR: compute-sanitizer not found in PATH. CUDA toolkit incomplete." + exit 2 + fi + echo "=== compute-sanitizer version ===" + compute-sanitizer --version | head -3 + + # --- Compile test binary + train_baseline_rl example --- + # train_baseline_rl is needed for multi_fold_convergence test which + # spawns it as a subprocess via `cargo run --example`. Pre-building + # avoids cargo doing it inside the sanitizer-instrumented run. + echo "=== Compiling test binary + train_baseline_rl (--release) ===" + cargo test -p ml --release --lib --no-run 2>&1 | tee /cargo-target/sanitizer-compile.log + cargo build -p ml --release --example train_baseline_rl 2>&1 | tee -a /cargo-target/sanitizer-compile.log + + # Locate the freshest ml-* test binary + TEST_BIN=$(ls -t target/release/deps/ml-* 2>/dev/null \ + | grep -vE '\.(d|rcgu|rmeta|rlib|so|json)$' \ + | head -1) + if [ -z "$TEST_BIN" ] || [ ! -x "$TEST_BIN" ]; then + echo "ERROR: could not locate compiled test binary in target/release/deps/" + ls -la target/release/deps/ | head -20 + exit 3 + fi + echo "=== Test binary: $TEST_BIN ===" + + # --- Verify the test exists in the binary --- + if ! "$TEST_BIN" --list 2>&1 | grep -q "$TEST_NAME"; then + echo "ERROR: test '$TEST_NAME' not found in binary." + echo "Available smoke tests:" + "$TEST_BIN" --list 2>&1 | grep smoke_tests | head -30 + exit 4 + fi + + # --- Run under compute-sanitizer --- + mkdir -p /tmp/sanitizer-output + LOG_FILE=/tmp/sanitizer-output/${SANITIZER_TOOL}.log + + echo "" + echo "===================================" + echo " Launching compute-sanitizer..." + echo "===================================" + START_TS=$(date +%s) + # --target-processes all so multi_fold_convergence's spawned + # `train_baseline_rl` subprocess is also instrumented (the test + # itself does no GPU work — the child binary does). + set +e + compute-sanitizer \ + --tool "$SANITIZER_TOOL" \ + --target-processes all \ + --launch-timeout 1200 \ + --error-exitcode 99 \ + --print-limit 200 \ + --log-file "$LOG_FILE" \ + ${EXTRA_ARGS} \ + -- "$TEST_BIN" "$TEST_NAME" --ignored --nocapture + EXIT_CODE=$? + set -e + END_TS=$(date +%s) + ELAPSED=$((END_TS - START_TS)) + + echo "" + echo "===================================" + echo " Sanitizer run complete" + echo "===================================" + echo " Exit code: $EXIT_CODE" + echo " Elapsed: ${ELAPSED}s" + echo " Log: $LOG_FILE ($(wc -l <"$LOG_FILE") lines)" + echo "" + + # --- Triage report --- + echo "=== Sanitizer error summary ===" + grep -E "ERROR SUMMARY|Internal Sanitizer Error" "$LOG_FILE" | tail -5 || echo " (no summary line)" + echo "" + + # Categorise errors. compute-sanitizer prints "ERROR SUMMARY: N errors" at the + # end. "Internal Sanitizer Error" is sanitizer-internal (allocation failure, + # tracking gap) — NOT a real bug in the application code. + INTERNAL_ERRORS=$(grep -c "Internal Sanitizer Error" "$LOG_FILE" || true) + REAL_ERROR_LINE=$(grep "ERROR SUMMARY:" "$LOG_FILE" | tail -1 || echo "") + REAL_ERRORS=$(echo "$REAL_ERROR_LINE" | grep -oE "[0-9]+ errors" | head -1 | grep -oE "[0-9]+" || echo "0") + + echo "=== Triage ===" + echo " Internal Sanitizer Errors (instrumentation gaps): $INTERNAL_ERRORS" + echo " Reported errors total: $REAL_ERRORS" + echo " Real (non-internal) errors: $((REAL_ERRORS - INTERNAL_ERRORS))" + echo "" + + # Show first 50 real (non-internal) errors with context + echo "=== First non-internal error excerpts ===" + grep -vE "Internal Sanitizer Error|^=========\s*$" "$LOG_FILE" \ + | grep -E "^=========" \ + | head -50 || echo " (none)" + + # Final pass/fail + if [ "$EXIT_CODE" -eq 99 ]; then + echo "" + echo "FAIL: compute-sanitizer detected real memory errors (--error-exitcode 99 fired)." + exit 1 + elif [ "$EXIT_CODE" -ne 0 ]; then + echo "" + echo "FAIL: test process exited with code $EXIT_CODE (test failure or sanitizer abort)." + exit "$EXIT_CODE" + else + echo "" + echo "PASS: zero real memory errors detected, test passed." + fi diff --git a/scripts/argo-nsys.sh b/scripts/argo-nsys.sh new file mode 100755 index 000000000..f6295cb10 --- /dev/null +++ b/scripts/argo-nsys.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Run a smoke test under Nsight Systems profiling on L40S via Argo Workflows. +# +# Output: .nsys-rep uploaded to MinIO at +# foxhunt-training-artifacts/profiles/smoke//profile-.nsys-rep +# +# Download + open: +# mc cp foxhunt/foxhunt-training-artifacts/profiles/smoke//profile.nsys-rep . +# nsys-ui profile.nsys-rep +# +# Why L40S not local: the laptop GPU has no nsys-ui counters bound and +# limited VRAM. L40S has all the counters and ample headroom; nsys overhead +# is typically ~5-10% so runtime stays close to native. +# +# Usage: +# ./scripts/argo-nsys.sh # performance::test_real_data_single_epoch +# ./scripts/argo-nsys.sh --test magnitude_distribution # different test +# ./scripts/argo-nsys.sh --multi-fold # exercise fold-boundary code +# ./scripts/argo-nsys.sh --watch +# +# Requires: argo CLI + +set -euo pipefail + +TEST_NAME="performance::test_real_data_single_epoch" +COMMIT_REF="HEAD" +GPU_POOL="" +EXTRA_ARGS="" +WATCH=false + +usage() { + cat < Smoke test name (default: $TEST_NAME) + Common values: + performance::test_real_data_single_epoch + iqn_quantile_monotonicity::iqn_multi_quantile_heads_produce_monotonic_estimates + multi_fold_convergence::test_multi_fold_convergence + magnitude_distribution::test_magnitude_distribution + --multi-fold Shortcut: --test multi_fold_convergence::test_multi_fold_convergence + Exercises fold-boundary code paths (IQN sync, aux Adam reset). + --ref Git ref to test (default: HEAD) + --gpu-pool GPU node pool (default: ci-training-l40s) + --extra-args Pass-through args to nsys profile. + Default: "--trace=cuda,nvtx,osrt --gpu-metrics-devices=0 + --gpu-metrics-set=ga10x" + For range capture: "--capture-range=cudaProfilerApi" + For more detail: "--cuda-memory-usage=true --cudabacktrace=all" + --watch Follow workflow logs + -h, --help Show this help +EOF + exit 0 +} + +while [[ $# -gt 0 ]]; do + case $1 in + --test) TEST_NAME="$2"; shift 2 ;; + --multi-fold) TEST_NAME="multi_fold_convergence::test_multi_fold_convergence"; shift ;; + --ref) COMMIT_REF="$2"; shift 2 ;; + --gpu-pool) GPU_POOL="$2"; shift 2 ;; + --extra-args) EXTRA_ARGS="$2"; shift 2 ;; + --watch) WATCH=true; shift ;; + -h|--help) usage ;; + *) echo "Unknown option: $1"; usage ;; + esac +done + +if [ "$COMMIT_REF" = "HEAD" ]; then + COMMIT_REF=$(git rev-parse HEAD) +fi + +CMD="argo submit -n foxhunt --from=wftmpl/nsys-test" +CMD="$CMD -p commit-ref=$COMMIT_REF" +CMD="$CMD -p test-name=$TEST_NAME" +[[ -n "$GPU_POOL" ]] && CMD="$CMD -p gpu-pool=$GPU_POOL" +[[ -n "$EXTRA_ARGS" ]] && CMD="$CMD -p nsys-extra-args=\"$EXTRA_ARGS\"" + +if $WATCH; then + CMD="$CMD --watch" +fi + +echo "Submitting nsys workflow..." +echo " commit: $(echo "$COMMIT_REF" | cut -c1-8)" +echo " test: $TEST_NAME" +[[ -n "$EXTRA_ARGS" ]] && echo " extra: $EXTRA_ARGS" +echo "" +eval "$CMD" diff --git a/scripts/argo-sanitizer.sh b/scripts/argo-sanitizer.sh new file mode 100755 index 000000000..92cded60b --- /dev/null +++ b/scripts/argo-sanitizer.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# Run a smoke test under compute-sanitizer on L40S via Argo Workflows. +# +# Why L40S not local: 4 GB laptop GPU can't fit memcheck instrumentation +# overhead alongside production training kernels (sanitizer falls back to +# "didn't track the launch"). L40S 48 GB has ample headroom. +# +# Usage: +# ./scripts/argo-sanitizer.sh # iqn_quantile_monotonicity, memcheck +# ./scripts/argo-sanitizer.sh --test magnitude_distribution # different test +# ./scripts/argo-sanitizer.sh --tool racecheck # different sanitizer tool +# ./scripts/argo-sanitizer.sh --watch # follow logs +# ./scripts/argo-sanitizer.sh --ref main --tool memcheck --watch +# +# Tool options: memcheck (OOB/leaks), racecheck (shared-mem races), +# synccheck (sync API misuse), initcheck (uninitialised reads) +# +# Requires: argo CLI + +set -euo pipefail + +TEST_NAME="iqn_quantile_monotonicity::iqn_multi_quantile_heads_produce_monotonic_estimates" +SANITIZER_TOOL="memcheck" +COMMIT_REF="HEAD" +GPU_POOL="" +EXTRA_ARGS="" +WATCH=false + +usage() { + cat < Smoke test name (default: $TEST_NAME) + Common values: + iqn_quantile_monotonicity::iqn_multi_quantile_heads_produce_monotonic_estimates + multi_fold_convergence::test_multi_fold_convergence + magnitude_distribution::test_magnitude_distribution + performance::test_real_data_single_epoch + --multi-fold Shortcut: --test multi_fold_convergence::test_multi_fold_convergence + Exercises fold-boundary code paths (IQN sync, aux Adam reset, + iqn_readiness reset, MSE clamp) that single-fold tests cannot. + Walk-forward needs ≥10 months of data — uses CI test-data PVC. + --tool Sanitizer tool (default: memcheck) + memcheck | racecheck | synccheck | initcheck + --ref Git ref to test (default: HEAD) + --gpu-pool GPU node pool (default: ci-training-l40s) + --extra-args Pass-through args to compute-sanitizer + e.g. "--leak-check full" for memcheck + --watch Follow workflow logs + -h, --help Show this help +EOF + exit 0 +} + +while [[ $# -gt 0 ]]; do + case $1 in + --test) TEST_NAME="$2"; shift 2 ;; + --multi-fold) TEST_NAME="multi_fold_convergence::test_multi_fold_convergence"; shift ;; + --tool) SANITIZER_TOOL="$2"; shift 2 ;; + --ref) COMMIT_REF="$2"; shift 2 ;; + --gpu-pool) GPU_POOL="$2"; shift 2 ;; + --extra-args) EXTRA_ARGS="$2"; shift 2 ;; + --watch) WATCH=true; shift ;; + -h|--help) usage ;; + *) echo "Unknown option: $1"; usage ;; + esac +done + +# Resolve HEAD to actual SHA so the workflow targets a stable commit +if [ "$COMMIT_REF" = "HEAD" ]; then + COMMIT_REF=$(git rev-parse HEAD) +fi + +# Validate sanitizer tool +case "$SANITIZER_TOOL" in + memcheck|racecheck|synccheck|initcheck) ;; + *) + echo "ERROR: invalid --tool '$SANITIZER_TOOL'" + echo "Allowed: memcheck | racecheck | synccheck | initcheck" + exit 1 + ;; +esac + +CMD="argo submit -n foxhunt --from=wftmpl/sanitizer-test" +CMD="$CMD -p commit-ref=$COMMIT_REF" +CMD="$CMD -p test-name=$TEST_NAME" +CMD="$CMD -p sanitizer-tool=$SANITIZER_TOOL" +[[ -n "$GPU_POOL" ]] && CMD="$CMD -p gpu-pool=$GPU_POOL" +[[ -n "$EXTRA_ARGS" ]] && CMD="$CMD -p sanitizer-extra-args=\"$EXTRA_ARGS\"" + +if $WATCH; then + CMD="$CMD --watch" +fi + +echo "Submitting compute-sanitizer workflow..." +echo " commit: $(echo "$COMMIT_REF" | cut -c1-8)" +echo " test: $TEST_NAME" +echo " tool: $SANITIZER_TOOL" +[[ -n "$EXTRA_ARGS" ]] && echo " extra: $EXTRA_ARGS" +echo "" +eval "$CMD"