2606506cd8
- argo-train.sh: --profile flag forces multi-seed render path so the nsys wrapper + foxhunt-training-artifacts upload step are visible in --dry-run YAML without cluster contact (test surface). - train-multi-seed-template.yaml: new `profile` parameter (default "false") gates the per-(seed, fold) `nsys profile --capture-range=cudaProfilerApi` wrapper and the `mc cp` upload to foxhunt-training-artifacts/profiles/<sha>/. mc binary fetched on-demand (ci-builder image lacks it). MinIO creds optional — upload warn-skips if absent. - Dockerfile.foxhunt-training-runtime: install nsight-systems-cli unpinned (pinning the stale 2024.4.1.61-1 from earlier plans breaks builds when apt index advances). - minio.yaml: add foxhunt-training-artifacts bucket to minio-init. - compare-nsys-profiles.py: V0 regression detector — compares cuda_gpu_kern_sum total_ns / epoch_count between two profiles; exits 1 on >20% slowdown. NVTX per-epoch ranges deferred to T5. - tests/test_nsys_harness.sh: dry-run grep test — verifies both required strings appear when --profile is set, and that the default (no --profile) path keeps profile=false in the rendered template. - dqn-wire-up-audit.md: Plan 5 Task 3 row added documenting the harness + the baseline-capture deferral to T5. Backward compat: test_multi_seed_harness.sh from P5T1 still PASS.
60 lines
2.5 KiB
Bash
Executable File
60 lines
2.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Test: argo-train.sh --profile renders an Argo workflow that contains the
|
|
# `nsys profile` wrapper and the foxhunt-training-artifacts upload step.
|
|
#
|
|
# Validation surface for Plan 5 Task 3 (A.4.1) — nsys profile harness.
|
|
# Verifies the harness without running the workflow against a live cluster
|
|
# (the dry-run path renders the multi-seed template file directly when
|
|
# --profile is set).
|
|
set -euo pipefail
|
|
|
|
# Resolve repo root from this script's location so the test runs from any cwd.
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
|
cd "${REPO_ROOT}"
|
|
|
|
DRY_OUT=$(mktemp -t nsys-dry.XXXXXX.yaml)
|
|
trap 'rm -f "$DRY_OUT"' EXIT
|
|
|
|
# 1) --profile --multi-seed 1 --folds 1 --dry-run must render YAML containing:
|
|
# - the `nsys profile` wrapper string (training-side instrumentation)
|
|
# - the `foxhunt-training-artifacts` MinIO bucket reference (artefact upload)
|
|
./scripts/argo-train.sh dqn --profile --multi-seed 1 --folds 1 --dry-run \
|
|
> "$DRY_OUT" 2>&1
|
|
|
|
if ! grep -q "nsys profile" "$DRY_OUT"; then
|
|
echo "FAIL: --profile flag does not produce nsys wrapper in rendered YAML"
|
|
echo "--- dry-run output (first 80 lines) ---"
|
|
head -n 80 "$DRY_OUT"
|
|
exit 1
|
|
fi
|
|
echo "PASS: --profile renders nsys profile wrapper"
|
|
|
|
if ! grep -q "foxhunt-training-artifacts" "$DRY_OUT"; then
|
|
echo "FAIL: --profile flag does not produce foxhunt-training-artifacts upload step"
|
|
echo "--- dry-run output (first 80 lines) ---"
|
|
head -n 80 "$DRY_OUT"
|
|
exit 1
|
|
fi
|
|
echo "PASS: --profile renders foxhunt-training-artifacts upload step"
|
|
|
|
# 2) Backward-compat: a default-flag run (no --profile) must NOT contain the
|
|
# nsys wrapper or artefact bucket — these are gated by the new flag only.
|
|
DRY_OUT_NO_PROF=$(mktemp -t nsys-dry-noprof.XXXXXX.yaml)
|
|
trap 'rm -f "$DRY_OUT" "$DRY_OUT_NO_PROF"' EXIT
|
|
|
|
./scripts/argo-train.sh dqn --multi-seed 2 --folds 1 --dry-run \
|
|
> "$DRY_OUT_NO_PROF" 2>&1
|
|
|
|
# The template DOES contain the literal `profile` parameter declaration with
|
|
# default value "false" (template-level conditional gate). Assert that the
|
|
# rendered workflow defaults to profile=false so a production no-profile run
|
|
# does not accidentally trigger nsys + upload.
|
|
if ! grep -q 'name: profile' "$DRY_OUT_NO_PROF"; then
|
|
echo "FAIL: profile parameter not declared in multi-seed template"
|
|
exit 1
|
|
fi
|
|
echo "PASS: multi-seed template declares profile parameter (default off)"
|
|
|
|
echo "ALL PASS: nsys profile harness wired correctly"
|