737c2c1305
Scaleway BSSD PVCs (cargo-target-*, sccache-*, feature-cache-pvc,
bin-cache, training-data-pvc, test-data-pvc, platform service PVCs) are
single-zone-locked at create time. The cluster's node pools in main.tf
set `region` but no `zone`, so each autoscale event picks a zone
arbitrarily. When the autoscaler spins up a node in a zone that doesn't
match an existing PVC, scheduling fails with
"1 node(s) didn't match PersistentVolume's node affinity"
until the autoscaler eventually retries in the right zone. We hit this
on the HM pool creation (fixed manually with zone=fr-par-2) and again
on this commit's ensure-binary autoscale.
Track 1 (this commit): add `topology.kubernetes.io/zone: fr-par-2` to
every k8s.scaleway.com/pool-name nodeSelector entry across the 11 argo
workflow templates (34 entries total). The Kubernetes scheduler AND
cluster-autoscaler both honor topology keys when deciding placement /
provisioning — so future autoscale events will only spin up fr-par-2
nodes, and PVC binding is guaranteed.
Track 2 (future, structural): add `zone = "${var.region}-2"` to each
scaleway_k8s_pool in infra/modules/kapsule/main.tf so the pools never
provision in any other zone. Requires terraform-state cleanup (the
GitLab http backend currently has no states; the HM pool was created
out-of-band) and drain/recreate of any existing mixed-zone nodes —
deferred.
Verification: pool=N / paired=N coverage report shows 1:1 pool-name
to topology.kubernetes.io/zone entries in every file. kubectl apply -f
of all 11 templates returned "configured" for each.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
319 lines
12 KiB
YAML
319 lines
12 KiB
YAML
# infra/k8s/argo/nsys-test-template.yaml
|
|
#
|
|
# One-shot Nsight Systems profiling run on L40S. Wraps a smoke test under
|
|
# `nsys profile` and uploads the .nsys-rep to MinIO at
|
|
# foxhunt-training-results/profiles/smoke/<short-sha>/.
|
|
#
|
|
# Mirrors the nsys integration in train-multi-seed-template (Plan 5 Task 3,
|
|
# A.4.1) but for smoke tests. Use this for rapid kernel-timing iteration
|
|
# (per-step bottleneck hunts, SOL %, occupancy diffs) without the full
|
|
# multi-seed Argo cycle.
|
|
#
|
|
# Why L40S not local: laptop RTX 3050 Ti has 4 GB VRAM; nsys on L40S (48 GB)
|
|
# avoids any contention and runs at near-native speed (typical ~5-10%
|
|
# slowdown vs uninstrumented).
|
|
#
|
|
# Usage:
|
|
# argo submit --watch -n foxhunt nsys-test-template.yaml \
|
|
# -p commit-ref=main \
|
|
# -p test-name=performance::test_real_data_single_epoch
|
|
#
|
|
# Or via the wrapper script: scripts/argo-nsys.sh
|
|
|
|
apiVersion: argoproj.io/v1alpha1
|
|
kind: WorkflowTemplate
|
|
metadata:
|
|
name: nsys-test
|
|
namespace: foxhunt
|
|
labels:
|
|
app.kubernetes.io/name: nsys-test
|
|
app.kubernetes.io/part-of: foxhunt
|
|
spec:
|
|
entrypoint: nsys-run
|
|
serviceAccountName: argo-workflow
|
|
podMetadata:
|
|
labels:
|
|
app.kubernetes.io/part-of: foxhunt
|
|
app.kubernetes.io/component: nsys-test
|
|
archiveLogs: true
|
|
podGC:
|
|
strategy: OnPodCompletion
|
|
ttlStrategy:
|
|
secondsAfterCompletion: 7200
|
|
activeDeadlineSeconds: 3600 # nsys overhead is small — 1h cap is generous
|
|
arguments:
|
|
parameters:
|
|
- name: commit-ref
|
|
value: HEAD
|
|
- name: gpu-pool
|
|
value: ci-training-l40s
|
|
- name: cuda-compute-cap
|
|
value: "89"
|
|
- name: test-name
|
|
value: "performance::test_real_data_single_epoch"
|
|
# nsys capture options. Defaults capture the full kernel timeline.
|
|
# GPU metrics counters (--gpu-metrics-devices) require elevated CUDA
|
|
# performance-counter privileges (NVGPUCTRPERM) which the container
|
|
# lacks; CUDA/NVTX/osrt traces alone still give per-kernel timing and
|
|
# call-graph context — sufficient for the bottleneck-hunt use case.
|
|
# Override via --extra-args if running on a node with relaxed perms.
|
|
- name: nsys-extra-args
|
|
value: "--trace=cuda,nvtx,osrt"
|
|
|
|
volumes:
|
|
- name: git-ssh-key
|
|
secret:
|
|
secretName: argo-git-ssh-key
|
|
defaultMode: 256
|
|
- name: cargo-target
|
|
persistentVolumeClaim:
|
|
claimName: cargo-target-cuda-test
|
|
# Use the same training-data-pvc as production training (full 27 months
|
|
# of OHLCV + MBP-10 + trades) instead of the smaller test-data-pvc.
|
|
- name: training-data
|
|
persistentVolumeClaim:
|
|
claimName: training-data-pvc
|
|
readOnly: true
|
|
|
|
templates:
|
|
- name: nsys-run
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}"
|
|
topology.kubernetes.io/zone: fr-par-2
|
|
tolerations:
|
|
- key: nvidia.com/gpu
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
- key: node.cilium.io/agent-not-ready
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
container:
|
|
image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest
|
|
command: ["/bin/bash", "-c"]
|
|
env:
|
|
- name: SQLX_OFFLINE
|
|
value: "true"
|
|
- name: CARGO_TERM_COLOR
|
|
value: always
|
|
- name: CARGO_TARGET_DIR
|
|
value: /cargo-target
|
|
- name: CARGO_HOME
|
|
value: /cargo-target/cargo-home
|
|
- name: CUDA_VISIBLE_DEVICES
|
|
value: "0"
|
|
- name: CUBLAS_WORKSPACE_CONFIG
|
|
value: ":4096:8"
|
|
- name: FOXHUNT_TEST_DATA
|
|
value: /data/futures-baseline
|
|
- name: TEST_DATA_DIR
|
|
value: /data/futures-baseline
|
|
# MBP-10 + trades are mandatory per feedback_mbp10_mandatory.md
|
|
# (OFI features are part of the production state vector).
|
|
- name: FOXHUNT_MBP10_DATA
|
|
value: /data/futures-baseline-mbp10
|
|
- name: FOXHUNT_TRADES_DATA
|
|
value: /data/futures-baseline-trades
|
|
- name: RUST_LOG
|
|
value: info
|
|
- name: LD_LIBRARY_PATH
|
|
value: /usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64
|
|
# MinIO upload — same secret + optional pattern as train-multi-seed.
|
|
# Both refs are `optional: true` so env mount succeeds on clusters
|
|
# that do not pre-create `minio-credentials` (upload then warn-fails).
|
|
- name: MINIO_ACCESS_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-credentials
|
|
key: access-key
|
|
optional: true
|
|
- name: MINIO_SECRET_KEY
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: minio-credentials
|
|
key: secret-key
|
|
optional: true
|
|
volumeMounts:
|
|
- name: git-ssh-key
|
|
mountPath: /etc/git-ssh
|
|
readOnly: true
|
|
- name: cargo-target
|
|
mountPath: /cargo-target
|
|
- name: training-data
|
|
mountPath: /data
|
|
readOnly: true
|
|
resources:
|
|
requests:
|
|
nvidia.com/gpu: "1"
|
|
cpu: "4"
|
|
memory: 32Gi
|
|
limits:
|
|
nvidia.com/gpu: "1"
|
|
cpu: "8"
|
|
memory: 64Gi
|
|
args:
|
|
- |
|
|
set -euo pipefail
|
|
|
|
REF="{{workflow.parameters.commit-ref}}"
|
|
TEST_NAME="{{workflow.parameters.test-name}}"
|
|
EXTRA_ARGS="{{workflow.parameters.nsys-extra-args}}"
|
|
|
|
echo "==================================="
|
|
echo " nsys profile L40S run"
|
|
echo "==================================="
|
|
echo " Ref: $REF"
|
|
echo " Test: $TEST_NAME"
|
|
echo " Extra: $EXTRA_ARGS"
|
|
echo " Pool: {{workflow.parameters.gpu-pool}}"
|
|
echo "==================================="
|
|
|
|
# --- SSH setup ---
|
|
mkdir -p ~/.ssh
|
|
cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519
|
|
chmod 600 ~/.ssh/id_ed25519
|
|
printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config
|
|
chmod 600 ~/.ssh/config
|
|
|
|
REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
|
|
BUILD="/cargo-target/src"
|
|
git config --global --add safe.directory "$BUILD"
|
|
|
|
# --- Persistent checkout on PVC ---
|
|
if [ -d "$BUILD/.git" ]; then
|
|
cd "$BUILD"
|
|
git fetch origin
|
|
if git rev-parse --verify "origin/$REF" >/dev/null 2>&1; then
|
|
TARGET=$(git rev-parse "origin/$REF")
|
|
else
|
|
TARGET=$(git rev-parse --verify "$REF" 2>/dev/null || echo "$REF")
|
|
fi
|
|
CURRENT=$(git rev-parse HEAD 2>/dev/null || echo "none")
|
|
if [ "$CURRENT" != "$TARGET" ]; then
|
|
echo "=== Updating checkout to $REF ($TARGET) ==="
|
|
git checkout --force --detach "$TARGET"
|
|
git clean -fd
|
|
fi
|
|
else
|
|
git clone --filter=blob:none "$REPO" "$BUILD"
|
|
cd "$BUILD"
|
|
git fetch origin
|
|
if git rev-parse --verify "origin/$REF" >/dev/null 2>&1; then
|
|
git checkout --force --detach "origin/$REF"
|
|
else
|
|
git checkout "$REF"
|
|
fi
|
|
fi
|
|
SHA=$(git rev-parse --short HEAD)
|
|
echo "Checked out $SHA"
|
|
|
|
export PATH="${CARGO_HOME}/bin:/usr/local/cuda/bin:${PATH}"
|
|
export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}}
|
|
|
|
bash scripts/ptx-cache-invalidate.sh "${CARGO_TARGET_DIR}/.ptx_cache"
|
|
|
|
# --- Verify nsys ---
|
|
NSYS_BIN=$(which nsys 2>/dev/null || echo /usr/local/cuda/bin/nsys)
|
|
if [ ! -x "$NSYS_BIN" ]; then
|
|
echo "ERROR: nsys not found at $NSYS_BIN"
|
|
exit 2
|
|
fi
|
|
echo "=== nsys version ==="
|
|
"$NSYS_BIN" --version | head -3
|
|
|
|
# --- Compile test binary + train_baseline_rl ---
|
|
echo "=== Compiling (--release --no-run + train_baseline_rl) ==="
|
|
cargo test -p ml --release --lib --no-run 2>&1 | tee /cargo-target/nsys-compile.log
|
|
cargo build -p ml --release --example train_baseline_rl 2>&1 | tee -a /cargo-target/nsys-compile.log
|
|
|
|
# Honour CARGO_TARGET_DIR — outputs go to ${CARGO_TARGET_DIR}/release/deps,
|
|
# NOT the repo-relative target/. (CI workflow sets CARGO_TARGET_DIR=/cargo-target.)
|
|
# Subshell + || true wraps the head -1 SIGPIPE so pipefail doesn't fire.
|
|
TARGET_DIR="${CARGO_TARGET_DIR:-target}"
|
|
TEST_BIN=$( (ls -t "$TARGET_DIR/release/deps/ml-"* 2>/dev/null \
|
|
| grep -vE '\.(d|rcgu|rmeta|rlib|so|json)$' \
|
|
| head -1) || true )
|
|
if [ -z "$TEST_BIN" ] || [ ! -x "$TEST_BIN" ]; then
|
|
echo "ERROR: could not locate compiled test binary in $TARGET_DIR/release/deps/"
|
|
ls -la "$TARGET_DIR/release/deps/" 2>&1 | head -20
|
|
exit 3
|
|
fi
|
|
echo "=== Test binary: $TEST_BIN ==="
|
|
|
|
# grep -q + pipefail = SIGPIPE on early exit. Capture once, grep
|
|
# without -q so the consumer reads all input.
|
|
TEST_LIST=$("$TEST_BIN" --list 2>&1 || true)
|
|
if ! echo "$TEST_LIST" | grep -F "$TEST_NAME" > /dev/null; then
|
|
echo "ERROR: test '$TEST_NAME' not found"
|
|
echo "$TEST_LIST" | grep smoke_tests | (head -30 || true)
|
|
exit 4
|
|
fi
|
|
|
|
# --- Run under nsys profile ---
|
|
mkdir -p /tmp/nsys-output
|
|
POD_NAME="${HOSTNAME:-pod}"
|
|
NSYS_OUT="/tmp/nsys-output/profile-${POD_NAME}.nsys-rep"
|
|
|
|
echo ""
|
|
echo "==================================="
|
|
echo " Launching nsys profile..."
|
|
echo "==================================="
|
|
START_TS=$(date +%s)
|
|
set +e
|
|
"$NSYS_BIN" profile \
|
|
--output="$NSYS_OUT" \
|
|
--force-overwrite=true \
|
|
--stats=true \
|
|
${EXTRA_ARGS} \
|
|
-- "$TEST_BIN" "$TEST_NAME" --ignored --nocapture
|
|
EXIT_CODE=$?
|
|
set -e
|
|
END_TS=$(date +%s)
|
|
ELAPSED=$((END_TS - START_TS))
|
|
|
|
echo ""
|
|
echo "==================================="
|
|
echo " nsys run complete"
|
|
echo "==================================="
|
|
echo " Exit code: $EXIT_CODE"
|
|
echo " Elapsed: ${ELAPSED}s"
|
|
ls -la "$NSYS_OUT" 2>/dev/null && \
|
|
echo " Profile: $(du -h "$NSYS_OUT" | cut -f1)"
|
|
echo ""
|
|
|
|
# --- Upload to MinIO (same pattern as train-multi-seed) ---
|
|
if [ -f "$NSYS_OUT" ]; then
|
|
echo "=== Uploading nsys profile to MinIO ==="
|
|
MC_BIN=$(which mc 2>/dev/null || echo "")
|
|
if [ -z "$MC_BIN" ]; then
|
|
MC_BIN=/tmp/mc
|
|
curl -fsSL https://dl.min.io/client/mc/release/linux-amd64/mc -o "$MC_BIN" || {
|
|
echo "WARN: failed to download mc — skipping upload"
|
|
echo "Profile available locally at: $NSYS_OUT (PVC: cargo-target)"
|
|
exit 0
|
|
}
|
|
chmod +x "$MC_BIN"
|
|
fi
|
|
"$MC_BIN" alias set foxhunt http://minio.foxhunt.svc.cluster.local:9000 \
|
|
"${MINIO_ACCESS_KEY:-}" "${MINIO_SECRET_KEY:-}" 2>/dev/null || true
|
|
# Smoke profiles go to a separate prefix to keep the
|
|
# production training profile bucket clean.
|
|
UPLOAD_PATH="foxhunt/foxhunt-training-results/profiles/smoke/${SHA}/profile-${POD_NAME}.nsys-rep"
|
|
if "$MC_BIN" cp "$NSYS_OUT" "$UPLOAD_PATH"; then
|
|
echo "=== nsys profile uploaded to ${UPLOAD_PATH} ==="
|
|
echo ""
|
|
echo "Download with:"
|
|
echo " mc cp ${UPLOAD_PATH} ./profile.nsys-rep"
|
|
echo "Open with: nsys-ui profile.nsys-rep"
|
|
else
|
|
echo "WARN: nsys upload failed — profile remains on PVC at $NSYS_OUT"
|
|
fi
|
|
fi
|
|
|
|
if [ "$EXIT_CODE" -ne 0 ]; then
|
|
echo ""
|
|
echo "FAIL: test exited with code $EXIT_CODE"
|
|
exit "$EXIT_CODE"
|
|
fi
|
|
echo ""
|
|
echo "PASS: nsys profile captured, test passed."
|