From 27f536ba6e5f0c387435a557b63d3e7db5e4ddf1 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 29 Apr 2026 07:52:11 +0200 Subject: [PATCH] infra(argo): plain smoke-test template + scripts/argo-smoke.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a no-wrapper L40S smoke runner alongside the nsys/sanitizer templates. Same compile + checkout + data-mount layout, no profiler binary, no sanitizer instrumentation, no MinIO artefact upload — just a plain test execution against the full training-data PVC. Use case: validate a smoke test passes on the real 27-month dataset when local data is too short (laptop only has 1 quarter of MBP-10/ trades, fxcache truncates below the 10-month minimum). - infra/k8s/argo/smoke-test-template.yaml: WorkflowTemplate smoke-test, entrypoint smoke-run, default test multi_fold_convergence::test_multi_fold_convergence. - infra/k8s/argo/kustomization.yaml: register the new template. - infra/k8s/argo/argo-workflow-netpol.yaml: extend the sanitizer/nsys NetworkPolicy podSelector to include smoke-test (identical egress requirements: git fetch, ci-builder pull, training-data PVC). - scripts/argo-smoke.sh: thin wrapper mirroring argo-nsys.sh / argo-sanitizer.sh (--multi-fold default, --test, --ref, --watch). Verified: kustomize dry-run clean, kubectl apply -k creates the template + reconfigures the netpol live in foxhunt namespace. --- infra/k8s/argo/argo-workflow-netpol.yaml | 3 +- infra/k8s/argo/kustomization.yaml | 1 + infra/k8s/argo/smoke-test-template.yaml | 246 +++++++++++++++++++++++ scripts/argo-smoke.sh | 81 ++++++++ 4 files changed, 330 insertions(+), 1 deletion(-) create mode 100644 infra/k8s/argo/smoke-test-template.yaml create mode 100755 scripts/argo-smoke.sh diff --git a/infra/k8s/argo/argo-workflow-netpol.yaml b/infra/k8s/argo/argo-workflow-netpol.yaml index 24493a63e..eb249659a 100644 --- a/infra/k8s/argo/argo-workflow-netpol.yaml +++ b/infra/k8s/argo/argo-workflow-netpol.yaml @@ -439,7 +439,7 @@ spec: matchLabels: app.kubernetes.io/name: mattermost --- -# ── sanitizer-test + nsys-test workflows: same egress as ci-pipeline ── +# ── sanitizer-test + nsys-test + smoke-test workflows: same egress as ci-pipeline ── # (git fetch over SSH, MinIO log archive + nsys-rep upload, registry image pull, # external HTTPS for mc download) apiVersion: networking.k8s.io/v1 @@ -455,6 +455,7 @@ spec: values: - sanitizer-test - nsys-test + - smoke-test policyTypes: - Egress egress: diff --git a/infra/k8s/argo/kustomization.yaml b/infra/k8s/argo/kustomization.yaml index b289f4e2a..67eb75321 100644 --- a/infra/k8s/argo/kustomization.yaml +++ b/infra/k8s/argo/kustomization.yaml @@ -8,6 +8,7 @@ resources: - build-ci-image-template.yaml - sanitizer-test-template.yaml - nsys-test-template.yaml + - smoke-test-template.yaml - argo-workflow-netpol.yaml - ci-deploy-rbac.yaml - archive-rbac.yaml diff --git a/infra/k8s/argo/smoke-test-template.yaml b/infra/k8s/argo/smoke-test-template.yaml new file mode 100644 index 000000000..22a323fd7 --- /dev/null +++ b/infra/k8s/argo/smoke-test-template.yaml @@ -0,0 +1,246 @@ +# infra/k8s/argo/smoke-test-template.yaml +# +# Plain smoke-test runner on L40S — no compute-sanitizer, no nsys profile, +# no MinIO artefact upload. Mirrors the compile + checkout + GPU + data-mount +# layout of nsys-test-template.yaml but executes the test binary directly. +# +# Use this for fast multi-fold / single-epoch validation runs against the +# full training-data PVC (27 months) when local laptop data is too short +# (laptop only has 1 quarter of MBP-10/trades). +# +# Why a separate template: the nsys/sanitizer wrappers add overhead and +# extract artefacts not needed for a straight "did the test pass on the +# real dataset" gate. Keeping the smoke variant lean keeps the iteration +# cycle short (no .nsys-rep upload, no sanitizer instrumentation slowdown). +# +# Usage: +# argo submit --watch -n foxhunt --from=wftmpl/smoke-test \ +# -p commit-ref=main \ +# -p test-name=multi_fold_convergence::test_multi_fold_convergence +# +# Or via the wrapper script: scripts/argo-smoke.sh + +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: smoke-test + namespace: foxhunt + labels: + app.kubernetes.io/name: smoke-test + app.kubernetes.io/part-of: foxhunt +spec: + entrypoint: smoke-run + serviceAccountName: argo-workflow + podMetadata: + labels: + app.kubernetes.io/part-of: foxhunt + app.kubernetes.io/component: smoke-test + archiveLogs: true + podGC: + strategy: OnPodCompletion + ttlStrategy: + secondsAfterCompletion: 7200 + activeDeadlineSeconds: 3600 + arguments: + parameters: + - name: commit-ref + value: HEAD + - name: gpu-pool + value: ci-training-l40s + - name: cuda-compute-cap + value: "89" + - name: test-name + value: "multi_fold_convergence::test_multi_fold_convergence" + + volumes: + - name: git-ssh-key + secret: + secretName: argo-git-ssh-key + defaultMode: 256 + - name: cargo-target + persistentVolumeClaim: + claimName: cargo-target-cuda-test + # Same training-data-pvc as production training (full 27 months of + # OHLCV + MBP-10 + trades) — laptop's 1-quarter MBP-10 truncates the + # fxcache below the smoke's 10-month minimum. + - name: training-data + persistentVolumeClaim: + claimName: training-data-pvc + readOnly: true + + templates: + - name: smoke-run + nodeSelector: + k8s.scaleway.com/pool-name: "{{workflow.parameters.gpu-pool}}" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + container: + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder:latest + command: ["/bin/bash", "-c"] + env: + - name: SQLX_OFFLINE + value: "true" + - name: CARGO_TERM_COLOR + value: always + - name: CARGO_TARGET_DIR + value: /cargo-target + - name: CARGO_HOME + value: /cargo-target/cargo-home + - name: CUDA_VISIBLE_DEVICES + value: "0" + - name: CUBLAS_WORKSPACE_CONFIG + value: ":4096:8" + - name: FOXHUNT_TEST_DATA + value: /data/futures-baseline + - name: TEST_DATA_DIR + value: /data/futures-baseline + # MBP-10 + trades are mandatory per feedback_mbp10_mandatory.md + # (OFI features are part of the production state vector). + - name: FOXHUNT_MBP10_DATA + value: /data/futures-baseline-mbp10 + - name: FOXHUNT_TRADES_DATA + value: /data/futures-baseline-trades + - name: RUST_LOG + value: info + - name: LD_LIBRARY_PATH + value: /usr/local/nvidia/lib64:/usr/local/cuda/lib64 + volumeMounts: + - name: git-ssh-key + mountPath: /etc/git-ssh + readOnly: true + - name: cargo-target + mountPath: /cargo-target + - name: training-data + mountPath: /data + readOnly: true + resources: + requests: + nvidia.com/gpu: "1" + cpu: "4" + memory: 32Gi + limits: + nvidia.com/gpu: "1" + cpu: "8" + memory: 64Gi + args: + - | + set -euo pipefail + + REF="{{workflow.parameters.commit-ref}}" + TEST_NAME="{{workflow.parameters.test-name}}" + + echo "===================================" + echo " Plain smoke L40S run" + echo "===================================" + echo " Ref: $REF" + echo " Test: $TEST_NAME" + echo " Pool: {{workflow.parameters.gpu-pool}}" + echo "===================================" + + # --- SSH setup --- + mkdir -p ~/.ssh + cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > ~/.ssh/config + chmod 600 ~/.ssh/config + + REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git" + BUILD="/cargo-target/src" + git config --global --add safe.directory "$BUILD" + + # --- Persistent checkout on PVC --- + if [ -d "$BUILD/.git" ]; then + cd "$BUILD" + git fetch origin + if git rev-parse --verify "origin/$REF" >/dev/null 2>&1; then + TARGET=$(git rev-parse "origin/$REF") + else + TARGET=$(git rev-parse --verify "$REF" 2>/dev/null || echo "$REF") + fi + CURRENT=$(git rev-parse HEAD 2>/dev/null || echo "none") + if [ "$CURRENT" != "$TARGET" ]; then + echo "=== Updating checkout to $REF ($TARGET) ===" + git checkout --force --detach "$TARGET" + git clean -fd + fi + else + git clone --filter=blob:none "$REPO" "$BUILD" + cd "$BUILD" + git fetch origin + if git rev-parse --verify "origin/$REF" >/dev/null 2>&1; then + git checkout --force --detach "origin/$REF" + else + git checkout "$REF" + fi + fi + SHA=$(git rev-parse --short HEAD) + echo "Checked out $SHA" + + export PATH="${CARGO_HOME}/bin:/usr/local/cuda/bin:${PATH}" + export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}} + + bash scripts/ptx-cache-invalidate.sh "${CARGO_TARGET_DIR}/.ptx_cache" + + # --- Compile test binary + train_baseline_rl --- + echo "=== Compiling (--release --no-run + train_baseline_rl) ===" + cargo test -p ml --release --lib --no-run 2>&1 | tee /cargo-target/smoke-compile.log + cargo build -p ml --release --example train_baseline_rl 2>&1 | tee -a /cargo-target/smoke-compile.log + + # Honour CARGO_TARGET_DIR — outputs go to ${CARGO_TARGET_DIR}/release/deps, + # NOT the repo-relative target/. Subshell + || true wraps the head -1 + # SIGPIPE so pipefail doesn't fire. + TARGET_DIR="${CARGO_TARGET_DIR:-target}" + TEST_BIN=$( (ls -t "$TARGET_DIR/release/deps/ml-"* 2>/dev/null \ + | grep -vE '\.(d|rcgu|rmeta|rlib|so|json)$' \ + | head -1) || true ) + if [ -z "$TEST_BIN" ] || [ ! -x "$TEST_BIN" ]; then + echo "ERROR: could not locate compiled test binary in $TARGET_DIR/release/deps/" + ls -la "$TARGET_DIR/release/deps/" 2>&1 | head -20 + exit 3 + fi + echo "=== Test binary: $TEST_BIN ===" + + # grep -q + pipefail = SIGPIPE on early exit. Capture once, grep + # without -q so the consumer reads all input. + TEST_LIST=$("$TEST_BIN" --list 2>&1 || true) + if ! echo "$TEST_LIST" | grep -F "$TEST_NAME" > /dev/null; then + echo "ERROR: test '$TEST_NAME' not found" + echo "$TEST_LIST" | grep smoke_tests | (head -30 || true) + exit 4 + fi + + # --- Run the test --- + echo "" + echo "===================================" + echo " Launching test..." + echo "===================================" + START_TS=$(date +%s) + set +e + "$TEST_BIN" "$TEST_NAME" --ignored --nocapture + EXIT_CODE=$? + set -e + END_TS=$(date +%s) + ELAPSED=$((END_TS - START_TS)) + + echo "" + echo "===================================" + echo " Smoke run complete" + echo "===================================" + echo " Exit code: $EXIT_CODE" + echo " Elapsed: ${ELAPSED}s" + echo " Commit: $SHA" + echo " Test: $TEST_NAME" + echo "===================================" + + if [ "$EXIT_CODE" -ne 0 ]; then + echo "" + echo "FAIL: test exited with code $EXIT_CODE" + exit "$EXIT_CODE" + fi + echo "" + echo "PASS: smoke completed." diff --git a/scripts/argo-smoke.sh b/scripts/argo-smoke.sh new file mode 100755 index 000000000..bbfecc417 --- /dev/null +++ b/scripts/argo-smoke.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Run a smoke test on L40S via Argo Workflows — no profiler, no sanitizer. +# +# Use this when you want to validate that a smoke test passes against the +# full training-data PVC (27 months of OHLCV + MBP-10 + trades) without +# nsys profile overhead or sanitizer instrumentation. Iteration cycle is +# faster than argo-nsys.sh / argo-sanitizer.sh because there's no artefact +# upload step. +# +# Why L40S not local: the laptop has only 1 quarter of MBP-10/trades data, +# which truncates the fxcache below the 10-month minimum that the +# multi_fold_convergence smoke needs. +# +# Usage: +# ./scripts/argo-smoke.sh # multi_fold_convergence (default) +# ./scripts/argo-smoke.sh --test magnitude_distribution # different test +# ./scripts/argo-smoke.sh --ref # specific commit +# ./scripts/argo-smoke.sh --watch # follow logs +# +# Requires: argo CLI + +set -euo pipefail + +TEST_NAME="multi_fold_convergence::test_multi_fold_convergence" +COMMIT_REF="HEAD" +GPU_POOL="" +WATCH=false + +usage() { + cat < Smoke test name (default: $TEST_NAME) + Common values: + multi_fold_convergence::test_multi_fold_convergence + magnitude_distribution::test_magnitude_distribution + performance::test_real_data_single_epoch + iqn_quantile_monotonicity::iqn_multi_quantile_heads_produce_monotonic_estimates + --multi-fold Shortcut: --test multi_fold_convergence::test_multi_fold_convergence + (default — exercises fold-boundary code paths) + --ref Git ref to test (default: HEAD) + --gpu-pool GPU node pool (default: ci-training-l40s) + --watch Follow workflow logs + -h, --help Show this help +EOF + exit 0 +} + +while [[ $# -gt 0 ]]; do + case $1 in + --test) TEST_NAME="$2"; shift 2 ;; + --multi-fold) TEST_NAME="multi_fold_convergence::test_multi_fold_convergence"; shift ;; + --ref) COMMIT_REF="$2"; shift 2 ;; + --gpu-pool) GPU_POOL="$2"; shift 2 ;; + --watch) WATCH=true; shift ;; + -h|--help) usage ;; + *) echo "Unknown option: $1"; usage ;; + esac +done + +if [ "$COMMIT_REF" = "HEAD" ]; then + COMMIT_REF=$(git rev-parse HEAD) +fi + +CMD="argo submit -n foxhunt --from=wftmpl/smoke-test" +CMD="$CMD -p commit-ref=$COMMIT_REF" +CMD="$CMD -p test-name=$TEST_NAME" +[[ -n "$GPU_POOL" ]] && CMD="$CMD -p gpu-pool=$GPU_POOL" + +if $WATCH; then + CMD="$CMD --watch" +fi + +echo "Submitting smoke workflow..." +echo " commit: $(echo "$COMMIT_REF" | cut -c1-8)" +echo " test: $TEST_NAME" +echo "" +eval "$CMD"