From 7e2eb708a7fc3a7cfa2f37d47a047ce94d20123a Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Wed, 29 Apr 2026 11:04:18 +0200
Subject: [PATCH] infra(argo): smoke-test --clean-cache parameter for
 build-cache-isolated bisect
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The L40S smoke template uses CARGO_TARGET_DIR=/cargo-target on a
persistent PVC, so every smoke probe builds incrementally on top of
artefacts left by prior probes. File deletions (e.g., regime_conditional.rs
in ff00af68a) can leave dangling rmeta/object references that perturb
downstream codegen between bisect runs — making the bisect result
contingent on which order probes were submitted in, not on the source.

Adds an optional `clean-cache` parameter (default `false`) which runs
`cargo clean -p ml -p ml-dqn --release` before the compile step. Other
crate artefacts (ml-core, ml-supervised, etc.) stay cached so the wipe
is bounded — fresh ml/ml-dqn compile in ~3-5 min vs ~30+ min full clean.

Use case: re-running a52d99613 + ff00af68a with `--clean-cache` to
verify the bisect under controlled build conditions. If the
broken/clean status flips with clean cache, the regression is
build-state-dependent rather than source-line; if it reproduces, the
source regression is real and bisect is definitive.

scripts/argo-smoke.sh exposes `--clean-cache` flag passing through
to the workflow parameter.
---
 infra/k8s/argo/smoke-test-template.yaml | 18 ++++++++++++++++++
 scripts/argo-smoke.sh                   |  7 +++++++
 2 files changed, 25 insertions(+)

diff --git a/infra/k8s/argo/smoke-test-template.yaml b/infra/k8s/argo/smoke-test-template.yaml
index 22a323fd7..96c36a2ef 100644
--- a/infra/k8s/argo/smoke-test-template.yaml
+++ b/infra/k8s/argo/smoke-test-template.yaml
@@ -51,6 +51,15 @@ spec:
         value: "89"
       - name: test-name
         value: "multi_fold_convergence::test_multi_fold_convergence"
+      # Wipe `/cargo-target/release` for the ml + ml-dqn crates before
+      # compile so the smoke binary is built from a known-clean state. The
+      # PVC's persistent target dir accumulates rmeta/object artefacts
+      # across probes; file deletions (e.g., `regime_conditional.rs` in
+      # ff00af68a) can leave dangling references that perturb downstream
+      # codegen between bisect runs. Defaults to "false" — only set when
+      # bisecting suspected build-cache contamination.
+      - name: clean-cache
+        value: "false"
 
   volumes:
     - name: git-ssh-key
@@ -186,6 +195,15 @@ spec:
 
             bash scripts/ptx-cache-invalidate.sh "${CARGO_TARGET_DIR}/.ptx_cache"
 
+            # --- Optional clean: ml + ml-dqn before compile ---
+            CLEAN_CACHE="{{workflow.parameters.clean-cache}}"
+            if [ "$CLEAN_CACHE" = "true" ]; then
+              echo "=== clean-cache=true — wiping ml + ml-dqn build artefacts ==="
+              cargo clean -p ml --release 2>&1 | tee /cargo-target/smoke-clean.log
+              cargo clean -p ml-dqn --release 2>&1 | tee -a /cargo-target/smoke-clean.log
+              echo "=== Cleaned. Forced fresh compile of ml/ml-dqn (deps stay cached) ==="
+            fi
+
             # --- Compile test binary + train_baseline_rl ---
             echo "=== Compiling (--release --no-run + train_baseline_rl) ==="
             cargo test -p ml --release --lib --no-run 2>&1 | tee /cargo-target/smoke-compile.log
diff --git a/scripts/argo-smoke.sh b/scripts/argo-smoke.sh
index bbfecc417..bedae47ae 100755
--- a/scripts/argo-smoke.sh
+++ b/scripts/argo-smoke.sh
@@ -25,6 +25,7 @@ TEST_NAME="multi_fold_convergence::test_multi_fold_convergence"
 COMMIT_REF="HEAD"
 GPU_POOL=""
 WATCH=false
+CLEAN_CACHE="false"
 
 usage() {
   cat <<EOF
@@ -43,6 +44,9 @@ Options:
                        (default — exercises fold-boundary code paths)
   --ref <ref>          Git ref to test (default: HEAD)
   --gpu-pool <pool>    GPU node pool (default: ci-training-l40s)
+  --clean-cache        Wipe ml + ml-dqn artefacts before compile
+                       (force fresh compile; deps stay cached). Use to
+                       rule out build-cache contamination during bisect.
   --watch              Follow workflow logs
   -h, --help           Show this help
 EOF
@@ -55,6 +59,7 @@ while [[ $# -gt 0 ]]; do
     --multi-fold)  TEST_NAME="multi_fold_convergence::test_multi_fold_convergence"; shift ;;
     --ref)         COMMIT_REF="$2"; shift 2 ;;
     --gpu-pool)    GPU_POOL="$2"; shift 2 ;;
+    --clean-cache) CLEAN_CACHE="true"; shift ;;
     --watch)       WATCH=true; shift ;;
     -h|--help)     usage ;;
     *)             echo "Unknown option: $1"; usage ;;
@@ -68,6 +73,7 @@ fi
 CMD="argo submit -n foxhunt --from=wftmpl/smoke-test"
 CMD="$CMD -p commit-ref=$COMMIT_REF"
 CMD="$CMD -p test-name=$TEST_NAME"
+CMD="$CMD -p clean-cache=$CLEAN_CACHE"
 [[ -n "$GPU_POOL" ]] && CMD="$CMD -p gpu-pool=$GPU_POOL"
 
 if $WATCH; then
@@ -77,5 +83,6 @@ fi
 echo "Submitting smoke workflow..."
 echo "  commit: $(echo "$COMMIT_REF" | cut -c1-8)"
 echo "  test:   $TEST_NAME"
+[[ "$CLEAN_CACHE" = "true" ]] && echo "  clean:  ml + ml-dqn target wipe before compile"
 echo ""
 eval "$CMD"