diff --git a/infra/k8s/argo/smoke-test-template.yaml b/infra/k8s/argo/smoke-test-template.yaml index 22a323fd7..96c36a2ef 100644 --- a/infra/k8s/argo/smoke-test-template.yaml +++ b/infra/k8s/argo/smoke-test-template.yaml @@ -51,6 +51,15 @@ spec: value: "89" - name: test-name value: "multi_fold_convergence::test_multi_fold_convergence" + # Wipe `/cargo-target/release` for the ml + ml-dqn crates before + # compile so the smoke binary is built from a known-clean state. The + # PVC's persistent target dir accumulates rmeta/object artefacts + # across probes; file deletions (e.g., `regime_conditional.rs` in + # ff00af68a) can leave dangling references that perturb downstream + # codegen between bisect runs. Defaults to "false" — only set when + # bisecting suspected build-cache contamination. + - name: clean-cache + value: "false" volumes: - name: git-ssh-key @@ -186,6 +195,15 @@ spec: bash scripts/ptx-cache-invalidate.sh "${CARGO_TARGET_DIR}/.ptx_cache" + # --- Optional clean: ml + ml-dqn before compile --- + CLEAN_CACHE="{{workflow.parameters.clean-cache}}" + if [ "$CLEAN_CACHE" = "true" ]; then + echo "=== clean-cache=true — wiping ml + ml-dqn build artefacts ===" + cargo clean -p ml --release 2>&1 | tee /cargo-target/smoke-clean.log + cargo clean -p ml-dqn --release 2>&1 | tee -a /cargo-target/smoke-clean.log + echo "=== Cleaned. Forced fresh compile of ml/ml-dqn (deps stay cached) ===" + fi + # --- Compile test binary + train_baseline_rl --- echo "=== Compiling (--release --no-run + train_baseline_rl) ===" cargo test -p ml --release --lib --no-run 2>&1 | tee /cargo-target/smoke-compile.log diff --git a/scripts/argo-smoke.sh b/scripts/argo-smoke.sh index bbfecc417..bedae47ae 100755 --- a/scripts/argo-smoke.sh +++ b/scripts/argo-smoke.sh @@ -25,6 +25,7 @@ TEST_NAME="multi_fold_convergence::test_multi_fold_convergence" COMMIT_REF="HEAD" GPU_POOL="" WATCH=false +CLEAN_CACHE="false" usage() { cat < Git ref to test (default: HEAD) --gpu-pool GPU node pool (default: ci-training-l40s) + --clean-cache Wipe ml + ml-dqn artefacts before compile + (force fresh compile; deps stay cached). Use to + rule out build-cache contamination during bisect. --watch Follow workflow logs -h, --help Show this help EOF @@ -55,6 +59,7 @@ while [[ $# -gt 0 ]]; do --multi-fold) TEST_NAME="multi_fold_convergence::test_multi_fold_convergence"; shift ;; --ref) COMMIT_REF="$2"; shift 2 ;; --gpu-pool) GPU_POOL="$2"; shift 2 ;; + --clean-cache) CLEAN_CACHE="true"; shift ;; --watch) WATCH=true; shift ;; -h|--help) usage ;; *) echo "Unknown option: $1"; usage ;; @@ -68,6 +73,7 @@ fi CMD="argo submit -n foxhunt --from=wftmpl/smoke-test" CMD="$CMD -p commit-ref=$COMMIT_REF" CMD="$CMD -p test-name=$TEST_NAME" +CMD="$CMD -p clean-cache=$CLEAN_CACHE" [[ -n "$GPU_POOL" ]] && CMD="$CMD -p gpu-pool=$GPU_POOL" if $WATCH; then @@ -77,5 +83,6 @@ fi echo "Submitting smoke workflow..." echo " commit: $(echo "$COMMIT_REF" | cut -c1-8)" echo " test: $TEST_NAME" +[[ "$CLEAN_CACHE" = "true" ]] && echo " clean: ml + ml-dqn target wipe before compile" echo "" eval "$CMD"