From 04df20bdc3ec9a89519f50832f2afff490bf08a8 Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Sat, 2 May 2026 23:30:06 +0200
Subject: [PATCH] infra(argo): smoke populates ensure-binary cache for next
 train run
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The smoke template already compiled `train_baseline_rl`. Now also build
`evaluate_baseline` + `precompute_features` and copy + strip all 3 to
`/data/bin/$SHORT_SHA/` after PASS — exactly the layout that
train-multi-seed-template.yaml's `ensure-binary` cache check
(lines 235-246) looks up by SHA. A smoke-then-train sequence at the same
SHA now hits the cache and skips the ~6-min compile, exiting in the
~1-min pod-startup baseline.

- Drop readOnly:true on training-data PVC mount (cargo writes binaries
  into /data/bin/$SHORT_SHA; read-side fxcache + market data unchanged)
- Build all 3 example binaries in a single cargo invocation (sccache-warm)
- Idempotent SHA-keyed dest dir; PASS-only so broken bins never cache

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 infra/k8s/argo/smoke-test-template.yaml | 53 ++++++++++++++++++++++---
 1 file changed, 48 insertions(+), 5 deletions(-)

diff --git a/infra/k8s/argo/smoke-test-template.yaml b/infra/k8s/argo/smoke-test-template.yaml
index 96c36a2ef..4de3efa3c 100644
--- a/infra/k8s/argo/smoke-test-template.yaml
+++ b/infra/k8s/argo/smoke-test-template.yaml
@@ -72,10 +72,15 @@ spec:
     # Same training-data-pvc as production training (full 27 months of
     # OHLCV + MBP-10 + trades) — laptop's 1-quarter MBP-10 truncates the
     # fxcache below the smoke's 10-month minimum.
+    #
+    # RW (not readOnly) so the smoke can populate `/data/bin/$SHORT_SHA/`
+    # with the train binaries it just built, letting the next train run's
+    # `ensure-binary` cache check (train-multi-seed-template.yaml:235-246)
+    # hit and skip the 6-min compile. Read-side (fxcache + raw market data)
+    # is unchanged.
     - name: training-data
       persistentVolumeClaim:
         claimName: training-data-pvc
-        readOnly: true
 
   templates:
     - name: smoke-run
@@ -126,7 +131,9 @@ spec:
             mountPath: /cargo-target
           - name: training-data
             mountPath: /data
-            readOnly: true
+            # RW so the post-PASS cache-population step can write to
+            # /data/bin/$SHORT_SHA/. Read-side paths (fxcache, MBP-10,
+            # trades) are unaffected.
         resources:
           requests:
             nvidia.com/gpu: "1"
@@ -204,10 +211,20 @@ spec:
               echo "=== Cleaned. Forced fresh compile of ml/ml-dqn (deps stay cached) ==="
             fi
 
-            # --- Compile test binary + train_baseline_rl ---
-            echo "=== Compiling (--release --no-run + train_baseline_rl) ==="
+            # --- Compile test binary + the 3 train binaries ---
+            #
+            # The 3 examples mirror exactly what `ensure-binary` produces
+            # (see train-multi-seed-template.yaml line ~268). Compiling
+            # them here lets the post-PASS step populate the
+            # `/data/bin/$SHORT_SHA/` cache so the next train run at the
+            # same SHA skips its own ensure-binary compile.
+            echo "=== Compiling (--release --no-run + train binaries) ==="
             cargo test -p ml --release --lib --no-run 2>&1 | tee /cargo-target/smoke-compile.log
-            cargo build -p ml --release --example train_baseline_rl 2>&1 | tee -a /cargo-target/smoke-compile.log
+            cargo build -p ml --release \
+              --example train_baseline_rl \
+              --example evaluate_baseline \
+              --example precompute_features \
+              2>&1 | tee -a /cargo-target/smoke-compile.log
 
             # Honour CARGO_TARGET_DIR — outputs go to ${CARGO_TARGET_DIR}/release/deps,
             # NOT the repo-relative target/. Subshell + || true wraps the head -1
@@ -262,3 +279,29 @@ spec:
             fi
             echo ""
             echo "PASS: smoke completed."
+
+            # --- Populate /data/bin/$SHORT_SHA/ for ensure-binary cache ---
+            # On PASS only — broken binaries must not be cached. Mirrors
+            # train-multi-seed-template.yaml:270-274 exactly: same
+            # destination layout, same strip step. Idempotent: writes are
+            # to a SHA-keyed dir so concurrent smokes at different SHAs
+            # don't collide; same-SHA writes are deterministic compile
+            # output and overwrite-safe.
+            FULL_SHA=$(git rev-parse HEAD)
+            SHORT_SHA=$(echo "$FULL_SHA" | cut -c1-9)
+            BIN_DIR="/data/bin/$SHORT_SHA"
+            BINARIES="train_baseline_rl evaluate_baseline precompute_features"
+            echo ""
+            echo "=== Populating ensure-binary cache at $BIN_DIR ==="
+            mkdir -p "$BIN_DIR"
+            for bin in $BINARIES; do
+              src="${CARGO_TARGET_DIR}/release/examples/$bin"
+              if [ ! -x "$src" ]; then
+                echo "WARN: $src not found (cache pop skipped for $bin)"
+                continue
+              fi
+              cp "$src" "$BIN_DIR/"
+              strip "$BIN_DIR/$bin" 2>/dev/null || true
+            done
+            ls -lh "$BIN_DIR/"
+            echo "=== Cache populated; next train@$SHORT_SHA hits ensure-binary cache ==="