diff --git a/infra/k8s/argo/smoke-test-template.yaml b/infra/k8s/argo/smoke-test-template.yaml index 96c36a2ef..4de3efa3c 100644 --- a/infra/k8s/argo/smoke-test-template.yaml +++ b/infra/k8s/argo/smoke-test-template.yaml @@ -72,10 +72,15 @@ spec: # Same training-data-pvc as production training (full 27 months of # OHLCV + MBP-10 + trades) — laptop's 1-quarter MBP-10 truncates the # fxcache below the smoke's 10-month minimum. + # + # RW (not readOnly) so the smoke can populate `/data/bin/$SHORT_SHA/` + # with the train binaries it just built, letting the next train run's + # `ensure-binary` cache check (train-multi-seed-template.yaml:235-246) + # hit and skip the 6-min compile. Read-side (fxcache + raw market data) + # is unchanged. - name: training-data persistentVolumeClaim: claimName: training-data-pvc - readOnly: true templates: - name: smoke-run @@ -126,7 +131,9 @@ spec: mountPath: /cargo-target - name: training-data mountPath: /data - readOnly: true + # RW so the post-PASS cache-population step can write to + # /data/bin/$SHORT_SHA/. Read-side paths (fxcache, MBP-10, + # trades) are unaffected. resources: requests: nvidia.com/gpu: "1" @@ -204,10 +211,20 @@ spec: echo "=== Cleaned. Forced fresh compile of ml/ml-dqn (deps stay cached) ===" fi - # --- Compile test binary + train_baseline_rl --- - echo "=== Compiling (--release --no-run + train_baseline_rl) ===" + # --- Compile test binary + the 3 train binaries --- + # + # The 3 examples mirror exactly what `ensure-binary` produces + # (see train-multi-seed-template.yaml line ~268). Compiling + # them here lets the post-PASS step populate the + # `/data/bin/$SHORT_SHA/` cache so the next train run at the + # same SHA skips its own ensure-binary compile. + echo "=== Compiling (--release --no-run + train binaries) ===" cargo test -p ml --release --lib --no-run 2>&1 | tee /cargo-target/smoke-compile.log - cargo build -p ml --release --example train_baseline_rl 2>&1 | tee -a /cargo-target/smoke-compile.log + cargo build -p ml --release \ + --example train_baseline_rl \ + --example evaluate_baseline \ + --example precompute_features \ + 2>&1 | tee -a /cargo-target/smoke-compile.log # Honour CARGO_TARGET_DIR — outputs go to ${CARGO_TARGET_DIR}/release/deps, # NOT the repo-relative target/. Subshell + || true wraps the head -1 @@ -262,3 +279,29 @@ spec: fi echo "" echo "PASS: smoke completed." + + # --- Populate /data/bin/$SHORT_SHA/ for ensure-binary cache --- + # On PASS only — broken binaries must not be cached. Mirrors + # train-multi-seed-template.yaml:270-274 exactly: same + # destination layout, same strip step. Idempotent: writes are + # to a SHA-keyed dir so concurrent smokes at different SHAs + # don't collide; same-SHA writes are deterministic compile + # output and overwrite-safe. + FULL_SHA=$(git rev-parse HEAD) + SHORT_SHA=$(echo "$FULL_SHA" | cut -c1-9) + BIN_DIR="/data/bin/$SHORT_SHA" + BINARIES="train_baseline_rl evaluate_baseline precompute_features" + echo "" + echo "=== Populating ensure-binary cache at $BIN_DIR ===" + mkdir -p "$BIN_DIR" + for bin in $BINARIES; do + src="${CARGO_TARGET_DIR}/release/examples/$bin" + if [ ! -x "$src" ]; then + echo "WARN: $src not found (cache pop skipped for $bin)" + continue + fi + cp "$src" "$BIN_DIR/" + strip "$BIN_DIR/$bin" 2>/dev/null || true + done + ls -lh "$BIN_DIR/" + echo "=== Cache populated; next train@$SHORT_SHA hits ensure-binary cache ==="