infra(argo): smoke populates ensure-binary cache for next train run
The smoke template already compiled `train_baseline_rl`. Now also build `evaluate_baseline` + `precompute_features` and copy + strip all 3 to `/data/bin/$SHORT_SHA/` after PASS — exactly the layout that train-multi-seed-template.yaml's `ensure-binary` cache check (lines 235-246) looks up by SHA. A smoke-then-train sequence at the same SHA now hits the cache and skips the ~6-min compile, exiting in the ~1-min pod-startup baseline. - Drop readOnly:true on training-data PVC mount (cargo writes binaries into /data/bin/$SHORT_SHA; read-side fxcache + market data unchanged) - Build all 3 example binaries in a single cargo invocation (sccache-warm) - Idempotent SHA-keyed dest dir; PASS-only so broken bins never cache Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -72,10 +72,15 @@ spec:
|
||||
# Same training-data-pvc as production training (full 27 months of
|
||||
# OHLCV + MBP-10 + trades) — laptop's 1-quarter MBP-10 truncates the
|
||||
# fxcache below the smoke's 10-month minimum.
|
||||
#
|
||||
# RW (not readOnly) so the smoke can populate `/data/bin/$SHORT_SHA/`
|
||||
# with the train binaries it just built, letting the next train run's
|
||||
# `ensure-binary` cache check (train-multi-seed-template.yaml:235-246)
|
||||
# hit and skip the 6-min compile. Read-side (fxcache + raw market data)
|
||||
# is unchanged.
|
||||
- name: training-data
|
||||
persistentVolumeClaim:
|
||||
claimName: training-data-pvc
|
||||
readOnly: true
|
||||
|
||||
templates:
|
||||
- name: smoke-run
|
||||
@@ -126,7 +131,9 @@ spec:
|
||||
mountPath: /cargo-target
|
||||
- name: training-data
|
||||
mountPath: /data
|
||||
readOnly: true
|
||||
# RW so the post-PASS cache-population step can write to
|
||||
# /data/bin/$SHORT_SHA/. Read-side paths (fxcache, MBP-10,
|
||||
# trades) are unaffected.
|
||||
resources:
|
||||
requests:
|
||||
nvidia.com/gpu: "1"
|
||||
@@ -204,10 +211,20 @@ spec:
|
||||
echo "=== Cleaned. Forced fresh compile of ml/ml-dqn (deps stay cached) ==="
|
||||
fi
|
||||
|
||||
# --- Compile test binary + train_baseline_rl ---
|
||||
echo "=== Compiling (--release --no-run + train_baseline_rl) ==="
|
||||
# --- Compile test binary + the 3 train binaries ---
|
||||
#
|
||||
# The 3 examples mirror exactly what `ensure-binary` produces
|
||||
# (see train-multi-seed-template.yaml line ~268). Compiling
|
||||
# them here lets the post-PASS step populate the
|
||||
# `/data/bin/$SHORT_SHA/` cache so the next train run at the
|
||||
# same SHA skips its own ensure-binary compile.
|
||||
echo "=== Compiling (--release --no-run + train binaries) ==="
|
||||
cargo test -p ml --release --lib --no-run 2>&1 | tee /cargo-target/smoke-compile.log
|
||||
cargo build -p ml --release --example train_baseline_rl 2>&1 | tee -a /cargo-target/smoke-compile.log
|
||||
cargo build -p ml --release \
|
||||
--example train_baseline_rl \
|
||||
--example evaluate_baseline \
|
||||
--example precompute_features \
|
||||
2>&1 | tee -a /cargo-target/smoke-compile.log
|
||||
|
||||
# Honour CARGO_TARGET_DIR — outputs go to ${CARGO_TARGET_DIR}/release/deps,
|
||||
# NOT the repo-relative target/. Subshell + || true wraps the head -1
|
||||
@@ -262,3 +279,29 @@ spec:
|
||||
fi
|
||||
echo ""
|
||||
echo "PASS: smoke completed."
|
||||
|
||||
# --- Populate /data/bin/$SHORT_SHA/ for ensure-binary cache ---
|
||||
# On PASS only — broken binaries must not be cached. Mirrors
|
||||
# train-multi-seed-template.yaml:270-274 exactly: same
|
||||
# destination layout, same strip step. Idempotent: writes are
|
||||
# to a SHA-keyed dir so concurrent smokes at different SHAs
|
||||
# don't collide; same-SHA writes are deterministic compile
|
||||
# output and overwrite-safe.
|
||||
FULL_SHA=$(git rev-parse HEAD)
|
||||
SHORT_SHA=$(echo "$FULL_SHA" | cut -c1-9)
|
||||
BIN_DIR="/data/bin/$SHORT_SHA"
|
||||
BINARIES="train_baseline_rl evaluate_baseline precompute_features"
|
||||
echo ""
|
||||
echo "=== Populating ensure-binary cache at $BIN_DIR ==="
|
||||
mkdir -p "$BIN_DIR"
|
||||
for bin in $BINARIES; do
|
||||
src="${CARGO_TARGET_DIR}/release/examples/$bin"
|
||||
if [ ! -x "$src" ]; then
|
||||
echo "WARN: $src not found (cache pop skipped for $bin)"
|
||||
continue
|
||||
fi
|
||||
cp "$src" "$BIN_DIR/"
|
||||
strip "$BIN_DIR/$bin" 2>/dev/null || true
|
||||
done
|
||||
ls -lh "$BIN_DIR/"
|
||||
echo "=== Cache populated; next train@$SHORT_SHA hits ensure-binary cache ==="
|
||||
|
||||
Reference in New Issue
Block a user