perf(ci): persistent checkout on PVC to preserve file mtimes
git clone gives all files the current timestamp, causing cargo to recompile every workspace crate even when source is unchanged. Switch to persistent checkout on the PVC: git fetch + git checkout --force only updates files that actually changed between commits, preserving mtimes on unchanged files so cargo correctly skips them. - compile-services: /cargo-target/src persistent checkout - compile-training: /cargo-target/src persistent checkout - compile-and-train: /cargo-target/src persistent checkout + CARGO_HOME - First run: full clone (fallback), subsequent: fetch + checkout Expected: ~40 crate recompile → only changed crates (~1-2 min) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
@@ -662,7 +662,7 @@ spec:
|
||||
SHA="{{workflow.parameters.commit-sha}}"
|
||||
SERVICE_PKGS="{{inputs.parameters.service-packages}}"
|
||||
|
||||
# Git clone (self-contained)
|
||||
# SSH setup
|
||||
mkdir -p ~/.ssh
|
||||
cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
@@ -670,10 +670,22 @@ spec:
|
||||
chmod 600 ~/.ssh/config
|
||||
|
||||
REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
|
||||
WORKSPACE="/tmp/workspace"
|
||||
git clone --no-checkout --filter=blob:none "$REPO" "$WORKSPACE"
|
||||
cd "$WORKSPACE"
|
||||
git checkout "$SHA"
|
||||
WORKSPACE="/cargo-target/src"
|
||||
|
||||
# Persistent checkout on PVC — git checkout only touches changed files,
|
||||
# preserving mtimes on unchanged files so cargo skips recompiling them
|
||||
if [ -d "$WORKSPACE/.git" ]; then
|
||||
echo "=== Updating persistent checkout ==="
|
||||
cd "$WORKSPACE"
|
||||
git fetch origin
|
||||
git checkout --force "$SHA"
|
||||
git clean -fdx
|
||||
else
|
||||
echo "=== Initial clone (first run) ==="
|
||||
git clone --filter=blob:none "$REPO" "$WORKSPACE"
|
||||
cd "$WORKSPACE"
|
||||
git checkout "$SHA"
|
||||
fi
|
||||
echo "Checked out $(git rev-parse --short HEAD)"
|
||||
|
||||
# Ensure cargo home registry is on PVC
|
||||
@@ -681,12 +693,12 @@ spec:
|
||||
|
||||
export FOXHUNT_BUILD_VERSION="{{inputs.parameters.tag}}"
|
||||
|
||||
# Prune target dir if it exceeds 25GB (prevents unbounded PVC growth)
|
||||
# Prune build artifacts if PVC exceeds 25GB (prevents unbounded growth)
|
||||
TARGET_SIZE_MB=$(du -sm "$CARGO_TARGET_DIR" 2>/dev/null | cut -f1 || echo 0)
|
||||
echo "Cargo target dir: ${TARGET_SIZE_MB}MB"
|
||||
echo "PVC usage: ${TARGET_SIZE_MB}MB"
|
||||
if [ "$TARGET_SIZE_MB" -gt 25000 ]; then
|
||||
echo "Target dir exceeds 25GB limit, running cargo clean..."
|
||||
cargo clean
|
||||
echo "PVC exceeds 25GB limit, pruning build artifacts..."
|
||||
rm -rf "$CARGO_TARGET_DIR/release" "$CARGO_TARGET_DIR/debug"
|
||||
fi
|
||||
|
||||
# Guard: empty package list would build entire workspace
|
||||
@@ -805,7 +817,7 @@ spec:
|
||||
SHA="{{workflow.parameters.commit-sha}}"
|
||||
TRAINING_EXAMPLES="{{inputs.parameters.training-examples}}"
|
||||
|
||||
# Git clone (self-contained — ephemeral node)
|
||||
# SSH setup
|
||||
mkdir -p ~/.ssh
|
||||
cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
@@ -813,10 +825,22 @@ spec:
|
||||
chmod 600 ~/.ssh/config
|
||||
|
||||
REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
|
||||
WORKSPACE="/tmp/workspace"
|
||||
git clone --no-checkout --filter=blob:none "$REPO" "$WORKSPACE"
|
||||
cd "$WORKSPACE"
|
||||
git checkout "$SHA"
|
||||
WORKSPACE="/cargo-target/src"
|
||||
|
||||
# Persistent checkout on PVC — git checkout only touches changed files,
|
||||
# preserving mtimes on unchanged files so cargo skips recompiling them
|
||||
if [ -d "$WORKSPACE/.git" ]; then
|
||||
echo "=== Updating persistent checkout ==="
|
||||
cd "$WORKSPACE"
|
||||
git fetch origin
|
||||
git checkout --force "$SHA"
|
||||
git clean -fdx
|
||||
else
|
||||
echo "=== Initial clone (first run) ==="
|
||||
git clone --filter=blob:none "$REPO" "$WORKSPACE"
|
||||
cd "$WORKSPACE"
|
||||
git checkout "$SHA"
|
||||
fi
|
||||
echo "Checked out $(git rev-parse --short HEAD)"
|
||||
|
||||
# Ensure cargo home registry is on PVC
|
||||
@@ -825,12 +849,12 @@ spec:
|
||||
export FOXHUNT_BUILD_VERSION="{{inputs.parameters.tag}}"
|
||||
export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}}
|
||||
|
||||
# Prune target dir if it exceeds 25GB (prevents unbounded PVC growth)
|
||||
# Prune build artifacts if PVC exceeds 25GB (prevents unbounded growth)
|
||||
TARGET_SIZE_MB=$(du -sm "$CARGO_TARGET_DIR" 2>/dev/null | cut -f1 || echo 0)
|
||||
echo "Cargo target dir: ${TARGET_SIZE_MB}MB"
|
||||
echo "PVC usage: ${TARGET_SIZE_MB}MB"
|
||||
if [ "$TARGET_SIZE_MB" -gt 25000 ]; then
|
||||
echo "Target dir exceeds 25GB limit, running cargo clean..."
|
||||
cargo clean
|
||||
echo "PVC exceeds 25GB limit, pruning build artifacts..."
|
||||
rm -rf "$CARGO_TARGET_DIR/release" "$CARGO_TARGET_DIR/debug"
|
||||
fi
|
||||
|
||||
# Separate ml examples from training_uploader (different build commands)
|
||||
|
||||
@@ -144,6 +144,8 @@ spec:
|
||||
value: always
|
||||
- name: CARGO_TARGET_DIR
|
||||
value: /cargo-target
|
||||
- name: CARGO_HOME
|
||||
value: /cargo-target/cargo-home
|
||||
- name: GITLAB_PAT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
@@ -167,7 +169,7 @@ spec:
|
||||
set -e
|
||||
SHA="{{workflow.parameters.commit-sha}}"
|
||||
|
||||
# Git clone
|
||||
# SSH setup
|
||||
mkdir -p ~/.ssh
|
||||
cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519
|
||||
chmod 600 ~/.ssh/id_ed25519
|
||||
@@ -175,21 +177,36 @@ spec:
|
||||
chmod 600 ~/.ssh/config
|
||||
|
||||
REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git"
|
||||
BUILD="/tmp/workspace"
|
||||
git clone --no-checkout --filter=blob:none "$REPO" "$BUILD"
|
||||
cd "$BUILD"
|
||||
git checkout "$SHA"
|
||||
BUILD="/cargo-target/src"
|
||||
|
||||
# Persistent checkout on PVC — git checkout only touches changed files,
|
||||
# preserving mtimes on unchanged files so cargo skips recompiling them
|
||||
if [ -d "$BUILD/.git" ]; then
|
||||
echo "=== Updating persistent checkout ==="
|
||||
cd "$BUILD"
|
||||
git fetch origin
|
||||
git checkout --force "$SHA"
|
||||
git clean -fdx
|
||||
else
|
||||
echo "=== Initial clone (first run) ==="
|
||||
git clone --filter=blob:none "$REPO" "$BUILD"
|
||||
cd "$BUILD"
|
||||
git checkout "$SHA"
|
||||
fi
|
||||
SHORT_SHA=$(git rev-parse --short HEAD)
|
||||
echo "Checked out ${SHORT_SHA}"
|
||||
|
||||
# Ensure cargo home registry is on PVC
|
||||
export PATH="${CARGO_HOME}/bin:${PATH}"
|
||||
|
||||
export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}}
|
||||
|
||||
# Prune target dir if it exceeds 25GB (prevents unbounded PVC growth)
|
||||
# Prune build artifacts if PVC exceeds 25GB (prevents unbounded growth)
|
||||
TARGET_SIZE_MB=$(du -sm "$CARGO_TARGET_DIR" 2>/dev/null | cut -f1 || echo 0)
|
||||
echo "Cargo target dir: ${TARGET_SIZE_MB}MB"
|
||||
echo "PVC usage: ${TARGET_SIZE_MB}MB"
|
||||
if [ "$TARGET_SIZE_MB" -gt 25000 ]; then
|
||||
echo "Target dir exceeds 25GB limit, running cargo clean..."
|
||||
cargo clean
|
||||
echo "PVC exceeds 25GB limit, pruning build artifacts..."
|
||||
rm -rf "$CARGO_TARGET_DIR/release" "$CARGO_TARGET_DIR/debug"
|
||||
fi
|
||||
|
||||
# Derive needed binaries from model parameter (only build what's used)
|
||||
|
||||
Reference in New Issue
Block a user