gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

perf(ci): replace sccache with persistent target dir for incremental compilation

Browse Source 
sccache forces CARGO_INCREMENTAL=0, causing all 37 workspace crates to
recompile from scratch every CI run (~20 min). Only upstream deps were
cached (635 hits); 109 workspace rlib crates were non-cacheable.

Changes:
- Add cargo-target-cpu and cargo-target-cuda PVCs (30Gi each)
- Mount persistent target dir at /cargo-target via CARGO_TARGET_DIR
- Drop RUSTC_WRAPPER=sccache and SCCACHE_DIR from both compile steps
- Drop hardcoded CARGO_BUILD_JOBS=14 (let cargo auto-detect from nproc)
- Add 25GB cleanup guard to prevent unbounded PVC growth
- Update binary copy paths to use $CARGO_TARGET_DIR/release/

First build (cold PVC) is same speed. Subsequent builds with typical
3-5 file changes should drop from ~20 min to ~2-3 min via incremental.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-10 21:22:17 +01:00
parent 3137064df2
commit 9278561ec5
3 changed files with 97 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
infra/k8s/argo/cargo-target-pvcs.yaml Normal file
View File

@@ -0,0 +1,43 @@
# Persistent cargo target directories for incremental compilation.
#
# Why: sccache can't cache workspace rlib crates (109 non-cacheable per build).
# Persisting target/ lets cargo's incremental compilation skip unchanged crates,
# reducing typical CI builds from ~20 min (full rebuild) to ~2-3 min.
#
# Two separate PVCs because compile-services (no cuda feature) and
# compile-training (cuda feature) produce incompatible artifacts.
#
# NOTE: Only one workflow should use each PVC at a time. ci-pipeline and
# compile-and-train must not run their compile steps concurrently.
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: cargo-target-cpu
namespace: foxhunt
labels:
app.kubernetes.io/name: cargo-target
app.kubernetes.io/component: ci-cache
app.kubernetes.io/part-of: foxhunt
spec:
accessModes: [ReadWriteOnce]
storageClassName: scw-bssd-retain
resources:
requests:
storage: 30Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: cargo-target-cuda
namespace: foxhunt
labels:
app.kubernetes.io/name: cargo-target
app.kubernetes.io/component: ci-cache
app.kubernetes.io/part-of: foxhunt
spec:
accessModes: [ReadWriteOnce]
storageClassName: scw-bssd-retain
resources:
requests:
storage: 30Gi

68
infra/k8s/argo/ci-pipeline-template.yaml
View File

@@ -30,12 +30,12 @@ spec:
items:
- key: .dockerconfigjson
path: config.json
- name: sccache-cpu
- name: cargo-target-cpu
persistentVolumeClaim:
claimName: sccache-cpu
- name: sccache-cuda
claimName: cargo-target-cpu
- name: cargo-target-cuda
persistentVolumeClaim:
claimName: sccache-cuda
claimName: cargo-target-cuda
- name: gitlab-pat
secret:
secretName: gitlab-pat
@@ -609,10 +609,8 @@ spec:
value: "true"
- name: CARGO_TERM_COLOR
value: always
- name: CARGO_BUILD_JOBS
value: "14"
- name: SCCACHE_DIR
value: /sccache
- name: CARGO_TARGET_DIR
value: /cargo-target
- name: GITLAB_PAT
valueFrom:
secretKeyRef:
@@ -629,8 +627,8 @@ spec:
- name: git-ssh-key
mountPath: /etc/git-ssh
readOnly: true
- name: sccache-cpu
mountPath: /sccache
- name: cargo-target-cpu
mountPath: /cargo-target
args:
- |
set -e
@@ -658,8 +656,14 @@ spec:
echo "Checked out $(git rev-parse --short HEAD)"
export FOXHUNT_BUILD_VERSION="{{inputs.parameters.tag}}"
export RUSTC_WRAPPER=sccache
sccache --zero-stats || true
# Prune target dir if it exceeds 25GB (prevents unbounded PVC growth)
TARGET_SIZE_MB=$(du -sm "$CARGO_TARGET_DIR" 2>/dev/null | cut -f1 || echo 0)
echo "Cargo target dir: ${TARGET_SIZE_MB}MB"
if [ "$TARGET_SIZE_MB" -gt 25000 ]; then
echo "Target dir exceeds 25GB limit, running cargo clean..."
cargo clean
fi
# Guard: empty package list would build entire workspace
if [ -z "$SERVICE_PKGS" ]; then
@@ -667,28 +671,27 @@ spec:
exit 1
fi
# Build only the affected service packages
# Build only the affected service packages (incremental via persistent target dir)
CARGO_ARGS=""
for pkg in $SERVICE_PKGS; do
CARGO_ARGS="$CARGO_ARGS -p $pkg"
done
echo "=== Building service binaries: $SERVICE_PKGS ==="
echo "=== Building service binaries: $SERVICE_PKGS (incremental) ==="
cargo build --release $CARGO_ARGS
# Collect built binaries
mkdir -p "$WORKSPACE/bin/services"
for pkg in $SERVICE_PKGS; do
bin_name=$(echo "$pkg" | tr '-' '_')
cp "target/release/$pkg" "$WORKSPACE/bin/services/" 2>/dev/null \
|| cp "target/release/$bin_name" "$WORKSPACE/bin/services/" 2>/dev/null \
|| { echo "Binary not found for $pkg"; ls target/release/; exit 1; }
cp "$CARGO_TARGET_DIR/release/$pkg" "$WORKSPACE/bin/services/" 2>/dev/null \
|| cp "$CARGO_TARGET_DIR/release/$bin_name" "$WORKSPACE/bin/services/" 2>/dev/null \
|| { echo "Binary not found for $pkg"; ls "$CARGO_TARGET_DIR/release/"; exit 1; }
done
strip "$WORKSPACE/bin/services/"*
echo "=== Service binaries ==="
ls -lh "$WORKSPACE/bin/services/"
sccache --show-stats || true
echo "=== Uploading service binaries to GitLab packages ==="
GITLAB="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"
@@ -726,7 +729,7 @@ spec:
echo "=== Service compile + upload done ($SERVICE_PKGS) ==="
# ── compile-training: selective per-binary CUDA build, sccache on local RWO PVC ──
# ── compile-training: selective per-binary CUDA build, incremental via persistent target dir ──
- name: compile-training
metadata:
labels:
@@ -750,10 +753,8 @@ spec:
value: "true"
- name: CARGO_TERM_COLOR
value: always
- name: CARGO_BUILD_JOBS
value: "14"
- name: SCCACHE_DIR
value: /sccache
- name: CARGO_TARGET_DIR
value: /cargo-target
- name: GITLAB_PAT
valueFrom:
secretKeyRef:
@@ -770,8 +771,8 @@ spec:
- name: git-ssh-key
mountPath: /etc/git-ssh
readOnly: true
- name: sccache-cuda
mountPath: /sccache
- name: cargo-target-cuda
mountPath: /cargo-target
args:
- |
set -e
@@ -799,9 +800,15 @@ spec:
echo "Checked out $(git rev-parse --short HEAD)"
export FOXHUNT_BUILD_VERSION="{{inputs.parameters.tag}}"
export RUSTC_WRAPPER=sccache
export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}}
sccache --zero-stats || true
# Prune target dir if it exceeds 25GB (prevents unbounded PVC growth)
TARGET_SIZE_MB=$(du -sm "$CARGO_TARGET_DIR" 2>/dev/null | cut -f1 || echo 0)
echo "Cargo target dir: ${TARGET_SIZE_MB}MB"
if [ "$TARGET_SIZE_MB" -gt 25000 ]; then
echo "Target dir exceeds 25GB limit, running cargo clean..."
cargo clean
fi
# Separate ml examples from training_uploader (different build commands)
ML_EXAMPLE_ARGS=""
@@ -823,7 +830,7 @@ spec:
exit 1
fi
echo "=== Building training binaries: $TRAINING_EXAMPLES (CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP) ==="
echo "=== Building training binaries: $TRAINING_EXAMPLES (CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP, incremental) ==="
if [ -n "$ML_EXAMPLE_ARGS" ]; then
cargo build --release -p ml --features ml/cuda $ML_EXAMPLE_ARGS
@@ -838,10 +845,10 @@ spec:
for ex in $TRAINING_EXAMPLES; do
case "$ex" in
training_uploader)
cp target/release/training_uploader "$WORKSPACE/bin/training/"
cp "$CARGO_TARGET_DIR/release/training_uploader" "$WORKSPACE/bin/training/"
;;
*)
cp target/release/examples/$ex "$WORKSPACE/bin/training/"
cp "$CARGO_TARGET_DIR/release/examples/$ex" "$WORKSPACE/bin/training/"
;;
esac
done
@@ -849,7 +856,6 @@ spec:
echo "=== Training binaries ==="
ls -lh "$WORKSPACE/bin/training/"
sccache --show-stats || true
echo "=== Uploading training binaries to GitLab packages ==="
GITLAB="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"

31
infra/k8s/argo/compile-and-train-template.yaml
View File

@@ -78,9 +78,9 @@ spec:
persistentVolumeClaim:
claimName: training-data-pvc
readOnly: true
- name: sccache-cuda
- name: cargo-target-cuda
persistentVolumeClaim:
claimName: sccache-cuda
claimName: cargo-target-cuda
volumeClaimTemplates:
- metadata:
@@ -142,10 +142,8 @@ spec:
value: "true"
- name: CARGO_TERM_COLOR
value: always
- name: CARGO_BUILD_JOBS
value: "14"
- name: SCCACHE_DIR
value: /sccache
- name: CARGO_TARGET_DIR
value: /cargo-target
- name: GITLAB_PAT
valueFrom:
secretKeyRef:
@@ -162,8 +160,8 @@ spec:
- name: git-ssh-key
mountPath: /etc/git-ssh
readOnly: true
- name: sccache-cuda
mountPath: /sccache
- name: cargo-target-cuda
mountPath: /cargo-target
args:
- |
set -e
@@ -188,9 +186,15 @@ spec:
SHORT_SHA=$(git rev-parse --short HEAD)
echo "Checked out ${SHORT_SHA}"
export RUSTC_WRAPPER=sccache
export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}}
sccache --zero-stats || true
# Prune target dir if it exceeds 25GB (prevents unbounded PVC growth)
TARGET_SIZE_MB=$(du -sm "$CARGO_TARGET_DIR" 2>/dev/null | cut -f1 || echo 0)
echo "Cargo target dir: ${TARGET_SIZE_MB}MB"
if [ "$TARGET_SIZE_MB" -gt 25000 ]; then
echo "Target dir exceeds 25GB limit, running cargo clean..."
cargo clean
fi
# Derive needed binaries from model parameter (only build what's used)
MODEL="{{workflow.parameters.model}}"
@@ -208,19 +212,18 @@ spec:
ML_EXAMPLE_ARGS="$ML_EXAMPLE_ARGS --example $ex"
done
echo "=== Building training binaries for $MODEL: $EXAMPLES ==="
echo " CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP, sccache → local PVC /sccache"
echo "=== Building training binaries for $MODEL: $EXAMPLES (incremental) ==="
echo " CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP, target dir on PVC /cargo-target"
cargo build --release -p ml --features ml/cuda $ML_EXAMPLE_ARGS
mkdir -p "$BUILD/bin/training"
for bin in $EXAMPLES; do
cp target/release/examples/$bin "$BUILD/bin/training/"
cp "$CARGO_TARGET_DIR/release/examples/$bin" "$BUILD/bin/training/"
done
strip "$BUILD/bin/training/"*
echo "=== Training binaries ==="
ls -lh "$BUILD/bin/training/"
sccache --show-stats || true
# Upload to GitLab packages under commit SHA
GITLAB="http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"