diff --git a/infra/k8s/argo/ci-pipeline-template.yaml b/infra/k8s/argo/ci-pipeline-template.yaml index c52a2cb7a..650fd3434 100644 --- a/infra/k8s/argo/ci-pipeline-template.yaml +++ b/infra/k8s/argo/ci-pipeline-template.yaml @@ -636,6 +636,16 @@ spec: value: always - name: CARGO_TARGET_DIR value: /cargo-target + - name: RUSTC_WRAPPER + value: /usr/local/bin/sccache + - name: SCCACHE_DIR + value: /cargo-target/sccache + - name: SCCACHE_CACHE_SIZE + value: 10G + - name: CARGO_HOME + value: /cargo-target/cargo-home + - name: CARGO_INCREMENTAL + value: "0" - name: GITLAB_PAT valueFrom: secretKeyRef: @@ -674,31 +684,39 @@ spec: git checkout "$SHA" echo "Checked out $(git rev-parse --short HEAD)" + # Ensure cargo home registry is on PVC + export PATH="${CARGO_HOME}/bin:${PATH}" + export FOXHUNT_BUILD_VERSION="{{inputs.parameters.tag}}" - # Prune target dir if it exceeds 25GB (prevents unbounded PVC growth) + # Prune sccache + target if it exceeds 25GB (prevents unbounded PVC growth) TARGET_SIZE_MB=$(du -sm "$CARGO_TARGET_DIR" 2>/dev/null | cut -f1 || echo 0) echo "Cargo target dir: ${TARGET_SIZE_MB}MB" if [ "$TARGET_SIZE_MB" -gt 25000 ]; then - echo "Target dir exceeds 25GB limit, running cargo clean..." - cargo clean + echo "Target dir exceeds 25GB limit, pruning..." + sccache --stop-server 2>/dev/null || true + rm -rf "$CARGO_TARGET_DIR/release" "$CARGO_TARGET_DIR/debug" fi + sccache --show-stats || true + # Guard: empty package list would build entire workspace if [ -z "$SERVICE_PKGS" ]; then echo "ERROR: service-packages is empty, refusing to build entire workspace" exit 1 fi - # Build only the affected service packages (incremental via persistent target dir) + # Build only the affected service packages (sccache for cross-build caching) CARGO_ARGS="" for pkg in $SERVICE_PKGS; do CARGO_ARGS="$CARGO_ARGS -p $pkg" done - echo "=== Building service binaries: $SERVICE_PKGS (incremental) ===" + echo "=== Building service binaries: $SERVICE_PKGS (sccache) ===" cargo build --release $CARGO_ARGS + sccache --show-stats + # Collect built binaries mkdir -p "$WORKSPACE/bin/services" for pkg in $SERVICE_PKGS; do @@ -774,6 +792,16 @@ spec: value: always - name: CARGO_TARGET_DIR value: /cargo-target + - name: RUSTC_WRAPPER + value: /usr/local/bin/sccache + - name: SCCACHE_DIR + value: /cargo-target/sccache + - name: SCCACHE_CACHE_SIZE + value: 10G + - name: CARGO_HOME + value: /cargo-target/cargo-home + - name: CARGO_INCREMENTAL + value: "0" - name: GITLAB_PAT valueFrom: secretKeyRef: @@ -812,17 +840,23 @@ spec: git checkout "$SHA" echo "Checked out $(git rev-parse --short HEAD)" + # Ensure cargo home registry is on PVC + export PATH="${CARGO_HOME}/bin:${PATH}" + export FOXHUNT_BUILD_VERSION="{{inputs.parameters.tag}}" export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}} - # Prune target dir if it exceeds 25GB (prevents unbounded PVC growth) + # Prune sccache + target if it exceeds 25GB (prevents unbounded PVC growth) TARGET_SIZE_MB=$(du -sm "$CARGO_TARGET_DIR" 2>/dev/null | cut -f1 || echo 0) echo "Cargo target dir: ${TARGET_SIZE_MB}MB" if [ "$TARGET_SIZE_MB" -gt 25000 ]; then - echo "Target dir exceeds 25GB limit, running cargo clean..." - cargo clean + echo "Target dir exceeds 25GB limit, pruning..." + sccache --stop-server 2>/dev/null || true + rm -rf "$CARGO_TARGET_DIR/release" "$CARGO_TARGET_DIR/debug" fi + sccache --show-stats || true + # Separate ml examples from training_uploader (different build commands) ML_EXAMPLE_ARGS="" BUILD_UPLOADER="false" @@ -843,7 +877,7 @@ spec: exit 1 fi - echo "=== Building training binaries: $TRAINING_EXAMPLES (CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP, incremental) ===" + echo "=== Building training binaries: $TRAINING_EXAMPLES (CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP, sccache) ===" if [ -n "$ML_EXAMPLE_ARGS" ]; then cargo build --release -p ml --features ml/cuda $ML_EXAMPLE_ARGS @@ -853,6 +887,8 @@ spec: cargo build --release -p training_uploader fi + sccache --show-stats + # Collect built binaries mkdir -p "$WORKSPACE/bin/training" for ex in $TRAINING_EXAMPLES; do