perf(ci): enable sccache + persistent CARGO_HOME for compile jobs

Both compile-services and compile-training were doing full rebuilds
every run because:
1. No RUSTC_WRAPPER=sccache — sccache was installed but never used
2. No persistent CARGO_HOME — crates.io registry re-downloaded each time
3. Incremental compilation doesn't work across git clones (mtime-based)

Fix: enable sccache (content-hash based, clone-agnostic), persist
CARGO_HOME on the PVC, disable incremental (conflicts with sccache).

Expected: first run populates cache, subsequent runs ~2-3min vs ~8min.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

infra/k8s/argo/ci-pipeline-template.yaml

@@ -636,6 +636,16 @@ spec:
             value: always
           - name: CARGO_TARGET_DIR
             value: /cargo-target
+          - name: RUSTC_WRAPPER
+            value: /usr/local/bin/sccache
+          - name: SCCACHE_DIR
+            value: /cargo-target/sccache
+          - name: SCCACHE_CACHE_SIZE
+            value: 10G
+          - name: CARGO_HOME
+            value: /cargo-target/cargo-home
+          - name: CARGO_INCREMENTAL
+            value: "0"
           - name: GITLAB_PAT
             valueFrom:
               secretKeyRef:
@@ -674,31 +684,39 @@ spec:
             git checkout "$SHA"
             echo "Checked out $(git rev-parse --short HEAD)"
 
+            # Ensure cargo home registry is on PVC
+            export PATH="${CARGO_HOME}/bin:${PATH}"
+
             export FOXHUNT_BUILD_VERSION="{{inputs.parameters.tag}}"
 
-            # Prune target dir if it exceeds 25GB (prevents unbounded PVC growth)
+            # Prune sccache + target if it exceeds 25GB (prevents unbounded PVC growth)
             TARGET_SIZE_MB=$(du -sm "$CARGO_TARGET_DIR" 2>/dev/null | cut -f1 || echo 0)
             echo "Cargo target dir: ${TARGET_SIZE_MB}MB"
             if [ "$TARGET_SIZE_MB" -gt 25000 ]; then
-              echo "Target dir exceeds 25GB limit, running cargo clean..."
-              cargo clean
+              echo "Target dir exceeds 25GB limit, pruning..."
+              sccache --stop-server 2>/dev/null || true
+              rm -rf "$CARGO_TARGET_DIR/release" "$CARGO_TARGET_DIR/debug"
             fi
 
+            sccache --show-stats || true
+
             # Guard: empty package list would build entire workspace
             if [ -z "$SERVICE_PKGS" ]; then
               echo "ERROR: service-packages is empty, refusing to build entire workspace"
               exit 1
             fi
 
-            # Build only the affected service packages (incremental via persistent target dir)
+            # Build only the affected service packages (sccache for cross-build caching)
             CARGO_ARGS=""
             for pkg in $SERVICE_PKGS; do
               CARGO_ARGS="$CARGO_ARGS -p $pkg"
             done
 
-            echo "=== Building service binaries: $SERVICE_PKGS (incremental) ==="
+            echo "=== Building service binaries: $SERVICE_PKGS (sccache) ==="
             cargo build --release $CARGO_ARGS
 
+            sccache --show-stats
+
             # Collect built binaries
             mkdir -p "$WORKSPACE/bin/services"
             for pkg in $SERVICE_PKGS; do
@@ -774,6 +792,16 @@ spec:
             value: always
           - name: CARGO_TARGET_DIR
             value: /cargo-target
+          - name: RUSTC_WRAPPER
+            value: /usr/local/bin/sccache
+          - name: SCCACHE_DIR
+            value: /cargo-target/sccache
+          - name: SCCACHE_CACHE_SIZE
+            value: 10G
+          - name: CARGO_HOME
+            value: /cargo-target/cargo-home
+          - name: CARGO_INCREMENTAL
+            value: "0"
           - name: GITLAB_PAT
             valueFrom:
               secretKeyRef:
@@ -812,17 +840,23 @@ spec:
             git checkout "$SHA"
             echo "Checked out $(git rev-parse --short HEAD)"
 
+            # Ensure cargo home registry is on PVC
+            export PATH="${CARGO_HOME}/bin:${PATH}"
+
             export FOXHUNT_BUILD_VERSION="{{inputs.parameters.tag}}"
             export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}}
 
-            # Prune target dir if it exceeds 25GB (prevents unbounded PVC growth)
+            # Prune sccache + target if it exceeds 25GB (prevents unbounded PVC growth)
             TARGET_SIZE_MB=$(du -sm "$CARGO_TARGET_DIR" 2>/dev/null | cut -f1 || echo 0)
             echo "Cargo target dir: ${TARGET_SIZE_MB}MB"
             if [ "$TARGET_SIZE_MB" -gt 25000 ]; then
-              echo "Target dir exceeds 25GB limit, running cargo clean..."
-              cargo clean
+              echo "Target dir exceeds 25GB limit, pruning..."
+              sccache --stop-server 2>/dev/null || true
+              rm -rf "$CARGO_TARGET_DIR/release" "$CARGO_TARGET_DIR/debug"
             fi
 
+            sccache --show-stats || true
+
             # Separate ml examples from training_uploader (different build commands)
             ML_EXAMPLE_ARGS=""
             BUILD_UPLOADER="false"
@@ -843,7 +877,7 @@ spec:
               exit 1
             fi
 
-            echo "=== Building training binaries: $TRAINING_EXAMPLES (CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP, incremental) ==="
+            echo "=== Building training binaries: $TRAINING_EXAMPLES (CUDA_COMPUTE_CAP=$CUDA_COMPUTE_CAP, sccache) ==="
 
             if [ -n "$ML_EXAMPLE_ARGS" ]; then
               cargo build --release -p ml --features ml/cuda $ML_EXAMPLE_ARGS
@@ -853,6 +887,8 @@ spec:
               cargo build --release -p training_uploader
             fi
 
+            sccache --show-stats
+
             # Collect built binaries
             mkdir -p "$WORKSPACE/bin/training"
             for ex in $TRAINING_EXAMPLES; do