diff --git a/infra/k8s/argo/ci-pipeline-template.yaml b/infra/k8s/argo/ci-pipeline-template.yaml index d9cbeb406..b79183da1 100644 --- a/infra/k8s/argo/ci-pipeline-template.yaml +++ b/infra/k8s/argo/ci-pipeline-template.yaml @@ -662,7 +662,7 @@ spec: SHA="{{workflow.parameters.commit-sha}}" SERVICE_PKGS="{{inputs.parameters.service-packages}}" - # Git clone (self-contained) + # SSH setup mkdir -p ~/.ssh cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519 chmod 600 ~/.ssh/id_ed25519 @@ -670,10 +670,22 @@ spec: chmod 600 ~/.ssh/config REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git" - WORKSPACE="/tmp/workspace" - git clone --no-checkout --filter=blob:none "$REPO" "$WORKSPACE" - cd "$WORKSPACE" - git checkout "$SHA" + WORKSPACE="/cargo-target/src" + + # Persistent checkout on PVC — git checkout only touches changed files, + # preserving mtimes on unchanged files so cargo skips recompiling them + if [ -d "$WORKSPACE/.git" ]; then + echo "=== Updating persistent checkout ===" + cd "$WORKSPACE" + git fetch origin + git checkout --force "$SHA" + git clean -fdx + else + echo "=== Initial clone (first run) ===" + git clone --filter=blob:none "$REPO" "$WORKSPACE" + cd "$WORKSPACE" + git checkout "$SHA" + fi echo "Checked out $(git rev-parse --short HEAD)" # Ensure cargo home registry is on PVC @@ -681,12 +693,12 @@ spec: export FOXHUNT_BUILD_VERSION="{{inputs.parameters.tag}}" - # Prune target dir if it exceeds 25GB (prevents unbounded PVC growth) + # Prune build artifacts if PVC exceeds 25GB (prevents unbounded growth) TARGET_SIZE_MB=$(du -sm "$CARGO_TARGET_DIR" 2>/dev/null | cut -f1 || echo 0) - echo "Cargo target dir: ${TARGET_SIZE_MB}MB" + echo "PVC usage: ${TARGET_SIZE_MB}MB" if [ "$TARGET_SIZE_MB" -gt 25000 ]; then - echo "Target dir exceeds 25GB limit, running cargo clean..." - cargo clean + echo "PVC exceeds 25GB limit, pruning build artifacts..." + rm -rf "$CARGO_TARGET_DIR/release" "$CARGO_TARGET_DIR/debug" fi # Guard: empty package list would build entire workspace @@ -805,7 +817,7 @@ spec: SHA="{{workflow.parameters.commit-sha}}" TRAINING_EXAMPLES="{{inputs.parameters.training-examples}}" - # Git clone (self-contained — ephemeral node) + # SSH setup mkdir -p ~/.ssh cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519 chmod 600 ~/.ssh/id_ed25519 @@ -813,10 +825,22 @@ spec: chmod 600 ~/.ssh/config REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git" - WORKSPACE="/tmp/workspace" - git clone --no-checkout --filter=blob:none "$REPO" "$WORKSPACE" - cd "$WORKSPACE" - git checkout "$SHA" + WORKSPACE="/cargo-target/src" + + # Persistent checkout on PVC — git checkout only touches changed files, + # preserving mtimes on unchanged files so cargo skips recompiling them + if [ -d "$WORKSPACE/.git" ]; then + echo "=== Updating persistent checkout ===" + cd "$WORKSPACE" + git fetch origin + git checkout --force "$SHA" + git clean -fdx + else + echo "=== Initial clone (first run) ===" + git clone --filter=blob:none "$REPO" "$WORKSPACE" + cd "$WORKSPACE" + git checkout "$SHA" + fi echo "Checked out $(git rev-parse --short HEAD)" # Ensure cargo home registry is on PVC @@ -825,12 +849,12 @@ spec: export FOXHUNT_BUILD_VERSION="{{inputs.parameters.tag}}" export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}} - # Prune target dir if it exceeds 25GB (prevents unbounded PVC growth) + # Prune build artifacts if PVC exceeds 25GB (prevents unbounded growth) TARGET_SIZE_MB=$(du -sm "$CARGO_TARGET_DIR" 2>/dev/null | cut -f1 || echo 0) - echo "Cargo target dir: ${TARGET_SIZE_MB}MB" + echo "PVC usage: ${TARGET_SIZE_MB}MB" if [ "$TARGET_SIZE_MB" -gt 25000 ]; then - echo "Target dir exceeds 25GB limit, running cargo clean..." - cargo clean + echo "PVC exceeds 25GB limit, pruning build artifacts..." + rm -rf "$CARGO_TARGET_DIR/release" "$CARGO_TARGET_DIR/debug" fi # Separate ml examples from training_uploader (different build commands) diff --git a/infra/k8s/argo/compile-and-train-template.yaml b/infra/k8s/argo/compile-and-train-template.yaml index a9e4d43a9..83f13bfc5 100644 --- a/infra/k8s/argo/compile-and-train-template.yaml +++ b/infra/k8s/argo/compile-and-train-template.yaml @@ -144,6 +144,8 @@ spec: value: always - name: CARGO_TARGET_DIR value: /cargo-target + - name: CARGO_HOME + value: /cargo-target/cargo-home - name: GITLAB_PAT valueFrom: secretKeyRef: @@ -167,7 +169,7 @@ spec: set -e SHA="{{workflow.parameters.commit-sha}}" - # Git clone + # SSH setup mkdir -p ~/.ssh cp /etc/git-ssh/ssh-privatekey ~/.ssh/id_ed25519 chmod 600 ~/.ssh/id_ed25519 @@ -175,21 +177,36 @@ spec: chmod 600 ~/.ssh/config REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git" - BUILD="/tmp/workspace" - git clone --no-checkout --filter=blob:none "$REPO" "$BUILD" - cd "$BUILD" - git checkout "$SHA" + BUILD="/cargo-target/src" + + # Persistent checkout on PVC — git checkout only touches changed files, + # preserving mtimes on unchanged files so cargo skips recompiling them + if [ -d "$BUILD/.git" ]; then + echo "=== Updating persistent checkout ===" + cd "$BUILD" + git fetch origin + git checkout --force "$SHA" + git clean -fdx + else + echo "=== Initial clone (first run) ===" + git clone --filter=blob:none "$REPO" "$BUILD" + cd "$BUILD" + git checkout "$SHA" + fi SHORT_SHA=$(git rev-parse --short HEAD) echo "Checked out ${SHORT_SHA}" + # Ensure cargo home registry is on PVC + export PATH="${CARGO_HOME}/bin:${PATH}" + export CUDA_COMPUTE_CAP={{workflow.parameters.cuda-compute-cap}} - # Prune target dir if it exceeds 25GB (prevents unbounded PVC growth) + # Prune build artifacts if PVC exceeds 25GB (prevents unbounded growth) TARGET_SIZE_MB=$(du -sm "$CARGO_TARGET_DIR" 2>/dev/null | cut -f1 || echo 0) - echo "Cargo target dir: ${TARGET_SIZE_MB}MB" + echo "PVC usage: ${TARGET_SIZE_MB}MB" if [ "$TARGET_SIZE_MB" -gt 25000 ]; then - echo "Target dir exceeds 25GB limit, running cargo clean..." - cargo clean + echo "PVC exceeds 25GB limit, pruning build artifacts..." + rm -rf "$CARGO_TARGET_DIR/release" "$CARGO_TARGET_DIR/debug" fi # Derive needed binaries from model parameter (only build what's used)