diff --git a/infra/docker/Dockerfile.ci-deps-cache b/infra/docker/Dockerfile.ci-deps-cache new file mode 100644 index 000000000..7d6549761 --- /dev/null +++ b/infra/docker/Dockerfile.ci-deps-cache @@ -0,0 +1,88 @@ +# Pre-built workspace third-party dependency cache. +# +# Built nightly by the refresh-deps-cache Argo WorkflowTemplate +# (infra/k8s/argo/refresh-deps-cache-template.yaml). Compile pods consume +# this image as an initContainer that rsyncs /cargo-target-prebuilt/ into +# the cargo-target-cpu PVC, dramatically cutting cold-cache compile time. +# +# Cold compile timing on a new node (PVC empty): +# without this image: 3-5 min compiling third-party deps +# with this image: ~30-60s rsync (in-cluster registry, fast layer) +# + delta-compile of workspace members only +# +# Cache invalidation: bumping DEPS_VERSION (build-arg) forces every +# compile pod to re-seed its PVC from this image's snapshot. Bump when: +# - Cargo.lock has churned significantly (most dep rlibs invalid) +# - rustc/toolchain version changes +# - workspace member structure changes meaningfully +# Otherwise leave DEPS_VERSION alone; cargo's own fingerprinting will +# discard stale rlibs and rebuild as needed. +# +# Stage 1: build atop ci-builder-cpu (same toolchain, same cuda). +# Stage 2: copy the snapshot into a thin layer for the runtime. +ARG BASE_TAG=latest +FROM gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder-cpu:${BASE_TAG} AS builder + +ARG DEPS_VERSION=1 +ENV DEPS_VERSION=$DEPS_VERSION +ENV CARGO_TARGET_DIR=/workspace/target +# Prebuild MUST mirror the CI compile step's env so generated rlib +# fingerprints match what the live build expects: +ENV CARGO_INCREMENTAL=0 +ENV SQLX_OFFLINE=true +# (We deliberately do NOT set RUSTC_WRAPPER=sccache here — sccache cache +# state lives on a different PVC; populating its cache from a transient +# image would be useless. The deps-cache image contains rlibs directly.) + +WORKDIR /workspace + +# Copy the entire workspace into the build stage. We rely on .dockerignore +# to skip target/, .git, etc. — see /workspace/.dockerignore in the repo. +# This is the "cargo-chef" approach inverted: instead of stubbing +# workspace members and building only deps, we build EVERYTHING (deps + +# workspace member rlibs). The downside is a slightly bigger image; the +# upside is zero special-case logic for build.rs files, examples, or +# weird workspace shapes. +COPY . . + +# Build the entire workspace in release mode. This populates target/ +# with: third-party crate rlibs (the expensive part), proc-macro .so +# files, build.rs outputs, and workspace member rlibs at this SHA. +# `|| true` so a transient compile error doesn't kill the cache build — +# we still get partial deps populated. The compile pod tolerates +# missing rlibs (cargo just rebuilds them). +RUN cargo build --locked --release --workspace || true + +# Snapshot just the directories cargo actually re-uses. We exclude: +# - target/release/build (build script outputs; cargo regenerates per-SHA) +# - target/release/incremental (per-build incremental, plus we set +# CARGO_INCREMENTAL=0 so this should be empty anyway) +# - target/release/examples / tests (not needed by the service compile) +# We DO ship: +# - target/release/deps (the big one — third-party + workspace rlibs) +# - target/release/.fingerprint (mandatory; cargo's freshness oracle) +# - target/release/{*.rmeta,*.d} (top-level workspace artifacts) +RUN mkdir -p /cargo-target-prebuilt/release \ + && cp -a /workspace/target/release/deps /cargo-target-prebuilt/release/ 2>/dev/null || true \ + && cp -a /workspace/target/release/.fingerprint /cargo-target-prebuilt/release/ 2>/dev/null || true \ + && cp -a /workspace/target/release/build /cargo-target-prebuilt/release/ 2>/dev/null || true \ + && du -sh /cargo-target-prebuilt /cargo-target-prebuilt/release/* 2>/dev/null || true \ + && echo "$DEPS_VERSION" > /cargo-target-prebuilt/.deps_version + +# ── Stage 2: thin runtime that just carries the snapshot + rsync ── +# rsync is needed by the initContainer in compile-and-deploy-template.yaml +# to copy this snapshot into the PVC with --ignore-existing. +FROM ubuntu:24.04 AS runtime +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get install -y --no-install-recommends rsync ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +ARG DEPS_VERSION=1 +ENV DEPS_VERSION=$DEPS_VERSION + +COPY --from=builder /cargo-target-prebuilt /cargo-target-prebuilt + +# Default command prints what's in the image; the real entrypoint is +# overridden by the initContainer spec in compile-and-deploy-template.yaml. +CMD ["sh", "-c", "echo 'foxhunt ci-builder-cpu-with-deps v'$DEPS_VERSION; du -sh /cargo-target-prebuilt"] diff --git a/infra/k8s/argo/README-deps-cache.md b/infra/k8s/argo/README-deps-cache.md new file mode 100644 index 000000000..d689b753b --- /dev/null +++ b/infra/k8s/argo/README-deps-cache.md @@ -0,0 +1,161 @@ +# Deps Cache Image — Cold-start Compile Acceleration + +This README documents the **`ci-builder-cpu-with-deps:nightly`** image +and the Argo plumbing that uses it. Created as part of the CI cold-start +optimization sweep (April 2026). + +## What it is + +A Docker image that carries a pre-built `target/` directory at +`/cargo-target-prebuilt/` containing the workspace's third-party +dependency rlibs. It's pulled by the `compile-services` step of +`compile-and-deploy-template.yaml` as an **initContainer** that rsyncs +its payload into the `cargo-target-cpu` PVC. + +## Why it exists + +On a fresh node (autoscaler scaled up, fresh PVC, or after a PVC purge), +`cargo build --release` would otherwise spend 3-5 minutes compiling +~1500 third-party crates before touching workspace member code. With +the deps cache image, the cold-start compile path is: + +1. **initContainer pulls image** from in-cluster registry (~30-60s) +2. **rsync prebuilt rlibs into PVC** (~30-60s, ~6-8 GB transfer over local FS) +3. **cargo build delta-compiles only workspace member code** (~1-2 min) + +Net cold-start saving: **2-4 min per compile pod**. + +## Files + +| File | Role | +|------|------| +| `infra/docker/Dockerfile.ci-deps-cache` | Builds the deps-cache image (multi-stage). Stage 1 atop ci-builder-cpu, runs `cargo build --release --workspace`. Stage 2 thin Ubuntu + rsync layer carrying just `/cargo-target-prebuilt/`. | +| `infra/k8s/argo/refresh-deps-cache-template.yaml` | WorkflowTemplate that drives a Kaniko build of the Dockerfile + push to GitLab registry. Includes a CronWorkflow that runs nightly at 03:00 UTC (suspended by default). | +| `infra/k8s/argo/compile-and-deploy-template.yaml` | The `compile-services` step has a `seed-deps-cache` initContainer that rsyncs `/cargo-target-prebuilt/` into the PVC, gated by a stamp file. | + +## How to refresh manually + +```bash +# Trigger a one-shot rebuild against current main HEAD. +argo submit --from workflowtemplate/refresh-deps-cache -n foxhunt +``` + +To target a specific commit or bump the cache version (forces re-seed +on every compile pod): + +```bash +argo submit --from workflowtemplate/refresh-deps-cache -n foxhunt \ + -p commit-sha=$(git rev-parse main) \ + -p deps-version=2 \ + -p image-tag=nightly +``` + +## Enabling the nightly cron + +The CronWorkflow ships **suspended** so you can validate manually first. +To enable: + +```bash +kubectl -n foxhunt patch cronworkflow refresh-deps-cache-nightly \ + --type=merge -p '{"spec":{"suspend":false}}' +``` + +To disable (return to manual-only): + +```bash +kubectl -n foxhunt patch cronworkflow refresh-deps-cache-nightly \ + --type=merge -p '{"spec":{"suspend":true}}' +``` + +## Bumping `DEPS_VERSION` + +The compile-services initContainer skips re-seeding when +`/cargo-target/cpu_deps_v${DEPS_VERSION}.stamp` exists on the PVC. +Bump `DEPS_VERSION` to force every running compile pod to re-seed +its PVC from a fresh image. Bump in two places (must match): + +1. `refresh-deps-cache-template.yaml` workflow `deps-version` parameter + default value +2. `compile-and-deploy-template.yaml` initContainer `DEPS_VERSION` env var + +Bump when: +- Cargo.lock has churned significantly (most prebuilt rlibs + fingerprint-mismatch cargo's freshness check) +- The Rust toolchain version changes (e.g. 1.89 -> 1.90) +- A workspace member is renamed or split (rare) + +You do **not** need to bump for routine commits — cargo's own +fingerprint check will discard stale rlibs and rebuild as needed. + +## Debugging + +**Symptom: compile-services is still slow on a fresh PVC.** + +Check the seed-deps-cache initContainer logs first: + +```bash +argo logs -n foxhunt -c seed-deps-cache +``` + +Look for: +- `WARN: /cargo-target-prebuilt missing in image` — the image was built + but the snapshot dir is empty. The inner `cargo build --workspace` + failed in the deps-cache image build. Check the + `refresh-deps-cache` workflow logs. +- `PVC already seeded for deps v1 (stamp present), skipping.` — + expected steady-state on warm pods. +- `=== Seeding cargo-target-cpu PVC from prebuilt deps cache ===` + + `=== Seed complete ===` — first run on this PVC, working as designed. + +**Symptom: cargo recompiles many third-party deps anyway despite the +seed completing.** + +Cargo's fingerprint check is sensitive to: rustc version, the entire +target.rustflags list, `RUSTFLAGS` env var, source code mtimes (on +build.rs), feature flags. The commit-SHA used to build the deps cache +image must match the live compile workflow's flag set. The most common +cause of full-rebuild is a `RUSTFLAGS` mismatch between the deps-cache +image build and the compile pod (e.g. mold vs wild linker swap). + +To diagnose, look at the cargo build output: lines like +`Compiling tokio v1.x.x` for crates that were obviously in the cache +indicate fingerprint mismatch. + +**Symptom: PVC fills up faster than before.** + +Each seed adds ~6-8 GB to the PVC. The compile-services step has a +prune-at-25GB guard that wipes `release/` and `debug/` if usage +exceeds 25GB — when that fires, the next run re-seeds (stamp file is +in PVC root, survives the prune). To raise the threshold, edit the +`TARGET_SIZE_MB -gt 25000` check in compile-and-deploy-template.yaml. + +## Assumptions that break the cache + +- **rustc toolchain version mismatch** between deps-cache image and + compile pod — cargo will full-rebuild. Resolution: rebuild the + deps-cache image (refresh-deps-cache). +- **Different `RUSTFLAGS` / `target..rustflags`** between + image build and live compile (e.g. switching CPU target features) + — full rebuild. Resolution: rebuild deps-cache image after the + rustflags change is committed. +- **Cargo.lock SHA divergence** > a few weeks old — most rlibs no + longer match. Resolution: rebuild deps-cache image (refresh nightly, + or trigger manually). +- **Linker swap (e.g. mold -> wild)** — link-output is cached at the + fingerprint level; changing linker doesn't trigger rebuild but may + produce different final binaries. Not a cache invalidation, just a + link-time difference. + +## When to delete this entirely + +If the underlying compile time drops to <30s in cold-start (e.g. via a +much faster registry or sccache covering 100% of deps), the +deps-cache complexity is no longer worth maintaining. To remove: + +1. Remove the `seed-deps-cache` initContainer block from + `compile-and-deploy-template.yaml` (the compile script tolerates + an unseeded PVC — cargo just compiles deps from scratch). +2. Delete `infra/docker/Dockerfile.ci-deps-cache` and + `infra/k8s/argo/refresh-deps-cache-template.yaml`. +3. Delete the published image: + `crane delete gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder-cpu-with-deps:nightly`. diff --git a/infra/k8s/argo/compile-and-deploy-template.yaml b/infra/k8s/argo/compile-and-deploy-template.yaml index ee4384bf9..8cf286d2e 100644 --- a/infra/k8s/argo/compile-and-deploy-template.yaml +++ b/infra/k8s/argo/compile-and-deploy-template.yaml @@ -169,6 +169,62 @@ spec: - key: node.cilium.io/agent-not-ready operator: Exists effect: NoSchedule + # ── seed-deps-cache: rsync prebuilt third-party rlibs into the PVC ── + # On a true cold PVC (e.g. fresh node, first run, or after a purge), + # this saves 2-4 min that would otherwise be spent compiling the + # ~1500 third-party crates in the workspace dep graph. The image is + # rebuilt nightly by refresh-deps-cache (see + # refresh-deps-cache-template.yaml). On a warm PVC, the stamp file + # short-circuits the whole step in ~50ms. + # + # Why rsync --ignore-existing instead of cp/overwrite: a warmer PVC + # may already have NEWER artifacts from a prior compile; we don't + # want to clobber them with potentially-stale prebuilt rlibs. Cargo + # will discard rlibs whose fingerprint mismatches anyway. + # + # If the deps-cache image isn't published yet (first deploy of this + # template), set imagePullPolicy: IfNotPresent + the initContainer + # is non-blocking-on-failure (`|| true`) so the compile-services + # step still runs without the cache prelude. + initContainers: + - name: seed-deps-cache + image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder-cpu-with-deps:nightly + imagePullPolicy: IfNotPresent + env: + - name: DEPS_VERSION + value: "1" + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: "2" + memory: 1Gi + volumeMounts: + - name: cargo-target-cpu + mountPath: /cargo-target + command: ["/bin/sh", "-c"] + args: + - | + set -e + STAMP="/cargo-target/cpu_deps_v${DEPS_VERSION}.stamp" + if [ -f "$STAMP" ]; then + echo "PVC already seeded for deps v${DEPS_VERSION} (stamp present), skipping." + exit 0 + fi + if [ ! -d /cargo-target-prebuilt ]; then + echo "WARN: /cargo-target-prebuilt missing in image — image not yet published?" + echo " Compile will run without prebuilt deps cache (slower cold-start)." + exit 0 + fi + echo "=== Seeding cargo-target-cpu PVC from prebuilt deps cache (v${DEPS_VERSION}) ===" + du -sh /cargo-target-prebuilt 2>/dev/null || true + # --ignore-existing: never clobber newer artifacts already on PVC. + # -a: preserve mtimes, perms, links — cargo's freshness check needs accurate mtimes. + rsync -a --ignore-existing /cargo-target-prebuilt/ /cargo-target/ + touch "$STAMP" + echo "=== Seed complete ===" + du -sh /cargo-target/release 2>/dev/null || true container: image: gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder-cpu:latest imagePullPolicy: Always diff --git a/infra/k8s/argo/kustomization.yaml b/infra/k8s/argo/kustomization.yaml index 67eb75321..74b3aadd2 100644 --- a/infra/k8s/argo/kustomization.yaml +++ b/infra/k8s/argo/kustomization.yaml @@ -9,6 +9,7 @@ resources: - sanitizer-test-template.yaml - nsys-test-template.yaml - smoke-test-template.yaml + - refresh-deps-cache-template.yaml - argo-workflow-netpol.yaml - ci-deploy-rbac.yaml - archive-rbac.yaml diff --git a/infra/k8s/argo/refresh-deps-cache-template.yaml b/infra/k8s/argo/refresh-deps-cache-template.yaml new file mode 100644 index 000000000..8d0b16869 --- /dev/null +++ b/infra/k8s/argo/refresh-deps-cache-template.yaml @@ -0,0 +1,176 @@ +# refresh-deps-cache: rebuild ci-builder-cpu-with-deps:nightly image +# +# Triggered: +# - nightly via CronWorkflow (see CronWorkflow at the bottom of this file) +# - manually via: argo submit --from workflowtemplate/refresh-deps-cache -n foxhunt +# +# What it does: clones the repo at HEAD of main, then runs Kaniko to build +# infra/docker/Dockerfile.ci-deps-cache and pushes to +# gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder-cpu-with-deps:nightly. +# +# The compile-and-deploy workflow's seed-deps-cache initContainer pulls +# this image and rsyncs its /cargo-target-prebuilt/ into the cargo-target-cpu PVC. +--- +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: refresh-deps-cache + namespace: foxhunt + labels: + app.kubernetes.io/name: refresh-deps-cache + app.kubernetes.io/part-of: foxhunt +spec: + activeDeadlineSeconds: 7200 # 2 hours; full workspace cargo build is slow + serviceAccountName: argo-workflow + entrypoint: build + podMetadata: + labels: + app.kubernetes.io/component: deps-cache-build + app.kubernetes.io/part-of: foxhunt + ttlStrategy: + secondsAfterCompletion: 7200 + arguments: + parameters: + - name: commit-sha + value: HEAD + - name: deps-version + value: "1" + - name: image-tag + value: nightly + + templates: + - name: build + inputs: + parameters: + - name: commit-sha + value: "{{workflow.parameters.commit-sha}}" + - name: deps-version + value: "{{workflow.parameters.deps-version}}" + - name: image-tag + value: "{{workflow.parameters.image-tag}}" + nodeSelector: + k8s.scaleway.com/pool-name: ci-compile-cpu + tolerations: + - key: node.cilium.io/agent-not-ready + operator: Exists + effect: NoSchedule + volumes: + - name: workspace + emptyDir: + sizeLimit: 30Gi # full workspace + target/release ~10GB + - name: git-ssh-key + secret: + secretName: argo-git-ssh-key + defaultMode: 256 + - name: registry-auth + secret: + secretName: gitlab-registry + items: + - key: .dockerconfigjson + path: config.json + initContainers: + - name: git-clone + image: alpine/git:latest + command: ["/bin/sh", "-c"] + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: "1" + memory: 512Mi + volumeMounts: + - name: git-ssh-key + mountPath: /etc/git-ssh + readOnly: true + - name: workspace + mountPath: /workspace + args: + - | + set -ex + mkdir -p /root/.ssh + cp /etc/git-ssh/ssh-privatekey /root/.ssh/id_ed25519 + chmod 600 /root/.ssh/id_ed25519 + printf 'Host *\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n' > /root/.ssh/config + chmod 600 /root/.ssh/config + + SHA="{{inputs.parameters.commit-sha}}" + REPO="ssh://git@gitlab-gitlab-shell.foxhunt.svc.cluster.local:2222/root/foxhunt.git" + git clone --no-checkout --filter=blob:none "$REPO" /workspace/src + cd /workspace/src + git checkout "$SHA" + echo "Checked out $(git rev-parse --short HEAD)" + container: + image: gcr.io/kaniko-project/executor:debug + command: ["/busybox/sh", "-c"] + env: + - name: DOCKER_CONFIG + value: /kaniko/.docker + # Larger than build-ci-image because the inner cargo build is heavy. + resources: + requests: + cpu: "8" + memory: 16Gi + limits: + cpu: "16" + memory: 32Gi + volumeMounts: + - name: registry-auth + mountPath: /kaniko/.docker + readOnly: true + - name: workspace + mountPath: /workspace + args: + - | + /kaniko/executor \ + --context=/workspace/src \ + --dockerfile=/workspace/src/infra/docker/Dockerfile.ci-deps-cache \ + --destination=gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/ci-builder-cpu-with-deps:{{inputs.parameters.image-tag}} \ + --build-arg=DEPS_VERSION={{inputs.parameters.deps-version}} \ + --build-arg=BASE_TAG=latest \ + --insecure-registry=gitlab-registry.foxhunt.svc.cluster.local:5000 \ + --skip-tls-verify-registry=gitlab-registry.foxhunt.svc.cluster.local:5000 \ + --cache=true \ + --cache-repo=gitlab-registry.foxhunt.svc.cluster.local:5000/root/foxhunt/cache \ + --snapshot-mode=redo + +--- +# Nightly cron — rebuilds the deps cache image at 03:00 UTC. +# Suspended by default; enable with: +# kubectl -n foxhunt patch cronworkflow refresh-deps-cache-nightly \ +# -p '{"spec":{"suspend":false}}' --type=merge +apiVersion: argoproj.io/v1alpha1 +kind: CronWorkflow +metadata: + name: refresh-deps-cache-nightly + namespace: foxhunt + labels: + app.kubernetes.io/name: refresh-deps-cache-nightly + app.kubernetes.io/part-of: foxhunt +spec: + schedule: "0 3 * * *" + timezone: UTC + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 3 + suspend: true + workflowSpec: + entrypoint: trigger + serviceAccountName: argo-workflow + ttlStrategy: + secondsAfterCompletion: 7200 + templates: + - name: trigger + steps: + - - name: refresh + templateRef: + name: refresh-deps-cache + template: build + arguments: + parameters: + - name: commit-sha + value: HEAD + - name: deps-version + value: "1" + - name: image-tag + value: nightly