ci: pre-baked CI builder image, pipeline prepare stage

- Add Dockerfile.ci-builder (CUDA 12.4 + Rust 1.89 + protoc + sccache + git + lld)
- Add prepare stage to build CI builder image via Kaniko (auto on Dockerfile change, manual otherwise)
- Replace before_script apt-get installs with pre-baked image in .rust-base
- Add git to Dockerfile.service for candle git dependency
- Add sccache stats output to check stage

.gitlab-ci.yml

@@ -1,8 +1,9 @@
 # GitLab CI/CD — Foxhunt
 # Runs on ci-build pool (GP1-XS, scale-to-zero)
-# Uses sccache for Rust build caching, Kaniko for Docker builds
+# Uses pre-baked CI builder image with CUDA, Rust, protoc, sccache
 
 stages:
+  - prepare
   - check
   - test
   - build
@@ -22,18 +23,42 @@ variables:
   INTERNAL_REGISTRY: gitlab-registry.foxhunt.svc.cluster.local:5000
   REGISTRY: ${INTERNAL_REGISTRY}/root/foxhunt
 
-# Base template for Rust jobs
+# --------------------------------------------------------------------------
+# Stage 0: Build CI builder image (auto when Dockerfile changes, manual otherwise)
+# --------------------------------------------------------------------------
+build-ci-builder:
+  stage: prepare
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
+  tags:
+    - kapsule
+    - docker
+  rules:
+    - changes:
+        - infra/docker/Dockerfile.ci-builder
+      when: on_success
+    - when: manual
+      allow_failure: true
+  before_script:
+    - mkdir -p /kaniko/.docker
+    - echo "{\"auths\":{\"${INTERNAL_REGISTRY}\":{\"auth\":\"$(printf '%s:%s' \"${CI_REGISTRY_USER}\" \"${CI_REGISTRY_PASSWORD}\" | base64 | tr -d '\\n')\"}}}" > /kaniko/.docker/config.json
+  script:
+    - /kaniko/executor
+      --context "${CI_PROJECT_DIR}"
+      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.ci-builder"
+      --insecure-registry "${INTERNAL_REGISTRY}"
+      --destination "${REGISTRY}/ci-builder:latest"
+
+# Base template for Rust jobs — uses pre-baked CI builder image
 .rust-base:
-  image: rust:1.89-slim
+  image: ${REGISTRY}/ci-builder:latest
   tags:
     - kapsule
     - rust
-  before_script:
-    - apt-get update && apt-get install -y git protobuf-compiler curl pkg-config libssl-dev
-    - curl -fsSL https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-x86_64-unknown-linux-musl.tar.gz
-      | tar xz --strip-components=1 -C /usr/local/bin sccache-v0.8.1-x86_64-unknown-linux-musl/sccache
-    - chmod +x /usr/local/bin/sccache
-    - rustup component add clippy
+  needs:
+    - job: build-ci-builder
+      optional: true
 
 # --------------------------------------------------------------------------
 # Stage 1: cargo check + clippy
@@ -42,8 +67,10 @@ check:
   extends: .rust-base
   stage: check
   script:
+    - sccache --zero-stats || true
     - cargo check --workspace
     - cargo clippy --workspace -- -D warnings
+    - sccache --show-stats || true
 
 # --------------------------------------------------------------------------
 # Stage 2: tests
@@ -51,7 +78,10 @@ check:
 test:
   extends: .rust-base
   stage: test
-  needs: [check]
+  needs:
+    - job: build-ci-builder
+      optional: true
+    - check
   script:
     - cargo test --workspace --lib
 
@@ -71,7 +101,7 @@ test:
     - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
   before_script:
     - mkdir -p /kaniko/.docker
-    - echo "{\"auths\":{\"${INTERNAL_REGISTRY}\":{\"auth\":\"$(printf '%s:%s' "${CI_REGISTRY_USER}" "${CI_REGISTRY_PASSWORD}" | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json
+    - echo "{\"auths\":{\"${INTERNAL_REGISTRY}\":{\"auth\":\"$(printf '%s:%s' \"${CI_REGISTRY_USER}\" \"${CI_REGISTRY_PASSWORD}\" | base64 | tr -d '\\n')\"}}}" > /kaniko/.docker/config.json
 
 build-trading-service:
   extends: .kaniko-base

infra/docker/Dockerfile.ci-builder

@@ -0,0 +1,44 @@
+# Pre-baked CI builder image for Foxhunt
+# Contains: CUDA 12.4, Rust 1.89, protoc, sccache, git, OpenSSL, lld, make
+# Build: docker build -f infra/docker/Dockerfile.ci-builder -t foxhunt-ci-builder .
+# Push:  docker tag foxhunt-ci-builder git.fxhnt.ai:5050/root/foxhunt/ci-builder:latest
+#        docker push git.fxhnt.ai:5050/root/foxhunt/ci-builder:latest
+
+FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    protobuf-compiler \
+    libprotobuf-dev \
+    curl \
+    ca-certificates \
+    pkg-config \
+    libssl-dev \
+    lld \
+    clang \
+    make \
+    perl \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Rust 1.89 stable
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.89.0
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# sccache for S3-backed build caching
+RUN curl -fsSL https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-unknown-linux-musl.tar.gz \
+    | tar xz -C /usr/local/bin --strip-components=1 sccache-v0.10.0-x86_64-unknown-linux-musl/sccache \
+    && chmod +x /usr/local/bin/sccache
+
+# clippy pre-installed
+RUN rustup component add clippy
+
+# CUDA env
+ENV CUDA_HOME=/usr/local/cuda
+ENV PATH="${CUDA_HOME}/bin:${PATH}"
+ENV LD_LIBRARY_PATH="${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}"
+
+# Verify
+RUN rustc --version && cargo --version && git --version && protoc --version && sccache --version && make --version && nvcc --version

infra/docker/Dockerfile.service

@@ -24,6 +24,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     perl \
     make \
     curl \
+    git \
     && rm -rf /var/lib/apt/lists/*
 
 # Conditionally install sccache when a bucket is configured