267240530d
- Add --cache=true --cache-repo to all 12 Kaniko builds - Cache Docker layers in Scaleway CR (rg.fr-par.scw.cloud/foxhunt-ci/cache) - Add Docker Hub auth to devcontainer + infra-runner prepare jobs - First build populates cache; subsequent builds skip base image pulls Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
108 lines
3.7 KiB
Docker
108 lines
3.7 KiB
Docker
# GPU training image for ML model training Jobs
|
|
# Usage: docker build -f infra/docker/Dockerfile.training .
|
|
# Run: docker run --gpus all foxhunt-training train_baseline_supervised --model kan [args...]
|
|
|
|
# =============================================================================
|
|
# Stage 1: Builder (CUDA dev image with Rust toolchain)
|
|
# =============================================================================
|
|
FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04 AS builder
|
|
|
|
ENV DEBIAN_FRONTEND=noninteractive
|
|
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
curl \
|
|
git \
|
|
build-essential \
|
|
pkg-config \
|
|
libssl-dev \
|
|
protobuf-compiler \
|
|
perl \
|
|
make \
|
|
ca-certificates \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
# Install Rust stable toolchain
|
|
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
|
|
ENV PATH="/root/.cargo/bin:${PATH}"
|
|
|
|
# Install sccache for build caching (no-op when SCCACHE_BUCKET is empty)
|
|
ARG SCCACHE_BUCKET=""
|
|
ARG AWS_ACCESS_KEY_ID=""
|
|
ARG AWS_SECRET_ACCESS_KEY=""
|
|
ARG SCCACHE_ENDPOINT=""
|
|
RUN if [ -n "$SCCACHE_BUCKET" ]; then \
|
|
curl -fsSL https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-x86_64-unknown-linux-musl.tar.gz \
|
|
| tar xz --strip-components=1 -C /usr/local/bin sccache-v0.8.1-x86_64-unknown-linux-musl/sccache \
|
|
&& chmod +x /usr/local/bin/sccache; \
|
|
fi
|
|
ENV RUSTC_WRAPPER=${SCCACHE_BUCKET:+/usr/local/bin/sccache}
|
|
ENV SCCACHE_BUCKET=${SCCACHE_BUCKET}
|
|
ENV SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT}
|
|
ENV SCCACHE_S3_USE_SSL=true
|
|
ENV AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
|
|
ENV AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
|
|
ENV AWS_REGION=${AWS_DEFAULT_REGION}
|
|
ENV SCCACHE_REGION=${AWS_DEFAULT_REGION}
|
|
|
|
WORKDIR /build
|
|
|
|
# Copy workspace manifests first for layer caching
|
|
COPY Cargo.toml Cargo.lock ./
|
|
COPY .sqlx ./.sqlx
|
|
|
|
# Copy workspace directories (post-restructure layout)
|
|
COPY crates ./crates
|
|
COPY bin ./bin
|
|
COPY services ./services
|
|
COPY testing ./testing
|
|
|
|
ENV SQLX_OFFLINE=true
|
|
ENV CUDA_COMPUTE_CAP=90
|
|
|
|
# Build training + hyperopt binaries with CUDA support
|
|
RUN cargo build --release -p ml --features ml/cuda \
|
|
--example train_baseline_rl \
|
|
--example train_baseline_supervised \
|
|
--example evaluate_baseline \
|
|
--example hyperopt_baseline_rl \
|
|
--example hyperopt_baseline_supervised \
|
|
&& mkdir -p /build/out \
|
|
&& for bin in train_baseline_rl train_baseline_supervised evaluate_baseline hyperopt_baseline_rl hyperopt_baseline_supervised; do \
|
|
cp target/release/examples/${bin} /build/out/ && strip /build/out/${bin}; \
|
|
done
|
|
|
|
# =============================================================================
|
|
# Stage 2: Runtime (CUDA runtime — no compiler, smaller image)
|
|
# =============================================================================
|
|
FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
|
|
|
|
ENV DEBIAN_FRONTEND=noninteractive
|
|
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
ca-certificates \
|
|
libssl3 \
|
|
curl \
|
|
unzip \
|
|
&& curl -fsSL https://downloads.rclone.org/v1.69.1/rclone-v1.69.1-linux-amd64.zip -o /tmp/rclone.zip \
|
|
&& unzip -j /tmp/rclone.zip '*/rclone' -d /usr/local/bin/ \
|
|
&& chmod +x /usr/local/bin/rclone \
|
|
&& rm /tmp/rclone.zip \
|
|
&& apt-get purge -y unzip \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
RUN groupadd -g 1000 foxhunt \
|
|
&& useradd -u 1000 -g foxhunt -m -s /bin/false foxhunt
|
|
|
|
COPY --from=builder /build/out/* /usr/local/bin/
|
|
|
|
# CUDA runtime environment
|
|
ENV NVIDIA_VISIBLE_DEVICES=all
|
|
ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
|
|
ENV CUDA_COMPUTE_CAP=90
|
|
|
|
USER foxhunt
|
|
|
|
WORKDIR /data
|
|
|
|
CMD ["echo", "Specify training command via K8s Job args"]
|