f0bbca23a5
Training image was recompiling from scratch inside Kaniko (~16+ min) without sccache. Now training binaries are built in compile-services with --features ml/cuda and PVC sccache (warm cache from service binaries), then packaged into a CUDA runtime image (~30s). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
47 lines
1.5 KiB
Docker
47 lines
1.5 KiB
Docker
# GPU training runtime image — packages pre-built CUDA binaries (no compilation)
|
|
# Binaries are compiled in CI compile-services job with --features ml/cuda
|
|
# Usage: kaniko --context dir://build-out --dockerfile Dockerfile.training-runtime
|
|
# Run: docker run --gpus all training train_baseline_supervised --model kan [args...]
|
|
|
|
FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
|
|
|
|
ENV DEBIAN_FRONTEND=noninteractive
|
|
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
ca-certificates \
|
|
libssl3 \
|
|
curl \
|
|
unzip \
|
|
&& curl -fsSL https://downloads.rclone.org/v1.69.1/rclone-v1.69.1-linux-amd64.zip -o /tmp/rclone.zip \
|
|
&& unzip -j /tmp/rclone.zip '*/rclone' -d /usr/local/bin/ \
|
|
&& chmod +x /usr/local/bin/rclone \
|
|
&& rm /tmp/rclone.zip \
|
|
&& apt-get purge -y unzip \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
RUN groupadd -g 1000 foxhunt \
|
|
&& useradd -u 1000 -g foxhunt -m -s /bin/false foxhunt
|
|
|
|
COPY train_baseline_rl \
|
|
train_baseline_supervised \
|
|
evaluate_baseline \
|
|
hyperopt_baseline_rl \
|
|
hyperopt_baseline_supervised \
|
|
/usr/local/bin/
|
|
|
|
RUN chmod +x /usr/local/bin/train_baseline_rl \
|
|
/usr/local/bin/train_baseline_supervised \
|
|
/usr/local/bin/evaluate_baseline \
|
|
/usr/local/bin/hyperopt_baseline_rl \
|
|
/usr/local/bin/hyperopt_baseline_supervised
|
|
|
|
# CUDA runtime environment
|
|
ENV NVIDIA_VISIBLE_DEVICES=all
|
|
ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
|
|
|
|
USER foxhunt
|
|
|
|
WORKDIR /data
|
|
|
|
CMD ["echo", "Specify training command via K8s Job args"]
|