c8f2dacc5f
- Fix container registry region nl-ams → fr-par across all CI jobs - Add sccache build-args to training image build (no-op fallback for local) - Add rclone to training Docker runtime for S3 output sync - Update train.sh: S3 sync on Job completion via rclone env-var config, --run-id tracking, evaluate preset for walk-forward evaluation - Add s3-credentials Secret template (.example, apply via kubectl) - Add design doc and implementation plan Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
133 lines
4.4 KiB
Docker
133 lines
4.4 KiB
Docker
# GPU training image for ML model training Jobs
|
|
# Usage: docker build -f infra/docker/Dockerfile.training .
|
|
# Run: docker run --gpus all foxhunt-training train_dqn [args...]
|
|
|
|
# =============================================================================
|
|
# Stage 1: Builder (CUDA dev image with Rust toolchain)
|
|
# =============================================================================
|
|
FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04 AS builder
|
|
|
|
ENV DEBIAN_FRONTEND=noninteractive
|
|
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
curl \
|
|
git \
|
|
build-essential \
|
|
pkg-config \
|
|
libssl-dev \
|
|
protobuf-compiler \
|
|
perl \
|
|
make \
|
|
ca-certificates \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
# Install Rust stable toolchain
|
|
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
|
|
ENV PATH="/root/.cargo/bin:${PATH}"
|
|
|
|
# Install sccache for build caching (no-op when SCCACHE_BUCKET is empty)
|
|
ARG SCCACHE_BUCKET=""
|
|
ARG AWS_ACCESS_KEY_ID=""
|
|
ARG AWS_SECRET_ACCESS_KEY=""
|
|
ARG SCCACHE_ENDPOINT=""
|
|
RUN if [ -n "$SCCACHE_BUCKET" ]; then \
|
|
curl -fsSL https://github.com/mozilla/sccache/releases/download/v0.8.1/sccache-v0.8.1-x86_64-unknown-linux-musl.tar.gz \
|
|
| tar xz --strip-components=1 -C /usr/local/bin sccache-v0.8.1-x86_64-unknown-linux-musl/sccache \
|
|
&& chmod +x /usr/local/bin/sccache; \
|
|
fi
|
|
ENV RUSTC_WRAPPER=${SCCACHE_BUCKET:+/usr/local/bin/sccache}
|
|
ENV SCCACHE_BUCKET=${SCCACHE_BUCKET}
|
|
ENV SCCACHE_ENDPOINT=${SCCACHE_ENDPOINT}
|
|
ENV SCCACHE_S3_USE_SSL=true
|
|
ENV AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
|
|
ENV AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
|
|
|
|
WORKDIR /build
|
|
|
|
# Copy workspace manifests first for layer caching
|
|
COPY Cargo.toml Cargo.lock ./
|
|
COPY .sqlx ./.sqlx
|
|
|
|
# Copy all workspace crate directories
|
|
COPY trading_engine ./trading_engine
|
|
COPY risk ./risk
|
|
COPY risk-data ./risk-data
|
|
COPY trading-data ./trading-data
|
|
COPY fxt ./fxt
|
|
COPY ml ./ml
|
|
COPY ml-data ./ml-data
|
|
COPY data ./data
|
|
COPY backtesting ./backtesting
|
|
COPY adaptive-strategy ./adaptive-strategy
|
|
COPY common ./common
|
|
COPY storage ./storage
|
|
COPY model_loader ./model_loader
|
|
COPY market-data ./market-data
|
|
COPY database ./database
|
|
COPY config ./config
|
|
COPY web-gateway ./web-gateway
|
|
COPY ctrader-openapi ./ctrader-openapi
|
|
COPY services ./services
|
|
COPY tests ./tests
|
|
|
|
ENV SQLX_OFFLINE=true
|
|
ENV CUDA_COMPUTE_CAP=90
|
|
|
|
# Build all training example binaries with CUDA support
|
|
RUN cargo build --release -p ml --features ml/cuda \
|
|
--example train_dqn_es_fut \
|
|
--example train_ppo_parquet \
|
|
--example train_tft_dbn \
|
|
--example train_mamba2_dbn \
|
|
--example train_liquid_dbn \
|
|
--example train_tggn_dbn \
|
|
--example train_kan_dbn \
|
|
--example train_xlstm_dbn \
|
|
--example train_diffusion_dbn \
|
|
--example train_tlob \
|
|
--example train_baseline \
|
|
--example evaluate_baseline \
|
|
--example hyperopt_dqn_demo \
|
|
--example hyperopt_ppo_demo \
|
|
--example hyperopt_tft_demo \
|
|
--example hyperopt_mamba2_demo \
|
|
&& mkdir -p /build/out \
|
|
&& for bin in train_dqn_es_fut train_ppo_parquet train_tft_dbn train_mamba2_dbn train_liquid_dbn train_tggn_dbn train_kan_dbn train_xlstm_dbn train_diffusion_dbn train_tlob train_baseline evaluate_baseline hyperopt_dqn_demo hyperopt_ppo_demo hyperopt_tft_demo hyperopt_mamba2_demo; do \
|
|
cp target/release/examples/${bin} /build/out/ && strip /build/out/${bin}; \
|
|
done
|
|
|
|
# =============================================================================
|
|
# Stage 2: Runtime (CUDA runtime — no compiler, smaller image)
|
|
# =============================================================================
|
|
FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
|
|
|
|
ENV DEBIAN_FRONTEND=noninteractive
|
|
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
ca-certificates \
|
|
libssl3 \
|
|
curl \
|
|
unzip \
|
|
&& curl -fsSL https://downloads.rclone.org/current/rclone-current-linux-amd64.zip -o /tmp/rclone.zip \
|
|
&& unzip -j /tmp/rclone.zip '*/rclone' -d /usr/local/bin/ \
|
|
&& chmod +x /usr/local/bin/rclone \
|
|
&& rm /tmp/rclone.zip \
|
|
&& apt-get purge -y unzip \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
RUN groupadd -g 1000 foxhunt \
|
|
&& useradd -u 1000 -g foxhunt -m -s /bin/false foxhunt
|
|
|
|
COPY --from=builder /build/out/* /usr/local/bin/
|
|
|
|
# CUDA runtime environment
|
|
ENV NVIDIA_VISIBLE_DEVICES=all
|
|
ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
|
|
ENV CUDA_COMPUTE_CAP=90
|
|
|
|
USER foxhunt
|
|
|
|
WORKDIR /data
|
|
|
|
CMD ["echo", "Specify training command via K8s Job args"]
|