5fb84a02f0
- Add standalone training binaries: TGGN, KAN, xLSTM, Diffusion (DBN data) - Update Dockerfile.training: 6 → 16 binaries (all 10 models + hyperopt + baseline) - Expand train.sh: 4 → 10 models, fix registry URL and GPU pool nodeSelector - Add GPU overlay manifests for trading-service and ml-training-service - Create training data PVC and upload pod manifests - Expand web-gateway model validation: 4 → 10 types (training + tune routes) - Extend dashboard: 10 model cards grouped by category (RL/Temporal/Graph/Generative) - Add training image build job to Gitea CI workflow - Update GPU taint controller to exclude inference pool from tainting - Fix job-template nodeSelector: gpu → gpu-training Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
110 lines
3.4 KiB
Docker
110 lines
3.4 KiB
Docker
# GPU training image for ML model training Jobs
|
|
# Usage: docker build -f infra/docker/Dockerfile.training .
|
|
# Run: docker run --gpus all foxhunt-training train_dqn [args...]
|
|
|
|
# =============================================================================
|
|
# Stage 1: Builder (CUDA dev image with Rust toolchain)
|
|
# =============================================================================
|
|
FROM nvidia/cuda:12.4.1-cudnn-devel-ubuntu22.04 AS builder
|
|
|
|
ENV DEBIAN_FRONTEND=noninteractive
|
|
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
curl \
|
|
git \
|
|
build-essential \
|
|
pkg-config \
|
|
libssl-dev \
|
|
protobuf-compiler \
|
|
perl \
|
|
make \
|
|
ca-certificates \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
# Install Rust stable toolchain
|
|
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
|
|
ENV PATH="/root/.cargo/bin:${PATH}"
|
|
|
|
WORKDIR /build
|
|
|
|
# Copy workspace manifests first for layer caching
|
|
COPY Cargo.toml Cargo.lock ./
|
|
COPY .sqlx ./.sqlx
|
|
|
|
# Copy all workspace crate directories
|
|
COPY trading_engine ./trading_engine
|
|
COPY risk ./risk
|
|
COPY risk-data ./risk-data
|
|
COPY trading-data ./trading-data
|
|
COPY fxt ./fxt
|
|
COPY ml ./ml
|
|
COPY ml-data ./ml-data
|
|
COPY data ./data
|
|
COPY backtesting ./backtesting
|
|
COPY adaptive-strategy ./adaptive-strategy
|
|
COPY common ./common
|
|
COPY storage ./storage
|
|
COPY model_loader ./model_loader
|
|
COPY market-data ./market-data
|
|
COPY database ./database
|
|
COPY config ./config
|
|
COPY web-gateway ./web-gateway
|
|
COPY ctrader-openapi ./ctrader-openapi
|
|
COPY services ./services
|
|
COPY tests ./tests
|
|
|
|
ENV SQLX_OFFLINE=true
|
|
ENV CUDA_COMPUTE_CAP=90
|
|
|
|
# Build all training example binaries with CUDA support
|
|
RUN cargo build --release -p ml --features ml/cuda \
|
|
--example train_dqn_es_fut \
|
|
--example train_ppo_parquet \
|
|
--example train_tft_dbn \
|
|
--example train_mamba2_dbn \
|
|
--example train_liquid_dbn \
|
|
--example train_tggn_dbn \
|
|
--example train_kan_dbn \
|
|
--example train_xlstm_dbn \
|
|
--example train_diffusion_dbn \
|
|
--example train_tlob \
|
|
--example train_baseline \
|
|
--example evaluate_baseline \
|
|
--example hyperopt_dqn_demo \
|
|
--example hyperopt_ppo_demo \
|
|
--example hyperopt_tft_demo \
|
|
--example hyperopt_mamba2_demo \
|
|
&& mkdir -p /build/out \
|
|
&& for bin in train_dqn_es_fut train_ppo_parquet train_tft_dbn train_mamba2_dbn train_liquid_dbn train_tggn_dbn train_kan_dbn train_xlstm_dbn train_diffusion_dbn train_tlob train_baseline evaluate_baseline hyperopt_dqn_demo hyperopt_ppo_demo hyperopt_tft_demo hyperopt_mamba2_demo; do \
|
|
cp target/release/examples/${bin} /build/out/ && strip /build/out/${bin}; \
|
|
done
|
|
|
|
# =============================================================================
|
|
# Stage 2: Runtime (CUDA runtime — no compiler, smaller image)
|
|
# =============================================================================
|
|
FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
|
|
|
|
ENV DEBIAN_FRONTEND=noninteractive
|
|
|
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
ca-certificates \
|
|
libssl3 \
|
|
curl \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
RUN groupadd -g 1000 foxhunt \
|
|
&& useradd -u 1000 -g foxhunt -m -s /bin/false foxhunt
|
|
|
|
COPY --from=builder /build/out/* /usr/local/bin/
|
|
|
|
# CUDA runtime environment
|
|
ENV NVIDIA_VISIBLE_DEVICES=all
|
|
ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
|
|
ENV CUDA_COMPUTE_CAP=90
|
|
|
|
USER foxhunt
|
|
|
|
WORKDIR /data
|
|
|
|
CMD ["echo", "Specify training command via K8s Job args"]
|