From f0bbca23a55ea8424e149d53f70928c2b434328d Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Thu, 26 Feb 2026 01:37:52 +0100
Subject: [PATCH] perf(ci): compile training binaries with CUDA in
 compile-services job

Training image was recompiling from scratch inside Kaniko (~16+ min)
without sccache. Now training binaries are built in compile-services
with --features ml/cuda and PVC sccache (warm cache from service
binaries), then packaged into a CUDA runtime image (~30s).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .gitlab-ci.yml                           | 23 +++++++++---
 infra/docker/Dockerfile.training-runtime | 46 ++++++++++++++++++++++++
 2 files changed, 65 insertions(+), 4 deletions(-)
 create mode 100644 infra/docker/Dockerfile.training-runtime

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 748543033..1a602d60a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -158,6 +158,7 @@ check:
         - infra/docker/Dockerfile.service
         - infra/docker/Dockerfile.web-gateway
         - infra/docker/Dockerfile.training
+        - infra/docker/Dockerfile.training-runtime
         - .gitlab-ci.yml
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
@@ -193,6 +194,7 @@ test:
         - infra/docker/Dockerfile.service
         - infra/docker/Dockerfile.web-gateway
         - infra/docker/Dockerfile.training
+        - infra/docker/Dockerfile.training-runtime
         - .gitlab-ci.yml
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       changes:
@@ -240,9 +242,11 @@ compile-services:
         - infra/docker/Dockerfile.service
         - infra/docker/Dockerfile.web-gateway
         - infra/docker/Dockerfile.training
+        - infra/docker/Dockerfile.training-runtime
         - .gitlab-ci.yml
   script:
     - sccache --zero-stats || true
+    # 1) Build service binaries (no CUDA feature)
     - cargo build --release
         -p trading_service
         -p api_gateway
@@ -252,6 +256,13 @@ compile-services:
         -p trading_agent_service
         -p data_acquisition_service
         -p web-gateway
+    # 2) Build training binaries with CUDA (sccache already warm from step 1)
+    - cargo build --release -p ml --features ml/cuda
+        --example train_baseline_rl
+        --example train_baseline_supervised
+        --example evaluate_baseline
+        --example hyperopt_baseline_rl
+        --example hyperopt_baseline_supervised
     - sccache --show-stats || true
     - mkdir -p build-out
     - |
@@ -259,6 +270,10 @@ compile-services:
         cp target/release/$bin build-out/
         strip build-out/$bin
       done
+      for bin in train_baseline_rl train_baseline_supervised evaluate_baseline hyperopt_baseline_rl hyperopt_baseline_supervised; do
+        cp target/release/examples/$bin build-out/
+        strip build-out/$bin
+      done
     - ls -lh build-out/
   artifacts:
     paths:
@@ -291,6 +306,7 @@ compile-services:
         - infra/docker/Dockerfile.web-gateway
         - infra/docker/Dockerfile.web-gateway-runtime
         - infra/docker/Dockerfile.training
+        - infra/docker/Dockerfile.training-runtime
         - .gitlab-ci.yml
   before_script:
     - mkdir -p /kaniko/.docker
@@ -382,14 +398,13 @@ build-web-gateway:
       --destination "${REGISTRY}/web-gateway:${CI_COMMIT_SHA}"
       --destination "${REGISTRY}/web-gateway:latest"
 
-# training: CUDA build (H100 target, full Kaniko — needs CUDA dev image)
+# training: pre-built CUDA binaries → CUDA runtime image (no recompilation)
 build-training:
   extends: .kaniko-base
   script:
     - /kaniko/executor
-      --context "${CI_PROJECT_DIR}"
-      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.training"
-      --cache=true --cache-repo="${REGISTRY}/cache"
+      --context dir://${CI_PROJECT_DIR}/build-out
+      --dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.training-runtime"
       --destination "${REGISTRY}/training:${CI_COMMIT_SHA}"
       --destination "${REGISTRY}/training:latest"
 
diff --git a/infra/docker/Dockerfile.training-runtime b/infra/docker/Dockerfile.training-runtime
new file mode 100644
index 000000000..f9bf09505
--- /dev/null
+++ b/infra/docker/Dockerfile.training-runtime
@@ -0,0 +1,46 @@
+# GPU training runtime image — packages pre-built CUDA binaries (no compilation)
+# Binaries are compiled in CI compile-services job with --features ml/cuda
+# Usage: kaniko --context dir://build-out --dockerfile Dockerfile.training-runtime
+# Run:   docker run --gpus all training train_baseline_supervised --model kan [args...]
+
+FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    libssl3 \
+    curl \
+    unzip \
+    && curl -fsSL https://downloads.rclone.org/v1.69.1/rclone-v1.69.1-linux-amd64.zip -o /tmp/rclone.zip \
+    && unzip -j /tmp/rclone.zip '*/rclone' -d /usr/local/bin/ \
+    && chmod +x /usr/local/bin/rclone \
+    && rm /tmp/rclone.zip \
+    && apt-get purge -y unzip \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN groupadd -g 1000 foxhunt \
+    && useradd -u 1000 -g foxhunt -m -s /bin/false foxhunt
+
+COPY train_baseline_rl \
+     train_baseline_supervised \
+     evaluate_baseline \
+     hyperopt_baseline_rl \
+     hyperopt_baseline_supervised \
+     /usr/local/bin/
+
+RUN chmod +x /usr/local/bin/train_baseline_rl \
+             /usr/local/bin/train_baseline_supervised \
+             /usr/local/bin/evaluate_baseline \
+             /usr/local/bin/hyperopt_baseline_rl \
+             /usr/local/bin/hyperopt_baseline_supervised
+
+# CUDA runtime environment
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
+
+USER foxhunt
+
+WORKDIR /data
+
+CMD ["echo", "Specify training command via K8s Job args"]