feat(infra): split training image into RL + supervised, rename ci-compile → ci-rl

RL models (DQN/PPO) only need basic CUDA runtime (~2GB), while supervised
models need cuDNN (~4GB). Splitting saves ~2GB pull time per RL job.
Rename the L4 GPU pool from ci-compile to ci-rl to reflect its actual use.
Add DaemonSet image pre-puller to cache training images on GPU nodes.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-28 10:16:42 +01:00
parent f9b57cdecb
commit c731bef759
13 changed files with 199 additions and 71 deletions

View File

@@ -9,7 +9,7 @@
# kapsule,rust → CPU runner → ci-compile-cpu pool (POP2-32C-128G, 32 vCPU)
# kapsule,gpu → GPU runner → ci-training (L40S)
# Override nodeSelector: KUBERNETES_NODE_SELECTOR_k8s.scaleway.com/pool-name: "<name>" (k=v format)
# Pools: ci-compile-cpu (POP2-32C-128G), ci-training (L40S), services, gitlab
# Pools: ci-compile-cpu (POP2-32C-128G), ci-rl (L4), ci-training (L40S), services, gitlab
# All compilation (services + training) runs on ci-compile-cpu. CUDA stubs in ci-builder image.
# Ensure pipeline is always created — individual job rules handle filtering.
@@ -538,31 +538,41 @@ build-web-gateway:
--destination "${REGISTRY}/web-gateway:${CI_COMMIT_SHA}"
--destination "${REGISTRY}/web-gateway:latest"
# training: pre-built CUDA binaries → CUDA runtime image (no recompilation)
build-training:
# training-rl: RL binaries → CUDA runtime image (no cuDNN, ~2GB)
build-training-rl:
extends: .kaniko-training-base
script:
- /kaniko/executor
--context dir://${CI_PROJECT_DIR}/build-out/training
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.training-runtime"
--destination "${REGISTRY}/training:${CI_COMMIT_SHA}"
--destination "${REGISTRY}/training:latest"
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.training-rl"
--destination "${REGISTRY}/training-rl:${CI_COMMIT_SHA}"
--destination "${REGISTRY}/training-rl:latest"
# training-supervised: supervised binaries → CUDA+cuDNN runtime image (~4GB)
build-training-supervised:
extends: .kaniko-training-base
script:
- /kaniko/executor
--context dir://${CI_PROJECT_DIR}/build-out/training
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.training-supervised"
--destination "${REGISTRY}/training-supervised:${CI_COMMIT_SHA}"
--destination "${REGISTRY}/training-supervised:latest"
# --------------------------------------------------------------------------
# Stage 3c: Training — RL on L4 (CPU-heavy), Supervised on L40S (GPU-heavy)
# --------------------------------------------------------------------------
# --------------------------------------------------------------------------
# RL training base: DQN + PPO → ci-compile pool (L4-1-24G)
# RL training base: DQN + PPO → ci-rl pool (L4-1-24G)
# RL hyperopt runs parallel PSO trials — CPU+GPU bound, low VRAM.
# --------------------------------------------------------------------------
.train-rl-base:
stage: train
image: ${REGISTRY}/training:${CI_COMMIT_SHA}
image: ${REGISTRY}/training-rl:${CI_COMMIT_SHA}
tags:
- kapsule
- gpu
# Runner default node_selector: k8s.scaleway.com/pool-name=ci-compile-cpu
# Runner default node_selector: k8s.scaleway.com/pool-name=ci-rl
variables:
# nvidia runtime provided by runner default (runtime_class_name = "nvidia")
# RL hyperopt is CPU-bound (parallel PSO trials). Use full L4 (8 vCPU).
@@ -589,7 +599,7 @@ build-training:
# --------------------------------------------------------------------------
.train-validate-base:
stage: train
image: ${REGISTRY}/training:${CI_COMMIT_SHA}
image: ${REGISTRY}/training-supervised:${CI_COMMIT_SHA}
tags:
- kapsule
- gpu
@@ -619,7 +629,7 @@ build-training:
train-validate-rl:
extends: .train-rl-base
needs:
- job: build-training
- job: build-training-rl
optional: true
script:
- |
@@ -651,7 +661,7 @@ train-validate-rl:
train-validate-tft:
extends: .train-validate-base
needs:
- job: build-training
- job: build-training-supervised
optional: true
script:
- /usr/local/bin/train_baseline_supervised
@@ -678,7 +688,7 @@ train-validate-tft:
hyperopt-ppo:
extends: .train-rl-base
needs:
- job: build-training
- job: build-training-rl
optional: true
timeout: 4h
script:
@@ -725,7 +735,7 @@ hyperopt-ppo:
train-validate-mamba2:
extends: .train-validate-base
needs:
- job: build-training
- job: build-training-supervised
optional: true
script:
- /usr/local/bin/train_baseline_supervised
@@ -752,7 +762,7 @@ train-validate-mamba2:
train-validate-liquid:
extends: .train-validate-base
needs:
- job: build-training
- job: build-training-supervised
optional: true
script:
- /usr/local/bin/train_baseline_supervised
@@ -779,7 +789,7 @@ train-validate-liquid:
train-validate-tggn:
extends: .train-validate-base
needs:
- job: build-training
- job: build-training-supervised
optional: true
script:
- /usr/local/bin/train_baseline_supervised
@@ -806,7 +816,7 @@ train-validate-tggn:
train-validate-tlob:
extends: .train-validate-base
needs:
- job: build-training
- job: build-training-supervised
optional: true
script:
- /usr/local/bin/train_baseline_supervised
@@ -833,7 +843,7 @@ train-validate-tlob:
train-validate-kan:
extends: .train-validate-base
needs:
- job: build-training
- job: build-training-supervised
optional: true
script:
- /usr/local/bin/train_baseline_supervised
@@ -860,7 +870,7 @@ train-validate-kan:
train-validate-xlstm:
extends: .train-validate-base
needs:
- job: build-training
- job: build-training-supervised
optional: true
script:
- /usr/local/bin/train_baseline_supervised
@@ -887,7 +897,7 @@ train-validate-xlstm:
train-validate-diffusion:
extends: .train-validate-base
needs:
- job: build-training
- job: build-training-supervised
optional: true
script:
- /usr/local/bin/train_baseline_supervised
@@ -916,7 +926,7 @@ train-validate-diffusion:
hyperopt-dqn:
extends: .train-rl-base
needs:
- job: build-training
- job: build-training-rl
optional: true
timeout: 4h
script:
@@ -968,7 +978,7 @@ hyperopt-dqn:
hyperopt-tft:
extends: .train-validate-base
needs:
- job: build-training
- job: build-training-supervised
optional: true
timeout: 4h
script:
@@ -1007,7 +1017,7 @@ hyperopt-tft:
hyperopt-mamba2:
extends: .train-validate-base
needs:
- job: build-training
- job: build-training-supervised
optional: true
timeout: 4h
script:
@@ -1046,7 +1056,7 @@ hyperopt-mamba2:
hyperopt-liquid:
extends: .train-validate-base
needs:
- job: build-training
- job: build-training-supervised
optional: true
timeout: 4h
script:
@@ -1085,7 +1095,7 @@ hyperopt-liquid:
hyperopt-tggn:
extends: .train-validate-base
needs:
- job: build-training
- job: build-training-supervised
optional: true
timeout: 4h
script:
@@ -1124,7 +1134,7 @@ hyperopt-tggn:
hyperopt-tlob:
extends: .train-validate-base
needs:
- job: build-training
- job: build-training-supervised
optional: true
timeout: 4h
script:
@@ -1163,7 +1173,7 @@ hyperopt-tlob:
hyperopt-kan:
extends: .train-validate-base
needs:
- job: build-training
- job: build-training-supervised
optional: true
timeout: 4h
script:
@@ -1202,7 +1212,7 @@ hyperopt-kan:
hyperopt-xlstm:
extends: .train-validate-base
needs:
- job: build-training
- job: build-training-supervised
optional: true
timeout: 4h
script:
@@ -1241,7 +1251,7 @@ hyperopt-xlstm:
hyperopt-diffusion:
extends: .train-validate-base
needs:
- job: build-training
- job: build-training-supervised
optional: true
timeout: 4h
script:
@@ -1412,7 +1422,9 @@ deploy:
optional: true
- job: build-web-gateway
optional: true
- job: build-training
- job: build-training-rl
optional: true
- job: build-training-supervised
optional: true
rules:
# Only deploy when source code changed (build produced new images)

View File

@@ -1,9 +1,9 @@
# GPU training runtime image — packages pre-built CUDA binaries (no compilation)
# Binaries are compiled in CI compile-services job with --features ml/cuda
# Usage: kaniko --context dir://build-out --dockerfile Dockerfile.training-runtime
# Run: docker run --gpus all training train_baseline_supervised --model kan [args...]
# RL training runtime image — DQN + PPO (no cuDNN needed, ~2GB)
# Binaries are compiled in CI compile-training job with --features ml/cuda
# Usage: kaniko --context dir://build-out/training --dockerfile Dockerfile.training-rl
# Run: docker run --gpus all training-rl train_baseline_rl --model dqn [args...]
FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
FROM nvidia/cuda:12.4.1-runtime-ubuntu22.04
ENV DEBIAN_FRONTEND=noninteractive
@@ -23,20 +23,14 @@ RUN groupadd -g 1000 foxhunt \
&& useradd -u 1000 -g foxhunt -m -s /bin/false foxhunt
COPY train_baseline_rl \
train_baseline_supervised \
evaluate_baseline \
evaluate_supervised \
hyperopt_baseline_rl \
hyperopt_baseline_supervised \
training_uploader \
/usr/local/bin/
RUN chmod +x /usr/local/bin/train_baseline_rl \
/usr/local/bin/train_baseline_supervised \
/usr/local/bin/evaluate_baseline \
/usr/local/bin/evaluate_supervised \
/usr/local/bin/hyperopt_baseline_rl \
/usr/local/bin/hyperopt_baseline_supervised \
/usr/local/bin/training_uploader
# CUDA runtime environment

View File

@@ -0,0 +1,45 @@
# Supervised training runtime image — TFT, Mamba2, TGGN, TLOB, Liquid, KAN, xLSTM, Diffusion (~4GB)
# Includes cuDNN for deep neural network operations (convolutions, attention)
# Binaries are compiled in CI compile-training job with --features ml/cuda
# Usage: kaniko --context dir://build-out/training --dockerfile Dockerfile.training-supervised
# Run: docker run --gpus all training-supervised train_baseline_supervised --model tft [args...]
FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
libssl3 \
curl \
unzip \
&& curl -fsSL https://downloads.rclone.org/v1.69.1/rclone-v1.69.1-linux-amd64.zip -o /tmp/rclone.zip \
&& unzip -j /tmp/rclone.zip '*/rclone' -d /usr/local/bin/ \
&& chmod +x /usr/local/bin/rclone \
&& rm /tmp/rclone.zip \
&& apt-get purge -y unzip \
&& rm -rf /var/lib/apt/lists/*
RUN groupadd -g 1000 foxhunt \
&& useradd -u 1000 -g foxhunt -m -s /bin/false foxhunt
COPY train_baseline_supervised \
evaluate_supervised \
hyperopt_baseline_supervised \
training_uploader \
/usr/local/bin/
RUN chmod +x /usr/local/bin/train_baseline_supervised \
/usr/local/bin/evaluate_supervised \
/usr/local/bin/hyperopt_baseline_supervised \
/usr/local/bin/training_uploader
# CUDA runtime environment
ENV NVIDIA_VISIBLE_DEVICES=all
ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
USER foxhunt
WORKDIR /data
CMD ["echo", "Specify training command via K8s Job args"]

View File

@@ -1,5 +1,5 @@
# GitLab Runner — RL training workloads (DQN, PPO)
# Runs on ci-compile pool (L4: 24GB VRAM, cheaper, CPU-heavy for env simulation)
# Runs on ci-rl pool (L4: 24GB VRAM, cheaper, CPU-heavy for env simulation)
# Mounts separate PVCs (training-data-l4-pvc, sccache-l4-pvc) to avoid RWO conflicts
gitlabUrl: http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181
@@ -51,7 +51,7 @@ runners:
helper_memory_limit = "512Mi"
image_pull_secrets = ["scw-registry", "gitlab-registry"]
[runners.kubernetes.node_selector]
"k8s.scaleway.com/pool-name" = "ci-compile"
"k8s.scaleway.com/pool-name" = "ci-rl"
[runners.kubernetes.node_tolerations]
"nvidia.com/gpu" = "NoSchedule"
"node.cilium.io/agent-not-ready" = "NoSchedule"

View File

@@ -1,6 +1,6 @@
# GitLab Runner — Kubernetes executor
# Runner manager pod lives on gitlab node pool
# Default: build pods spawn on ci-compile pool (L4: 24GB VRAM, cheaper)
# Default: build pods spawn on ci-rl pool (L4: 24GB VRAM, cheaper)
# Training jobs override to ci-training pool (L40S: 48GB VRAM) via node_selector
gitlabUrl: http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181
@@ -38,11 +38,11 @@ runners:
memory_limit_overwrite_max_allowed = "80Gi"
# L40S scale-to-zero needs ~3-5 min to provision; default 180s times out
poll_timeout = 600
# All GPU node pools (ci-compile L4, ci-training L40S) have nvidia GPU
# All GPU node pools (ci-rl L4, ci-training L40S) have nvidia GPU
# Setting runtimeClassName=nvidia on all pods so CUDA works everywhere
runtime_class_name = "nvidia"
# Default resource limits — balanced for both Rust compile and Kaniko builds
# L4 node: 8 vCPU / 30Gi RAM, ci-compile pool max_size=2
# L4 node: 8 vCPU / 30Gi RAM, ci-rl pool max_size=2
# Compile/test: runs 2 in parallel (1 per node), needs ~20Gi for ml+CUDA linking
# Kaniko: runs after compile, needs ~200m/512Mi but gets these defaults
# With 3500m/12Gi request: 2 pods fit per L4 node = 4 concurrent Kaniko builds
@@ -57,7 +57,7 @@ runners:
image_pull_secrets = ["scw-registry", "gitlab-registry"]
# Sub-tables must come AFTER all scalar values (TOML rule)
[runners.kubernetes.node_selector]
"k8s.scaleway.com/pool-name" = "ci-compile"
"k8s.scaleway.com/pool-name" = "ci-rl"
[runners.kubernetes.node_tolerations]
"nvidia.com/gpu" = "NoSchedule"
# Cilium CNI agent takes ~30s to initialize on fresh scale-from-zero nodes

View File

@@ -94,7 +94,7 @@
4. Repeat for gitlab pool (if not autoscaling, same process).
5. Autoscaling pools (ci-training, ci-compile, gpu-dev):
5. Autoscaling pools (ci-training, ci-rl, gpu-dev):
- No action needed — next scale-up automatically uses private IP.
6. Verify Tailscale still works:

View File

@@ -1,5 +1,5 @@
# NVIDIA DCGM Exporter — GPU metrics for Cockpit
# Runs on all GPU node pools (ci-training, ci-compile, gpu-dev)
# Runs on all GPU node pools (ci-training, ci-rl, gpu-dev)
# Alloy auto-discovers and scrapes these pods via annotations
---
apiVersion: v1

View File

@@ -0,0 +1,71 @@
# DaemonSet image pre-puller — keeps training images cached on GPU nodes
# Runs on ci-rl (L4) and ci-training (L40S) pools so training jobs skip the pull.
# Init containers pull :latest tags, then the main container sleeps forever.
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: image-prepuller
namespace: foxhunt
labels:
app: image-prepuller
app.kubernetes.io/part-of: foxhunt
spec:
selector:
matchLabels:
app: image-prepuller
template:
metadata:
labels:
app: image-prepuller
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: k8s.scaleway.com/pool-name
operator: In
values:
- ci-rl
- ci-training
tolerations:
- key: nvidia.com/gpu
operator: Exists
effect: NoSchedule
- key: node.cilium.io/agent-not-ready
operator: Exists
effect: NoSchedule
imagePullSecrets:
- name: scw-registry
initContainers:
- name: pull-training-rl
image: rg.fr-par.scw.cloud/foxhunt-ci/training-rl:latest
command: ["echo", "training-rl image pulled"]
resources:
requests:
cpu: 10m
memory: 16Mi
limits:
cpu: 10m
memory: 16Mi
- name: pull-training-supervised
image: rg.fr-par.scw.cloud/foxhunt-ci/training-supervised:latest
command: ["echo", "training-supervised image pulled"]
resources:
requests:
cpu: 10m
memory: 16Mi
limits:
cpu: 10m
memory: 16Mi
containers:
- name: pause
image: registry.k8s.io/pause:3.10
resources:
requests:
cpu: 10m
memory: 16Mi
limits:
cpu: 10m
memory: 16Mi

View File

@@ -31,9 +31,10 @@ spec:
- name: scw-registry
restartPolicy: Never
# Native sidecar (K8s 1.28+) — uploads artifacts on training completion
# Use training-rl:latest for DQN/PPO, training-supervised:latest for others
initContainers:
- name: uploader
image: rg.fr-par.scw.cloud/foxhunt-ci/training:latest
image: rg.fr-par.scw.cloud/foxhunt-ci/training-supervised:latest
restartPolicy: Always
command: ["/usr/local/bin/training_uploader"]
env:
@@ -74,7 +75,7 @@ spec:
memory: 256Mi
containers:
- name: training
image: rg.fr-par.scw.cloud/foxhunt-ci/training:latest
image: rg.fr-par.scw.cloud/foxhunt-ci/training-supervised:latest
# Available binaries in /usr/local/bin/:
# train_baseline_rl (for dqn, ppo)
# train_baseline_supervised (for tft, mamba2, tggn, tlob, liquid, kan, xlstm, diffusion)

View File

@@ -16,10 +16,10 @@ inputs = {
enable_gitlab_pool = true
gitlab_type = "GP1-XS"
# CI compile pool (L4 for cargo check/test/build — CUDA CC 89, 24GB VRAM)
enable_ci_compile_pool = true
ci_compile_type = "L4-1-24G"
ci_compile_max_size = 1
# CI RL pool (L4 for DQN/PPO training + hyperopt — CUDA CC 89, 24GB VRAM)
enable_ci_rl_pool = true
ci_rl_type = "L4-1-24G"
ci_rl_max_size = 1
# CPU compile pool (POP2-32C-128G for cargo check/build — 32 vCPU, 128GB RAM, no GPU)
enable_ci_compile_cpu_pool = true

View File

@@ -11,6 +11,11 @@ moved {
to = scaleway_k8s_pool.ci_training
}
moved {
from = scaleway_k8s_pool.ci_compile
to = scaleway_k8s_pool.ci_rl
}
resource "scaleway_vpc_private_network" "foxhunt" {
name = "${var.cluster_name}-pn"
region = var.region
@@ -71,16 +76,16 @@ resource "scaleway_k8s_pool" "gitlab" {
region = var.region
}
# CI compile pool (L4 — 24GB VRAM, CUDA CC 89, for RL training + hyperopt)
# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-compile
resource "scaleway_k8s_pool" "ci_compile" {
count = var.enable_ci_compile_pool ? 1 : 0
# CI RL pool (L4 — 24GB VRAM, CUDA CC 89, for RL training + hyperopt)
# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-rl
resource "scaleway_k8s_pool" "ci_rl" {
count = var.enable_ci_rl_pool ? 1 : 0
cluster_id = scaleway_k8s_cluster.foxhunt.id
name = "ci-compile"
node_type = var.ci_compile_type
name = "ci-rl"
node_type = var.ci_rl_type
size = 1
min_size = 0
max_size = var.ci_compile_max_size
max_size = var.ci_rl_max_size
autoscaling = true
autohealing = true
public_ip_disabled = var.public_ip_disabled

View File

@@ -24,9 +24,9 @@ output "gitlab_pool_id" {
value = var.enable_gitlab_pool ? scaleway_k8s_pool.gitlab[0].id : ""
}
output "ci_compile_pool_id" {
description = "ID of the CI compile node pool (L4)"
value = var.enable_ci_compile_pool ? scaleway_k8s_pool.ci_compile[0].id : ""
output "ci_rl_pool_id" {
description = "ID of the CI RL node pool (L4)"
value = var.enable_ci_rl_pool ? scaleway_k8s_pool.ci_rl[0].id : ""
}
output "ci_compile_cpu_pool_id" {

View File

@@ -39,20 +39,20 @@ variable "gitlab_type" {
default = "DEV1-L"
}
variable "enable_ci_compile_pool" {
description = "Create L4 GPU node pool for CI compilation + testing"
variable "enable_ci_rl_pool" {
description = "Create L4 GPU node pool for RL training + hyperopt"
type = bool
default = false
}
variable "ci_compile_type" {
description = "Instance type for the CI compile pool (L4 for CUDA compilation)"
variable "ci_rl_type" {
description = "Instance type for the CI RL pool (L4 for DQN/PPO training)"
type = string
default = "L4-1-24G"
}
variable "ci_compile_max_size" {
description = "Maximum number of nodes in the CI compile pool"
variable "ci_rl_max_size" {
description = "Maximum number of nodes in the CI RL pool"
type = number
default = 1
}