feat(infra): split training image into RL + supervised, rename ci-compile → ci-rl
RL models (DQN/PPO) only need basic CUDA runtime (~2GB), while supervised models need cuDNN (~4GB). Splitting saves ~2GB pull time per RL job. Rename the L4 GPU pool from ci-compile to ci-rl to reflect its actual use. Add DaemonSet image pre-puller to cache training images on GPU nodes. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -9,7 +9,7 @@
|
||||
# kapsule,rust → CPU runner → ci-compile-cpu pool (POP2-32C-128G, 32 vCPU)
|
||||
# kapsule,gpu → GPU runner → ci-training (L40S)
|
||||
# Override nodeSelector: KUBERNETES_NODE_SELECTOR_k8s.scaleway.com/pool-name: "<name>" (k=v format)
|
||||
# Pools: ci-compile-cpu (POP2-32C-128G), ci-training (L40S), services, gitlab
|
||||
# Pools: ci-compile-cpu (POP2-32C-128G), ci-rl (L4), ci-training (L40S), services, gitlab
|
||||
# All compilation (services + training) runs on ci-compile-cpu. CUDA stubs in ci-builder image.
|
||||
|
||||
# Ensure pipeline is always created — individual job rules handle filtering.
|
||||
@@ -538,31 +538,41 @@ build-web-gateway:
|
||||
--destination "${REGISTRY}/web-gateway:${CI_COMMIT_SHA}"
|
||||
--destination "${REGISTRY}/web-gateway:latest"
|
||||
|
||||
# training: pre-built CUDA binaries → CUDA runtime image (no recompilation)
|
||||
build-training:
|
||||
# training-rl: RL binaries → CUDA runtime image (no cuDNN, ~2GB)
|
||||
build-training-rl:
|
||||
extends: .kaniko-training-base
|
||||
script:
|
||||
- /kaniko/executor
|
||||
--context dir://${CI_PROJECT_DIR}/build-out/training
|
||||
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.training-runtime"
|
||||
--destination "${REGISTRY}/training:${CI_COMMIT_SHA}"
|
||||
--destination "${REGISTRY}/training:latest"
|
||||
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.training-rl"
|
||||
--destination "${REGISTRY}/training-rl:${CI_COMMIT_SHA}"
|
||||
--destination "${REGISTRY}/training-rl:latest"
|
||||
|
||||
# training-supervised: supervised binaries → CUDA+cuDNN runtime image (~4GB)
|
||||
build-training-supervised:
|
||||
extends: .kaniko-training-base
|
||||
script:
|
||||
- /kaniko/executor
|
||||
--context dir://${CI_PROJECT_DIR}/build-out/training
|
||||
--dockerfile "${CI_PROJECT_DIR}/infra/docker/Dockerfile.training-supervised"
|
||||
--destination "${REGISTRY}/training-supervised:${CI_COMMIT_SHA}"
|
||||
--destination "${REGISTRY}/training-supervised:latest"
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Stage 3c: Training — RL on L4 (CPU-heavy), Supervised on L40S (GPU-heavy)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# RL training base: DQN + PPO → ci-compile pool (L4-1-24G)
|
||||
# RL training base: DQN + PPO → ci-rl pool (L4-1-24G)
|
||||
# RL hyperopt runs parallel PSO trials — CPU+GPU bound, low VRAM.
|
||||
# --------------------------------------------------------------------------
|
||||
.train-rl-base:
|
||||
stage: train
|
||||
image: ${REGISTRY}/training:${CI_COMMIT_SHA}
|
||||
image: ${REGISTRY}/training-rl:${CI_COMMIT_SHA}
|
||||
tags:
|
||||
- kapsule
|
||||
- gpu
|
||||
# Runner default node_selector: k8s.scaleway.com/pool-name=ci-compile-cpu
|
||||
# Runner default node_selector: k8s.scaleway.com/pool-name=ci-rl
|
||||
variables:
|
||||
# nvidia runtime provided by runner default (runtime_class_name = "nvidia")
|
||||
# RL hyperopt is CPU-bound (parallel PSO trials). Use full L4 (8 vCPU).
|
||||
@@ -589,7 +599,7 @@ build-training:
|
||||
# --------------------------------------------------------------------------
|
||||
.train-validate-base:
|
||||
stage: train
|
||||
image: ${REGISTRY}/training:${CI_COMMIT_SHA}
|
||||
image: ${REGISTRY}/training-supervised:${CI_COMMIT_SHA}
|
||||
tags:
|
||||
- kapsule
|
||||
- gpu
|
||||
@@ -619,7 +629,7 @@ build-training:
|
||||
train-validate-rl:
|
||||
extends: .train-rl-base
|
||||
needs:
|
||||
- job: build-training
|
||||
- job: build-training-rl
|
||||
optional: true
|
||||
script:
|
||||
- |
|
||||
@@ -651,7 +661,7 @@ train-validate-rl:
|
||||
train-validate-tft:
|
||||
extends: .train-validate-base
|
||||
needs:
|
||||
- job: build-training
|
||||
- job: build-training-supervised
|
||||
optional: true
|
||||
script:
|
||||
- /usr/local/bin/train_baseline_supervised
|
||||
@@ -678,7 +688,7 @@ train-validate-tft:
|
||||
hyperopt-ppo:
|
||||
extends: .train-rl-base
|
||||
needs:
|
||||
- job: build-training
|
||||
- job: build-training-rl
|
||||
optional: true
|
||||
timeout: 4h
|
||||
script:
|
||||
@@ -725,7 +735,7 @@ hyperopt-ppo:
|
||||
train-validate-mamba2:
|
||||
extends: .train-validate-base
|
||||
needs:
|
||||
- job: build-training
|
||||
- job: build-training-supervised
|
||||
optional: true
|
||||
script:
|
||||
- /usr/local/bin/train_baseline_supervised
|
||||
@@ -752,7 +762,7 @@ train-validate-mamba2:
|
||||
train-validate-liquid:
|
||||
extends: .train-validate-base
|
||||
needs:
|
||||
- job: build-training
|
||||
- job: build-training-supervised
|
||||
optional: true
|
||||
script:
|
||||
- /usr/local/bin/train_baseline_supervised
|
||||
@@ -779,7 +789,7 @@ train-validate-liquid:
|
||||
train-validate-tggn:
|
||||
extends: .train-validate-base
|
||||
needs:
|
||||
- job: build-training
|
||||
- job: build-training-supervised
|
||||
optional: true
|
||||
script:
|
||||
- /usr/local/bin/train_baseline_supervised
|
||||
@@ -806,7 +816,7 @@ train-validate-tggn:
|
||||
train-validate-tlob:
|
||||
extends: .train-validate-base
|
||||
needs:
|
||||
- job: build-training
|
||||
- job: build-training-supervised
|
||||
optional: true
|
||||
script:
|
||||
- /usr/local/bin/train_baseline_supervised
|
||||
@@ -833,7 +843,7 @@ train-validate-tlob:
|
||||
train-validate-kan:
|
||||
extends: .train-validate-base
|
||||
needs:
|
||||
- job: build-training
|
||||
- job: build-training-supervised
|
||||
optional: true
|
||||
script:
|
||||
- /usr/local/bin/train_baseline_supervised
|
||||
@@ -860,7 +870,7 @@ train-validate-kan:
|
||||
train-validate-xlstm:
|
||||
extends: .train-validate-base
|
||||
needs:
|
||||
- job: build-training
|
||||
- job: build-training-supervised
|
||||
optional: true
|
||||
script:
|
||||
- /usr/local/bin/train_baseline_supervised
|
||||
@@ -887,7 +897,7 @@ train-validate-xlstm:
|
||||
train-validate-diffusion:
|
||||
extends: .train-validate-base
|
||||
needs:
|
||||
- job: build-training
|
||||
- job: build-training-supervised
|
||||
optional: true
|
||||
script:
|
||||
- /usr/local/bin/train_baseline_supervised
|
||||
@@ -916,7 +926,7 @@ train-validate-diffusion:
|
||||
hyperopt-dqn:
|
||||
extends: .train-rl-base
|
||||
needs:
|
||||
- job: build-training
|
||||
- job: build-training-rl
|
||||
optional: true
|
||||
timeout: 4h
|
||||
script:
|
||||
@@ -968,7 +978,7 @@ hyperopt-dqn:
|
||||
hyperopt-tft:
|
||||
extends: .train-validate-base
|
||||
needs:
|
||||
- job: build-training
|
||||
- job: build-training-supervised
|
||||
optional: true
|
||||
timeout: 4h
|
||||
script:
|
||||
@@ -1007,7 +1017,7 @@ hyperopt-tft:
|
||||
hyperopt-mamba2:
|
||||
extends: .train-validate-base
|
||||
needs:
|
||||
- job: build-training
|
||||
- job: build-training-supervised
|
||||
optional: true
|
||||
timeout: 4h
|
||||
script:
|
||||
@@ -1046,7 +1056,7 @@ hyperopt-mamba2:
|
||||
hyperopt-liquid:
|
||||
extends: .train-validate-base
|
||||
needs:
|
||||
- job: build-training
|
||||
- job: build-training-supervised
|
||||
optional: true
|
||||
timeout: 4h
|
||||
script:
|
||||
@@ -1085,7 +1095,7 @@ hyperopt-liquid:
|
||||
hyperopt-tggn:
|
||||
extends: .train-validate-base
|
||||
needs:
|
||||
- job: build-training
|
||||
- job: build-training-supervised
|
||||
optional: true
|
||||
timeout: 4h
|
||||
script:
|
||||
@@ -1124,7 +1134,7 @@ hyperopt-tggn:
|
||||
hyperopt-tlob:
|
||||
extends: .train-validate-base
|
||||
needs:
|
||||
- job: build-training
|
||||
- job: build-training-supervised
|
||||
optional: true
|
||||
timeout: 4h
|
||||
script:
|
||||
@@ -1163,7 +1173,7 @@ hyperopt-tlob:
|
||||
hyperopt-kan:
|
||||
extends: .train-validate-base
|
||||
needs:
|
||||
- job: build-training
|
||||
- job: build-training-supervised
|
||||
optional: true
|
||||
timeout: 4h
|
||||
script:
|
||||
@@ -1202,7 +1212,7 @@ hyperopt-kan:
|
||||
hyperopt-xlstm:
|
||||
extends: .train-validate-base
|
||||
needs:
|
||||
- job: build-training
|
||||
- job: build-training-supervised
|
||||
optional: true
|
||||
timeout: 4h
|
||||
script:
|
||||
@@ -1241,7 +1251,7 @@ hyperopt-xlstm:
|
||||
hyperopt-diffusion:
|
||||
extends: .train-validate-base
|
||||
needs:
|
||||
- job: build-training
|
||||
- job: build-training-supervised
|
||||
optional: true
|
||||
timeout: 4h
|
||||
script:
|
||||
@@ -1412,7 +1422,9 @@ deploy:
|
||||
optional: true
|
||||
- job: build-web-gateway
|
||||
optional: true
|
||||
- job: build-training
|
||||
- job: build-training-rl
|
||||
optional: true
|
||||
- job: build-training-supervised
|
||||
optional: true
|
||||
rules:
|
||||
# Only deploy when source code changed (build produced new images)
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
# GPU training runtime image — packages pre-built CUDA binaries (no compilation)
|
||||
# Binaries are compiled in CI compile-services job with --features ml/cuda
|
||||
# Usage: kaniko --context dir://build-out --dockerfile Dockerfile.training-runtime
|
||||
# Run: docker run --gpus all training train_baseline_supervised --model kan [args...]
|
||||
# RL training runtime image — DQN + PPO (no cuDNN needed, ~2GB)
|
||||
# Binaries are compiled in CI compile-training job with --features ml/cuda
|
||||
# Usage: kaniko --context dir://build-out/training --dockerfile Dockerfile.training-rl
|
||||
# Run: docker run --gpus all training-rl train_baseline_rl --model dqn [args...]
|
||||
|
||||
FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
|
||||
FROM nvidia/cuda:12.4.1-runtime-ubuntu22.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
@@ -23,20 +23,14 @@ RUN groupadd -g 1000 foxhunt \
|
||||
&& useradd -u 1000 -g foxhunt -m -s /bin/false foxhunt
|
||||
|
||||
COPY train_baseline_rl \
|
||||
train_baseline_supervised \
|
||||
evaluate_baseline \
|
||||
evaluate_supervised \
|
||||
hyperopt_baseline_rl \
|
||||
hyperopt_baseline_supervised \
|
||||
training_uploader \
|
||||
/usr/local/bin/
|
||||
|
||||
RUN chmod +x /usr/local/bin/train_baseline_rl \
|
||||
/usr/local/bin/train_baseline_supervised \
|
||||
/usr/local/bin/evaluate_baseline \
|
||||
/usr/local/bin/evaluate_supervised \
|
||||
/usr/local/bin/hyperopt_baseline_rl \
|
||||
/usr/local/bin/hyperopt_baseline_supervised \
|
||||
/usr/local/bin/training_uploader
|
||||
|
||||
# CUDA runtime environment
|
||||
45
infra/docker/Dockerfile.training-supervised
Normal file
45
infra/docker/Dockerfile.training-supervised
Normal file
@@ -0,0 +1,45 @@
|
||||
# Supervised training runtime image — TFT, Mamba2, TGGN, TLOB, Liquid, KAN, xLSTM, Diffusion (~4GB)
|
||||
# Includes cuDNN for deep neural network operations (convolutions, attention)
|
||||
# Binaries are compiled in CI compile-training job with --features ml/cuda
|
||||
# Usage: kaniko --context dir://build-out/training --dockerfile Dockerfile.training-supervised
|
||||
# Run: docker run --gpus all training-supervised train_baseline_supervised --model tft [args...]
|
||||
|
||||
FROM nvidia/cuda:12.4.1-cudnn-runtime-ubuntu22.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
libssl3 \
|
||||
curl \
|
||||
unzip \
|
||||
&& curl -fsSL https://downloads.rclone.org/v1.69.1/rclone-v1.69.1-linux-amd64.zip -o /tmp/rclone.zip \
|
||||
&& unzip -j /tmp/rclone.zip '*/rclone' -d /usr/local/bin/ \
|
||||
&& chmod +x /usr/local/bin/rclone \
|
||||
&& rm /tmp/rclone.zip \
|
||||
&& apt-get purge -y unzip \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN groupadd -g 1000 foxhunt \
|
||||
&& useradd -u 1000 -g foxhunt -m -s /bin/false foxhunt
|
||||
|
||||
COPY train_baseline_supervised \
|
||||
evaluate_supervised \
|
||||
hyperopt_baseline_supervised \
|
||||
training_uploader \
|
||||
/usr/local/bin/
|
||||
|
||||
RUN chmod +x /usr/local/bin/train_baseline_supervised \
|
||||
/usr/local/bin/evaluate_supervised \
|
||||
/usr/local/bin/hyperopt_baseline_supervised \
|
||||
/usr/local/bin/training_uploader
|
||||
|
||||
# CUDA runtime environment
|
||||
ENV NVIDIA_VISIBLE_DEVICES=all
|
||||
ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
|
||||
|
||||
USER foxhunt
|
||||
|
||||
WORKDIR /data
|
||||
|
||||
CMD ["echo", "Specify training command via K8s Job args"]
|
||||
@@ -1,5 +1,5 @@
|
||||
# GitLab Runner — RL training workloads (DQN, PPO)
|
||||
# Runs on ci-compile pool (L4: 24GB VRAM, cheaper, CPU-heavy for env simulation)
|
||||
# Runs on ci-rl pool (L4: 24GB VRAM, cheaper, CPU-heavy for env simulation)
|
||||
# Mounts separate PVCs (training-data-l4-pvc, sccache-l4-pvc) to avoid RWO conflicts
|
||||
|
||||
gitlabUrl: http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181
|
||||
@@ -51,7 +51,7 @@ runners:
|
||||
helper_memory_limit = "512Mi"
|
||||
image_pull_secrets = ["scw-registry", "gitlab-registry"]
|
||||
[runners.kubernetes.node_selector]
|
||||
"k8s.scaleway.com/pool-name" = "ci-compile"
|
||||
"k8s.scaleway.com/pool-name" = "ci-rl"
|
||||
[runners.kubernetes.node_tolerations]
|
||||
"nvidia.com/gpu" = "NoSchedule"
|
||||
"node.cilium.io/agent-not-ready" = "NoSchedule"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# GitLab Runner — Kubernetes executor
|
||||
# Runner manager pod lives on gitlab node pool
|
||||
# Default: build pods spawn on ci-compile pool (L4: 24GB VRAM, cheaper)
|
||||
# Default: build pods spawn on ci-rl pool (L4: 24GB VRAM, cheaper)
|
||||
# Training jobs override to ci-training pool (L40S: 48GB VRAM) via node_selector
|
||||
|
||||
gitlabUrl: http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181
|
||||
@@ -38,11 +38,11 @@ runners:
|
||||
memory_limit_overwrite_max_allowed = "80Gi"
|
||||
# L40S scale-to-zero needs ~3-5 min to provision; default 180s times out
|
||||
poll_timeout = 600
|
||||
# All GPU node pools (ci-compile L4, ci-training L40S) have nvidia GPU
|
||||
# All GPU node pools (ci-rl L4, ci-training L40S) have nvidia GPU
|
||||
# Setting runtimeClassName=nvidia on all pods so CUDA works everywhere
|
||||
runtime_class_name = "nvidia"
|
||||
# Default resource limits — balanced for both Rust compile and Kaniko builds
|
||||
# L4 node: 8 vCPU / 30Gi RAM, ci-compile pool max_size=2
|
||||
# L4 node: 8 vCPU / 30Gi RAM, ci-rl pool max_size=2
|
||||
# Compile/test: runs 2 in parallel (1 per node), needs ~20Gi for ml+CUDA linking
|
||||
# Kaniko: runs after compile, needs ~200m/512Mi but gets these defaults
|
||||
# With 3500m/12Gi request: 2 pods fit per L4 node = 4 concurrent Kaniko builds
|
||||
@@ -57,7 +57,7 @@ runners:
|
||||
image_pull_secrets = ["scw-registry", "gitlab-registry"]
|
||||
# Sub-tables must come AFTER all scalar values (TOML rule)
|
||||
[runners.kubernetes.node_selector]
|
||||
"k8s.scaleway.com/pool-name" = "ci-compile"
|
||||
"k8s.scaleway.com/pool-name" = "ci-rl"
|
||||
[runners.kubernetes.node_tolerations]
|
||||
"nvidia.com/gpu" = "NoSchedule"
|
||||
# Cilium CNI agent takes ~30s to initialize on fresh scale-from-zero nodes
|
||||
|
||||
@@ -94,7 +94,7 @@
|
||||
|
||||
4. Repeat for gitlab pool (if not autoscaling, same process).
|
||||
|
||||
5. Autoscaling pools (ci-training, ci-compile, gpu-dev):
|
||||
5. Autoscaling pools (ci-training, ci-rl, gpu-dev):
|
||||
- No action needed — next scale-up automatically uses private IP.
|
||||
|
||||
6. Verify Tailscale still works:
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# NVIDIA DCGM Exporter — GPU metrics for Cockpit
|
||||
# Runs on all GPU node pools (ci-training, ci-compile, gpu-dev)
|
||||
# Runs on all GPU node pools (ci-training, ci-rl, gpu-dev)
|
||||
# Alloy auto-discovers and scrapes these pods via annotations
|
||||
---
|
||||
apiVersion: v1
|
||||
|
||||
71
infra/k8s/training/image-prepuller.yaml
Normal file
71
infra/k8s/training/image-prepuller.yaml
Normal file
@@ -0,0 +1,71 @@
|
||||
# DaemonSet image pre-puller — keeps training images cached on GPU nodes
|
||||
# Runs on ci-rl (L4) and ci-training (L40S) pools so training jobs skip the pull.
|
||||
# Init containers pull :latest tags, then the main container sleeps forever.
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: image-prepuller
|
||||
namespace: foxhunt
|
||||
labels:
|
||||
app: image-prepuller
|
||||
app.kubernetes.io/part-of: foxhunt
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: image-prepuller
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: image-prepuller
|
||||
spec:
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: k8s.scaleway.com/pool-name
|
||||
operator: In
|
||||
values:
|
||||
- ci-rl
|
||||
- ci-training
|
||||
tolerations:
|
||||
- key: nvidia.com/gpu
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
- key: node.cilium.io/agent-not-ready
|
||||
operator: Exists
|
||||
effect: NoSchedule
|
||||
imagePullSecrets:
|
||||
- name: scw-registry
|
||||
initContainers:
|
||||
- name: pull-training-rl
|
||||
image: rg.fr-par.scw.cloud/foxhunt-ci/training-rl:latest
|
||||
command: ["echo", "training-rl image pulled"]
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 16Mi
|
||||
limits:
|
||||
cpu: 10m
|
||||
memory: 16Mi
|
||||
- name: pull-training-supervised
|
||||
image: rg.fr-par.scw.cloud/foxhunt-ci/training-supervised:latest
|
||||
command: ["echo", "training-supervised image pulled"]
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 16Mi
|
||||
limits:
|
||||
cpu: 10m
|
||||
memory: 16Mi
|
||||
containers:
|
||||
- name: pause
|
||||
image: registry.k8s.io/pause:3.10
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 16Mi
|
||||
limits:
|
||||
cpu: 10m
|
||||
memory: 16Mi
|
||||
@@ -31,9 +31,10 @@ spec:
|
||||
- name: scw-registry
|
||||
restartPolicy: Never
|
||||
# Native sidecar (K8s 1.28+) — uploads artifacts on training completion
|
||||
# Use training-rl:latest for DQN/PPO, training-supervised:latest for others
|
||||
initContainers:
|
||||
- name: uploader
|
||||
image: rg.fr-par.scw.cloud/foxhunt-ci/training:latest
|
||||
image: rg.fr-par.scw.cloud/foxhunt-ci/training-supervised:latest
|
||||
restartPolicy: Always
|
||||
command: ["/usr/local/bin/training_uploader"]
|
||||
env:
|
||||
@@ -74,7 +75,7 @@ spec:
|
||||
memory: 256Mi
|
||||
containers:
|
||||
- name: training
|
||||
image: rg.fr-par.scw.cloud/foxhunt-ci/training:latest
|
||||
image: rg.fr-par.scw.cloud/foxhunt-ci/training-supervised:latest
|
||||
# Available binaries in /usr/local/bin/:
|
||||
# train_baseline_rl (for dqn, ppo)
|
||||
# train_baseline_supervised (for tft, mamba2, tggn, tlob, liquid, kan, xlstm, diffusion)
|
||||
|
||||
@@ -16,10 +16,10 @@ inputs = {
|
||||
enable_gitlab_pool = true
|
||||
gitlab_type = "GP1-XS"
|
||||
|
||||
# CI compile pool (L4 for cargo check/test/build — CUDA CC 89, 24GB VRAM)
|
||||
enable_ci_compile_pool = true
|
||||
ci_compile_type = "L4-1-24G"
|
||||
ci_compile_max_size = 1
|
||||
# CI RL pool (L4 for DQN/PPO training + hyperopt — CUDA CC 89, 24GB VRAM)
|
||||
enable_ci_rl_pool = true
|
||||
ci_rl_type = "L4-1-24G"
|
||||
ci_rl_max_size = 1
|
||||
|
||||
# CPU compile pool (POP2-32C-128G for cargo check/build — 32 vCPU, 128GB RAM, no GPU)
|
||||
enable_ci_compile_cpu_pool = true
|
||||
|
||||
@@ -11,6 +11,11 @@ moved {
|
||||
to = scaleway_k8s_pool.ci_training
|
||||
}
|
||||
|
||||
moved {
|
||||
from = scaleway_k8s_pool.ci_compile
|
||||
to = scaleway_k8s_pool.ci_rl
|
||||
}
|
||||
|
||||
resource "scaleway_vpc_private_network" "foxhunt" {
|
||||
name = "${var.cluster_name}-pn"
|
||||
region = var.region
|
||||
@@ -71,16 +76,16 @@ resource "scaleway_k8s_pool" "gitlab" {
|
||||
region = var.region
|
||||
}
|
||||
|
||||
# CI compile pool (L4 — 24GB VRAM, CUDA CC 89, for RL training + hyperopt)
|
||||
# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-compile
|
||||
resource "scaleway_k8s_pool" "ci_compile" {
|
||||
count = var.enable_ci_compile_pool ? 1 : 0
|
||||
# CI RL pool (L4 — 24GB VRAM, CUDA CC 89, for RL training + hyperopt)
|
||||
# Nodes auto-labeled: k8s.scaleway.com/pool-name=ci-rl
|
||||
resource "scaleway_k8s_pool" "ci_rl" {
|
||||
count = var.enable_ci_rl_pool ? 1 : 0
|
||||
cluster_id = scaleway_k8s_cluster.foxhunt.id
|
||||
name = "ci-compile"
|
||||
node_type = var.ci_compile_type
|
||||
name = "ci-rl"
|
||||
node_type = var.ci_rl_type
|
||||
size = 1
|
||||
min_size = 0
|
||||
max_size = var.ci_compile_max_size
|
||||
max_size = var.ci_rl_max_size
|
||||
autoscaling = true
|
||||
autohealing = true
|
||||
public_ip_disabled = var.public_ip_disabled
|
||||
|
||||
@@ -24,9 +24,9 @@ output "gitlab_pool_id" {
|
||||
value = var.enable_gitlab_pool ? scaleway_k8s_pool.gitlab[0].id : ""
|
||||
}
|
||||
|
||||
output "ci_compile_pool_id" {
|
||||
description = "ID of the CI compile node pool (L4)"
|
||||
value = var.enable_ci_compile_pool ? scaleway_k8s_pool.ci_compile[0].id : ""
|
||||
output "ci_rl_pool_id" {
|
||||
description = "ID of the CI RL node pool (L4)"
|
||||
value = var.enable_ci_rl_pool ? scaleway_k8s_pool.ci_rl[0].id : ""
|
||||
}
|
||||
|
||||
output "ci_compile_cpu_pool_id" {
|
||||
|
||||
@@ -39,20 +39,20 @@ variable "gitlab_type" {
|
||||
default = "DEV1-L"
|
||||
}
|
||||
|
||||
variable "enable_ci_compile_pool" {
|
||||
description = "Create L4 GPU node pool for CI compilation + testing"
|
||||
variable "enable_ci_rl_pool" {
|
||||
description = "Create L4 GPU node pool for RL training + hyperopt"
|
||||
type = bool
|
||||
default = false
|
||||
}
|
||||
|
||||
variable "ci_compile_type" {
|
||||
description = "Instance type for the CI compile pool (L4 for CUDA compilation)"
|
||||
variable "ci_rl_type" {
|
||||
description = "Instance type for the CI RL pool (L4 for DQN/PPO training)"
|
||||
type = string
|
||||
default = "L4-1-24G"
|
||||
}
|
||||
|
||||
variable "ci_compile_max_size" {
|
||||
description = "Maximum number of nodes in the CI compile pool"
|
||||
variable "ci_rl_max_size" {
|
||||
description = "Maximum number of nodes in the CI RL pool"
|
||||
type = number
|
||||
default = 1
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user