gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ci): revert RL training back to L4 pool

Browse Source 
RL (DQN/PPO) stays on L4-1-24G — CPU-bound, low VRAM (<1GB).
Supervised stays on L40S-1-48G — GPU-bound, high VRAM.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-27 22:32:58 +01:00
parent b160afe038
commit a7d18f4f7c
1 changed files with 7 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
.gitlab-ci.yml
View File

@@ -424,12 +424,12 @@ build-training:
--destination "${REGISTRY}/training:latest"
# --------------------------------------------------------------------------
# Stage 3c: Training — all models run on L40S (48GB VRAM, 8 vCPU)
# Stage 3c: Training — RL on L4 (CPU-heavy), Supervised on L40S (GPU-heavy)
# --------------------------------------------------------------------------
# --------------------------------------------------------------------------
# RL training base: DQN + PPO → ci-training pool (L40S)
# RL hyperopt runs parallel PSO trials — CPU+GPU bound.
# RL training base: DQN + PPO → ci-compile pool (L4-1-24G)
# RL hyperopt runs parallel PSO trials — CPU+GPU bound, low VRAM.
# --------------------------------------------------------------------------
.train-rl-base:
stage: train
@@ -437,15 +437,14 @@ build-training:
tags:
- kapsule
- gpu
# Runner default node_selector: pool=ci-compile (L4-1-24G)
variables:
# nvidia runtime provided by runner default (runtime_class_name = "nvidia")
# Route to L40S pool — all training on one GPU type for simplicity
KUBERNETES_NODE_SELECTOR_pool: "pool=ci-training"
# RL hyperopt is CPU-bound (parallel PSO trials). Use full node (8 vCPU).
# RL hyperopt is CPU-bound (parallel PSO trials). Use full L4 (8 vCPU).
KUBERNETES_CPU_REQUEST: "6000m"
KUBERNETES_CPU_LIMIT: "7800m"
KUBERNETES_MEMORY_REQUEST: "8Gi"
KUBERNETES_MEMORY_LIMIT: "40Gi"
KUBERNETES_MEMORY_LIMIT: "20Gi"
rules:
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
when: manual
@@ -461,7 +460,7 @@ build-training:
# --------------------------------------------------------------------------
# Supervised training base: TFT, Mamba2, TGGN, TLOB, Liquid, KAN, xLSTM, Diffusion → L40S
# Supervised models are GPU-bound — low CPU, high VRAM. Same pool as RL.
# Supervised models are GPU-bound — low CPU, high VRAM.
# --------------------------------------------------------------------------
.train-validate-base:
stage: train