diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0ae635928..53c0b795b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -424,12 +424,12 @@ build-training: --destination "${REGISTRY}/training:latest" # -------------------------------------------------------------------------- -# Stage 3c: Training — all models run on L40S (48GB VRAM, 8 vCPU) +# Stage 3c: Training — RL on L4 (CPU-heavy), Supervised on L40S (GPU-heavy) # -------------------------------------------------------------------------- # -------------------------------------------------------------------------- -# RL training base: DQN + PPO → ci-training pool (L40S) -# RL hyperopt runs parallel PSO trials — CPU+GPU bound. +# RL training base: DQN + PPO → ci-compile pool (L4-1-24G) +# RL hyperopt runs parallel PSO trials — CPU+GPU bound, low VRAM. # -------------------------------------------------------------------------- .train-rl-base: stage: train @@ -437,15 +437,14 @@ build-training: tags: - kapsule - gpu + # Runner default node_selector: pool=ci-compile (L4-1-24G) variables: # nvidia runtime provided by runner default (runtime_class_name = "nvidia") - # Route to L40S pool — all training on one GPU type for simplicity - KUBERNETES_NODE_SELECTOR_pool: "pool=ci-training" - # RL hyperopt is CPU-bound (parallel PSO trials). Use full node (8 vCPU). + # RL hyperopt is CPU-bound (parallel PSO trials). Use full L4 (8 vCPU). KUBERNETES_CPU_REQUEST: "6000m" KUBERNETES_CPU_LIMIT: "7800m" KUBERNETES_MEMORY_REQUEST: "8Gi" - KUBERNETES_MEMORY_LIMIT: "40Gi" + KUBERNETES_MEMORY_LIMIT: "20Gi" rules: - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push" when: manual @@ -461,7 +460,7 @@ build-training: # -------------------------------------------------------------------------- # Supervised training base: TFT, Mamba2, TGGN, TLOB, Liquid, KAN, xLSTM, Diffusion → L40S -# Supervised models are GPU-bound — low CPU, high VRAM. Same pool as RL. +# Supervised models are GPU-bound — low CPU, high VRAM. # -------------------------------------------------------------------------- .train-validate-base: stage: train