gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(ci): tune RL resources to actual usage — 2000m CPU, concurrent=2

Browse Source 
RL env simulation is single-threaded (~1 core actual). Lowering from
7000m to 2000m request allows DQN + PPO hyperopt to run simultaneously
on one L4 node.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-27 18:46:16 +01:00
parent 694e10c7a6
commit 20e67d4d64
2 changed files with 15 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
.gitlab-ci.yml
View File

@@ -423,12 +423,11 @@ build-training:
# Picked up by gitlab-runner-rl which mounts training-data-l4-pvc + sccache-l4-pvc
# Runner default node_selector = ci-compile (L4-1-24G)
variables:
# RL is CPU-bound — full L4 node (7800m CPU, 45Gi RAM, 1 GPU)
# concurrent=1 on RL runner → one job gets everything for fastest iteration
KUBERNETES_CPU_REQUEST: "7000m"
KUBERNETES_CPU_LIMIT: "7800m"
KUBERNETES_MEMORY_REQUEST: "16Gi"
KUBERNETES_MEMORY_LIMIT: "40Gi"
# RL is single-threaded (~1 core actual), 2000m allows 2 concurrent on L4
KUBERNETES_CPU_REQUEST: "2000m"
KUBERNETES_CPU_LIMIT: "3800m"
KUBERNETES_MEMORY_REQUEST: "4Gi"
KUBERNETES_MEMORY_LIMIT: "8Gi"
rules:
- if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
when: manual

21
infra/k8s/gitlab/runner-rl-values.yaml
View File

@@ -34,18 +34,17 @@ runners:
image = "rust:1.89-slim"
privileged = false
node_selector_overwrite_allowed = ".*"
cpu_request_overwrite_max_allowed = "7800m"
cpu_limit_overwrite_max_allowed = "7800m"
memory_request_overwrite_max_allowed = "40Gi"
memory_limit_overwrite_max_allowed = "40Gi"
cpu_request_overwrite_max_allowed = "4000m"
cpu_limit_overwrite_max_allowed = "4000m"
memory_request_overwrite_max_allowed = "16Gi"
memory_limit_overwrite_max_allowed = "16Gi"
poll_timeout = 600
runtime_class_name = "nvidia"
# RL jobs are CPU-bound — give full L4 node (7800m alloc, 45Gi RAM, 1 GPU)
# concurrent=1 so one job gets everything for fastest iteration
cpu_request = "7000m"
cpu_limit = "7800m"
memory_request = "16Gi"
memory_limit = "40Gi"
# RL is single-threaded (~1 core actual), 2000m request allows 2 concurrent jobs
cpu_request = "2000m"
cpu_limit = "3800m"
memory_request = "4Gi"
memory_limit = "8Gi"
helper_cpu_request = "100m"
helper_cpu_limit = "500m"
helper_memory_request = "128Mi"
@@ -70,4 +69,4 @@ runners:
tags: "kapsule-rl,gpu"
concurrent: 1
concurrent: 2