c731bef759
RL models (DQN/PPO) only need basic CUDA runtime (~2GB), while supervised models need cuDNN (~4GB). Splitting saves ~2GB pull time per RL job. Rename the L4 GPU pool from ci-compile to ci-rl to reflect its actual use. Add DaemonSet image pre-puller to cache training images on GPU nodes. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
73 lines
2.4 KiB
YAML
73 lines
2.4 KiB
YAML
# GitLab Runner — RL training workloads (DQN, PPO)
|
|
# Runs on ci-rl pool (L4: 24GB VRAM, cheaper, CPU-heavy for env simulation)
|
|
# Mounts separate PVCs (training-data-l4-pvc, sccache-l4-pvc) to avoid RWO conflicts
|
|
|
|
gitlabUrl: http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181
|
|
# runnerToken set via --set at install time
|
|
|
|
replicas: 1
|
|
|
|
# Reuse the existing gitlab-runner SA (has pods/secrets/configmaps RBAC)
|
|
rbac:
|
|
create: false
|
|
serviceAccount:
|
|
create: false
|
|
name: gitlab-runner
|
|
|
|
nodeSelector:
|
|
k8s.scaleway.com/pool-name: gitlab
|
|
tolerations:
|
|
- key: gitlab
|
|
operator: Equal
|
|
value: "true"
|
|
effect: NoSchedule
|
|
|
|
runners:
|
|
cloneUrl: http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181
|
|
config: |
|
|
[[runners]]
|
|
clone_url = "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"
|
|
tag_list = ["kapsule-rl", "gpu"]
|
|
[runners.kubernetes]
|
|
namespace = "foxhunt"
|
|
service_account = "gitlab-runner"
|
|
image = "rust:1.89-slim"
|
|
privileged = false
|
|
node_selector_overwrite_allowed = ".*"
|
|
cpu_request_overwrite_max_allowed = "4000m"
|
|
cpu_limit_overwrite_max_allowed = "4000m"
|
|
memory_request_overwrite_max_allowed = "16Gi"
|
|
memory_limit_overwrite_max_allowed = "16Gi"
|
|
poll_timeout = 600
|
|
runtime_class_name = "nvidia"
|
|
# RL is single-threaded (~1 core actual), 2000m request allows 2 concurrent jobs
|
|
cpu_request = "2000m"
|
|
cpu_limit = "3800m"
|
|
memory_request = "4Gi"
|
|
memory_limit = "8Gi"
|
|
helper_cpu_request = "100m"
|
|
helper_cpu_limit = "500m"
|
|
helper_memory_request = "128Mi"
|
|
helper_memory_limit = "512Mi"
|
|
image_pull_secrets = ["scw-registry", "gitlab-registry"]
|
|
[runners.kubernetes.node_selector]
|
|
"k8s.scaleway.com/pool-name" = "ci-rl"
|
|
[runners.kubernetes.node_tolerations]
|
|
"nvidia.com/gpu" = "NoSchedule"
|
|
"node.cilium.io/agent-not-ready" = "NoSchedule"
|
|
[runners.kubernetes.pod_labels]
|
|
"app.kubernetes.io/part-of" = "foxhunt-ci"
|
|
# L4-specific PVCs — separate from L40S to avoid RWO Multi-Attach errors
|
|
[[runners.kubernetes.volumes.pvc]]
|
|
name = "training-data-l4-pvc"
|
|
mount_path = "/mnt/training-data"
|
|
read_only = true
|
|
[[runners.kubernetes.volumes.pvc]]
|
|
name = "sccache-l4-pvc"
|
|
mount_path = "/mnt/sccache"
|
|
read_only = false
|
|
|
|
tags: "kapsule-rl,gpu"
|
|
|
|
concurrent: 2
|