feat(ci): add H100 GPU pool for hyperopt jobs

- Create ci-training-h100 runner (runner-h100-values.yaml)
- Route hyperopt-dqn and hyperopt-ppo to H100 via kapsule,h100 tags
- H100 is ~2x faster and cheaper per run vs L40S for hyperopt workloads

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-04 09:41:37 +01:00
parent 1ebbfea8f6
commit a563c64dd3
2 changed files with 89 additions and 1 deletions

View File

@@ -8,8 +8,9 @@
# Runner routing (by tags):
# kapsule,rust → CPU runner → ci-compile-cpu pool (POP2-32C-128G, 32 vCPU)
# kapsule,gpu → GPU runner → ci-training (L40S)
# kapsule,h100 → H100 runner → ci-training-h100 (H100 80GB)
# Override nodeSelector: KUBERNETES_NODE_SELECTOR_k8s.scaleway.com/pool-name: "<name>" (k=v format)
# Pools: ci-compile-cpu (POP2-32C-128G), ci-training (L40S training), services, gitlab
# Pools: ci-compile-cpu (POP2-32C-128G), ci-training (L40S), ci-training-h100 (H100), services, gitlab
# All compilation (services + training) runs on ci-compile-cpu. CUDA stubs in ci-builder image.
# Ensure pipeline is always created — individual job rules handle filtering.
@@ -685,6 +686,13 @@ train-validate-tft:
hyperopt-ppo:
extends: .train-rl-base
tags:
- kapsule
- h100
variables:
KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-training-h100"
KUBERNETES_MEMORY_REQUEST: "16Gi"
KUBERNETES_MEMORY_LIMIT: "48Gi"
needs:
- job: compile-training
optional: true
@@ -924,6 +932,13 @@ train-validate-diffusion:
hyperopt-dqn:
extends: .train-rl-base
tags:
- kapsule
- h100
variables:
KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-training-h100"
KUBERNETES_MEMORY_REQUEST: "16Gi"
KUBERNETES_MEMORY_LIMIT: "48Gi"
needs:
- job: compile-training
optional: true

View File

@@ -0,0 +1,73 @@
# GitLab Runner — H100 GPU training workloads (hyperopt, walk-forward)
# Runs on ci-training-h100 pool (H100: 80GB VRAM)
# Mounts separate PVCs per GPU node to avoid RWO conflicts.
gitlabUrl: http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181
# runnerToken set via --set at install time
replicas: 1
# Reuse the existing gitlab-runner SA (has pods/secrets/configmaps RBAC)
rbac:
create: false
serviceAccount:
create: false
name: gitlab-runner
nodeSelector:
k8s.scaleway.com/pool-name: gitlab
tolerations:
- key: gitlab
operator: Equal
value: "true"
effect: NoSchedule
runners:
cloneUrl: http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181
config: |
[[runners]]
clone_url = "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"
tag_list = ["kapsule", "h100"]
[runners.kubernetes]
namespace = "foxhunt"
service_account = "gitlab-runner"
image = "rust:1.89-slim"
privileged = false
node_selector_overwrite_allowed = ".*"
cpu_request_overwrite_max_allowed = "8000m"
cpu_limit_overwrite_max_allowed = "8000m"
memory_request_overwrite_max_allowed = "96Gi"
memory_limit_overwrite_max_allowed = "96Gi"
poll_timeout = 600
runtime_class_name = "nvidia"
pod_annotations_overwrite_allowed = ".*"
# Default resources for H100 training
cpu_request = "2000m"
cpu_limit = "3800m"
memory_request = "4Gi"
memory_limit = "8Gi"
helper_cpu_request = "100m"
helper_cpu_limit = "500m"
helper_memory_request = "128Mi"
helper_memory_limit = "512Mi"
image_pull_secrets = ["scw-registry", "gitlab-registry"]
[runners.kubernetes.node_selector]
"k8s.scaleway.com/pool-name" = "ci-training-h100"
[runners.kubernetes.node_tolerations]
"nvidia.com/gpu" = "NoSchedule"
"node.cilium.io/agent-not-ready" = "NoSchedule"
[runners.kubernetes.pod_labels]
"app.kubernetes.io/part-of" = "foxhunt-ci"
# H100-specific PVCs (separate from L40S to avoid RWO conflicts)
[[runners.kubernetes.volumes.pvc]]
name = "training-data-h100-pvc"
mount_path = "/mnt/training-data"
read_only = true
[[runners.kubernetes.volumes.pvc]]
name = "sccache-h100-pvc"
mount_path = "/mnt/sccache"
read_only = false
tags: "kapsule,h100"
concurrent: 2