From a563c64dd39b8a26e6cd6ed024b624f3ac0ef6ff Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Wed, 4 Mar 2026 09:41:37 +0100
Subject: [PATCH] feat(ci): add H100 GPU pool for hyperopt jobs

- Create ci-training-h100 runner (runner-h100-values.yaml)
- Route hyperopt-dqn and hyperopt-ppo to H100 via kapsule,h100 tags
- H100 is ~2x faster and cheaper per run vs L40S for hyperopt workloads

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .gitlab-ci.yml                           | 17 +++++-
 infra/k8s/gitlab/runner-h100-values.yaml | 73 ++++++++++++++++++++++++
 2 files changed, 89 insertions(+), 1 deletion(-)
 create mode 100644 infra/k8s/gitlab/runner-h100-values.yaml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d394839ec..ec53fd7ea 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -8,8 +8,9 @@
 # Runner routing (by tags):
 #   kapsule,rust → CPU runner → ci-compile-cpu pool (POP2-32C-128G, 32 vCPU)
 #   kapsule,gpu  → GPU runner → ci-training (L40S)
+#   kapsule,h100 → H100 runner → ci-training-h100 (H100 80GB)
 # Override nodeSelector: KUBERNETES_NODE_SELECTOR_k8s.scaleway.com/pool-name: "<name>" (k=v format)
-# Pools: ci-compile-cpu (POP2-32C-128G), ci-training (L40S training), services, gitlab
+# Pools: ci-compile-cpu (POP2-32C-128G), ci-training (L40S), ci-training-h100 (H100), services, gitlab
 # All compilation (services + training) runs on ci-compile-cpu. CUDA stubs in ci-builder image.
 
 # Ensure pipeline is always created — individual job rules handle filtering.
@@ -685,6 +686,13 @@ train-validate-tft:
 
 hyperopt-ppo:
   extends: .train-rl-base
+  tags:
+    - kapsule
+    - h100
+  variables:
+    KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-training-h100"
+    KUBERNETES_MEMORY_REQUEST: "16Gi"
+    KUBERNETES_MEMORY_LIMIT: "48Gi"
   needs:
     - job: compile-training
       optional: true
@@ -924,6 +932,13 @@ train-validate-diffusion:
 
 hyperopt-dqn:
   extends: .train-rl-base
+  tags:
+    - kapsule
+    - h100
+  variables:
+    KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-training-h100"
+    KUBERNETES_MEMORY_REQUEST: "16Gi"
+    KUBERNETES_MEMORY_LIMIT: "48Gi"
   needs:
     - job: compile-training
       optional: true
diff --git a/infra/k8s/gitlab/runner-h100-values.yaml b/infra/k8s/gitlab/runner-h100-values.yaml
new file mode 100644
index 000000000..2629abd04
--- /dev/null
+++ b/infra/k8s/gitlab/runner-h100-values.yaml
@@ -0,0 +1,73 @@
+# GitLab Runner — H100 GPU training workloads (hyperopt, walk-forward)
+# Runs on ci-training-h100 pool (H100: 80GB VRAM)
+# Mounts separate PVCs per GPU node to avoid RWO conflicts.
+
+gitlabUrl: http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181
+# runnerToken set via --set at install time
+
+replicas: 1
+
+# Reuse the existing gitlab-runner SA (has pods/secrets/configmaps RBAC)
+rbac:
+  create: false
+serviceAccount:
+  create: false
+  name: gitlab-runner
+
+nodeSelector:
+  k8s.scaleway.com/pool-name: gitlab
+tolerations:
+  - key: gitlab
+    operator: Equal
+    value: "true"
+    effect: NoSchedule
+
+runners:
+  cloneUrl: http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181
+  config: |
+    [[runners]]
+      clone_url = "http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181"
+      tag_list = ["kapsule", "h100"]
+      [runners.kubernetes]
+        namespace = "foxhunt"
+        service_account = "gitlab-runner"
+        image = "rust:1.89-slim"
+        privileged = false
+        node_selector_overwrite_allowed = ".*"
+        cpu_request_overwrite_max_allowed = "8000m"
+        cpu_limit_overwrite_max_allowed = "8000m"
+        memory_request_overwrite_max_allowed = "96Gi"
+        memory_limit_overwrite_max_allowed = "96Gi"
+        poll_timeout = 600
+        runtime_class_name = "nvidia"
+        pod_annotations_overwrite_allowed = ".*"
+        # Default resources for H100 training
+        cpu_request = "2000m"
+        cpu_limit = "3800m"
+        memory_request = "4Gi"
+        memory_limit = "8Gi"
+        helper_cpu_request = "100m"
+        helper_cpu_limit = "500m"
+        helper_memory_request = "128Mi"
+        helper_memory_limit = "512Mi"
+        image_pull_secrets = ["scw-registry", "gitlab-registry"]
+      [runners.kubernetes.node_selector]
+        "k8s.scaleway.com/pool-name" = "ci-training-h100"
+      [runners.kubernetes.node_tolerations]
+        "nvidia.com/gpu" = "NoSchedule"
+        "node.cilium.io/agent-not-ready" = "NoSchedule"
+      [runners.kubernetes.pod_labels]
+        "app.kubernetes.io/part-of" = "foxhunt-ci"
+      # H100-specific PVCs (separate from L40S to avoid RWO conflicts)
+      [[runners.kubernetes.volumes.pvc]]
+        name = "training-data-h100-pvc"
+        mount_path = "/mnt/training-data"
+        read_only = true
+      [[runners.kubernetes.volumes.pvc]]
+        name = "sccache-h100-pvc"
+        mount_path = "/mnt/sccache"
+        read_only = false
+
+  tags: "kapsule,h100"
+
+concurrent: 2