ci: move runner to H100 pool, add training validation stage

- Runner node selector: ci-build (L4) → gpu-training (H100) for faster
  compilation (2x+ vCPUs) and native GPU training support
- Resource limits bumped: 20 vCPU / 128Gi (was 10 vCPU / 38Gi)
- New 'train' stage with manual DQN and TFT validation jobs
  (50-step smoke tests to verify CUDA + model correctness)
- Requires helm upgrade of gitlab-runner with new values

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

.gitlab-ci.yml

@@ -1,5 +1,5 @@
 # GitLab CI/CD — Foxhunt
-# Runs on ci-build pool (L4 or H100, scale-to-zero), linker: clang+lld
+# Runs on gpu-training pool (H100, scale-to-zero), linker: clang+lld
 # CI builder image: CUDA 12.4 + Rust 1.89 + protoc + sccache (hosted on Scaleway CR)
 # Service images: pushed to Scaleway Container Registry (Kaniko)
 
@@ -20,6 +20,7 @@ stages:
   - test
   - compile
   - build
+  - train
   - deploy
 
 variables:
@@ -433,6 +434,51 @@ build-training:
       --destination "${REGISTRY}/training:${CI_COMMIT_SHA}"
       --destination "${REGISTRY}/training:latest"
 
+# --------------------------------------------------------------------------
+# Stage 3c: Training validation — quick smoke test on GPU
+# Runs the training binary for a few steps to verify CUDA + model correctness.
+# Triggered manually or on schedule (full ensemble training uses train.sh).
+# --------------------------------------------------------------------------
+.train-validate-base:
+  stage: train
+  image: ${REGISTRY}/training:${CI_COMMIT_SHA}
+  tags:
+    - kapsule
+    - gpu
+  rules:
+    - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
+      when: manual
+      allow_failure: true
+    - if: $CI_PIPELINE_SOURCE == "schedule" && $TRAIN_VALIDATE == "true"
+  before_script:
+    - export LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v stubs | tr '\n' ':' | sed 's/:$//')
+    - nvidia-smi
+    - mkdir -p /output
+
+train-validate-dqn:
+  extends: .train-validate-base
+  needs: [build-training]
+  script:
+    - /usr/local/bin/train_baseline_rl
+        --model dqn
+        --symbol ES.FUT
+        --data-dir /mnt/training-data/futures-baseline
+        --output-dir /output
+        --max-steps-per-epoch 50
+    - echo "DQN validation passed"
+
+train-validate-tft:
+  extends: .train-validate-base
+  needs: [build-training]
+  script:
+    - /usr/local/bin/train_baseline_supervised
+        --model tft
+        --symbol ES.FUT
+        --data-dir /mnt/training-data/futures-baseline
+        --output-dir /output
+        --max-steps-per-epoch 50
+    - echo "TFT validation passed"
+
 # --------------------------------------------------------------------------
 # IaC: Terragrunt plan on MR (runs on gitlab pool)
 # --------------------------------------------------------------------------

infra/k8s/gitlab/runner-values.yaml

@@ -1,6 +1,6 @@
-# GitLab Runner — Kubernetes executor targeting ci-build pool (L4)
+# GitLab Runner — Kubernetes executor targeting gpu-training pool (H100)
 # Runner manager pod lives on gitlab node pool
-# Build pods spawn on ci-build pool (L4-1-24G, scale-to-zero — €0.75/hr vs H100 €3.50+/hr)
+# Build pods spawn on gpu-training pool (H100-1-80G, scale-to-zero)
 
 gitlabUrl: http://gitlab-webservice-default.foxhunt.svc.cluster.local:8181
 # runnerToken set via --set at install time
@@ -26,13 +26,13 @@ runners:
         namespace = "foxhunt"
         image = "rust:1.89-slim"
         privileged = false
-        # L4 scale-to-zero needs ~3-5 min to provision; default 180s times out
+        # GPU pool scale-to-zero needs ~3-5 min to provision; default 180s times out
         poll_timeout = 600
-        # Resource limits for build pods (L4 node: 12 vCPU, 45.5GB)
-        cpu_request = "4000m"
-        cpu_limit = "10000m"
-        memory_request = "16Gi"
-        memory_limit = "38Gi"
+        # Resource limits for build pods (H100-1-80G node)
+        cpu_request = "8000m"
+        cpu_limit = "20000m"
+        memory_request = "32Gi"
+        memory_limit = "128Gi"
         helper_cpu_request = "100m"
         helper_cpu_limit = "500m"
         helper_memory_request = "128Mi"
@@ -42,7 +42,7 @@ runners:
         runtime_class_name = "nvidia"
       # Sub-tables must come AFTER all scalar values (TOML rule)
       [runners.kubernetes.node_selector]
-        "k8s.scaleway.com/pool-name" = "ci-build"
+        "k8s.scaleway.com/pool-name" = "gpu-training"
       [runners.kubernetes.node_tolerations]
         "nvidia.com/gpu" = "NoSchedule"
         # Cilium CNI agent takes ~30s to initialize on fresh scale-from-zero nodes