From d537ee87e59b7f67a2ad0977a725d55a58d1f8a3 Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Wed, 4 Mar 2026 22:47:29 +0100
Subject: [PATCH] ci: increase compile-training pod timeout to 20 min

When all 8 per-service compile jobs fire simultaneously (e.g. on
Cargo.toml change), the single POP2-32C-128G node is fully loaded.
Training compile pod can't schedule until service jobs finish.
KUBERNETES_POLL_TIMEOUT=1200 gives it time to wait.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .gitlab-ci.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 38d5fc6fb..e078f9a7c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -599,6 +599,9 @@ compile-training:
     # CUDA stubs from ci-builder image — compiles CUDA kernels without GPU hardware.
     # L4 GPU = compute capability 8.9 → candle-kernels generates sm_89 PTX
     CUDA_COMPUTE_CAP: "89"
+    # When all 8 service jobs fire (Cargo.toml change), the node is fully loaded.
+    # Increase pod scheduling timeout so training waits for service jobs to finish.
+    KUBERNETES_POLL_TIMEOUT: "1200"
   rules:
     - if: $CI_COMMIT_BRANCH == "main" && $CI_PIPELINE_SOURCE == "push"
       changes: