From 4b2f9e8133b1099c2b3af13af2bcb35bb83efc9a Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Mon, 2 Mar 2026 19:55:54 +0100
Subject: [PATCH] =?UTF-8?q?fix(ml):=20GPU-aware=20PSO=20thread=20cap=20?=
 =?UTF-8?q?=E2=80=94=203=E2=86=927=20parallel=20trials=20on=20L40S?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The auto-detect heuristic used cpus/2 ("smart cap"), designed for
CPU-bound workloads.  DQN/PPO trials are GPU-bound — each rayon
thread submits CUDA kernels and waits on cudaDeviceSynchronize(),
using minimal CPU.  cpus-1 is the correct cap.

On L40S-1-48G (8 vCPU): 3 threads → 7 threads (2.3× more trials).
Also bumps CI CPU limit 7500m→8000m to expose all 8 cores.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .gitlab-ci.yml                             |  4 ++--
 crates/ml/examples/hyperopt_baseline_rl.rs | 20 ++++++++++----------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c69e06fdf..52e5e5af0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -554,8 +554,8 @@ build-web-dashboard:
     # Route to ci-training pool (L40S GPU)
     KUBERNETES_NODE_SELECTOR_POOL: "k8s.scaleway.com/pool-name=ci-training"
     KUBERNETES_RUNTIME_CLASS_NAME: "nvidia"
-    KUBERNETES_CPU_REQUEST: "6000m"
-    KUBERNETES_CPU_LIMIT: "7500m"
+    KUBERNETES_CPU_REQUEST: "7000m"
+    KUBERNETES_CPU_LIMIT: "8000m"
     KUBERNETES_MEMORY_REQUEST: "8Gi"
     KUBERNETES_MEMORY_LIMIT: "20Gi"
     CUBLAS_WORKSPACE_CONFIG: ":4096:8"
diff --git a/crates/ml/examples/hyperopt_baseline_rl.rs b/crates/ml/examples/hyperopt_baseline_rl.rs
index 86ae022a9..52a258dc1 100644
--- a/crates/ml/examples/hyperopt_baseline_rl.rs
+++ b/crates/ml/examples/hyperopt_baseline_rl.rs
@@ -106,7 +106,7 @@ struct Args {
 
     /// Number of parallel trial evaluations.
     /// Each trial uses ~1 CPU core + shared GPU for forward/backward.
-    /// 0 = auto-detect (available CPUs - 2 reserved for CUDA driver).
+    /// 0 = auto-detect (CPUs - 1; GPU-bound trials need minimal CPU).
     /// 1 = sequential. N = N concurrent trials.
     #[arg(long, default_value = "0")]
     parallel: usize,
@@ -307,13 +307,13 @@ fn main() -> Result<()> {
     info!("Device: CUDA GPU");
 
     // Resolve parallel: 0 = auto-detect.
-    // Smart cap: on GPU instances with few vCPUs (e.g. L40S-1-48G has 8 vCPU),
-    // more threads than vCPU/2 causes CPU contention that starves GPU.
-    let cpu_cap = cpus.saturating_sub(2).max(1);
-    #[allow(clippy::integer_division)]
-    let cpu_smart_cap = (cpus / 2).max(1);
-    let parallel = if args.parallel == 0 {
-        cpu_cap.min(cpu_smart_cap)
+    // DQN/PPO trials are GPU-bound: each rayon thread submits CUDA kernels
+    // for forward/backward passes and spends most of its time waiting on
+    // cudaDeviceSynchronize(). CPU usage per thread is minimal (data feeding,
+    // reward calculation), so cpus-1 threads is safe — reserve 1 core for
+    // the CUDA driver event loop, OS, and the OTLP batch exporter.
+    let cpu_threads = if args.parallel == 0 {
+        cpus.saturating_sub(1).max(1)
     } else {
         args.parallel
     };
@@ -321,8 +321,8 @@ fn main() -> Result<()> {
     // Cap by VRAM budget with corrected constants (DQN ~50MB overhead, ~0.0005MB/sample)
     let budget = ml::hyperopt::HardwareBudget::detect();
     let vram_cap = budget.plan_hyperopt(50.0, 0.0005, 64.0, 4096.0).max_concurrent_trials;
-    let num_threads = parallel.min(vram_cap);
-    info!("Parallel: {} threads (CPU: {}, CPU smart cap: {}, VRAM cap: {})", num_threads, cpus, cpu_smart_cap, vram_cap);
+    let num_threads = cpu_threads.min(vram_cap);
+    info!("Parallel: {} threads (CPU: {}, CPU cap: {}, VRAM cap: {})", num_threads, cpus, cpu_threads, vram_cap);
 
     // Configure rayon thread pool for parallel trial evaluation
     if num_threads > 1 {