From 10b86b43191ba41d82c7fdba9bb9b2e7b75e7859 Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Thu, 19 Mar 2026 20:20:30 +0100
Subject: [PATCH] fix(ci): enforce --test-threads=1 for ALL GPU integration
 tests
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Root cause: parallel test execution within a single cargo test binary
corrupts the CUDA primary context (cuDevicePrimaryCtxRetain race).

The dqn-smoke tests ran in parallel threads — 3 GPU tests using a
shared OnceLock<MlDevice> raced with smoke_e2e_dqn_training_loop
which creates a fresh CudaContext::new(0). The parallel context
init/teardown left the primary context in an error state, causing
subsequent cuInit(0) to fail silently.

Lib tests passed because they already had --test-threads=1.
Integration tests (dqn-smoke, dqn-smoke-train, ppo-barrier, etc.)
were missing it.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 infra/k8s/argo/gpu-test-pipeline-template.yaml | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/infra/k8s/argo/gpu-test-pipeline-template.yaml b/infra/k8s/argo/gpu-test-pipeline-template.yaml
index 5b75e6072..8653870ef 100644
--- a/infra/k8s/argo/gpu-test-pipeline-template.yaml
+++ b/infra/k8s/argo/gpu-test-pipeline-template.yaml
@@ -277,7 +277,7 @@ spec:
             # --test-threads=1 for all GPU lib tests: prevents concurrent cuBLAS
             # handle creation that causes CUBLAS_STATUS_NOT_INITIALIZED cascades.
             run_tests "core-lib" cargo test -p ml-core --features cuda --lib -- --test-threads=1
-            run_tests "bayesian" cargo test -p ml --features cuda --test bayesian_changepoint_test
+            run_tests "bayesian" cargo test -p ml --features cuda --test bayesian_changepoint_test -- --test-threads=1
 
             # --- Per-model tests ---
             IFS=',' read -ra MODEL_LIST <<< "$MODELS"
@@ -293,11 +293,14 @@ spec:
                     run_tests "dqn-ml-lib" cargo test -p ml --features cuda --lib -- --test-threads=1 dqn
                   fi
                   if [ "$SCOPE" = "integration" ] || [ "$SCOPE" = "all" ]; then
-                    run_tests "dqn-smoke" cargo test -p ml --features cuda --test smoke_test_real_data
+                    # --test-threads=1 for ALL GPU integration tests: parallel execution
+                    # corrupts the CUDA primary context (cuDevicePrimaryCtxRetain fails
+                    # when multiple threads race on context init/teardown).
+                    run_tests "dqn-smoke" cargo test -p ml --features cuda --test smoke_test_real_data -- --test-threads=1
                     run_tests "dqn-pipeline" cargo test -p ml --features cuda --test dqn_training_pipeline_test -- --test-threads=1
-                    run_tests "dqn-smoke-train" cargo test -p ml --features cuda --test dqn_training_smoke_test
+                    run_tests "dqn-smoke-train" cargo test -p ml --features cuda --test dqn_training_smoke_test -- --test-threads=1
                     run_tests "dqn-early-stop" cargo test -p ml --features cuda --test dqn_early_stopping_termination_test -- --test-threads=1
-                    run_tests "dqn-collapse" cargo test -p ml --features cuda --test dqn_action_collapse_fix_test
+                    run_tests "dqn-collapse" cargo test -p ml --features cuda --test dqn_action_collapse_fix_test -- --test-threads=1
                   fi
                   ;;
                 ppo)
@@ -305,7 +308,7 @@ spec:
                     run_tests "ppo-lib" cargo test -p ml --features cuda --lib -- --test-threads=1 ppo
                   fi
                   if [ "$SCOPE" = "integration" ] || [ "$SCOPE" = "all" ]; then
-                    run_tests "ppo-barrier" cargo test -p ml --features cuda --test barrier_optimization_test
+                    run_tests "ppo-barrier" cargo test -p ml --features cuda --test barrier_optimization_test -- --test-threads=1
                   fi
                   ;;
                 *)
@@ -317,10 +320,10 @@ spec:
                     run_tests "${MODEL}-lib" cargo test -p ml --features cuda --lib -- --test-threads=1 "$LIB_FILTER"
                   fi
                   if [ "$SCOPE" = "integration" ] || [ "$SCOPE" = "all" ]; then
-                    run_tests "${MODEL}-gpu" cargo test -p ml --features cuda --test supervised_gpu_smoke_test -- "test_${MODEL}_gpu_smoke"
+                    run_tests "${MODEL}-gpu" cargo test -p ml --features cuda --test supervised_gpu_smoke_test -- --test-threads=1 "test_${MODEL}_gpu_smoke"
                     # Also run model-specific integration tests if they exist
                     if cargo test -p ml --features cuda --test "${MODEL}_integration" --no-run 2>/dev/null; then
-                      run_tests "${MODEL}-integ" cargo test -p ml --features cuda --test "${MODEL}_integration"
+                      run_tests "${MODEL}-integ" cargo test -p ml --features cuda --test "${MODEL}_integration" -- --test-threads=1
                     fi
                   fi
                   ;;