diff --git a/infra/k8s/argo/gpu-test-pipeline-template.yaml b/infra/k8s/argo/gpu-test-pipeline-template.yaml index 5b75e6072..8653870ef 100644 --- a/infra/k8s/argo/gpu-test-pipeline-template.yaml +++ b/infra/k8s/argo/gpu-test-pipeline-template.yaml @@ -277,7 +277,7 @@ spec: # --test-threads=1 for all GPU lib tests: prevents concurrent cuBLAS # handle creation that causes CUBLAS_STATUS_NOT_INITIALIZED cascades. run_tests "core-lib" cargo test -p ml-core --features cuda --lib -- --test-threads=1 - run_tests "bayesian" cargo test -p ml --features cuda --test bayesian_changepoint_test + run_tests "bayesian" cargo test -p ml --features cuda --test bayesian_changepoint_test -- --test-threads=1 # --- Per-model tests --- IFS=',' read -ra MODEL_LIST <<< "$MODELS" @@ -293,11 +293,14 @@ spec: run_tests "dqn-ml-lib" cargo test -p ml --features cuda --lib -- --test-threads=1 dqn fi if [ "$SCOPE" = "integration" ] || [ "$SCOPE" = "all" ]; then - run_tests "dqn-smoke" cargo test -p ml --features cuda --test smoke_test_real_data + # --test-threads=1 for ALL GPU integration tests: parallel execution + # corrupts the CUDA primary context (cuDevicePrimaryCtxRetain fails + # when multiple threads race on context init/teardown). + run_tests "dqn-smoke" cargo test -p ml --features cuda --test smoke_test_real_data -- --test-threads=1 run_tests "dqn-pipeline" cargo test -p ml --features cuda --test dqn_training_pipeline_test -- --test-threads=1 - run_tests "dqn-smoke-train" cargo test -p ml --features cuda --test dqn_training_smoke_test + run_tests "dqn-smoke-train" cargo test -p ml --features cuda --test dqn_training_smoke_test -- --test-threads=1 run_tests "dqn-early-stop" cargo test -p ml --features cuda --test dqn_early_stopping_termination_test -- --test-threads=1 - run_tests "dqn-collapse" cargo test -p ml --features cuda --test dqn_action_collapse_fix_test + run_tests "dqn-collapse" cargo test -p ml --features cuda --test dqn_action_collapse_fix_test -- --test-threads=1 fi ;; ppo) @@ -305,7 +308,7 @@ spec: run_tests "ppo-lib" cargo test -p ml --features cuda --lib -- --test-threads=1 ppo fi if [ "$SCOPE" = "integration" ] || [ "$SCOPE" = "all" ]; then - run_tests "ppo-barrier" cargo test -p ml --features cuda --test barrier_optimization_test + run_tests "ppo-barrier" cargo test -p ml --features cuda --test barrier_optimization_test -- --test-threads=1 fi ;; *) @@ -317,10 +320,10 @@ spec: run_tests "${MODEL}-lib" cargo test -p ml --features cuda --lib -- --test-threads=1 "$LIB_FILTER" fi if [ "$SCOPE" = "integration" ] || [ "$SCOPE" = "all" ]; then - run_tests "${MODEL}-gpu" cargo test -p ml --features cuda --test supervised_gpu_smoke_test -- "test_${MODEL}_gpu_smoke" + run_tests "${MODEL}-gpu" cargo test -p ml --features cuda --test supervised_gpu_smoke_test -- --test-threads=1 "test_${MODEL}_gpu_smoke" # Also run model-specific integration tests if they exist if cargo test -p ml --features cuda --test "${MODEL}_integration" --no-run 2>/dev/null; then - run_tests "${MODEL}-integ" cargo test -p ml --features cuda --test "${MODEL}_integration" + run_tests "${MODEL}-integ" cargo test -p ml --features cuda --test "${MODEL}_integration" -- --test-threads=1 fi fi ;;