From 3112e07a5a784a90f6902ff174ce137210356513 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 13 Mar 2026 19:52:04 +0100 Subject: [PATCH] perf(ci): serialize dqn-pipeline tests to avoid GPU contention 4 concurrent DQN trainers on a shared H100 causes severe GPU contention (28-39% utilization, 42 GB VRAM). Serializing with --test-threads=1 gives each test full GPU access, reducing total wallclock time despite sequential execution. Co-Authored-By: Claude Opus 4.6 --- infra/k8s/argo/gpu-test-pipeline-template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/k8s/argo/gpu-test-pipeline-template.yaml b/infra/k8s/argo/gpu-test-pipeline-template.yaml index 3af0be826..64498f028 100644 --- a/infra/k8s/argo/gpu-test-pipeline-template.yaml +++ b/infra/k8s/argo/gpu-test-pipeline-template.yaml @@ -263,7 +263,7 @@ spec: fi if [ "$SCOPE" = "integration" ] || [ "$SCOPE" = "all" ]; then run_tests "dqn-smoke" cargo test -p ml --features cuda --test smoke_test_real_data - run_tests "dqn-pipeline" cargo test -p ml --features cuda --test dqn_training_pipeline_test + run_tests "dqn-pipeline" cargo test -p ml --features cuda --test dqn_training_pipeline_test -- --test-threads=1 run_tests "dqn-smoke-train" cargo test -p ml --features cuda --test dqn_training_smoke_test run_tests "dqn-early-stop" cargo test -p ml --features cuda --test dqn_early_stopping_termination_test run_tests "dqn-collapse" cargo test -p ml --features cuda --test dqn_action_collapse_fix_test