diff --git a/infra/k8s/argo/train-multi-seed-template.yaml b/infra/k8s/argo/train-multi-seed-template.yaml index c3dc602ff..1d6d43f35 100644 --- a/infra/k8s/argo/train-multi-seed-template.yaml +++ b/infra/k8s/argo/train-multi-seed-template.yaml @@ -58,12 +58,17 @@ spec: value: HEAD - name: git-branch value: main + # Default sm_89 / ci-training-l40s as of 2026-05-14 per + # `feedback_default_to_l40s_pool.md`. argo-train.sh's + # CUDA_COMPUTE_CAP derivation passes "89" by default to match; + # explicit --gpu-pool ci-training-h100 overrides both back to + # sm_90 + H100. - name: cuda-compute-cap - value: "90" + value: "89" - name: model value: dqn - name: gpu-pool - value: ci-training-h100 + value: ci-training-l40s - name: hyperopt-trials value: "0" # Multi-seed runs typically skip hyperopt (re-use baseline params) - name: hyperopt-epochs diff --git a/infra/k8s/argo/train-template.yaml b/infra/k8s/argo/train-template.yaml index 53df5adec..e424ef886 100644 --- a/infra/k8s/argo/train-template.yaml +++ b/infra/k8s/argo/train-template.yaml @@ -45,8 +45,16 @@ spec: value: "90" - name: model value: dqn + # gpu-pool default: ci-training-l40s (set 2026-05-14 per + # `feedback_default_to_l40s_pool.md` — SP-chain training has been + # standardising on L40S since 2026-05-09; the prior ci-training-h100 + # default required every SP-run invocation to pass an explicit + # `--gpu-pool ci-training-l40s` override. H100 remains opt-in via + # `--gpu-pool ci-training-h100` for runs that genuinely need + # 80 GB VRAM or sm_90 features). Compute-cap derivation in + # `argo-train.sh` matches this default to sm_89 (Ada Lovelace). - name: gpu-pool - value: ci-training-h100 + value: ci-training-l40s - name: hyperopt-trials value: "20" - name: hyperopt-epochs diff --git a/scripts/argo-test.sh b/scripts/argo-test.sh index 14f2d5d44..987668412 100755 --- a/scripts/argo-test.sh +++ b/scripts/argo-test.sh @@ -25,7 +25,8 @@ Options: Use "all" for all 10 models --scope lib, integration, or all (default: all) --ref Git ref to test (default: HEAD) - --gpu-pool GPU node pool (default: ci-training-h100) + --gpu-pool GPU node pool (default: ci-training-l40s; opt into + ci-training-h100 for 80 GB / sm_90 workloads) --watch Follow workflow logs -h, --help Show this help EOF diff --git a/scripts/argo-train.sh b/scripts/argo-train.sh index 7ab881dd4..2164a29a2 100755 --- a/scripts/argo-train.sh +++ b/scripts/argo-train.sh @@ -2,10 +2,10 @@ # Train a model via Argo Workflows. # # Usage: -# ./scripts/argo-train.sh dqn # defaults: HEAD, H100, 50 epochs +# ./scripts/argo-train.sh dqn # defaults: HEAD, L40S, 50 epochs # ./scripts/argo-train.sh dqn --sha abc1234 # specific commit # ./scripts/argo-train.sh dqn --epochs 100 --trials 40 # override training params -# ./scripts/argo-train.sh ppo --gpu-pool ci-training # L40S instead of H100 +# ./scripts/argo-train.sh ppo --gpu-pool ci-training-h100 # opt into H100 (80 GB) # ./scripts/argo-train.sh dqn --baseline # skip hyperopt # ./scripts/argo-train.sh dqn --watch # follow logs # @@ -49,7 +49,8 @@ Options: --branch Git branch (default: main) --trials Hyperopt trials (default: 20) --epochs Training epochs (default: 50) - --gpu-pool GPU node pool (default: ci-training-h100) + --gpu-pool GPU node pool (default: ci-training-l40s; opt into + ci-training-h100 for 80 GB / sm_90 workloads) --symbol Trading symbol (default: ES.FUT) --capital Initial capital (default: 35000) --baseline Skip hyperopt (trials=0) @@ -125,16 +126,18 @@ if ! [[ "$FOLDS" =~ ^[0-9]+$ ]] || [[ "$FOLDS" -lt 1 ]]; then fi # Auto-derive cuda-compute-cap from GPU pool — cubins must match device sm_XX. -# Default pool is ci-training-h100 (sm_90). Override for other architectures: +# Default pool is ci-training-l40s (sm_89) as of 2026-05-14 per +# `feedback_default_to_l40s_pool.md`. Override for other architectures: # ci-training-h100* → sm_90 (Hopper) -# ci-training-l40s → sm_89 (Ada Lovelace) +# ci-training-l40s → sm_89 (Ada Lovelace, current default) # ci-training → sm_89 (alias for L40S — pool is named bare in some # clusters; fixed 2026-05-04 after train-mnpf7 deployed # with sm_90 cubins on L40S device, requiring terminate # + resubmit with explicit --gpu-pool ci-training-l40s). -case "${GPU_POOL:-ci-training-h100}" in +case "${GPU_POOL:-ci-training-l40s}" in + *h100*) CUDA_COMPUTE_CAP="90" ;; # Hopper (opt-in) *l40s*|ci-training) CUDA_COMPUTE_CAP="89" ;; - *h100*|*) CUDA_COMPUTE_CAP="90" ;; # default Hopper + *) CUDA_COMPUTE_CAP="89" ;; # default Ada Lovelace esac # ── Route: single-job (existing template) vs multi-seed DAG (new template) ──