feat: L40S GPU profile + auto-derive cuda-compute-cap from GPU pool

argo-train.sh now auto-selects cuda-compute-cap based on --gpu-pool:
  - ci-training-h100* → sm_90 (Hopper)
  - ci-training-l40s  → sm_89 (Ada Lovelace)

Added config/gpu/l40s.toml:
  - batch_size=4096 (between H100's 8192 and A100's 2048)
  - buffer_size=300K (scaled for 48GB VRAM)
  - gpu_timesteps_per_episode=2000 (bandwidth-limited)
  - gpu_n_episodes=2048 (scaled from H100's 4096)

GPU profile loader maps "L40S" → "l40s" (was "a100" fallback).

Also fixed pre-existing test drift: num_atoms=52 in h100.toml/a100.toml
was 51 in test expectations (padding alignment for C51 kernels).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-20 16:59:06 +02:00
parent 733b2c32ec
commit 904185004c
5 changed files with 59 additions and 7 deletions

View File

@@ -0,0 +1 @@
{"sessionId":"4d4aa47f-4eb8-44d0-9d38-840da6e33fc0","pid":543045,"acquiredAt":1776692195986}

View File

@@ -1,7 +1,7 @@
# A100 (40-80GB VRAM)
[training]
batch_size = 2048
num_atoms = 51
num_atoms = 52
buffer_size = 200000
hidden_dim_base = 256
replay_buffer_vram_fraction = 0.70

16
config/gpu/l40s.toml Normal file
View File

@@ -0,0 +1,16 @@
# L40S (48GB GDDR6, Ada Lovelace sm_89) — datacenter GPU, lower bandwidth than H100
# 864 GB/s memory bandwidth vs H100's 3.35 TB/s (4x lower)
# 18,176 CUDA cores, 568 tensor cores (4th gen), FP8 support
[training]
batch_size = 4096 # between H100's 8192 and A100's 2048 — 48GB VRAM allows this
num_atoms = 52
buffer_size = 300000 # scaled for 48GB VRAM (H100=500K at 80GB)
hidden_dim_base = 256
replay_buffer_vram_fraction = 0.0 # exact sizing, consistent with H100
[experience]
gpu_timesteps_per_episode = 2000 # between H100's 5000 and A100's 500 — bandwidth-limited
gpu_n_episodes = 2048 # 18176 CUDA cores scaled down from H100's 4096
[cuda]
cuda_stack_bytes = 65536 # 64KB (same as H100/A100)

View File

@@ -20,6 +20,7 @@ const DEFAULT_TOML: &str = include_str!("../../../../config/gpu/default.toml");
const RTX3050_TOML: &str = include_str!("../../../../config/gpu/rtx3050.toml");
const H100_TOML: &str = include_str!("../../../../config/gpu/h100.toml");
const A100_TOML: &str = include_str!("../../../../config/gpu/a100.toml");
const L40S_TOML: &str = include_str!("../../../../config/gpu/l40s.toml");
/// GPU configuration profile with training, experience collection, and CUDA settings.
///
@@ -158,7 +159,9 @@ impl GpuProfile {
"rtx3050"
} else if upper.contains("H100") || upper.contains("H200") || upper.contains("GH200") {
"h100"
} else if upper.contains("A100") || upper.contains("L40") {
} else if upper.contains("L40") {
"l40s"
} else if upper.contains("A100") {
"a100"
} else {
"default"
@@ -171,6 +174,7 @@ impl GpuProfile {
"rtx3050" => RTX3050_TOML,
"h100" => H100_TOML,
"a100" => A100_TOML,
"l40s" => L40S_TOML,
_ => DEFAULT_TOML,
}
}
@@ -255,7 +259,7 @@ mod tests {
fn test_profile_name_for_l40s() {
assert_eq!(
GpuProfile::profile_name_for_device("NVIDIA L40S"),
"a100"
"l40s"
);
}
@@ -286,10 +290,10 @@ mod tests {
fn test_embedded_h100_parses() {
let profile: GpuProfile = toml::from_str(H100_TOML).unwrap();
assert_eq!(profile.training.batch_size, 8192);
assert_eq!(profile.training.num_atoms, 51);
assert_eq!(profile.training.num_atoms, 52);
assert_eq!(profile.training.buffer_size, 500_000);
assert_eq!(profile.experience.gpu_timesteps_per_episode, 5000);
assert_eq!(profile.experience.gpu_n_episodes, Some(1024));
assert_eq!(profile.experience.gpu_n_episodes, Some(4096));
assert_eq!(profile.cuda.cuda_stack_bytes, 65536);
}
@@ -297,12 +301,23 @@ mod tests {
fn test_embedded_a100_parses() {
let profile: GpuProfile = toml::from_str(A100_TOML).unwrap();
assert_eq!(profile.training.batch_size, 2048);
assert_eq!(profile.training.num_atoms, 51);
assert_eq!(profile.training.num_atoms, 52);
assert_eq!(profile.training.buffer_size, 200_000);
assert_eq!(profile.experience.gpu_timesteps_per_episode, 500);
assert_eq!(profile.cuda.cuda_stack_bytes, 65536);
}
#[test]
fn test_embedded_l40s_parses() {
let profile: GpuProfile = toml::from_str(L40S_TOML).unwrap();
assert_eq!(profile.training.batch_size, 4096);
assert_eq!(profile.training.num_atoms, 52);
assert_eq!(profile.training.buffer_size, 300_000);
assert_eq!(profile.experience.gpu_timesteps_per_episode, 2000);
assert_eq!(profile.experience.gpu_n_episodes, Some(2048));
assert_eq!(profile.cuda.cuda_stack_bytes, 65536);
}
#[test]
fn test_embedded_default_parses() {
let profile: GpuProfile = toml::from_str(DEFAULT_TOML).unwrap();
@@ -325,7 +340,16 @@ mod tests {
fn test_load_for_device_h100() {
let profile = GpuProfile::load_for_device("NVIDIA H100 80GB HBM3");
assert_eq!(profile.training.batch_size, 8192);
assert_eq!(profile.training.num_atoms, 51);
assert_eq!(profile.training.num_atoms, 52);
assert_eq!(profile.cuda.cuda_stack_bytes, 65536);
}
#[test]
fn test_load_for_device_l40s() {
let profile = GpuProfile::load_for_device("NVIDIA L40S");
assert_eq!(profile.training.batch_size, 4096);
assert_eq!(profile.training.num_atoms, 52);
assert_eq!(profile.experience.gpu_n_episodes, Some(2048));
assert_eq!(profile.cuda.cuda_stack_bytes, 65536);
}

View File

@@ -77,10 +77,20 @@ while [[ $# -gt 0 ]]; do
esac
done
# Auto-derive cuda-compute-cap from GPU pool — cubins must match device sm_XX.
# Default pool is ci-training-h100 (sm_90). Override for other architectures:
# ci-training-h100* → sm_90 (Hopper)
# ci-training-l40s → sm_89 (Ada Lovelace)
case "${GPU_POOL:-ci-training-h100}" in
*l40s*) CUDA_COMPUTE_CAP="89" ;;
*h100*|*) CUDA_COMPUTE_CAP="90" ;; # default Hopper
esac
CMD="argo submit -n foxhunt --from=wftmpl/train"
CMD="$CMD -p commit-sha=$SHA"
CMD="$CMD -p git-branch=$BRANCH"
CMD="$CMD -p model=$MODEL"
CMD="$CMD -p cuda-compute-cap=$CUDA_COMPUTE_CAP"
[[ -n "$TRIALS" ]] && CMD="$CMD -p hyperopt-trials=$TRIALS"
[[ -n "$EPOCHS" ]] && CMD="$CMD -p train-epochs=$EPOCHS"
@@ -100,5 +110,6 @@ $BASELINE && echo " mode: baseline (no hyperopt)"
[[ -n "$TRIALS" ]] && echo " trials: $TRIALS"
[[ -n "$EPOCHS" ]] && echo " epochs: $EPOCHS"
[[ -n "$GPU_POOL" ]] && echo " gpu: $GPU_POOL"
echo " sm: $CUDA_COMPUTE_CAP"
echo ""
eval "$CMD"