diff --git a/.claude/scheduled_tasks.lock b/.claude/scheduled_tasks.lock new file mode 100644 index 000000000..c003233a1 --- /dev/null +++ b/.claude/scheduled_tasks.lock @@ -0,0 +1 @@ +{"sessionId":"4d4aa47f-4eb8-44d0-9d38-840da6e33fc0","pid":543045,"acquiredAt":1776692195986} \ No newline at end of file diff --git a/config/gpu/a100.toml b/config/gpu/a100.toml index 693861986..a023599fd 100644 --- a/config/gpu/a100.toml +++ b/config/gpu/a100.toml @@ -1,7 +1,7 @@ # A100 (40-80GB VRAM) [training] batch_size = 2048 -num_atoms = 51 +num_atoms = 52 buffer_size = 200000 hidden_dim_base = 256 replay_buffer_vram_fraction = 0.70 diff --git a/config/gpu/l40s.toml b/config/gpu/l40s.toml new file mode 100644 index 000000000..b64149fbc --- /dev/null +++ b/config/gpu/l40s.toml @@ -0,0 +1,16 @@ +# L40S (48GB GDDR6, Ada Lovelace sm_89) — datacenter GPU, lower bandwidth than H100 +# 864 GB/s memory bandwidth vs H100's 3.35 TB/s (4x lower) +# 18,176 CUDA cores, 568 tensor cores (4th gen), FP8 support +[training] +batch_size = 4096 # between H100's 8192 and A100's 2048 — 48GB VRAM allows this +num_atoms = 52 +buffer_size = 300000 # scaled for 48GB VRAM (H100=500K at 80GB) +hidden_dim_base = 256 +replay_buffer_vram_fraction = 0.0 # exact sizing, consistent with H100 + +[experience] +gpu_timesteps_per_episode = 2000 # between H100's 5000 and A100's 500 — bandwidth-limited +gpu_n_episodes = 2048 # 18176 CUDA cores scaled down from H100's 4096 + +[cuda] +cuda_stack_bytes = 65536 # 64KB (same as H100/A100) diff --git a/crates/ml-core/src/gpu/profile.rs b/crates/ml-core/src/gpu/profile.rs index 89b0897f8..836fe73f5 100644 --- a/crates/ml-core/src/gpu/profile.rs +++ b/crates/ml-core/src/gpu/profile.rs @@ -20,6 +20,7 @@ const DEFAULT_TOML: &str = include_str!("../../../../config/gpu/default.toml"); const RTX3050_TOML: &str = include_str!("../../../../config/gpu/rtx3050.toml"); const H100_TOML: &str = include_str!("../../../../config/gpu/h100.toml"); const A100_TOML: &str = include_str!("../../../../config/gpu/a100.toml"); +const L40S_TOML: &str = include_str!("../../../../config/gpu/l40s.toml"); /// GPU configuration profile with training, experience collection, and CUDA settings. /// @@ -158,7 +159,9 @@ impl GpuProfile { "rtx3050" } else if upper.contains("H100") || upper.contains("H200") || upper.contains("GH200") { "h100" - } else if upper.contains("A100") || upper.contains("L40") { + } else if upper.contains("L40") { + "l40s" + } else if upper.contains("A100") { "a100" } else { "default" @@ -171,6 +174,7 @@ impl GpuProfile { "rtx3050" => RTX3050_TOML, "h100" => H100_TOML, "a100" => A100_TOML, + "l40s" => L40S_TOML, _ => DEFAULT_TOML, } } @@ -255,7 +259,7 @@ mod tests { fn test_profile_name_for_l40s() { assert_eq!( GpuProfile::profile_name_for_device("NVIDIA L40S"), - "a100" + "l40s" ); } @@ -286,10 +290,10 @@ mod tests { fn test_embedded_h100_parses() { let profile: GpuProfile = toml::from_str(H100_TOML).unwrap(); assert_eq!(profile.training.batch_size, 8192); - assert_eq!(profile.training.num_atoms, 51); + assert_eq!(profile.training.num_atoms, 52); assert_eq!(profile.training.buffer_size, 500_000); assert_eq!(profile.experience.gpu_timesteps_per_episode, 5000); - assert_eq!(profile.experience.gpu_n_episodes, Some(1024)); + assert_eq!(profile.experience.gpu_n_episodes, Some(4096)); assert_eq!(profile.cuda.cuda_stack_bytes, 65536); } @@ -297,12 +301,23 @@ mod tests { fn test_embedded_a100_parses() { let profile: GpuProfile = toml::from_str(A100_TOML).unwrap(); assert_eq!(profile.training.batch_size, 2048); - assert_eq!(profile.training.num_atoms, 51); + assert_eq!(profile.training.num_atoms, 52); assert_eq!(profile.training.buffer_size, 200_000); assert_eq!(profile.experience.gpu_timesteps_per_episode, 500); assert_eq!(profile.cuda.cuda_stack_bytes, 65536); } + #[test] + fn test_embedded_l40s_parses() { + let profile: GpuProfile = toml::from_str(L40S_TOML).unwrap(); + assert_eq!(profile.training.batch_size, 4096); + assert_eq!(profile.training.num_atoms, 52); + assert_eq!(profile.training.buffer_size, 300_000); + assert_eq!(profile.experience.gpu_timesteps_per_episode, 2000); + assert_eq!(profile.experience.gpu_n_episodes, Some(2048)); + assert_eq!(profile.cuda.cuda_stack_bytes, 65536); + } + #[test] fn test_embedded_default_parses() { let profile: GpuProfile = toml::from_str(DEFAULT_TOML).unwrap(); @@ -325,7 +340,16 @@ mod tests { fn test_load_for_device_h100() { let profile = GpuProfile::load_for_device("NVIDIA H100 80GB HBM3"); assert_eq!(profile.training.batch_size, 8192); - assert_eq!(profile.training.num_atoms, 51); + assert_eq!(profile.training.num_atoms, 52); + assert_eq!(profile.cuda.cuda_stack_bytes, 65536); + } + + #[test] + fn test_load_for_device_l40s() { + let profile = GpuProfile::load_for_device("NVIDIA L40S"); + assert_eq!(profile.training.batch_size, 4096); + assert_eq!(profile.training.num_atoms, 52); + assert_eq!(profile.experience.gpu_n_episodes, Some(2048)); assert_eq!(profile.cuda.cuda_stack_bytes, 65536); } diff --git a/scripts/argo-train.sh b/scripts/argo-train.sh index 55cf04882..79fb4943a 100755 --- a/scripts/argo-train.sh +++ b/scripts/argo-train.sh @@ -77,10 +77,20 @@ while [[ $# -gt 0 ]]; do esac done +# Auto-derive cuda-compute-cap from GPU pool — cubins must match device sm_XX. +# Default pool is ci-training-h100 (sm_90). Override for other architectures: +# ci-training-h100* → sm_90 (Hopper) +# ci-training-l40s → sm_89 (Ada Lovelace) +case "${GPU_POOL:-ci-training-h100}" in + *l40s*) CUDA_COMPUTE_CAP="89" ;; + *h100*|*) CUDA_COMPUTE_CAP="90" ;; # default Hopper +esac + CMD="argo submit -n foxhunt --from=wftmpl/train" CMD="$CMD -p commit-sha=$SHA" CMD="$CMD -p git-branch=$BRANCH" CMD="$CMD -p model=$MODEL" +CMD="$CMD -p cuda-compute-cap=$CUDA_COMPUTE_CAP" [[ -n "$TRIALS" ]] && CMD="$CMD -p hyperopt-trials=$TRIALS" [[ -n "$EPOCHS" ]] && CMD="$CMD -p train-epochs=$EPOCHS" @@ -100,5 +110,6 @@ $BASELINE && echo " mode: baseline (no hyperopt)" [[ -n "$TRIALS" ]] && echo " trials: $TRIALS" [[ -n "$EPOCHS" ]] && echo " epochs: $EPOCHS" [[ -n "$GPU_POOL" ]] && echo " gpu: $GPU_POOL" +echo " sm: $CUDA_COMPUTE_CAP" echo "" eval "$CMD"