gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(gpu): cap auto-scaled episode count at GPU buffer limit (4096)

Browse Source 
optimal_n_episodes() returns 8192 on H100 (132 SMs) but the GPU
experience collector pre-allocates buffers for MAX_EPISODES_LIMIT=4096.
Exceeding that caused "Episode count exceeds allocated buffer size"
and silent fallback to CPU. Now all 3 auto-scaling sites clamp at 4096.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-09 22:43:34 +01:00
parent 6a8f8d7b50
commit dd075ce02a
1 changed files with 6 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
crates/ml/src/trainers/dqn/trainer.rs
View File

@@ -1850,10 +1850,11 @@ impl DQNTrainer {
let configured = self.hyperparams.gpu_n_episodes;
if configured >= 128 {
match detect_gpu_hardware() {
// Cap at GPU experience collector's buffer limit (4096)
Ok(hw) => configured.max(hw.optimal_n_episodes(
state_dim,
self.hyperparams.gpu_timesteps_per_episode,
)),
)).min(4096),
Err(_) => configured,
}
} else {
@@ -1892,10 +1893,11 @@ impl DQNTrainer {
let configured = self.hyperparams.gpu_n_episodes;
if configured >= 128 {
match detect_gpu_hardware() {
// Cap at GPU experience collector's buffer limit (4096)
Ok(hw) => configured.max(hw.optimal_n_episodes(
state_dim,
self.hyperparams.gpu_timesteps_per_episode,
)),
)).min(4096),
Err(_) => configured,
}
} else {
@@ -2001,7 +2003,8 @@ impl DQNTrainer {
aligned_sd,
self.hyperparams.gpu_timesteps_per_episode,
);
let chosen = configured.max(optimal);
// Cap at GPU experience collector's buffer limit (4096)
let chosen = configured.max(optimal).min(4096);
if chosen != configured {
info!(
"GPU auto-scaled n_episodes: {} → {} (SMs={}, VRAM={:.0}MB)",