fix(dqn): replace hardcoded PER memory cap with dynamic VRAM-proportional budget

The GPU PER replay buffer had a hardcoded 4 GB MAX_BYTES limit that
rejected the auto-sizer's 10M-entry proposal on H100 (80 GB VRAM).
Now per_max_buffer_bytes() computes 20% of total VRAM (min 1 GB) and
flows through OptimalReplayConfig → DQNConfig → GpuReplayBufferConfig
so both subsystems agree on the budget.

Also fixes misleading regime detection log (indices 211/219 → 40/41)
and renames dqn_config_2025 → dqn_default_config.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

crates/ml-core/src/memory_optimization/auto_batch_size.rs

@@ -427,6 +427,19 @@ pub struct GpuMemoryInfo {
     pub used_memory_mb: f64,
 }
 
+/// Bundled output from [`GpuHardwareInfo::optimal_replay_config`].
+///
+/// Ensures callers always receive both the capacity **and** the PER memory
+/// budget together, preventing the temporal-coupling bug where the auto-sizer
+/// proposes a capacity that the PER pre-flight check rejects.
+#[derive(Debug, Clone, Copy)]
+pub struct OptimalReplayConfig {
+    /// VRAM-optimal replay buffer capacity (clamped 100K..10M).
+    pub capacity: usize,
+    /// Maximum PER buffer allocation in bytes (20% of total VRAM, min 1 GB).
+    pub per_max_buffer_bytes: usize,
+}
+
 /// Full GPU hardware info for dynamic scaling decisions.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct GpuHardwareInfo {
@@ -492,7 +505,7 @@ impl GpuHardwareInfo {
         aligned.min(MAX_EPISODES)
     }
 
-    /// Calculate VRAM-optimal replay buffer capacity.
+    /// Calculate VRAM-optimal replay buffer capacity and PER memory budget.
     ///
     /// Allocates `vram_fraction` of free GPU memory to the replay buffer.
     /// Each transition stores: 2 × state_dim × f32 (state + next_state)
@@ -501,7 +514,12 @@ impl GpuHardwareInfo {
     /// Enforces guardrails:
     ///   - MIN_REPLAY_CAPACITY = 100_000 (below this, sample efficiency degrades)
     ///   - MAX_REPLAY_CAPACITY = 10_000_000 (above this, priority staleness dominates)
-    pub fn optimal_replay_capacity(&self, state_dim: usize, vram_fraction: f64) -> usize {
+    ///   - PER memory budget cap: capacity must fit within `per_max_buffer_bytes()` so
+    ///     the GPU PER pre-flight check never rejects the auto-sizer's output.
+    ///
+    /// Returns an [`OptimalReplayConfig`] bundling the capacity with the PER
+    /// memory budget so callers cannot forget to pass the budget downstream.
+    pub fn optimal_replay_config(&self, state_dim: usize, vram_fraction: f64) -> OptimalReplayConfig {
         const MIN_REPLAY_CAPACITY: usize = 100_000;
         const MAX_REPLAY_CAPACITY: usize = 10_000_000;
 
@@ -509,7 +527,23 @@ impl GpuHardwareInfo {
         let budget_bytes = (self.free_memory_mb * vram_fraction * 1024.0 * 1024.0) as usize;
         let raw_capacity = budget_bytes / bytes_per_transition.max(1);
 
-        raw_capacity.clamp(MIN_REPLAY_CAPACITY, MAX_REPLAY_CAPACITY)
+        // Cap by PER buffer VRAM budget (proportional to total VRAM).
+        // Uses the same bytes-per-transition formula as GpuReplayBuffer's pre-flight
+        // check, so the auto-sizer never proposes a capacity the PER would reject.
+        let per_max = self.per_max_buffer_bytes();
+        let per_cap = per_max / bytes_per_transition.max(1);
+
+        let capacity = raw_capacity.min(per_cap).clamp(MIN_REPLAY_CAPACITY, MAX_REPLAY_CAPACITY);
+        OptimalReplayConfig { capacity, per_max_buffer_bytes: per_max }
+    }
+
+    /// Maximum PER replay buffer allocation (bytes) proportional to total VRAM.
+    ///
+    /// Formula: 20% of total GPU memory (min 1 GB).
+    /// No upper cap — scales with available hardware.
+    pub fn per_max_buffer_bytes(&self) -> usize {
+        let max_mb = (self.total_memory_mb * 0.20).max(1024.0);
+        (max_mb * 1024.0 * 1024.0) as usize
     }
 }
 
@@ -1073,9 +1107,10 @@ mod tests {
             free_memory_mb: 72_000.0,
             sm_count: 132,
         };
-        let cap = hw.optimal_replay_capacity(56, 0.70);
-        assert!(cap >= 500_000, "H100 should get at least 500K capacity, got {}", cap);
-        assert!(cap <= 10_000_000, "Should be capped at max, got {}", cap);
+        let cfg = hw.optimal_replay_config(56, 0.70);
+        assert!(cfg.capacity >= 500_000, "H100 should get at least 500K capacity, got {}", cfg.capacity);
+        assert!(cfg.capacity <= 10_000_000, "Should be capped at max, got {}", cfg.capacity);
+        assert!(cfg.per_max_buffer_bytes > 0, "PER budget must be positive");
     }
 
     #[test]
@@ -1086,7 +1121,9 @@ mod tests {
             free_memory_mb: 2_000.0,
             sm_count: 20,
         };
-        let cap = hw.optimal_replay_capacity(48, 0.70);
-        assert!(cap >= 100_000, "Small GPU should get at least min capacity");
+        let cfg = hw.optimal_replay_config(48, 0.70);
+        assert!(cfg.capacity >= 100_000, "Small GPU should get at least min capacity");
+        // 20% of 4096 MB = ~819 MB, but min is 1 GB
+        assert_eq!(cfg.per_max_buffer_bytes, 1024 * 1024 * 1024, "Small GPU PER budget should be min 1 GB");
     }
 }

crates/ml-core/src/memory_optimization/mod.rs

@@ -11,7 +11,8 @@ pub mod quantization;
 
 pub use auto_batch_size::{
     detect_gpu_hardware, detect_gpu_memory, sm_count_from_device_name, AutoBatchSizer,
-    BatchSizeConfig, GpuHardwareInfo, GpuMemoryInfo, ModelPrecision, OptimizerType,
+    BatchSizeConfig, GpuHardwareInfo, GpuMemoryInfo, ModelPrecision, OptimalReplayConfig,
+    OptimizerType,
 };
 pub use lazy_loader::{LazyCheckpointLoader, LoadStrategy};
 pub use oom_detection::{extract_oom_size, is_oom_error};