From ede797daa021010ee6785fe97865d41ad98c427b Mon Sep 17 00:00:00 2001
From: jgrusewski <jgrusewski@localhost>
Date: Tue, 7 Apr 2026 23:36:01 +0200
Subject: [PATCH] =?UTF-8?q?feat(v8):=20pessimistic=20Q-value=20initializat?=
 =?UTF-8?q?ion=20=E2=80=94=20b=5Fv2=20bias=20set=20to=20-0.1?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Forces exploration by starting value head biases at -0.1 for all atoms,
so the model must discover positive-value states through experience rather
than confidently exploiting random positives from zero-init.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 .../ml/src/cuda_pipeline/gpu_dqn_trainer.rs   | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs
index a3b41be7b..739f55633 100644
--- a/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs
+++ b/crates/ml/src/cuda_pipeline/gpu_dqn_trainer.rs
@@ -2794,6 +2794,42 @@ impl GpuDqnTrainer {
             .map_err(|e| MLError::ModelError(format!("nan_flags alloc: {e}")))?;
         let exposure_aux_scratch = stream.alloc_zeros::<f32>(b * config.branch_0_size * config.num_atoms)
             .map_err(|e| MLError::ModelError(format!("alloc exposure_aux_scratch: {e}")))?;
+
+        // v8: Pessimistic Q-value initialization — shift value head bias to -0.1
+        // Forces exploration: model starts believing all states are slightly negative,
+        // must DISCOVER positive-value states through experience.
+        {
+            let sizes = compute_param_sizes(&config);
+            let b_v2_offset: usize = sizes[..7].iter().sum(); // [0..6] = w_s1,b_s1,w_s2,b_s2,w_v1,b_v1,w_v2 → [7] = b_v2
+            let pessimistic_val = -0.1_f32;
+            let bias_data: Vec<f32> = vec![pessimistic_val; config.num_atoms];
+            let bias_bf16: Vec<half::bf16> = bias_data.iter().map(|&v| half::bf16::from_f32(v)).collect();
+
+            // Write to f32 master params buffer
+            unsafe {
+                cudarc::driver::sys::cuMemcpyHtoDAsync_v2(
+                    params_buf.raw_ptr() + (b_v2_offset * std::mem::size_of::<f32>()) as u64,
+                    bias_data.as_ptr().cast(),
+                    bias_data.len() * std::mem::size_of::<f32>(),
+                    stream.cu_stream(),
+                );
+            }
+
+            // Write to bf16 shadow params buffer
+            unsafe {
+                cudarc::driver::sys::cuMemcpyHtoDAsync_v2(
+                    params_bf16.raw_ptr() + (b_v2_offset * std::mem::size_of::<half::bf16>()) as u64,
+                    bias_bf16.as_ptr().cast(),
+                    bias_bf16.len() * std::mem::size_of::<half::bf16>(),
+                    stream.cu_stream(),
+                );
+            }
+            tracing::info!(
+                "v8: Pessimistic Q-init: b_v2 bias set to {} ({} atoms)",
+                pessimistic_val, config.num_atoms
+            );
+        }
+
         Ok(Self {
             config,
             stream,