feat(v8): pessimistic Q-value initialization — b_v2 bias set to -0.1

Forces exploration by starting value head biases at -0.1 for all atoms,
so the model must discover positive-value states through experience rather
than confidently exploiting random positives from zero-init.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-07 23:36:01 +02:00
parent d39786df72
commit ede797daa0

View File

@@ -2794,6 +2794,42 @@ impl GpuDqnTrainer {
.map_err(|e| MLError::ModelError(format!("nan_flags alloc: {e}")))?;
let exposure_aux_scratch = stream.alloc_zeros::<f32>(b * config.branch_0_size * config.num_atoms)
.map_err(|e| MLError::ModelError(format!("alloc exposure_aux_scratch: {e}")))?;
// v8: Pessimistic Q-value initialization — shift value head bias to -0.1
// Forces exploration: model starts believing all states are slightly negative,
// must DISCOVER positive-value states through experience.
{
let sizes = compute_param_sizes(&config);
let b_v2_offset: usize = sizes[..7].iter().sum(); // [0..6] = w_s1,b_s1,w_s2,b_s2,w_v1,b_v1,w_v2 → [7] = b_v2
let pessimistic_val = -0.1_f32;
let bias_data: Vec<f32> = vec![pessimistic_val; config.num_atoms];
let bias_bf16: Vec<half::bf16> = bias_data.iter().map(|&v| half::bf16::from_f32(v)).collect();
// Write to f32 master params buffer
unsafe {
cudarc::driver::sys::cuMemcpyHtoDAsync_v2(
params_buf.raw_ptr() + (b_v2_offset * std::mem::size_of::<f32>()) as u64,
bias_data.as_ptr().cast(),
bias_data.len() * std::mem::size_of::<f32>(),
stream.cu_stream(),
);
}
// Write to bf16 shadow params buffer
unsafe {
cudarc::driver::sys::cuMemcpyHtoDAsync_v2(
params_bf16.raw_ptr() + (b_v2_offset * std::mem::size_of::<half::bf16>()) as u64,
bias_bf16.as_ptr().cast(),
bias_bf16.len() * std::mem::size_of::<half::bf16>(),
stream.cu_stream(),
);
}
tracing::info!(
"v8: Pessimistic Q-init: b_v2 bias set to {} ({} atoms)",
pessimistic_val, config.num_atoms
);
}
Ok(Self {
config,
stream,