feat(v8): pessimistic Q-value initialization — b_v2 bias set to -0.1
Forces exploration by starting value head biases at -0.1 for all atoms, so the model must discover positive-value states through experience rather than confidently exploiting random positives from zero-init. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -2794,6 +2794,42 @@ impl GpuDqnTrainer {
|
||||
.map_err(|e| MLError::ModelError(format!("nan_flags alloc: {e}")))?;
|
||||
let exposure_aux_scratch = stream.alloc_zeros::<f32>(b * config.branch_0_size * config.num_atoms)
|
||||
.map_err(|e| MLError::ModelError(format!("alloc exposure_aux_scratch: {e}")))?;
|
||||
|
||||
// v8: Pessimistic Q-value initialization — shift value head bias to -0.1
|
||||
// Forces exploration: model starts believing all states are slightly negative,
|
||||
// must DISCOVER positive-value states through experience.
|
||||
{
|
||||
let sizes = compute_param_sizes(&config);
|
||||
let b_v2_offset: usize = sizes[..7].iter().sum(); // [0..6] = w_s1,b_s1,w_s2,b_s2,w_v1,b_v1,w_v2 → [7] = b_v2
|
||||
let pessimistic_val = -0.1_f32;
|
||||
let bias_data: Vec<f32> = vec![pessimistic_val; config.num_atoms];
|
||||
let bias_bf16: Vec<half::bf16> = bias_data.iter().map(|&v| half::bf16::from_f32(v)).collect();
|
||||
|
||||
// Write to f32 master params buffer
|
||||
unsafe {
|
||||
cudarc::driver::sys::cuMemcpyHtoDAsync_v2(
|
||||
params_buf.raw_ptr() + (b_v2_offset * std::mem::size_of::<f32>()) as u64,
|
||||
bias_data.as_ptr().cast(),
|
||||
bias_data.len() * std::mem::size_of::<f32>(),
|
||||
stream.cu_stream(),
|
||||
);
|
||||
}
|
||||
|
||||
// Write to bf16 shadow params buffer
|
||||
unsafe {
|
||||
cudarc::driver::sys::cuMemcpyHtoDAsync_v2(
|
||||
params_bf16.raw_ptr() + (b_v2_offset * std::mem::size_of::<half::bf16>()) as u64,
|
||||
bias_bf16.as_ptr().cast(),
|
||||
bias_bf16.len() * std::mem::size_of::<half::bf16>(),
|
||||
stream.cu_stream(),
|
||||
);
|
||||
}
|
||||
tracing::info!(
|
||||
"v8: Pessimistic Q-init: b_v2 bias set to {} ({} atoms)",
|
||||
pessimistic_val, config.num_atoms
|
||||
);
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
config,
|
||||
stream,
|
||||
|
||||
Reference in New Issue
Block a user