diag(dqn): persistent picked_action_history — full-window kernel pick histogram
The `val_picked_dir_dist` reader added in 89ece2e36 read `chunked_actions_buf`
which is overwritten each chunk, so the diagnostic only saw the last ~64 bars
of the 214 654-bar val window. The cluster's run-after-run identical
distribution (`s=0.0001 h=0.20 l=0.0000 f=0.80` post-physics; `s=0.18 h=0.18
l=0.41 f=0.22` last-chunk picks) was therefore inconclusive: the post-physics
flatness covers the full window, but the pick-side diversity may only hold for
the last 64 bars while the first 99.97% of bars produce a different
distribution.
Add `picked_action_history_buf` — a window-major `[n_windows, max_len]` i32
buffer parallel to `actions_history_buf`. Filled by an additional
`scatter_intent_chunk` launch immediately after the existing intent-mag
scatter (same kernel handle, different src/dst — `chunked_actions_buf` →
`picked_action_history_buf`). One extra kernel launch per chunk; the launch
config and grid sizing are identical to the existing intent scatter.
The reader `read_chunked_actions_direction_distribution` now reads this
buffer instead of the chunk-local one. The HEALTH_DIAG line stays at
`val_picked_dir_dist [short=... hold=... long=... flat=...]` but now
reflects the full val window.
Decision matrix once the cluster reports the new diagnostic:
both `val_dir_dist` and `val_picked_dir_dist` show ~80% Flat
→ kernel itself produces collapsed picks across the window;
Q-values must be near-uniform for most bars; the eval-collapse is
a learning problem (network can't differentiate states).
`val_picked_dir_dist` diverse but `val_dir_dist` ~80% Flat
→ kernel is diverse, env_step drains active picks; the eval-collapse
is a physics problem (Kelly cap or another gate I haven't found).
Build clean at 11-warning baseline. No new kernel source, no determinism
contract change.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -318,6 +318,19 @@ pub struct GpuBacktestEvaluator {
|
||||
/// `None` until `ensure_action_select_ready` allocates it.
|
||||
intent_mag_buf: Option<CudaSlice<i32>>,
|
||||
|
||||
/// Persistent per-step history of the policy's RAW factored action picks
|
||||
/// (PRE-env_step). `[n_windows, max_len]` window-major, parallel layout to
|
||||
/// `actions_history_buf`. Filled by scattering `chunked_actions_buf` into
|
||||
/// the corresponding step slots after each chunk's `experience_action_select`
|
||||
/// (reuses the existing `scatter_intent_chunk` kernel — same scatter
|
||||
/// semantics, different source/destination buffers). Diagnostic-only:
|
||||
/// pairs with `actions_history_buf` (POST-env_step `actual_dir`) so the
|
||||
/// reader can compare the kernel's intent vs the env's realisation
|
||||
/// across the full val window — localises whether eval-mode collapse is
|
||||
/// upstream (kernel produces collapsed picks) or downstream (env_step
|
||||
/// drains active picks to Flat). Reset to zero by `reset_evaluation_state`.
|
||||
picked_action_history_buf: Option<CudaSlice<i32>>,
|
||||
|
||||
// Gather kernel output buffer (overwritten every step)
|
||||
states_buf: CudaSlice<f32>, // [n_windows * state_dim] (8-aligned, f32 for cublasSgemm)
|
||||
|
||||
@@ -643,6 +656,7 @@ impl GpuBacktestEvaluator {
|
||||
actions_buf,
|
||||
actions_history_buf,
|
||||
intent_mag_buf: None,
|
||||
picked_action_history_buf: None,
|
||||
states_buf,
|
||||
metrics_buf,
|
||||
n_windows,
|
||||
@@ -1165,17 +1179,22 @@ impl GpuBacktestEvaluator {
|
||||
}
|
||||
|
||||
/// Read the per-direction histogram of the policy's RAW Boltzmann picks
|
||||
/// from the most recent chunk (the last `chunk_len` bars of the eval
|
||||
/// rollout). Reads `chunked_actions_buf` BEFORE env_step touches it —
|
||||
/// `actions_history_buf` records the POST-physics `actual_dir`, so a
|
||||
/// divergence between the two distributions localises whether the eval
|
||||
/// collapse is upstream (kernel produces degenerate picks) or downstream
|
||||
/// (env_step / Kelly cap drains active picks to Flat).
|
||||
/// across the FULL val window. Reads `picked_action_history_buf` —
|
||||
/// scattered from `chunked_actions_buf` after each chunk's
|
||||
/// `experience_action_select`, so the buffer accumulates the kernel's
|
||||
/// raw action_idx writes for every bar before env_step touches anything.
|
||||
///
|
||||
/// Returns `[short, hold, long, flat]` summing to 1.0 over the chunk.
|
||||
/// `Ok([0.0; 4])` if `chunked_actions_buf` has not been allocated yet.
|
||||
/// `actions_history_buf` records the POST-physics `actual_dir`, so a
|
||||
/// divergence between this reader and `read_eval_action_distribution_per_direction`
|
||||
/// localises whether eval collapse is upstream (kernel produces collapsed
|
||||
/// picks → both flat) or downstream (kernel diverse, env_step / Kelly cap
|
||||
/// drains active picks to Flat → only post-physics flat).
|
||||
///
|
||||
/// Returns `[short, hold, long, flat]` summing to 1.0 over the full window.
|
||||
/// `Ok([0.0; 4])` if `picked_action_history_buf` has not been allocated yet
|
||||
/// (first call before `ensure_action_select_ready` has run).
|
||||
pub fn read_chunked_actions_direction_distribution(&self) -> Result<[f32; 4], MLError> {
|
||||
let buf = match self.chunked_actions_buf.as_ref() {
|
||||
let buf = match self.picked_action_history_buf.as_ref() {
|
||||
Some(b) => b,
|
||||
None => return Ok([0.0; 4]),
|
||||
};
|
||||
@@ -1183,10 +1202,17 @@ impl GpuBacktestEvaluator {
|
||||
let mut host = vec![0_i32; len];
|
||||
self.stream
|
||||
.memcpy_dtoh(buf, &mut host)
|
||||
.map_err(|e| MLError::ModelError(format!("chunked_actions dtoh: {e}")))?;
|
||||
.map_err(|e| MLError::ModelError(format!("picked_action_history dtoh: {e}")))?;
|
||||
let mut counts = [0_u64; 4];
|
||||
let mut total = 0_u64;
|
||||
for &a in &host {
|
||||
// action_idx is always non-negative; the only invalid entry is
|
||||
// a kernel-side bug. Slots beyond the window length still get
|
||||
// written by experience_action_select (it runs for the full
|
||||
// chunk_len × n_windows batch regardless of done_flags), so
|
||||
// every slot in [0, max_len) carries a real Boltzmann pick.
|
||||
// action_idx = 0 (Short Quarter Market Normal) is a legitimate
|
||||
// pick; the Short bucket includes it.
|
||||
if a < 0 {
|
||||
continue;
|
||||
}
|
||||
@@ -1235,6 +1261,11 @@ impl GpuBacktestEvaluator {
|
||||
self.stream.memset_zeros(intent)
|
||||
.map_err(|e| MLError::ModelError(format!("reset intent_mag_buf: {e}")))?;
|
||||
}
|
||||
// Zero the picked-action history for the same reason.
|
||||
if let Some(ref mut picked) = self.picked_action_history_buf {
|
||||
self.stream.memset_zeros(picked)
|
||||
.map_err(|e| MLError::ModelError(format!("reset picked_action_history_buf: {e}")))?;
|
||||
}
|
||||
|
||||
// Reset Kelly stats — each evaluation starts cold. Kelly priors +
|
||||
// warmup floor in trade_physics.cuh handle the cold-start period.
|
||||
@@ -1287,6 +1318,9 @@ impl GpuBacktestEvaluator {
|
||||
let intent_history = self.intent_mag_buf.as_ref().ok_or_else(|| {
|
||||
MLError::ModelError("intent_mag_buf unexpectedly None".to_owned())
|
||||
})?;
|
||||
let picked_action_history = self.picked_action_history_buf.as_ref().ok_or_else(|| {
|
||||
MLError::ModelError("picked_action_history_buf unexpectedly None".to_owned())
|
||||
})?;
|
||||
let scatter_intent = self.scatter_intent_kernel.as_ref().ok_or_else(|| {
|
||||
MLError::ModelError("scatter_intent_kernel unexpectedly None".to_owned())
|
||||
})?;
|
||||
@@ -1444,6 +1478,34 @@ impl GpuBacktestEvaluator {
|
||||
)))?;
|
||||
}
|
||||
|
||||
// ── Phase 4c: Parallel scatter of raw factored action picks ──
|
||||
//
|
||||
// Same scatter kernel, different src/dst — copies the kernel's
|
||||
// RAW action_idx writes (chunked_actions_buf, before env_step
|
||||
// touches anything) into picked_action_history_buf at the
|
||||
// current chunk's step range. Pairs with actions_history_buf
|
||||
// (POST-env_step actual_dir) so a downstream reader can compare
|
||||
// the kernel's intent with the env's realisation across the
|
||||
// full val window. Diagnostic-only.
|
||||
unsafe {
|
||||
self.stream
|
||||
.launch_builder(scatter_intent)
|
||||
.arg(ch_actions)
|
||||
.arg(picked_action_history)
|
||||
.arg(&(n as i32))
|
||||
.arg(&(self.max_len as i32))
|
||||
.arg(&start_step_i32)
|
||||
.arg(&chunk_len_i32)
|
||||
.launch(LaunchConfig {
|
||||
grid_dim: (scatter_blocks.max(1), 1, 1),
|
||||
block_dim: (256, 1, 1),
|
||||
shared_mem_bytes: 0,
|
||||
})
|
||||
.map_err(|e| MLError::ModelError(format!(
|
||||
"scatter picked_actions chunk_start={chunk_start}: {e}"
|
||||
)))?;
|
||||
}
|
||||
|
||||
// ── Phase 5: Batched env_step (single launch for entire chunk) ──
|
||||
//
|
||||
// All chunk_len steps processed in one kernel launch. The batched
|
||||
@@ -1809,6 +1871,14 @@ impl GpuBacktestEvaluator {
|
||||
let intent_mag_buf = self.stream.alloc_zeros::<i32>(n * self.max_len)
|
||||
.map_err(|e| MLError::ModelError(format!("alloc intent_mag_buf: {e}")))?;
|
||||
|
||||
// Picked-action history: parallel layout to actions_history_buf,
|
||||
// captures the kernel's RAW factored action_idx PRE-env_step. Filled
|
||||
// by scattering chunked_actions_buf into per-step slots (reuses the
|
||||
// scatter_intent_chunk kernel — same scatter semantics, different
|
||||
// src/dst). Diagnostic-only.
|
||||
let picked_action_history_buf = self.stream.alloc_zeros::<i32>(n * self.max_len)
|
||||
.map_err(|e| MLError::ModelError(format!("alloc picked_action_history_buf: {e}")))?;
|
||||
|
||||
let ch_q_gaps = self.stream.alloc_zeros::<f32>(cn)
|
||||
.map_err(|e| MLError::ModelError(format!("alloc chunked q_gaps: {e}")))?;
|
||||
|
||||
@@ -1835,6 +1905,7 @@ impl GpuBacktestEvaluator {
|
||||
self.chunked_intent_mag_buf = Some(ch_intent_mag_buf);
|
||||
self.chunked_q_gaps_buf = Some(ch_q_gaps);
|
||||
self.intent_mag_buf = Some(intent_mag_buf);
|
||||
self.picked_action_history_buf = Some(picked_action_history_buf);
|
||||
self.scatter_intent_kernel = Some(scatter_intent);
|
||||
self.chunked_conviction_buf = Some(ch_conviction);
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user