gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

diag(crt-2): per-horizon alpha-input EMA test — hypothesis investigation

Browse Source 
Driven by ffr59 (commit b44a97ff9) findings: all 5 horizons flip
direction every 2.5 events; win rate flat 24% across conviction; mean
PnL anti-correlated with conviction. Hypothesis: per-event alpha output
is high-frequency noise on top of a slower signal. If true, smoothing
input alpha BEFORE the conviction formula should reduce direction flips
and recover signal.

Adds Wiener-α adaptive EMA on raw alpha_probs[h] for each horizon,
applied BEFORE the multi-horizon conviction formula. Floor at 0.1
(stronger than the 0.4 floor on the output-side conviction EMA — this
tests whether INPUT smoothing has different impact than OUTPUT smoothing).

Three new device slots:
  - alpha_ema_per_b_per_h (per-horizon EMA state)
  - alpha_diff_var_per_b_per_h (variance of changes)
  - alpha_sample_var_per_b_per_h (variance of value)

The CRT.diag Group A direction-flip counter still reads RAW alpha_probs
so we have a head-to-head comparison: raw flip rate vs smoothed flip rate.
Group E adds the smoothed-direction counter + mean run length.

End-of-run log adds one line per horizon:
  crt_diag h<X> smoothed: flips=Y mean_run_len=Z events (vs raw F / M)

If smoothed mean_run_len >> raw mean_run_len: hypothesis is RIGHT, the
input had signal under the noise. Next step would be to make this an
operational EMA in the controller.

If smoothed and raw are similar: hypothesis is WRONG, per-event output
is genuinely noisy. Next step would be to investigate model training
(horizon collapse) OR the AUC=0.66 measurement definition.

Either way, definitive result from one smoke run.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-05-20 22:34:35 +02:00
parent b44a97ff94
commit 0e17fd4f2d
3 changed files with 329 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
crates/ml-backtesting/src/harness.rs
View File

@@ -392,6 +392,27 @@ impl BacktestHarness {
hist_str.push_str(&format!(" {}:{}", bucket_labels[bk], count));
}
eprintln!("crt_diag h{} run_length_hist:{}", horizons[h], hist_str);
// CRT.diag.2 Group E: SMOOTHED-direction flip rate head-to-head
// vs raw. If smoothed mean_run_len ≫ raw mean_run_len, the
// hypothesis (per-event output noise masks a slower signal)
// is supported. If similar, hypothesis is falsified.
let smoothed_flips: u64 = (0..n_b)
.map(|b| diag.smoothed_flip_count[b * n_horizons + h] as u64)
.sum();
let smoothed_sum_run: u64 = (0..n_b)
.map(|b| diag.smoothed_sum_run_length[b * n_horizons + h])
.sum();
let smoothed_mean_run = if smoothed_flips > 0 {
smoothed_sum_run as f64 / smoothed_flips as f64
} else {
f64::NAN
};
eprintln!(
"crt_diag h{} smoothed: flips={} mean_run_len={:.1} events (vs raw {} / {:.1})",
horizons[h], smoothed_flips, smoothed_mean_run,
flips, mean_run,
);
}
// Group B — smoothed-conviction histogram.

111
crates/ml-backtesting/src/sim/mod.rs
View File

@@ -124,6 +124,14 @@ pub struct CrtDiagnostics {
/// Win count per entry-conviction bucket (segment_realized > 0).
/// Layout: `[backtest * 10 + bucket]`.
pub outcome_n_wins: Vec<u32>,
/// CRT.diag.2 Group E: per-horizon SMOOTHED-direction flip count.
/// Layout: `[backtest * N_HORIZONS + h]`. Compare head-to-head with
/// `flip_count` (which counts RAW alpha_probs direction flips).
pub smoothed_flip_count: Vec<u32>,
/// CRT.diag.2 Group E: per-horizon cumulative SMOOTHED run length over
/// completed runs (u64 to avoid overflow at multi-million-event smoke).
/// Layout: `[backtest * N_HORIZONS + h]`.
pub smoothed_sum_run_length: Vec<u64>,
}
/// S2.1/S2.2: max_hold enforcement diagnostic counters.
@@ -313,6 +321,28 @@ pub struct LobSimCuda {
pub(crate) diag_outcome_n_d: CudaSlice<u32>, // [n_backtests * 10]
pub(crate) diag_outcome_sum_pnl_d: CudaSlice<f32>, // [n_backtests * 10]
pub(crate) diag_outcome_n_wins_d: CudaSlice<u32>, // [n_backtests * 10]
// CRT.diag.2: per-horizon Wiener-α EMA on raw alpha_probs. Smoothed
// alpha enters the multi-horizon §4.4 conviction formula instead of
// the raw broadcast values — hypothesis test for whether per-event
// output noise is hiding a slower signal. All zero-init: sentinel "no
// observation yet" so the first decision-kernel call bootstraps per
// pearl_first_observation_bootstrap. Floor on Wiener α = 0.1 (stronger
// smoothing than the 0.4 output-side floor; testing input vs output
// smoothing differ). Touched by both decision_policy_default and
// decision_policy_program every event; no host roundtrip.
pub(crate) alpha_ema_per_b_per_h_d: CudaSlice<f32>, // [n_backtests * N_HORIZONS]
pub(crate) alpha_diff_var_per_b_per_h_d: CudaSlice<f32>, // [n_backtests * N_HORIZONS]
pub(crate) alpha_sample_var_per_b_per_h_d: CudaSlice<f32>, // [n_backtests * N_HORIZONS]
// CRT.diag.2 Group E: per-horizon SMOOTHED direction-flip + mean-run-length.
// Mirrors Group A but reading alpha_smoothed[h] instead of alpha_probs[h].
// Skipping the 5-bucket histogram (Group A keeps it). Head-to-head
// comparison: raw flip rate vs smoothed flip rate, per horizon.
pub(crate) diag_smoothed_flip_count_d: CudaSlice<u32>, // [n_backtests * N_HORIZONS]
pub(crate) diag_smoothed_current_run_length_d: CudaSlice<u32>, // [n_backtests * N_HORIZONS]
pub(crate) diag_smoothed_sum_run_length_d: CudaSlice<u64>, // [n_backtests * N_HORIZONS]
pub(crate) diag_smoothed_prev_dir_d: CudaSlice<i8>, // [n_backtests * N_HORIZONS]
}
impl LobSimCuda {
@@ -611,6 +641,35 @@ impl LobSimCuda {
.alloc_zeros::<u32>(n_backtests * 10)
.context("alloc diag_outcome_n_wins_d")?;
// CRT.diag.2: per-horizon alpha-input EMA state. Zero-init = "no
// observation yet" — first decision-kernel call bootstraps per
// pearl_first_observation_bootstrap. Same pattern as the
// conviction-level Wiener-α EMA above (zero-init = sentinel).
let alpha_ema_per_b_per_h_d = stream
.alloc_zeros::<f32>(n_backtests * N_HORIZONS)
.context("alloc alpha_ema_per_b_per_h_d")?;
let alpha_diff_var_per_b_per_h_d = stream
.alloc_zeros::<f32>(n_backtests * N_HORIZONS)
.context("alloc alpha_diff_var_per_b_per_h_d")?;
let alpha_sample_var_per_b_per_h_d = stream
.alloc_zeros::<f32>(n_backtests * N_HORIZONS)
.context("alloc alpha_sample_var_per_b_per_h_d")?;
// CRT.diag.2 Group E: per-horizon SMOOTHED direction-flip counters.
// Same shape as Group A's flip/run-length slots (minus the histogram).
let diag_smoothed_flip_count_d = stream
.alloc_zeros::<u32>(n_backtests * N_HORIZONS)
.context("alloc diag_smoothed_flip_count_d")?;
let diag_smoothed_current_run_length_d = stream
.alloc_zeros::<u32>(n_backtests * N_HORIZONS)
.context("alloc diag_smoothed_current_run_length_d")?;
let diag_smoothed_sum_run_length_d = stream
.alloc_zeros::<u64>(n_backtests * N_HORIZONS)
.context("alloc diag_smoothed_sum_run_length_d")?;
let diag_smoothed_prev_dir_d = stream
.alloc_zeros::<i8>(n_backtests * N_HORIZONS)
.context("alloc diag_smoothed_prev_dir_d")?;
// CRT Phase A0.5 corrective: on-device conviction history buffer.
// Capacity = 5M decisions × 4B/f32 = 20MB. Matches the prior
// host-side `Vec::with_capacity(3_000_000)` plus headroom for
@@ -715,6 +774,13 @@ impl LobSimCuda {
diag_outcome_n_d,
diag_outcome_sum_pnl_d,
diag_outcome_n_wins_d,
alpha_ema_per_b_per_h_d,
alpha_diff_var_per_b_per_h_d,
alpha_sample_var_per_b_per_h_d,
diag_smoothed_flip_count_d,
diag_smoothed_current_run_length_d,
diag_smoothed_sum_run_length_d,
diag_smoothed_prev_dir_d,
})
}
@@ -953,14 +1019,17 @@ impl LobSimCuda {
/// touches none of these.
pub fn read_diagnostics(&self) -> Result<CrtDiagnostics> {
let n_b = self.n_backtests;
let mut flip_count = vec![0u32; n_b * N_HORIZONS];
let mut sum_run_length = vec![0u64; n_b * N_HORIZONS];
let mut run_length_hist = vec![0u32; n_b * N_HORIZONS * 5];
let mut conv_hist = vec![0u32; n_b * 10];
let mut hold_hist = vec![0u32; n_b * 6];
let mut outcome_n = vec![0u32; n_b * 10];
let mut outcome_sum_pnl = vec![0f32; n_b * 10];
let mut outcome_n_wins = vec![0u32; n_b * 10];
let mut flip_count = vec![0u32; n_b * N_HORIZONS];
let mut sum_run_length = vec![0u64; n_b * N_HORIZONS];
let mut run_length_hist = vec![0u32; n_b * N_HORIZONS * 5];
let mut conv_hist = vec![0u32; n_b * 10];
let mut hold_hist = vec![0u32; n_b * 6];
let mut outcome_n = vec![0u32; n_b * 10];
let mut outcome_sum_pnl = vec![0f32; n_b * 10];
let mut outcome_n_wins = vec![0u32; n_b * 10];
// CRT.diag.2 Group E: smoothed-direction flip/run-length.
let mut smoothed_flip_count = vec![0u32; n_b * N_HORIZONS];
let mut smoothed_sum_run_length = vec![0u64; n_b * N_HORIZONS];
self.stream.memcpy_dtoh(&self.diag_flip_count_d, flip_count.as_mut_slice())
.context("diag_flip_count DtoH")?;
self.stream.memcpy_dtoh(&self.diag_sum_run_length_d, sum_run_length.as_mut_slice())
@@ -977,6 +1046,12 @@ impl LobSimCuda {
.context("diag_outcome_sum_pnl DtoH")?;
self.stream.memcpy_dtoh(&self.diag_outcome_n_wins_d, outcome_n_wins.as_mut_slice())
.context("diag_outcome_n_wins DtoH")?;
self.stream.memcpy_dtoh(&self.diag_smoothed_flip_count_d,
smoothed_flip_count.as_mut_slice())
.context("diag_smoothed_flip_count DtoH")?;
self.stream.memcpy_dtoh(&self.diag_smoothed_sum_run_length_d,
smoothed_sum_run_length.as_mut_slice())
.context("diag_smoothed_sum_run_length DtoH")?;
Ok(CrtDiagnostics {
flip_count,
sum_run_length,
@@ -986,6 +1061,8 @@ impl LobSimCuda {
outcome_n,
outcome_sum_pnl,
outcome_n_wins,
smoothed_flip_count,
smoothed_sum_run_length,
})
}
@@ -1392,6 +1469,15 @@ impl LobSimCuda {
.arg(&mut self.diag_run_length_hist_d)
.arg(&mut self.diag_prev_dir_signed_d)
.arg(&mut self.diag_conv_hist_d)
// CRT.diag.2: per-horizon alpha-input EMA state.
.arg(&mut self.alpha_ema_per_b_per_h_d)
.arg(&mut self.alpha_diff_var_per_b_per_h_d)
.arg(&mut self.alpha_sample_var_per_b_per_h_d)
// CRT.diag.2 Group E: smoothed-direction flip counters.
.arg(&mut self.diag_smoothed_flip_count_d)
.arg(&mut self.diag_smoothed_current_run_length_d)
.arg(&mut self.diag_smoothed_sum_run_length_d)
.arg(&mut self.diag_smoothed_prev_dir_d)
.arg(&n)
.launch(cfg)?;
}
@@ -1432,6 +1518,15 @@ impl LobSimCuda {
.arg(&mut self.diag_run_length_hist_d)
.arg(&mut self.diag_prev_dir_signed_d)
.arg(&mut self.diag_conv_hist_d)
// CRT.diag.2: per-horizon alpha-input EMA state.
.arg(&mut self.alpha_ema_per_b_per_h_d)
.arg(&mut self.alpha_diff_var_per_b_per_h_d)
.arg(&mut self.alpha_sample_var_per_b_per_h_d)
// CRT.diag.2 Group E: smoothed-direction flip counters.
.arg(&mut self.diag_smoothed_flip_count_d)
.arg(&mut self.diag_smoothed_current_run_length_d)
.arg(&mut self.diag_smoothed_sum_run_length_d)
.arg(&mut self.diag_smoothed_prev_dir_d)
.arg(&n)
.launch(cfg)?;
}