perf(diag): add --diag-every flag + fix PnL accumulation bug

- --diag-every N (default 100): skip device readback on non-diag steps.
  Eliminates ~10 stream.sync per step → ~2× throughput at large batch.
- Fix PnL bug: was accumulating raw_rewards (shaped, all batches) instead
  of rewards/scale on done steps only.
- Wire diag-every=100 in Argo template (diag JSONL every 100 steps).

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-05-25 23:56:21 +02:00
parent 7b983e5c4f
commit 3541ba7352
2 changed files with 22 additions and 3 deletions

View File

@@ -177,6 +177,12 @@ struct Cli {
#[arg(long, default_value_t = 100)]
log_every: usize,
/// Write diag JSONL every N steps (default: 1 = every step). Higher
/// values skip per-step device readback, eliminating stream.sync
/// overhead for ~2× throughput at large batch sizes.
#[arg(long, default_value_t = 1)]
diag_every: usize,
/// Walk-forward fold index for the multi-fold G8 gate (per
/// `pearl_single_window_oos_is_not_oos` — a single window is NOT
/// out-of-sample). Slices the MBP-10 file list into K equal-sized
@@ -562,6 +568,9 @@ fn main() -> Result<()> {
}
// ── Per-step diag dump (mapped-pinned reads). ────────────────
// Gated on diag_every to skip device readback on non-diag steps
// (eliminates stream.sync overhead for ~2× throughput).
if step % cli.diag_every == 0 || step == cli.n_steps - 1 {
// Per `feedback_no_htod_htoh_only_mapped_pinned`: raw
// `stream.memcpy_dtoh` on a regular `&mut [T]` is forbidden;
// the only permitted CPU↔GPU path is mapped-pinned staging
@@ -662,12 +671,18 @@ fn main() -> Result<()> {
let done_count: u32 = dones_host.iter().map(|&d| if d > 0.5 { 1 } else { 0 }).sum();
// Per-trade cumulative stats (surfer validation).
// Use post-scale reward / current_scale to recover approximate USD.
let current_scale = trainer.isv_host[ml_alpha::rl::isv_slots::RL_REWARD_SCALE_INDEX];
for b in 0..cli.n_backtests {
if dones_host[b] > 0.5 {
let pnl = raw_rewards_host[b] as f64;
pnl_cum_usd += pnl;
let pnl_usd = if current_scale > 1e-9 {
rewards_host[b] as f64 / current_scale as f64
} else {
rewards_host[b] as f64
};
pnl_cum_usd += pnl_usd;
total_trades += 1;
if pnl > 0.0 { win_count += 1; }
if pnl_usd > 0.0 { win_count += 1; }
hold_time_sum += trade_duration_host[b] as f64;
}
}
@@ -1122,6 +1137,7 @@ fn main() -> Result<()> {
elapsed,
);
}
} // end if step % diag_every
last_stats = Some(stats);
summary.n_steps_completed = step + 1;
}

View File

@@ -49,6 +49,8 @@ spec:
value: "32"
- name: per-capacity
value: "32768"
- name: diag-every
value: "100"
- name: instrument-mode
value: "all"
- name: log-every
@@ -172,6 +174,7 @@ spec:
--seq-len {{workflow.parameters.seq-len}} \
--n-backtests {{workflow.parameters.n-backtests}} \
--per-capacity {{workflow.parameters.per-capacity}} \
--diag-every {{workflow.parameters.diag-every}} \
--seed {{workflow.parameters.seed}} \
--instrument-mode "{{workflow.parameters.instrument-mode}}" \
--fold-idx {{workflow.parameters.fold-idx}} \