perf(rl): eliminate 2 of 3 per-step lobsim syncs — 21ms → 10ms/step
Remove unnecessary stream.synchronize() from: 1. step_fill_from_market_targets (lobsim): stream ordering handles the dependency — step_pnl_track launches on the same stream 2. step_pnl_track (lobsim): downstream kernels read pos_d via stream ordering, no host read needed per step Defer pos_fraction readback by one step (same pattern as loss readback) — eliminates the third 7ms sync. pos_fraction is a dataset-level statistic that barely changes step to step. nsys: 3.0 → 1.0 slow syncs/step. Sync time: 21.8 → 9.65 ms/step. The remaining sync is the training graph event wait (actual compute). Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -6864,12 +6864,12 @@ impl IntegratedTrainer {
|
||||
)
|
||||
.context("step_with_lobsim_gpu: gather_pos_fraction")?;
|
||||
|
||||
// Sync to make the pos_fraction host-readable (small cost;
|
||||
// only needed for the clamped pw computation).
|
||||
unsafe {
|
||||
raw_stream_sync(self.raw_stream)
|
||||
.map_err(|e| anyhow::anyhow!("pos_fraction sync: {:?}", e))?;
|
||||
}
|
||||
// One-step deferred read: the gather kernel wrote pos_fraction
|
||||
// on the stream THIS step, but we read PREVIOUS step's data
|
||||
// (already host-visible from prior stream work). One-step lag
|
||||
// is acceptable — pos_fraction is a dataset-level class balance
|
||||
// statistic that barely changes step to step. Eliminates a
|
||||
// 7ms host-blocking stream sync.
|
||||
let pf_slice = pf_staging.read_all();
|
||||
|
||||
// Run supervised training step.
|
||||
|
||||
@@ -1300,7 +1300,8 @@ impl LobSimCuda {
|
||||
.arg(&n)
|
||||
.launch(cfg)?;
|
||||
}
|
||||
self.stream.synchronize()?;
|
||||
// No sync needed — step_pnl_track launches on the SAME stream,
|
||||
// so CUDA stream ordering guarantees submit_market completes first.
|
||||
|
||||
self.step_pnl_track(current_ts_ns)?;
|
||||
Ok(())
|
||||
@@ -1473,7 +1474,9 @@ impl LobSimCuda {
|
||||
.arg(&mut self.diag_outcome_n_wins_d)
|
||||
.launch(cfg)?;
|
||||
}
|
||||
self.stream.synchronize()?;
|
||||
// No sync — downstream kernels on the same stream (fused_reward_pipeline,
|
||||
// rl_per_push) read pos_d via stream ordering. Host-side trade log reads
|
||||
// (read_trade_records) must sync explicitly when called.
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user