gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(rl): graph capture state machine — gate end_capture on begin_capture

Browse Source 
The warmup step set graph_warmup_done=true but never called
begin_capture; the end_capture check then fired prematurely, causing
CUDA_ERROR_ILLEGAL_STATE. Fixed by tracking a local `capturing_prefill`
/ `capturing_postfill` flag that's true only when begin_capture ran.

Also adds Graph A2 (post-snapshot) capture for 7 kernels:
session_risk → min_hold → trail_decay → trail_mutate →
trail_stop → position_heat → actions_to_market_targets.

Removes unused buf_len variable.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-05-25 22:24:53 +02:00
parent 4c94366f63
commit 2c9aefe19e
1 changed files with 34 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
crates/ml-alpha/src/trainer/integrated.rs
View File

@@ -4166,9 +4166,11 @@ impl IntegratedTrainer {
.launch()
.context("prefill graph launch")?;
} else {
let capturing_prefill = self.graph_warmup_done;
if !self.graph_warmup_done {
self.graph_warmup_done = true;
} else {
}
if capturing_prefill {
self.stream
.begin_capture(CUstreamCaptureMode::CU_STREAM_CAPTURE_MODE_RELAXED)
.map_err(|e| anyhow::anyhow!("prefill begin_capture: {e}"))?;
@@ -4494,9 +4496,7 @@ impl IntegratedTrainer {
}
}
// End of prefill kernel sequence. If we were capturing,
// finalize the graph.
if self.prefill_graph.is_none() && self.graph_warmup_done {
if capturing_prefill {
let graph = self
.stream
.end_capture(
@@ -4535,6 +4535,24 @@ impl IntegratedTrainer {
let pos_bytes_i = lobsim.pos_bytes() as i32;
let b_size_i = b_size as i32;
// ── Graph A2: post-snapshot / pre-fill kernel pipeline ─────────
// 7 kernels from session_risk through actions_to_market_targets.
// Same three-state machine as prefill — captures after warmup,
// replays on third+ step. Shares the warmup counter with Graph A.
if self.postfill_graph.is_some() {
self.postfill_graph
.as_ref()
.unwrap()
.launch()
.context("postfill graph launch")?;
} else {
let capturing_postfill = self.prefill_graph.is_some();
if capturing_postfill {
self.stream
.begin_capture(CUstreamCaptureMode::CU_STREAM_CAPTURE_MODE_RELAXED)
.map_err(|e| anyhow::anyhow!("postfill begin_capture: {e}"))?;
}
// Session risk circuit breaker — block opening actions when
// cumulative PnL EMA drops below threshold.
{
@@ -4728,6 +4746,18 @@ impl IntegratedTrainer {
}
}
if capturing_postfill {
let graph = self
.stream
.end_capture(
CUgraphInstantiate_flags::CUDA_GRAPH_INSTANTIATE_FLAG_AUTO_FREE_ON_LAUNCH,
)
.context("postfill end_capture")?
.ok_or_else(|| anyhow::anyhow!("postfill end_capture returned None"))?;
self.postfill_graph = Some(graph);
}
} // end else (postfill warmup / capture dispatch)
// Run the fill + pnl_track on whatever's now in
// market_targets_d.
lobsim