From 6f973b3a3f2c4bc1fd4e5aa98ad01688c6af045f Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Tue, 26 May 2026 00:37:30 +0200 Subject: [PATCH] =?UTF-8?q?spec:=20bidirectional=20HER=20=E2=80=94=20backw?= =?UTF-8?q?ard=20peak=20+=20forward=20continuation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 --- .../specs/2026-05-26-gpu-hindsight-replay.md | 57 +++++++++++++++---- 1 file changed, 47 insertions(+), 10 deletions(-) diff --git a/docs/superpowers/specs/2026-05-26-gpu-hindsight-replay.md b/docs/superpowers/specs/2026-05-26-gpu-hindsight-replay.md index a9a842152..c9178b1e4 100644 --- a/docs/superpowers/specs/2026-05-26-gpu-hindsight-replay.md +++ b/docs/superpowers/specs/2026-05-26-gpu-hindsight-replay.md @@ -20,11 +20,17 @@ The agent learns exit timing 10-100× faster because it sees the counterfactual --- -## Architecture: Backward-Looking HER (no future data needed) +## Architecture: Bidirectional HER (backward + forward) -The old DQN trainer used FORWARD-looking hindsight (lookahead into future bars). This requires future price data that doesn't exist at push time in the step-by-step RL loop. +Two complementary hindsight signals, both GPU-native: -**This design uses BACKWARD-looking HER:** at trade close, we look at the prices DURING the trade (from entry to exit) and find the peak. This is always available — we just need a per-batch mid-price ring buffer that accumulates during the trade. +**Backward HER (immediate at close):** Peak unrealized PnL DURING the trade. "You held past the peak — should have exited at step Z." Teaches anti-wipeout timing. + +**Forward HER (delayed by LOOKAHEAD steps):** What happened AFTER exit. "Price kept rising 10 steps after you exited — should have held." Teaches patience / wave-riding. + +Both are available in the sequential training loop: +- Backward: fires at trade close (peak is in the mid_ring) +- Forward: fires LOOKAHEAD steps after a trade close (subsequent steps reveal post-exit price action) ### Data flow @@ -61,23 +67,32 @@ The agent learns: "states where mid was rising rapidly → keep holding. States ## Device-Resident Data Structures ```rust -// In GpuReplayBuffer or as separate HER struct: pub struct GpuHindsight { + // ── Backward HER (peak during trade) ──────────────────────── /// Per-batch mid-price ring during current trade. - /// Ring fills from entry to exit; resets on done. pub mid_ring_d: CudaSlice, // [B × HINDSIGHT_RING_LEN] - - /// Per-batch write index into mid_ring (0-based from trade entry). + /// Per-batch write index into mid_ring. pub ring_write_idx_d: CudaSlice, // [B] - - /// Per-batch peak mid during current trade (running max). + /// Per-batch running peak mid during current trade. pub peak_mid_d: CudaSlice, // [B] - /// Per-batch entry mid (captured at trade open). pub entry_mid_d: CudaSlice, // [B] + + // ── Forward HER (post-exit continuation) ──────────────────── + /// Recently-closed trades waiting for forward evaluation. + /// Ring of (entry_price, exit_step, direction, actual_pnl, batch_idx). + pub closed_ring_d: CudaSlice, // [CLOSED_RING_SIZE × 5] + /// h_t at entry for each closed trade (for synthetic transition). + pub closed_h_t_d: CudaSlice, // [CLOSED_RING_SIZE × HIDDEN_DIM] + /// Write head into closed ring. + pub closed_write_head_d: CudaSlice, // [1] + /// Count of valid entries in closed ring. + pub closed_count_d: CudaSlice, // [1] } pub const HINDSIGHT_RING_LEN: usize = 512; +pub const CLOSED_RING_SIZE: usize = 256; // last 256 closed trades +pub const FORWARD_LOOKAHEAD: usize = 50; // evaluate 50 steps after close ``` The ring length of 512 covers trades up to 512 steps long. At the surfer's min-hold of 100 steps, typical trades run 100-300 steps. 512 is generous. @@ -130,6 +145,28 @@ Per thread (one per batch): 9. Write to replay at write_head (shared-mem prefix-sum for multi-batch coordination) 10. Reset: ring_write_idx[b] = 0, peak_mid[b] = 0 +### `rl_hindsight_forward.cu` — Forward HER: "should have held longer" + +Runs every step. Scans the closed_ring for trades that closed FORWARD_LOOKAHEAD steps ago. For each, computes what the PnL would be NOW if the trade had been held. + +``` +Grid=(ceil(CLOSED_RING_SIZE/32)), Block=(32) +``` + +Per thread (one per closed_ring entry): +1. If entry is empty or age < FORWARD_LOOKAHEAD: return +2. Read entry_price, direction from closed_ring +3. current_mid = 0.5 × (bid_px[0] + ask_px[0]) +4. forward_pnl = direction × (current_mid - entry_price) +5. actual_pnl = closed_ring[i].actual_pnl +6. IF forward_pnl > actual_pnl × HINDSIGHT_THRESHOLD: + - Synthetic transition: h_t = closed_h_t[i], action = Hold, reward = forward_pnl + - Push to replay with boosted priority + - Mark entry as consumed (clear slot) +7. IF age > FORWARD_LOOKAHEAD × 2: expire entry (clear slot regardless) + +This teaches: "at this entry state, holding past your exit point would have yielded +X more." + ### ISV Slots ```