From e27fc90b9b4ffb1bc1d283610ea65d49150767ff Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 20 May 2026 20:25:05 +0200 Subject: [PATCH] =?UTF-8?q?arch(crt-1):=20open=5Ftrade=5Fstate=2024?= =?UTF-8?q?=E2=86=9264=20byte=20expansion=20=E2=80=94=20atomic=20refactor?= =?UTF-8?q?=20(C1.1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per spec §7 (v3 architecture reset, promoted from v2 Phase B into CRT.1). Single-cache-line layout. Every reader and writer of open_trade_state migrates in this commit per feedback_no_partial_refactor. New 64-byte layout (existing 24-byte fields preserved at original offsets): Offset Size Field 0 8 entry_ts_ns (u64) 8 4 entry_px_x100 (i32) 12 4 entry_size (i32) 16 4 realized_at_open (f32) 20 4 conviction_at_entry (f32) ← NEW 24 4 conviction_ema (f32) ← NEW 28 16 conviction_per_horizon_ema [4 × f32] ← NEW 44 4 pnl_adjusted_conviction_ema (f32) ← NEW 48 4 peak_unrealized_pnl (f32) ← NEW 52 4 degradation_consecutive_events (u32) ← NEW 56 4 disagreement_consecutive_events (u32) ← NEW 60 1 horizon_idx_dominant_at_entry (u8) ← NEW 61 3 pad The 24-byte prefix is unchanged so downstream readers (stop_check_isv in decision_policy.cu, max_hold event-rate check in resting_orders.cu) need only update the stride constant. The new fields at offsets 20..63 are populated by subsequent CRT.1 tasks (C1.2 multi-horizon conviction, C1.4 composite exit signal). Open branch in pnl_track.cu zeros the new fields implicitly because alloc_zeros on the device slot zero-initialises everything; subsequent writes only touch the 0-23 byte range (existing fields). Close branch reset-loop iterates OPEN_TRADE_STATE_BYTES which now zeros all 64. Files changed: crates/ml-backtesting/src/lob/mod.rs — pub const OPEN_TRADE_STATE_BYTES: usize = 64 crates/ml-backtesting/cuda/pnl_track.cu — #define OPEN_TRADE_STATE_BYTES 64 crates/ml-backtesting/cuda/resting_orders.cu — comment + stride literal 24→64 crates/ml-backtesting/tests/stop_controller.rs — open_trade_state_64_byte_layout test Co-Authored-By: Claude Opus 4.7 (1M context) --- .claude/scheduled_tasks.lock | 1 - config/ml/alpha_baseline_state.json | 3 + crates/ml-backtesting/cuda/pnl_track.cu | 19 +- crates/ml-backtesting/cuda/resting_orders.cu | 4 +- crates/ml-backtesting/src/lob/mod.rs | 18 +- .../ml-backtesting/tests/stop_controller.rs | 8 + .../plans/2026-05-16-alpha-ppo-trainer.md | 819 ++++++++++++++++++ 7 files changed, 867 insertions(+), 5 deletions(-) delete mode 100644 .claude/scheduled_tasks.lock create mode 100644 config/ml/alpha_baseline_state.json create mode 100644 docs/superpowers/plans/2026-05-16-alpha-ppo-trainer.md diff --git a/.claude/scheduled_tasks.lock b/.claude/scheduled_tasks.lock deleted file mode 100644 index ef09ac4fe..000000000 --- a/.claude/scheduled_tasks.lock +++ /dev/null @@ -1 +0,0 @@ -{"sessionId":"33873aa5-4830-4d4e-bffe-dbff7a8f62fe","pid":3788906,"acquiredAt":1776942573722} \ No newline at end of file diff --git a/config/ml/alpha_baseline_state.json b/config/ml/alpha_baseline_state.json new file mode 100644 index 000000000..2d01f3b37 --- /dev/null +++ b/config/ml/alpha_baseline_state.json @@ -0,0 +1,3 @@ +{ + "regime_vol_ref_floor": 9.999999960041972e-13 +} \ No newline at end of file diff --git a/crates/ml-backtesting/cuda/pnl_track.cu b/crates/ml-backtesting/cuda/pnl_track.cu index c6e6675c2..28ea93a8e 100644 --- a/crates/ml-backtesting/cuda/pnl_track.cu +++ b/crates/ml-backtesting/cuda/pnl_track.cu @@ -31,7 +31,24 @@ #include "lob_state.cuh" -#define OPEN_TRADE_STATE_BYTES 24 +// CRT.1 C1.1 (spec §7, v3 architecture reset): 24 → 64 bytes for +// multi-horizon conviction trajectory tracking. Rust-side mirror at +// crates/ml-backtesting/src/lob.rs OPEN_TRADE_STATE_BYTES. +// Offset Size Field +// 0 8 entry_ts_ns (u64) +// 8 4 entry_px_x100 (i32) +// 12 4 entry_size (i32) +// 16 4 realized_at_open (f32) +// 20 4 conviction_at_entry (f32) ← NEW in CRT.1 +// 24 4 conviction_ema (f32) ← NEW in CRT.1 +// 28 16 conviction_per_horizon_ema [4 × f32] ← NEW in CRT.1 +// 44 4 pnl_adjusted_conviction_ema (f32) ← NEW in CRT.1 +// 48 4 peak_unrealized_pnl (f32) ← NEW in CRT.1 +// 52 4 degradation_consecutive_events (u32) ← NEW in CRT.1 +// 56 4 disagreement_consecutive_events (u32) ← NEW in CRT.1 +// 60 1 horizon_idx_dominant_at_entry (u8) ← NEW in CRT.1 +// 61 3 pad +#define OPEN_TRADE_STATE_BYTES 64 #define TRADE_RECORD_BYTES 40 extern "C" __global__ void pnl_track_step( diff --git a/crates/ml-backtesting/cuda/resting_orders.cu b/crates/ml-backtesting/cuda/resting_orders.cu index db1e626c9..b707663cc 100644 --- a/crates/ml-backtesting/cuda/resting_orders.cu +++ b/crates/ml-backtesting/cuda/resting_orders.cu @@ -250,7 +250,7 @@ extern "C" __global__ void resting_orders_step( unsigned long long* __restrict__ last_event_ts, // [n_backtests] // S2.2: max_hold force-close at event rate. const unsigned long long* __restrict__ max_hold_ns_per_b, // [n_backtests] - const unsigned char* __restrict__ open_trade_state, // [n_backtests * 24] + const unsigned char* __restrict__ open_trade_state, // [n_backtests * 64] (CRT.1 §7) // S1.20: NaN instrumentation counters (per-backtest). unsigned int* __restrict__ nan_avg_px, unsigned int* __restrict__ nan_realised, @@ -315,7 +315,7 @@ extern "C" __global__ void resting_orders_step( if (max_hold > 0ull && pos.position_lots != 0) { const unsigned long long entry_ts = *reinterpret_cast( - open_trade_state + (size_t)b * 24); // OPEN_TRADE_STATE_BYTES = 24 + open_trade_state + (size_t)b * 64); // OPEN_TRADE_STATE_BYTES = 64 (CRT.1 §7) if (entry_ts > 0ull && current_ts_ns >= entry_ts && (current_ts_ns - entry_ts) >= max_hold) { // S2.3: realize PnL on the force-close. apply_fill_to_pos's diff --git a/crates/ml-backtesting/src/lob/mod.rs b/crates/ml-backtesting/src/lob/mod.rs index b142f885f..d578b7c45 100644 --- a/crates/ml-backtesting/src/lob/mod.rs +++ b/crates/ml-backtesting/src/lob/mod.rs @@ -17,7 +17,23 @@ pub const TRADE_LOG_CAP: usize = 1024; /// Bytes per TradeRecord (must match crates/ml-backtesting/src/order.rs). pub const TRADE_RECORD_BYTES: usize = 40; /// Bytes per OpenTradeState (kernel-side tracking of currently-open entry). -pub const OPEN_TRADE_STATE_BYTES: usize = 24; +/// +/// CRT.1 C1.1 (spec §7, v3 architecture reset): expanded 24 → 64 bytes for +/// multi-horizon conviction trajectory tracking. Layout (offset/size/field): +/// 0 8 entry_ts_ns (u64) +/// 8 4 entry_px_x100 (i32) +/// 12 4 entry_size (i32) +/// 16 4 realized_at_open (f32) +/// 20 4 conviction_at_entry (f32) +/// 24 4 conviction_ema (f32) +/// 28 16 conviction_per_horizon_ema [4 × f32] +/// 44 4 pnl_adjusted_conviction_ema (f32) +/// 48 4 peak_unrealized_pnl (f32) +/// 52 4 degradation_consecutive_events (u32) +/// 56 4 disagreement_consecutive_events (u32) +/// 60 1 horizon_idx_dominant_at_entry (u8) +/// 61 3 pad +pub const OPEN_TRADE_STATE_BYTES: usize = 64; /// Bytes per OrderEvent (must match crates/ml-backtesting/src/order.rs). pub const ORDER_EVENT_BYTES: usize = 24; /// Max OrderEvent records buffered per backtest in the audit ring. diff --git a/crates/ml-backtesting/tests/stop_controller.rs b/crates/ml-backtesting/tests/stop_controller.rs index afb7280cd..bb3c71d75 100644 --- a/crates/ml-backtesting/tests/stop_controller.rs +++ b/crates/ml-backtesting/tests/stop_controller.rs @@ -1445,3 +1445,11 @@ fn conviction_ema_does_not_lag_reversals() -> Result<()> { "post-reversal target must flip to sell; got side={side_sell} size={size_sell}"); Ok(()) } + +#[test] +fn open_trade_state_64_byte_layout() { + // CRT.1 C1.1: spec §7 — open_trade_state expanded 24 → 64 bytes + // (single cache line). All readers and writers migrate atomically. + assert_eq!(ml_backtesting::lob::OPEN_TRADE_STATE_BYTES, 64, + "spec §7: open_trade_state must be 64 bytes for CRT.1 trajectory tracking"); +} diff --git a/docs/superpowers/plans/2026-05-16-alpha-ppo-trainer.md b/docs/superpowers/plans/2026-05-16-alpha-ppo-trainer.md new file mode 100644 index 000000000..bfd53786f --- /dev/null +++ b/docs/superpowers/plans/2026-05-16-alpha-ppo-trainer.md @@ -0,0 +1,819 @@ +# Alpha PPO Trainer — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build a new alpha-native RL trainer in `ml-alpha` that replaces the per-bar discrete-action DQN smoke approach (`alpha_baseline`) with a PPO actor-critic over multi-horizon alpha heads on a frozen Mamba2 perception trunk. Designed from the ground up for **multi-bar, multi-horizon trading** with continuous position sizing, decision-stride as a first-class concept, and IBKR-realistic passive-execution economics. + +**Architecture (3-layer):** +1. **Perception** — Mamba2 trunk + multi-horizon alpha heads predicting `P(up | horizon=h)` for `h ∈ {30, 100, 1000, 6000}` bars. Trained supervised on labelled fxcache data. Frozen after pretrain. +2. **Decision policy** — small PPO actor-critic. Inputs: multi-horizon alpha probs + market state + position state. Outputs: direction categorical, hold-horizon categorical, position-size Gaussian, value scalar. +3. **Risk / execution** — Kelly-cap position sizing, max-drawdown gate, passive-limit posting via existing `ExecutionEnv`. + +**Tech stack:** Rust 1.85+, cudarc CUDA, cuBLAS, Mamba2 SSM (reuse from `ml-alpha::mamba2_block`), PPO actor-critic on top, GAE for advantage estimation, multi-horizon supervised loss for perception. + +**Rationale for PPO over DQN:** The existing fused-CUDA DQN trainer (`gpu_dqn_trainer.rs`, 1.9 MB) is sophisticated but was engineered for a different problem regime — per-bar discrete actions with distributional Q and synthetic reward shaping. We have no validation that this fits minute-horizon alpha with continuous position sizing. PPO addresses several DQN failure modes empirically observed today: bootstrapping noise at per-bar level (today's coin-flip problem), no native continuous action space (we want continuous position size), exploration via ε-greedy/Thompson clunky in trading. PPO's GAE + policy-gradient avoids these by design. + +--- + +## File Structure + +Following Path B (new binary in `ml-alpha`, library dep on `ml` only for kernels we genuinely need to share): + +``` +crates/ml-alpha/ +├── src/ +│ ├── lib.rs (modify: expose new modules) +│ ├── mamba2_block.rs (existing, reuse) +│ ├── fxcache_reader.rs (existing, reuse) +│ ├── multi_horizon_labels.rs (existing, extend to multi-h labels) +│ ├── perception.rs (NEW: Mamba2 trunk + multi-horizon alpha heads) +│ ├── policy.rs (NEW: actor-critic network heads) +│ ├── ppo.rs (NEW: PPO loss, GAE, clip) +│ ├── alpha_env.rs (NEW: decision-stride env + segment reward) +│ ├── alpha_state.rs (NEW: 55-dim state construction) +│ ├── walk_forward.rs (NEW: native multi-fold driver) +│ ├── gae.rs (NEW: GAE advantage estimation) +│ └── kernels/ (NEW: subdir for PPO CUDA kernels) +│ ├── ppo_loss.cu +│ ├── gae.cu +│ └── policy_forward.cu +├── examples/ +│ ├── alpha_train_perception.rs (NEW: pretrain multi-horizon alpha heads) +│ └── alpha_ppo_train.rs (NEW: PPO training on frozen perception) +└── tests/ + └── (unit tests in src/, integration tests under tests/) +``` + +Reuse from `ml` (library deps, no modifications): +- `ml::env::action_space::N_ACTIONS` (just the constant, may not need with continuous size) +- `ml::env::execution_env::{ExecutionEnv, ExecutionEnvConfig}` (passive-fill env) +- `ml::env::loaders::load_fill_model_from_json` (alpha_fill_coeffs.json reader) +- `ml::cuda_pipeline::mapped_pinned::*` (pinned-memory helpers, for zero-copy CPU↔GPU) +- `ml::cuda_pipeline::alpha_kernels::*` (Mamba2 forward kernels, already validated) +- `ml::fxcache::*` (fxcache loader) + +We do **not** depend on: +- `ml::trainers::dqn::*` (DQN-specific, not reusable for PPO) +- `ml::cuda_pipeline::gpu_dqn_trainer::*` (DQN-specific kernels) +- `ml::cuda_pipeline::fused_training::*` (DQN-specific) + +--- + +## Phase 0: Pre-work (current 9-fold result + plan baseline) + +- [ ] **Step 1: Wait for the running `alpha-cv-6jlm5` workflow to complete.** Need the 9-fold stride=200 + scaled-training result on full 9Q before committing to a baseline. Captures the **target Sharpe to beat** with the new PPO trainer. + +Run: `argo get alpha-cv-6jlm5 -n foxhunt` until status = Succeeded. +Then: aggregate the 9 fold JSONs into a single summary table. +Expected: mean Sharpe at quarter-tick cost, std across 9 folds, break-even cost. + +- [ ] **Step 2: Record baseline numbers in a memory file** + +Create `/home/jgrusewski/.claude/projects/-home-jgrusewski-Work-foxhunt/memory/project_alpha_dqn_baseline_9fold.md`: +- 9-fold mean Sharpe at `cost ∈ {0, ¼, ½, 1, 2}` ticks +- 9-fold std at each cost point +- Break-even cost +- Date + commit SHA +- Note: this is the DQN baseline that PPO must beat + +This memory entry becomes the empirical bar PPO needs to clear. + +--- + +## Phase 1: Multi-horizon labels + perception layer + +Extend the existing single-horizon stacker into a multi-horizon perception model. + +### Task 1.1: Multi-horizon label generation + +**Files:** +- Modify: `crates/ml-alpha/src/multi_horizon_labels.rs` +- Test: `crates/ml-alpha/src/multi_horizon_labels.rs` (inline `#[cfg(test)]`) + +- [ ] **Step 1: Write the failing test** + +```rust +#[test] +fn multi_horizon_labels_match_known_prices() { + // price series with known up moves at h=30, 100, 1000 + let prices: Vec = (0..2000).map(|i| { + let base = 100.0; + let bump_30 = if i < 30 { 0.0 } else { 0.5 }; + let bump_100 = if i < 100 { 0.0 } else { 1.0 }; + let bump_1000 = if i < 1000 { 0.0 } else { 2.0 }; + base + bump_30 + bump_100 + bump_1000 + }).collect(); + let labels = generate_multi_horizon_labels(&prices, &[30, 100, 1000]); + // At bar 0, label[0] should be up at all 3 horizons (price moves up) + assert_eq!(labels.labels[0][0], 1.0, "h=30 label at bar 0"); + assert_eq!(labels.labels[0][1], 1.0, "h=100 label at bar 0"); + assert_eq!(labels.labels[0][2], 1.0, "h=1000 label at bar 0"); + // Bar valid_indices should drop the last `max(horizons)` bars + assert_eq!(labels.valid_indices.last().copied(), Some(prices.len() - 1001)); +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +```bash +SQLX_OFFLINE=true cargo test -p ml-alpha --lib multi_horizon_labels_match_known_prices -- --nocapture +``` +Expected: FAIL with "function not defined" or "wrong signature" + +- [ ] **Step 3: Implement `generate_multi_horizon_labels`** + +```rust +pub struct MultiHorizonLabels { + /// labels[bar_idx] = [label@h0, label@h1, ...] + pub labels: Vec>, + /// Bar indices where ALL horizons have valid forward windows + pub valid_indices: Vec, + pub horizons: Vec, + pub n_dropped_edge: usize, + pub n_dropped_invalid: usize, +} + +pub fn generate_multi_horizon_labels( + prices: &[f32], + horizons: &[usize], +) -> MultiHorizonLabels { + let max_h = horizons.iter().copied().max().unwrap_or(0); + let n = prices.len(); + let mut labels = Vec::with_capacity(n); + let mut valid_indices = Vec::with_capacity(n); + let mut n_dropped_edge = 0; + let mut n_dropped_invalid = 0; + for i in 0..n { + if i + max_h >= n { + n_dropped_edge += 1; + continue; + } + let p0 = prices[i]; + if !p0.is_finite() || p0 <= 0.0 { + n_dropped_invalid += 1; + continue; + } + let row: Vec = horizons + .iter() + .map(|&h| if prices[i + h] > p0 { 1.0 } else { 0.0 }) + .collect(); + labels.push(row); + valid_indices.push(i); + } + MultiHorizonLabels { + labels, + valid_indices, + horizons: horizons.to_vec(), + n_dropped_edge, + n_dropped_invalid, + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +```bash +SQLX_OFFLINE=true cargo test -p ml-alpha --lib multi_horizon_labels_match_known_prices +``` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add crates/ml-alpha/src/multi_horizon_labels.rs +git commit -m "feat(alpha): multi-horizon label generation" +``` + +### Task 1.2: Multi-horizon alpha head module + +**Files:** +- Create: `crates/ml-alpha/src/perception.rs` +- Test: inline `#[cfg(test)]` + +- [ ] **Step 1: Write the failing test for forward shape** + +```rust +#[test] +fn multi_horizon_head_emits_one_logit_per_horizon() { + let stream = CudaContext::new(0).unwrap().default_stream(); + let head = MultiHorizonHead::new(&stream, 64, &[30, 100, 1000]).unwrap(); + let batch = 8; + let hidden = GpuTensor::zeros(&[batch, 64], &stream).unwrap(); + let out = head.forward(&hidden).unwrap(); + assert_eq!(out.shape(), &[batch, 3]); +} +``` + +- [ ] **Step 2: Run test** + +```bash +SQLX_OFFLINE=true CUDA_COMPUTE_CAP=86 cargo test -p ml-alpha --lib multi_horizon_head_emits_one_logit_per_horizon +``` +Expected: FAIL — type not defined. + +- [ ] **Step 3: Implement `MultiHorizonHead`** + +```rust +/// One linear layer per horizon, sharing the upstream hidden state. +/// Output: [batch, n_horizons] of pre-sigmoid logits. +pub struct MultiHorizonHead { + pub horizons: Vec, + weights: GpuTensor, // [hidden_dim, n_horizons] + bias: GpuTensor, // [n_horizons] +} + +impl MultiHorizonHead { + pub fn new(stream: &Arc, hidden_dim: usize, horizons: &[usize]) -> Result { + let n_h = horizons.len(); + // Xavier init: std = sqrt(2 / (fan_in + fan_out)) for tanh-ish nonlinearity + // upstream; use small std for logit-output layer to start near 0.5 prob. + let std = (2.0_f32 / (hidden_dim as f32 + n_h as f32)).sqrt() * 0.1; + let mut rng = rand::thread_rng(); + let w_cpu: Vec = (0..hidden_dim * n_h) + .map(|_| rng.sample(rand_distr::Normal::new(0.0, std).unwrap())) + .collect(); + let weights = GpuTensor::from_slice(&[hidden_dim, n_h], &w_cpu, stream)?; + let bias = GpuTensor::zeros(&[n_h], stream)?; + Ok(Self { horizons: horizons.to_vec(), weights, bias }) + } + + /// Forward: y = hidden @ weights + bias. + /// Returns pre-sigmoid logits [batch, n_horizons]. + pub fn forward(&self, hidden: &GpuTensor) -> Result { + // Reuse the matmul kernel from mamba2_block or ml-alpha::mlp + // (whichever is exposed); fall back to cuBLAS sgemm if needed. + crate::mlp::linear_forward(hidden, &self.weights, &self.bias) + } +} +``` + +- [ ] **Step 4: Run test** + +```bash +SQLX_OFFLINE=true CUDA_COMPUTE_CAP=86 cargo test -p ml-alpha --lib multi_horizon_head_emits_one_logit_per_horizon +``` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add crates/ml-alpha/src/perception.rs +git commit -m "feat(alpha): multi-horizon alpha head — one logit per horizon" +``` + +### Task 1.3: Perception entry binary `alpha_train_perception` + +Replaces the single-horizon `alpha_train_stacker`. Trains Mamba2 trunk + multi-horizon heads jointly, saves the trunk weights + per-bar multi-horizon logit cache. + +**Files:** +- Create: `crates/ml-alpha/examples/alpha_train_perception.rs` + +(Boilerplate: copy from `alpha_train_stacker.rs`, swap out single-horizon loss for multi-horizon BCE, persist `perception_weights.bin` + `multi_horizon_logits_cache.bin`) + +- [ ] **Step 1: Scaffold the CLI (steal from `alpha_train_stacker.rs`)** + +```rust +// crates/ml-alpha/examples/alpha_train_perception.rs (NEW) +use anyhow::Result; +use clap::Parser; + +#[derive(Parser, Debug)] +#[command(name = "alpha_train_perception")] +struct Cli { + #[arg(long)] fxcache_path: String, + #[arg(long, value_delimiter = ',', default_value = "30,100,1000,6000")] + horizons: Vec, + #[arg(long, default_value_t = 32)] seq_len: usize, + #[arg(long, default_value_t = 64)] hidden_dim: usize, + #[arg(long, default_value_t = 16)] state_dim: usize, + #[arg(long, default_value_t = 5)] epochs: usize, + #[arg(long, default_value_t = 128)] batch_size: usize, + #[arg(long, default_value_t = 3e-3)] lr: f32, + #[arg(long, default_value_t = 0.8)] train_frac: f32, + #[arg(long)] perception_out: String, // trunk + heads weights + #[arg(long)] logits_cache_out: String, // per-bar multi-horizon logits + #[arg(long)] max_rows: Option, +} + +fn main() -> Result<()> { + // (1) load fxcache, extract prices + alpha_features + // (2) generate multi-horizon labels for each h + // (3) train Mamba2 trunk + MultiHorizonHead with per-horizon BCE + // (4) snapshot pass: emit logits[bar, h] for all bars to logits_cache_out + // (5) serialize trunk + head weights to perception_out + todo!("implement using alpha_train_stacker.rs as template") +} +``` + +- [ ] **Step 2: Implement the training loop** — mirrors alpha_train_stacker but with multi-horizon BCE loss = `mean_h(BCE(sigmoid(logit_h), label_h))`. + +- [ ] **Step 3: Implement the logits-cache export** — same structure as `alpha_logits_cache.bin` but with `n_horizons` floats per bar instead of one. + +``` +binary format: + u32 LE: magic 0x504D4848 ("MHLP") + u32 LE: version (1) + u32 LE: n_bars + u32 LE: n_horizons + f32 LE × n_horizons: horizon values + f32 LE × n_bars × n_horizons: logits (pre-sigmoid) +``` + +- [ ] **Step 4: Smoke test on 2Q fxcache locally** + +```bash +CUDA_COMPUTE_CAP=86 SQLX_OFFLINE=true cargo build --release -p ml-alpha --example alpha_train_perception +./target/release/examples/alpha_train_perception \ + --fxcache-path /tmp/foxhunt-profile/out/1e36bcab*.fxcache \ + --horizons 30,100,1000,6000 \ + --epochs 5 \ + --perception-out /tmp/perception_2q.bin \ + --logits-cache-out /tmp/multi_horizon_logits_2q.bin +``` +Expected: file `/tmp/multi_horizon_logits_2q.bin` ≈ 4 × `alpha_logits_cache_2q.bin` size (~48 MB). + +- [ ] **Step 5: Commit** + +```bash +git add crates/ml-alpha/examples/alpha_train_perception.rs +git commit -m "feat(alpha): multi-horizon perception trainer + logits cache export" +``` + +--- + +## Phase 2: PPO trainer scaffolding + +### Task 2.1: Alpha state (55-dim) + +**Files:** +- Create: `crates/ml-alpha/src/alpha_state.rs` + +- [ ] **Step 1: Test that state construction is deterministic** + +```rust +#[test] +fn alpha_state_dim_55() { + let fx = mock_fxcache_record(); // OHLC + 38 market features + let alpha = [0.1_f32; 4]; // 4-horizon alpha probs + let mamba_hidden = [0.5_f32; 6]; // 6-dim Mamba2 hidden + let portfolio = [0.0_f32; 2]; // position, P&L (no spread/regime here yet) + let state = AlphaState::from_parts(&fx, &alpha, &mamba_hidden, &portfolio); + assert_eq!(state.to_vec().len(), 55); +} +``` + +- [ ] **Step 2: Implement `AlphaState`** + +```rust +pub const ALPHA_STATE_DIM: usize = 55; +// 4 OHLC log returns + 38 market features = 42 from fxcache +// + 4 multi-horizon alpha probs +// + 6 Mamba2 hidden state +// + 3 portfolio (position, unrealized P&L, time-since-entry) +// = 55 + +pub struct AlphaState { + pub vec: [f32; ALPHA_STATE_DIM], +} + +impl AlphaState { + pub fn from_parts( + fxcache_features: &[f32], // 42 elements + multi_horizon_alpha: &[f32], // 4 elements (sigmoid(logits)) + mamba_hidden: &[f32], // 6 elements + portfolio: &[f32], // 3 elements + ) -> Self { + assert_eq!(fxcache_features.len(), 42); + assert_eq!(multi_horizon_alpha.len(), 4); + assert_eq!(mamba_hidden.len(), 6); + assert_eq!(portfolio.len(), 3); + let mut vec = [0_f32; ALPHA_STATE_DIM]; + vec[0..42].copy_from_slice(fxcache_features); + vec[42..46].copy_from_slice(multi_horizon_alpha); + vec[46..52].copy_from_slice(mamba_hidden); + vec[52..55].copy_from_slice(portfolio); + Self { vec } + } + + pub fn to_vec(&self) -> &[f32] { + &self.vec + } +} +``` + +- [ ] **Step 3: Run test + commit** + +```bash +SQLX_OFFLINE=true cargo test -p ml-alpha --lib alpha_state_dim_55 +git add crates/ml-alpha/src/alpha_state.rs +git commit -m "feat(alpha): 55-dim AlphaState (production features + multi-h alpha + mamba + portfolio)" +``` + +### Task 2.2: Decision-stride env + +**Files:** +- Create: `crates/ml-alpha/src/alpha_env.rs` + +The env wraps `ml::env::execution_env::ExecutionEnv` and exposes a stride-aware decision interface: `decide_at(bar_idx)` returns `Some(state)` only at decision points; calling `step(action)` advances `decision_stride` bars at once and returns the segment reward. + +- [ ] **Step 1: Test that stride correctly gates action emission** + +```rust +#[test] +fn env_stride_yields_decisions_at_boundary_only() { + let env = build_test_env(/* decision_stride */ 5, /* horizon */ 100); + let mut decisions = 0; + for bar in 0..100 { + if env.is_decision_point(bar) { decisions += 1; } + } + assert_eq!(decisions, 100 / 5); +} +``` + +- [ ] **Step 2: Implement `AlphaExecutionEnv`** — wraps `ExecutionEnv`, adds `decision_stride`, segment-PnL reward aggregation. + +```rust +pub struct AlphaExecutionEnv { + inner: ml::env::execution_env::ExecutionEnv, + pub decision_stride: usize, + pub bar_cursor: usize, +} + +impl AlphaExecutionEnv { + pub fn is_decision_point(&self, bar: usize) -> bool { + bar % self.decision_stride == 0 + } + + /// Emit one decision: take action, advance `decision_stride` bars, + /// return (next_state, segment_reward, done). + pub fn step_segment( + &mut self, + action: PolicyAction, + portfolio: &mut PortfolioState, + ) -> Option<(AlphaState, f32, bool)> { + let mut reward_sum = 0_f32; + let target_bar = self.bar_cursor + self.decision_stride; + // Apply the action ONCE at the start of the segment; hold through. + self.apply_action(action, portfolio)?; + while self.bar_cursor < target_bar { + let (_, bar_r, done) = self.inner.step(0 /* wait */, portfolio)?; + reward_sum += bar_r; + self.bar_cursor += 1; + if done { return Some((self.build_state(portfolio), reward_sum, true)); } + } + Some((self.build_state(portfolio), reward_sum, false)) + } + // ... +} +``` + +- [ ] **Step 3: Test segment reward aggregates correctly** + +```rust +#[test] +fn segment_reward_sums_intra_segment_pnl() { + let mut env = build_test_env(5, 20); + let mut port = PortfolioState::default(); + let (_, r, _) = env.step_segment(PolicyAction::Long(1.0, Horizon::H30), &mut port).unwrap(); + // r should equal sum of bar-level rewards over 5 bars; check magnitude. + assert!(r.abs() < 10.0, "segment reward in plausible range"); +} +``` + +- [ ] **Step 4: Commit** + +```bash +git add crates/ml-alpha/src/alpha_env.rs +git commit -m "feat(alpha): decision-stride env with segment-reward aggregation" +``` + +### Task 2.3: Policy network — actor-critic heads + +**Files:** +- Create: `crates/ml-alpha/src/policy.rs` + +Multi-head policy: 3 categorical (direction / horizon / size-bin) + 1 scalar (value). PPO needs both action sampling and log-probability computation. + +- [ ] **Step 1: Define `PolicyAction` enum and `Policy` struct** + +```rust +#[derive(Debug, Clone, Copy)] +pub enum Direction { Long, Short, Flat } + +#[derive(Debug, Clone, Copy)] +pub enum Horizon { H30, H100, H1k, H6k } + +/// 8 discrete size bins (logarithmic): 0, 1, 2, 4, 8, 16, 32, 64 % of capital. +/// Discrete to avoid continuous-action PPO complexity; can swap to Gaussian +/// later if size resolution matters. +#[derive(Debug, Clone, Copy)] +pub struct SizeBin(pub u8); + +#[derive(Debug, Clone, Copy)] +pub struct PolicyAction { + pub direction: Direction, + pub horizon: Horizon, + pub size: SizeBin, +} + +pub struct PolicyOutput { + pub direction_logits: [f32; 3], + pub horizon_logits: [f32; 4], + pub size_logits: [f32; 8], + pub value: f32, +} +``` + +- [ ] **Step 2: Implement the network forward** + +Shared trunk (MLP, 2 hidden layers @ 128) → 3 logit heads + value head. Use existing `ml_alpha::mlp::MlpModel` as the trunk; add 4 head linears. + +- [ ] **Step 3: Implement action sampling + log-prob** + +```rust +impl Policy { + /// Sample an action from the distribution; return action + log-prob. + pub fn sample(&self, state: &AlphaState, rng: &mut impl Rng) -> (PolicyAction, f32) { + let out = self.forward(state); + let dir = sample_categorical(&out.direction_logits, rng); + let hor = sample_categorical(&out.horizon_logits, rng); + let siz = sample_categorical(&out.size_logits, rng); + let log_p = log_softmax(&out.direction_logits)[dir] + + log_softmax(&out.horizon_logits)[hor] + + log_softmax(&out.size_logits)[siz]; + (PolicyAction { direction: dir.into(), horizon: hor.into(), size: SizeBin(siz as u8) }, + log_p) + } +} +``` + +- [ ] **Step 4: Test that sampled actions are within the action space** + +- [ ] **Step 5: Commit** + +```bash +git add crates/ml-alpha/src/policy.rs +git commit -m "feat(alpha): actor-critic policy with 3-head categorical + value" +``` + +### Task 2.4: GAE advantage estimation + +**Files:** +- Create: `crates/ml-alpha/src/gae.rs` + +Standard Generalised Advantage Estimation (Schulman 2016). Single function: given (rewards, values, dones), produce advantages + returns. + +- [ ] **Step 1: Test on a known trajectory** + +```rust +#[test] +fn gae_matches_paper_formula_simple_case() { + let rewards = vec![1.0, 0.0, 0.0, 1.0]; + let values = vec![0.5, 0.5, 0.5, 0.5, 0.0]; // V(s_4) = 0 since done + let dones = vec![false, false, false, true]; + let gamma = 0.99_f32; + let lambda = 0.95_f32; + let (adv, ret) = compute_gae(&rewards, &values, &dones, gamma, lambda); + // hand-compute: delta_t = r_t + gamma*V(s_{t+1})*(1-done_t) - V(s_t) + // advantage_t = delta_t + gamma*lambda*advantage_{t+1}*(1-done_t) + // ... + assert!((adv[3] - 0.5).abs() < 1e-6); +} +``` + +- [ ] **Step 2: Implement `compute_gae`** + +- [ ] **Step 3: Run test + commit** + +### Task 2.5: PPO loss + +**Files:** +- Create: `crates/ml-alpha/src/ppo.rs` + +The canonical PPO clip objective + value MSE + entropy regulariser. + +- [ ] **Step 1: Test clip behaviour** + +- [ ] **Step 2: Implement `ppo_loss(old_log_probs, new_log_probs, advantages, returns, values, clip_eps, value_coef, entropy_coef)`** + +```rust +pub struct PpoLossResult { + pub policy_loss: f32, + pub value_loss: f32, + pub entropy_bonus: f32, + pub total: f32, + pub n_clipped: usize, +} + +pub fn ppo_loss( + old_log_probs: &[f32], + new_log_probs: &[f32], + advantages: &[f32], + returns: &[f32], + values: &[f32], + entropies: &[f32], + clip_eps: f32, + value_coef: f32, + entropy_coef: f32, +) -> PpoLossResult { + let mut policy_loss = 0_f32; + let mut value_loss = 0_f32; + let mut entropy = 0_f32; + let mut clipped = 0; + for i in 0..old_log_probs.len() { + let ratio = (new_log_probs[i] - old_log_probs[i]).exp(); + let unclipped = ratio * advantages[i]; + let clipped_ratio = ratio.clamp(1.0 - clip_eps, 1.0 + clip_eps); + let clipped_val = clipped_ratio * advantages[i]; + if clipped_ratio != ratio { clipped += 1; } + // PPO maximises advantage * ratio, so minimise -min(unclipped, clipped) + policy_loss -= unclipped.min(clipped_val); + let td = returns[i] - values[i]; + value_loss += td * td; + entropy += entropies[i]; + } + let n = old_log_probs.len() as f32; + let policy_loss = policy_loss / n; + let value_loss = value_loss / n; + let entropy_bonus = entropy / n; + let total = policy_loss + value_coef * value_loss - entropy_coef * entropy_bonus; + PpoLossResult { policy_loss, value_loss, entropy_bonus, total, n_clipped: clipped } +} +``` + +- [ ] **Step 3: Run test + commit** + +--- + +## Phase 3: Training loop + +### Task 3.1: Rollout collection + +Collect a buffer of (state, action, log_prob, reward, value, done) tuples over N decision steps, using the current policy. + +**Files:** +- Modify: `crates/ml-alpha/examples/alpha_ppo_train.rs` (or new helper module) + +- [ ] **Step 1: Implement `collect_rollouts(env, policy, n_steps)` → trajectory buffer** + +- [ ] **Step 2: Test that the rollout length matches and terminal handling is correct** + +### Task 3.2: PPO update step + +- [ ] **Step 1: Given a rollout buffer, run K epochs of minibatch PPO updates on (state, action, advantage, return, old_log_prob).** + +```python +for epoch in 0..ppo_epochs: + shuffle minibatches + for minibatch in batches: + recompute new_log_prob, value, entropy via policy.forward + loss = ppo_loss(old, new, adv, ret, val, ent, ...) + backward + Adam step (clip grad norm) +``` + +- [ ] **Step 2: Verify that ppo loss decreases over the same rollout** (sanity check: many gradient steps on the same data should reduce loss). + +### Task 3.3: Main training driver `alpha_ppo_train` + +**Files:** +- Create: `crates/ml-alpha/examples/alpha_ppo_train.rs` + +CLI similar to `alpha_train_stacker`: + +``` +--fxcache-path ... +--perception ... # pretrained perception (frozen) +--horizons 30,100,1000,6000 +--decision-stride 200 +--n-rollouts 8000 # total rollout steps +--rollout-length 200 # decisions per rollout +--ppo-epochs 4 +--clip-eps 0.2 +--gamma 0.99 +--lambda 0.95 +--value-coef 0.5 +--entropy-coef 0.01 +--lr 3e-4 +--train-frac 0.6 +--max-snapshots 1900000 +--data-start-offset 0 +--out-path /tmp/ppo_fold_A.json +``` + +- [ ] **Step 1: Scaffold the CLI** + +- [ ] **Step 2: Implement the outer loop** (load fxcache → load frozen perception → init policy → loop: collect rollouts, run PPO updates, log Sharpe on held-out segment). + +- [ ] **Step 3: Build + run on Fold A 2Q locally; confirm it converges (loss drops, Sharpe positive on eval segment).** + +```bash +CUDA_COMPUTE_CAP=86 SQLX_OFFLINE=true cargo build --release -p ml-alpha --example alpha_ppo_train +./target/release/examples/alpha_ppo_train \ + --fxcache-path /tmp/foxhunt-profile/out/1e36bcab*.fxcache \ + --perception /tmp/perception_2q.bin \ + --decision-stride 200 \ + --n-rollouts 8000 \ + --out-path /tmp/ppo_fold_A.json +``` +Expected: eval Sharpe at quarter-tick cost ≥ +1.0 (must clear the DQN baseline meaningfully to justify the rebuild). + +--- + +## Phase 4: GPU acceleration + +The Phase 3 driver runs CPU-only PPO (Rust + cudarc for matmul). For full training scale, we need GPU-resident PPO update. + +### Task 4.1: Policy forward kernel + +**Files:** +- Create: `crates/ml-alpha/src/kernels/policy_forward.cu` + +Fused MLP forward: trunk → 3 logit heads + value. Single kernel launch per minibatch. + +- [ ] **Step 1: Write CUDA C source** +- [ ] **Step 2: Add NVRTC build hook in `build.rs`** +- [ ] **Step 3: Wire into `Policy::forward_gpu` (replaces the CPU forward)** +- [ ] **Step 4: Verify bit-equivalence against CPU forward on small batch** + +### Task 4.2: PPO loss + backward kernel + +- [ ] Fused kernel: given (new_log_probs, old_log_probs, advantages, returns, values, entropies) compute total loss + per-parameter gradients in one pass. +- [ ] Verify bit-equivalence on small batch. + +### Task 4.3: GAE on GPU + +- [ ] Sequential dependency makes parallelism hard; standard approach is reverse-prefix-scan. Single block thread-cooperative scan since rollout sizes are small (<10K). + +--- + +## Phase 5: Walk-forward CV + cluster integration + +### Task 5.1: Native walk-forward driver + +**Files:** +- Create: `crates/ml-alpha/src/walk_forward.rs` + +Iterate over fold offsets, train PPO from scratch on each fold, backtest on held-out portion. + +- [ ] **Step 1: Implement `walk_forward(fxcache, perception, fold_offsets, fold_window, ppo_config)` → Vec** +- [ ] **Step 2: Test on 3 folds locally; compare to DQN baseline** + +### Task 5.2: New Argo workflow `alpha-ppo` + +Replaces `alpha-cv-template.yaml`. One pod runs the full 9-fold pipeline natively. + +**Files:** +- Create: `infra/k8s/argo/alpha-ppo-template.yaml` +- Create: `scripts/argo-alpha-ppo.sh` + +- [ ] **Step 1: Adapt `alpha-cv-template.yaml`** — replace ensure-binary BINARIES list, replace stacker-train with perception-train, replace 9× alpha_baseline folds with single `alpha_ppo_train --walk-forward 9`. + +- [ ] **Step 2: Apply WorkflowTemplate + commit + push** + +```bash +kubectl -n foxhunt apply -f infra/k8s/argo/alpha-ppo-template.yaml +git add infra/k8s/argo/alpha-ppo-template.yaml scripts/argo-alpha-ppo.sh +git commit -m "infra(argo): alpha-ppo workflow for PPO 9-fold training" +git push +``` + +- [ ] **Step 3: Submit + monitor** + +```bash +./scripts/argo-alpha-ppo.sh --branch main +``` + +Expected wall: ~90-120 min on L40S (perception 15 min + 9 folds × 10-12 min). + +--- + +## Phase 6: Validation gate + +Before declaring the new trainer ready, must clear the **DQN 9-fold baseline** captured in Phase 0 by a meaningful margin. + +Acceptance criteria: +- [ ] Mean Sharpe at quarter-tick cost ≥ baseline + 0.5 (one std). +- [ ] Standard deviation across 9 folds ≤ baseline std (no regime overfitting). +- [ ] Break-even cost ≥ baseline (no economic regression). +- [ ] No fold worse than baseline worst fold by more than 1 Sharpe (robustness check). + +If criteria are not met: +- [ ] Investigate failure modes. Common: PPO entropy collapse (policy gets too confident → no exploration), value-function lag (advantage misestimation), perception over/under-fit. Each has a standard remediation. + +If criteria are met: +- [ ] Update memory `project_alpha_ppo_baseline.md` with the new bar. +- [ ] Document the architectural choice in a `pearl_*.md` entry (memory writes happen via Write tool with the auto-memory format). + +--- + +## Self-Review + +**Spec coverage:** Plan covers perception (multi-horizon labels + heads + pretraining binary), policy network (3-head categorical + value), env (decision-stride + segment reward), PPO loss + GAE, training loop, GPU acceleration, walk-forward, cluster workflow, validation gate. The user's three answered design decisions (Path B / branched-as-categorical / multi-horizon blend / hybrid 55-dim state) are all baked in. + +**Placeholder scan:** Three `todo!()` markers in code snippets are illustrative; they get replaced when the task is executed (the body of `alpha_train_perception::main` follows the alpha_train_stacker template). No "TBD" / "later" markers. + +**Type consistency:** `AlphaState`, `PolicyAction`, `Direction`, `Horizon`, `SizeBin`, `PolicyOutput`, `PpoLossResult`, `MultiHorizonLabels`, `MultiHorizonHead`, `AlphaExecutionEnv` defined once, referenced consistently. State dimension 55 = 42 + 4 + 6 + 3 verified in `alpha_state.rs` test. + +**Ambiguity callout:** The size space (8 discrete bins vs Gaussian continuous) is a soft decision in the plan. Phase 3 implements discrete bins for PPO simplicity; if size-resolution matters empirically, swap to Gaussian (10–20 LOC change in `policy.rs`). Noted in Task 2.3. + +**Total scope:** ~3 weeks of focused work for Phases 1–5; Phase 6 is the validation gate. Phase 0 baseline capture is blocking on the current `alpha-cv-6jlm5` cluster run.