From ddb7b170bc0fbdb91e32d6b73d0a6168be40ea01 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 17 Apr 2026 00:02:02 +0200 Subject: [PATCH] =?UTF-8?q?plan:=20Tick=20Microstructure=20Intelligence=20?= =?UTF-8?q?=E2=80=94=204=20tasks,=20OFI=5FDIM=208=E2=86=9220?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task 1: IncrementalMicrostructureCalculator (12 new features, O(1)/tick) Task 2: fxcache OFI_DIM 8→20 atomic update (15 files, 30+ locations) Task 3: Extend precompute pipeline with MicrostructureState Task 4: Regenerate fxcache + smoke test + compute-sanitizer Co-Authored-By: Claude Opus 4.6 (1M context) --- ...-04-16-tick-microstructure-intelligence.md | 635 ++++++++++++++++++ 1 file changed, 635 insertions(+) create mode 100644 docs/superpowers/plans/2026-04-16-tick-microstructure-intelligence.md diff --git a/docs/superpowers/plans/2026-04-16-tick-microstructure-intelligence.md b/docs/superpowers/plans/2026-04-16-tick-microstructure-intelligence.md new file mode 100644 index 000000000..f4657ea51 --- /dev/null +++ b/docs/superpowers/plans/2026-04-16-tick-microstructure-intelligence.md @@ -0,0 +1,635 @@ +# Tick-Level Microstructure Intelligence Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Extract 12 new predictive features from raw MBP-10 tick data, expanding OFI_DIM from 8→20 and feature vector from 50→62 dimensions, giving the DQN model sub-bar microstructure awareness. + +**Architecture:** Extend `OFICalculator` with `IncrementalMicrostructureCalculator` (all O(1) per tick). Update fxcache format (OFI_DIM 8→20, single format, regenerate all files). Update 15+ files with hardcoded `8` literals atomically. Same calculator struct used for both precompute (training) and live streaming (inference). + +**Tech Stack:** Rust 1.85, ml-features crate, fxcache binary format, Databento MBP-10/Trades, CUDA experience collector + +--- + +## Task 1: IncrementalMicrostructureCalculator — 12 New Features + +Extend the existing `OFICalculator` with 12 new tick-level features. All O(1) per tick. + +**Files:** +- Modify: `crates/ml-features/src/ofi_calculator.rs` + +- [ ] **Step 1: Add `MicrostructureState` struct for running statistics** + +At the end of `crates/ml-features/src/ofi_calculator.rs`, add: + +```rust +/// Running state for 12 tick-level microstructure features (indices 8-19). +/// All computations are O(1) per tick via EMAs, counters, and running sums. +/// Used by both batch precompute (training) and live streaming (inference). +#[derive(Debug, Clone)] +pub struct MicrostructureState { + // [8] OFI Trajectory: online linear regression of OFI_L1 over time + ofi_traj_sum_t: f64, // sum of t values + ofi_traj_sum_t2: f64, // sum of t² values + ofi_traj_sum_ofi: f64, // sum of OFI values + ofi_traj_sum_t_ofi: f64, // sum of t*OFI values + ofi_traj_n: u64, // count + + // [9] Realized Variance: sum of squared log returns + rv_sum_sq: f64, + prev_mid: f64, + + // [10] Trade Arrival Intensity (Hawkes EMA) + hawkes_intensity: f64, + last_trade_ns: u64, + hawkes_mu: f64, // baseline intensity + hawkes_alpha: f64, // excitation + hawkes_beta: f64, // decay rate + + // [11] Book Pressure Gradient + book_pressure_ema: f64, + + // [12] Spread Dynamics + spread_min: f64, + spread_max: f64, + spread_sum: f64, + spread_count: u64, + + // [13] Aggression Ratio + trades_at_ask: u64, + trades_total: u64, + + // [14] Queue Depletion Asymmetry + prev_bid_sz_top3: u64, + prev_ask_sz_top3: u64, + bid_depletion_ema: f64, + ask_depletion_ema: f64, + + // [15] Order Count Flux (approximation from MBP-10 snapshots) + prev_bid_ct_top3: u32, + prev_ask_ct_top3: u32, + ct_increase_sum: u64, + ct_decrease_sum: u64, + + // [16] Intra-Bar Momentum (Welford online covariance) + half1_sum: f64, half1_sum_sq: f64, half1_n: u64, + half2_sum: f64, half2_sum_sq: f64, half2_n: u64, + bar_start_ns: u64, + bar_mid_ns: u64, + + // [17] Microstructure Regime Score (composite) + // Derived from aggression, spread, book thickness at snapshot time + + // [18] OFI Acceleration (EMA of OFI deltas) + prev_ofi_l1: f64, + ofi_delta_ema: f64, + + // [19] Toxicity Gradient (EMA of VPIN deltas) + prev_vpin: f64, + vpin_delta_ema: f64, + + tick_count: u64, +} +``` + +- [ ] **Step 2: Implement `MicrostructureState` methods** + +```rust +impl MicrostructureState { + pub fn new(bar_start_ns: u64, bar_duration_ns: u64) -> Self { + Self { + ofi_traj_sum_t: 0.0, ofi_traj_sum_t2: 0.0, + ofi_traj_sum_ofi: 0.0, ofi_traj_sum_t_ofi: 0.0, ofi_traj_n: 0, + rv_sum_sq: 0.0, prev_mid: 0.0, + hawkes_intensity: 0.0, last_trade_ns: 0, hawkes_mu: 1.0, + hawkes_alpha: 0.5, hawkes_beta: 1.0, + book_pressure_ema: 0.0, + spread_min: f64::MAX, spread_max: f64::MIN, spread_sum: 0.0, spread_count: 0, + trades_at_ask: 0, trades_total: 0, + prev_bid_sz_top3: 0, prev_ask_sz_top3: 0, + bid_depletion_ema: 0.0, ask_depletion_ema: 0.0, + prev_bid_ct_top3: 0, prev_ask_ct_top3: 0, + ct_increase_sum: 0, ct_decrease_sum: 0, + half1_sum: 0.0, half1_sum_sq: 0.0, half1_n: 0, + half2_sum: 0.0, half2_sum_sq: 0.0, half2_n: 0, + bar_start_ns, bar_mid_ns: bar_start_ns + bar_duration_ns / 2, + prev_ofi_l1: 0.0, ofi_delta_ema: 0.0, + prev_vpin: 0.0, vpin_delta_ema: 0.0, + tick_count: 0, + } + } + + /// Update from MBP-10 snapshot. O(1) per call. + pub fn update_snapshot(&mut self, snapshot: &Mbp10Snapshot) { + let alpha = 0.05; + self.tick_count += 1; + + // Mid price + let best = &snapshot.levels[0]; + let bid_f = BidAskPair::price_to_f64(best.bid_px); + let ask_f = BidAskPair::price_to_f64(best.ask_px); + let mid = (bid_f + ask_f) / 2.0; + + // [9] Realized Variance + if self.prev_mid > 0.0 && mid > 0.0 { + let log_ret = (mid / self.prev_mid).ln(); + self.rv_sum_sq += log_ret * log_ret; + + // [16] Intra-bar momentum + if snapshot.timestamp < self.bar_mid_ns { + self.half1_sum += log_ret; + self.half1_sum_sq += log_ret * log_ret; + self.half1_n += 1; + } else { + self.half2_sum += log_ret; + self.half2_sum_sq += log_ret * log_ret; + self.half2_n += 1; + } + } + self.prev_mid = mid; + + // [11] Book Pressure Gradient: weighted imbalance across 10 levels + let mut weighted_imb = 0.0; + let mut total_sz = 0.0; + for (l, pair) in snapshot.levels.iter().enumerate().take(10) { + let w = (-0.3 * l as f64).exp(); + weighted_imb += w * (pair.bid_sz as f64 - pair.ask_sz as f64); + total_sz += pair.bid_sz as f64 + pair.ask_sz as f64; + } + let pressure = if total_sz > 0.0 { weighted_imb / total_sz } else { 0.0 }; + self.book_pressure_ema = (1.0 - alpha) * self.book_pressure_ema + alpha * pressure; + + // [12] Spread Dynamics + let spread = ask_f - bid_f; + if spread > 0.0 { + self.spread_min = self.spread_min.min(spread); + self.spread_max = self.spread_max.max(spread); + self.spread_sum += spread; + self.spread_count += 1; + } + + // [14] Queue Depletion + let bid_sz_top3: u64 = snapshot.levels.iter().take(3).map(|p| p.bid_sz as u64).sum(); + let ask_sz_top3: u64 = snapshot.levels.iter().take(3).map(|p| p.ask_sz as u64).sum(); + if self.prev_bid_sz_top3 > 0 { + let bid_dep = (self.prev_bid_sz_top3 as f64 - bid_sz_top3 as f64).max(0.0); + let ask_dep = (self.prev_ask_sz_top3 as f64 - ask_sz_top3 as f64).max(0.0); + self.bid_depletion_ema = (1.0 - alpha) * self.bid_depletion_ema + alpha * bid_dep; + self.ask_depletion_ema = (1.0 - alpha) * self.ask_depletion_ema + alpha * ask_dep; + } + self.prev_bid_sz_top3 = bid_sz_top3; + self.prev_ask_sz_top3 = ask_sz_top3; + + // [15] Order Count Flux + let bid_ct_top3: u32 = snapshot.levels.iter().take(3).map(|p| p.bid_ct).sum(); + let ask_ct_top3: u32 = snapshot.levels.iter().take(3).map(|p| p.ask_ct).sum(); + if self.prev_bid_ct_top3 > 0 { + let delta = (bid_ct_top3 as i64 - self.prev_bid_ct_top3 as i64) + + (ask_ct_top3 as i64 - self.prev_ask_ct_top3 as i64); + if delta > 0 { self.ct_increase_sum += delta as u64; } + else { self.ct_decrease_sum += (-delta) as u64; } + } + self.prev_bid_ct_top3 = bid_ct_top3; + self.prev_ask_ct_top3 = ask_ct_top3; + } + + /// Update from trade event. O(1) per call. + pub fn update_trade(&mut self, price: f64, _volume: u64, is_buy: bool, timestamp_ns: u64) { + let alpha = 0.05; + + // [10] Hawkes intensity + if self.last_trade_ns > 0 { + let dt = (timestamp_ns - self.last_trade_ns) as f64 / 1e9; // seconds + self.hawkes_intensity = self.hawkes_mu + + self.hawkes_alpha * (self.hawkes_intensity - self.hawkes_mu) + * (-self.hawkes_beta * dt).exp() + + self.hawkes_alpha; + } + self.last_trade_ns = timestamp_ns; + + // [13] Aggression Ratio + if is_buy { self.trades_at_ask += 1; } + self.trades_total += 1; + + // Update prev_mid for realized variance if we don't have a snapshot mid + if self.prev_mid == 0.0 { self.prev_mid = price; } + } + + /// Update OFI-derived features. Call after OFICalculator.calculate(). + pub fn update_ofi_derived(&mut self, ofi_l1: f64, vpin: f64) { + let alpha = 0.1; + let t = self.tick_count as f64; + + // [8] OFI Trajectory (online linear regression) + self.ofi_traj_n += 1; + self.ofi_traj_sum_t += t; + self.ofi_traj_sum_t2 += t * t; + self.ofi_traj_sum_ofi += ofi_l1; + self.ofi_traj_sum_t_ofi += t * ofi_l1; + + // [18] OFI Acceleration + let ofi_delta = ofi_l1 - self.prev_ofi_l1; + self.ofi_delta_ema = (1.0 - alpha) * self.ofi_delta_ema + alpha * ofi_delta; + self.prev_ofi_l1 = ofi_l1; + + // [19] Toxicity Gradient + let vpin_delta = vpin - self.prev_vpin; + self.vpin_delta_ema = (1.0 - alpha) * self.vpin_delta_ema + alpha * vpin_delta; + self.prev_vpin = vpin; + } + + /// Snapshot all 12 features. Pure read, non-mutating, idempotent. + pub fn snapshot(&self) -> [f64; 12] { + let n = self.ofi_traj_n as f64; + + // [8] OFI Trajectory slope + let ofi_slope = if n > 1.0 { + let denom = n * self.ofi_traj_sum_t2 - self.ofi_traj_sum_t * self.ofi_traj_sum_t; + if denom.abs() > 1e-12 { + (n * self.ofi_traj_sum_t_ofi - self.ofi_traj_sum_t * self.ofi_traj_sum_ofi) / denom + } else { 0.0 } + } else { 0.0 }; + + // [9] Realized Variance (already accumulated) + let rv = self.rv_sum_sq; + + // [10] Hawkes intensity + let hawkes = self.hawkes_intensity.min(100.0); + + // [11] Book Pressure Gradient (EMA) + let book_pressure = self.book_pressure_ema; + + // [12] Spread Dynamics + let spread_dyn = if self.spread_count > 0 && self.spread_sum > 0.0 { + let mean = self.spread_sum / self.spread_count as f64; + (self.spread_max - self.spread_min) / mean.max(1e-12) + } else { 0.0 }; + + // [13] Aggression Ratio + let aggression = if self.trades_total > 0 { + self.trades_at_ask as f64 / self.trades_total as f64 + } else { 0.5 }; + + // [14] Queue Depletion Asymmetry + let max_dep = self.bid_depletion_ema.max(self.ask_depletion_ema).max(1e-12); + let queue_asym = (self.bid_depletion_ema - self.ask_depletion_ema) / max_dep; + + // [15] Order Count Flux + let total_flux = (self.ct_increase_sum + self.ct_decrease_sum) as f64; + let flux = if total_flux > 0.0 { + self.ct_increase_sum as f64 / total_flux + } else { 0.5 }; + + // [16] Intra-Bar Momentum (correlation proxy via sign of product of means) + let h1_mean = if self.half1_n > 0 { self.half1_sum / self.half1_n as f64 } else { 0.0 }; + let h2_mean = if self.half2_n > 0 { self.half2_sum / self.half2_n as f64 } else { 0.0 }; + let momentum = (h1_mean * h2_mean).signum() * (h1_mean * h2_mean).abs().sqrt().min(1.0); + + // [17] Microstructure Regime Score + let thin_book = 1.0 / (self.book_pressure_ema.abs() + 1.0); + let regime_score = 1.0 / (1.0 + (-5.0 * (aggression - 0.5) * spread_dyn * thin_book).exp()); + + // [18] OFI Acceleration + let ofi_accel = self.ofi_delta_ema; + + // [19] Toxicity Gradient + let tox_grad = self.vpin_delta_ema; + + [ofi_slope, rv, hawkes, book_pressure, spread_dyn, aggression, + queue_asym, flux, momentum, regime_score, ofi_accel, tox_grad] + } + + /// Reset for next bar. + pub fn reset(&mut self, bar_start_ns: u64, bar_duration_ns: u64) { + *self = Self::new(bar_start_ns, bar_duration_ns); + } +} +``` + +- [ ] **Step 3: Add `ExtendedOFIFeatures` struct** + +```rust +/// Extended OFI features: original 8 + 12 microstructure = 20 total. +#[derive(Debug, Clone)] +pub struct ExtendedOFIFeatures { + pub base: OFIFeatures, + pub micro: [f64; 12], +} + +impl ExtendedOFIFeatures { + pub fn to_array(&self) -> [f64; 20] { + let mut out = [0.0; 20]; + let base = self.base.to_array(); + out[..8].copy_from_slice(&base); + out[8..20].copy_from_slice(&self.micro); + out + } + + pub fn zeros() -> Self { + Self { base: OFIFeatures::zeros(), micro: [0.0; 12] } + } +} +``` + +- [ ] **Step 4: Verify compilation** + +```bash +SQLX_OFFLINE=true cargo check -p ml-features 2>&1 | tail -5 +``` + +- [ ] **Step 5: Commit** + +```bash +cd /home/jgrusewski/Work/foxhunt && git add crates/ml-features/src/ofi_calculator.rs && git commit -m "feat(tick): IncrementalMicrostructureCalculator — 12 new tick-level features + +MicrostructureState: OFI trajectory (online LinReg), realized variance, +Hawkes trade intensity, book pressure gradient, spread dynamics, +aggression ratio, queue depletion, order count flux, intra-bar momentum +(Welford), microstructure regime score, OFI acceleration, toxicity +gradient. All O(1) per tick, no allocation. ExtendedOFIFeatures [20]. + +Co-Authored-By: Claude Opus 4.6 (1M context) " +``` + +--- + +## Task 2: fxcache OFI_DIM 8→20 — Single Atomic Update + +Update ALL hardcoded OFI_DIM=8 literals across the codebase. This MUST be a single commit — partial updates break the build. + +**Files (ALL modified in this task):** +- Modify: `crates/ml/src/fxcache.rs` +- Modify: `crates/ml/examples/precompute_features.rs` +- Modify: `crates/ml/src/cuda_pipeline/gpu_experience_collector.rs` +- Modify: `crates/ml/src/cuda_pipeline/mod.rs` +- Modify: `crates/ml/src/cuda_pipeline/gpu_walk_forward.rs` +- Modify: `crates/ml/src/cuda_pipeline/backtest_gather_kernel.cu` +- Modify: `crates/ml/src/trainers/dqn/trainer/mod.rs` +- Modify: `crates/ml/src/trainers/dqn/trainer/constructor.rs` +- Modify: `crates/ml/src/trainers/dqn/trainer/metrics.rs` +- Modify: `crates/ml/src/trainers/dqn/data_loading.rs` +- Modify: `crates/ml/src/hyperopt/adapters/dqn.rs` +- Modify: `crates/ml/src/hyperopt/adapters/ppo.rs` +- Modify: `crates/ml/examples/train_baseline_rl.rs` +- Modify: `crates/ml/tests/fxcache_roundtrip_test.rs` +- Modify: `crates/ml/tests/smoke_test_real_data.rs` + +- [ ] **Step 1: Export OFI_DIM from fxcache.rs** + +In `crates/ml/src/fxcache.rs`, change constants: + +```rust +/// OFI vector dimensionality (20 MBP-10 microstructure features). +pub const OFI_DIM: usize = 20; + +/// Total f64 values per record: features + targets + OFI = 42 + 4 + 20 = 66. +const RECORD_F64_COUNT: usize = FEAT_DIM + TARGET_DIM + OFI_DIM; +``` + +Make `OFI_DIM` public (`pub const`) so other crates can import it. + +Also update the doc comment at line 16: `ofi_dim u16 = 20`. + +- [ ] **Step 2: Update all `[f64; 8]` → `[f64; 20]` in fxcache.rs** + +In `fxcache.rs`, change: +- Line 195: `pub ofi: Vec<[f64; OFI_DIM]>` — already uses constant, no change needed if OFI_DIM is updated +- Line 426: `vec![[0.5_f64; 8]; 10]` → `vec![[0.5_f64; OFI_DIM]; 10]` +- Line 442: `vec![[0.0_f64; 8]; 10]` → `vec![[0.0_f64; OFI_DIM]; 10]` +- Line 593: `vec![[0.0_f64; 8]; 5]` → `vec![[0.0_f64; OFI_DIM]; 5]` + +Note: `[f64; OFI_DIM]` works because OFI_DIM is a const. + +- [ ] **Step 3: Update precompute_features.rs** + +- Line 329: `Vec<[f64; 8]>` → `Vec<[f64; 20]>` (or import and use `OFI_DIM`) +- Line 417: `[0.0; 8]` → `[0.0; 20]` +- Line 420: `[0.0; 8]` → `[0.0; 20]` +- Line 500: `"8-dim"` → `"20-dim"` + +For now, the precompute still only fills 8 features (from existing OFICalculator). The remaining 12 are zero-padded. Task 3 will wire the new calculator. + +- [ ] **Step 4: Update gpu_experience_collector.rs** + +- Line 747: `if state_dim >= market_dim + portfolio_dim + 8 { 8 }` → `if state_dim >= market_dim + portfolio_dim + 20 { 20 }` + +- [ ] **Step 5: Update cuda_pipeline/mod.rs** + +- Line 301: `ofi: &[[f64; 8]]` → `ofi: &[[f64; 20]]` +- Line 342: `[0.0_f32; 8]` → `[0.0_f32; 20]` +- Line 366: `ofi_data: &[[f64; 8]]` → `ofi_data: &[[f64; 20]]` +- Line 402: `if self.ofi_features.is_some() { 8 }` → `{ 20 }` +- Line 477: same → `{ 20 }` +- Line 516: same → `{ 20 }` + +- [ ] **Step 6: Update gpu_walk_forward.rs** + +- Line 554: `ofi_data: Option<&[[f64; 8]]>` → `Option<&[[f64; 20]]>` +- Line 602: `[0.0_f32; 8]` → `[0.0_f32; 20]` + +- [ ] **Step 7: Update backtest_gather_kernel.cu** + +- Line 45: comment `// 0 = no OFI, 8 = standard OFI features` → `// 0 = no OFI, 20 = extended microstructure` + +- [ ] **Step 8: Update trainers/dqn/trainer/mod.rs** + +- Line 411: `pub ofi_features: Option>` → `Arc<[[f64; 20]]>` +- Line 739: `Vec<[f64; 8]>` → `Vec<[f64; 20]>` +- Line 1215: `ofi: &[[f64; 8]]` → `ofi: &[[f64; 20]]` + +- [ ] **Step 9: Update constructor.rs** + +- Line 53: `if ofi_pre { 80 } else { 72 }` → `if ofi_pre { 88 } else { 72 }` (42+8+16+20=86, aligned (86+7)&!7=88) +- Line 255: `if ofi_enabled { 74 } else { 66 }` → `if ofi_enabled { 86 } else { 66 }` (42+8+16+20=86) +- Line 256: alignment stays `(raw_state_dim + 7) & !7` + +- [ ] **Step 10: Update metrics.rs** + +- Line 488: `ofi_dim: if ofi_enabled { 8 }` → `{ 20 }` + +- [ ] **Step 11: Update data_loading.rs** + +- Line 402: `[0.0; 8]` → `[0.0; 20]` +- Line 405: `[0.0; 8]` → `[0.0; 20]` +- Line 408: `[0.0; 8]` → `[0.0; 20]` + +- [ ] **Step 12: Update hyperopt/adapters/dqn.rs** + +- Line 538: `Option>` → `Arc<[[f64; 20]]>` +- Line 920: `Vec<[f64; 8]>` → `Vec<[f64; 20]>` +- Line 1208: `Option>` → `Arc<[[f64; 20]]>` +- Line 1437: `ofi_dim: if ofi_enabled { 8 }` → `{ 20 }` + +- [ ] **Step 13: Update hyperopt/adapters/ppo.rs** + +- Line 1498: `Option>` → `Option>` + +- [ ] **Step 14: Update train_baseline_rl.rs** + +- Line 594: `vec![[0.0_f64; 8]; n]` → `vec![[0.0_f64; 20]; n]` + +- [ ] **Step 15: Update test files** + +- `tests/fxcache_roundtrip_test.rs:38`: `fn make_ofi(n: usize) -> Vec<[f64; 8]>` → `[f64; 20]` +- `tests/fxcache_roundtrip_test.rs:190`: `vec![[0.0_f64; 8]; ...]` → `[0.0_f64; 20]` +- `tests/smoke_test_real_data.rs:444`: `Option>` → `[f64; 20]` + +- [ ] **Step 16: Verify compilation** + +```bash +SQLX_OFFLINE=true cargo check --workspace 2>&1 | tail -10 +``` + +Fix ANY remaining `[f64; 8]` or literal `8` references related to OFI. + +- [ ] **Step 17: Run tests** + +```bash +SQLX_OFFLINE=true cargo test -p ml --lib 2>&1 | tail -5 +``` + +Note: fxcache roundtrip tests may fail if they create fxcache files with old OFI_DIM=8. The test code was updated in step 15 — verify they pass with the new dimension. + +- [ ] **Step 18: Commit** + +```bash +cd /home/jgrusewski/Work/foxhunt && git add -A && git commit -m "feat(tick): OFI_DIM 8→20 — atomic update across 15 files + +fxcache: OFI_DIM=20, RECORD_F64_COUNT=66, 272 bytes/bar. +constructor: state_dim 74→86 (aligned 88) with OFI. +experience collector: ofi_dim detection 8→20. +All [f64; 8] → [f64; 20]. All literal 8 → 20. +Existing 8 features preserved at indices 0-7. +New 12 features zero-padded until precompute is extended. + +Co-Authored-By: Claude Opus 4.6 (1M context) " +``` + +--- + +## Task 3: Extend Precompute Pipeline + +Wire `MicrostructureState` into `precompute_features.rs` to compute all 20 OFI features per bar from tick data. + +**Files:** +- Modify: `crates/ml/examples/precompute_features.rs` + +- [ ] **Step 1: Import MicrostructureState** + +Add import at top of `precompute_features.rs`: + +```rust +use ml_features::ofi_calculator::MicrostructureState; +``` + +- [ ] **Step 2: Wire into per-bar OFI computation** + +Find the OFI computation loop (around line 329). Currently it creates `OFICalculator`, processes snapshots per bar, and outputs `[f64; 8]`. Change to: + +For each bar: +1. Create `MicrostructureState::new(bar_start_ns, 60_000_000_000)` (60s bar) +2. For each MBP-10 snapshot within the bar: call `micro_state.update_snapshot(snapshot)` +3. For each trade within the bar: call `micro_state.update_trade(price, volume, is_buy, ts)` +4. After OFI calculation: call `micro_state.update_ofi_derived(ofi.ofi_level1, ofi.vpin)` +5. Get `micro_12 = micro_state.snapshot()` +6. Combine: `base_8 ++ micro_12 = [f64; 20]` + +- [ ] **Step 3: Verify compilation** + +```bash +SQLX_OFFLINE=true cargo check -p ml 2>&1 | tail -5 +``` + +- [ ] **Step 4: Commit** + +```bash +cd /home/jgrusewski/Work/foxhunt && git add crates/ml/examples/precompute_features.rs && git commit -m "feat(tick): precompute pipeline extended — 20 OFI features per bar + +MicrostructureState processes all MBP-10 snapshots + trades per bar. +12 new tick-level features: OFI trajectory, realized variance, Hawkes +intensity, book pressure, spread dynamics, aggression, queue depletion, +order count flux, intra-bar momentum, regime score, OFI acceleration, +toxicity gradient. Zero-padded when MBP-10 data unavailable. + +Co-Authored-By: Claude Opus 4.6 (1M context) " +``` + +--- + +## Task 4: Regenerate fxcache on PVC + Smoke Test + +Regenerate all fxcache files with 20-dim OFI, verify model loads correctly. + +**Files:** +- No code changes — operational task + +- [ ] **Step 1: Regenerate local test data fxcache** + +```bash +SQLX_OFFLINE=true cargo run --release --example precompute_features -- --data-dir test_data/futures-baseline 2>&1 | tail -10 +``` + +- [ ] **Step 2: Verify fxcache header** + +```bash +xxd test_data/futures-baseline/*.fxcache | head -5 +``` + +Check that `ofi_dim` field (bytes 14-15) is `0x14 0x00` (20 in little-endian). + +- [ ] **Step 3: Run smoke test** + +```bash +SQLX_OFFLINE=true FOXHUNT_TEST_DATA=test_data/futures-baseline cargo test -p ml --lib -- test_generalization_components_smoke --include-ignored --nocapture 2>&1 | tail -20 +``` + +- [ ] **Step 4: Run compute-sanitizer** + +```bash +FOXHUNT_TEST_DATA=test_data/futures-baseline compute-sanitizer --tool memcheck --print-limit 5 target/debug/deps/ml-* "test_generalization_components_smoke" --test-threads=1 --include-ignored 2>&1 | grep "ERROR SUMMARY" +``` + +**Target: 0 errors.** + +- [ ] **Step 5: Run full test suite** + +```bash +SQLX_OFFLINE=true cargo test -p ml --lib 2>&1 | tail -5 +``` + +**Target: 899+ passed, 0 failed.** + +- [ ] **Step 6: Commit test data** + +```bash +cd /home/jgrusewski/Work/foxhunt && git add test_data/ && git commit -m "chore(tick): regenerate test fxcache with 20-dim OFI + +Smoke test passes. compute-sanitizer: 0 errors. 899+ tests pass. +Feature vector: 42 + 20 = 62 dimensions. + +Co-Authored-By: Claude Opus 4.6 (1M context) " +``` + +--- + +## Self-Review + +**Spec coverage:** +- ✅ 12 new features in MicrostructureState — Task 1 +- ✅ fxcache OFI_DIM 8→20 — Task 2 +- ✅ All hardcoded `8` literals enumerated — Task 2 (15 files) +- ✅ Precompute pipeline extension — Task 3 +- ✅ constructor state_dim 74→86 — Task 2 Step 9 +- ✅ Experience collector ofi_dim detection — Task 2 Step 4 +- ✅ Atomic deploy (single commit for all OFI changes) — Task 2 +- ✅ Smoke test + compute-sanitizer — Task 4 +- ✅ O(1) per tick, no allocation — MicrostructureState uses only scalar running stats +- ✅ snapshot() is pure read, non-mutating — const &self + +**Not in this plan (separate future tasks):** +- Databento live integration (TICK Task 5) +- H100 between-bar speculative compute (TICK Task 6) +- PVC regeneration via Argo workflow (operational) + +**Placeholder scan:** No TBD, TODO, or "implement later" found. + +**Type consistency:** `[f64; 20]` used consistently. `MicrostructureState` name consistent. `ExtendedOFIFeatures` used in Task 1, referenced in Task 3.