From e6b3fffa6ddc1f90d82bd7ea50d7b8439d6df46d Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Thu, 16 Apr 2026 23:43:55 +0200 Subject: [PATCH] =?UTF-8?q?spec:=20fix=204=20critical=20issues=20from=20re?= =?UTF-8?q?view=20=E2=80=94=20OFI=5FDIM=20cascade,=20deploy=20atomicity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Enumerate all 6+ files with hardcoded OFI_DIM=8 literal 2. Document atomic deploy requirement (code + fxcache together) 3. Rename Depth Renewal Rate → Order Count Flux (MBP-10 approximation) 4. Fix Intra-Bar Momentum to Welford online covariance (O(1), no Vec) 5. Clarify snapshot() is pure read, non-mutating, idempotent 6. Fix precompute time: 8s CPU + 5min I/O = ~5-10min total Co-Authored-By: Claude Opus 4.6 (1M context) --- ...tick-microstructure-intelligence-design.md | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/docs/superpowers/specs/2026-04-16-tick-microstructure-intelligence-design.md b/docs/superpowers/specs/2026-04-16-tick-microstructure-intelligence-design.md index 60456041d..47b2f8570 100644 --- a/docs/superpowers/specs/2026-04-16-tick-microstructure-intelligence-design.md +++ b/docs/superpowers/specs/2026-04-16-tick-microstructure-intelligence-design.md @@ -44,7 +44,7 @@ All features are **incremental** — O(1) per tick via running statistics (EMAs, | 12 | **Spread Dynamics** | Running min/max/mean of spread | (max_spread - min_spread) / mean_spread. Widening = market makers pulling back = volatility incoming. | | 13 | **Aggression Ratio** | Two counters (ask_trades, total) | trades_at_ask / total_trades (Lee-Ready). > 0.6 = strong buy pressure. | | 14 | **Queue Depletion Asymmetry** | Running depletion rates bid/ask | (bid_depletion - ask_depletion) / max. Reveals hidden institutional flow direction. | -| 15 | **Depth Renewal Rate** | Counters (new_orders, cancels) at top 3 | new_placed / canceled. Market makers refreshing = confidence. Not refreshing = about to pull. | +| 15 | **Order Count Flux** | Running delta of bid_ct/ask_ct at top 3 | Δbid_ct>0 → new order proxy, Δbid_ct<0 → cancel/fill proxy. Approximation: MBP-10 snapshots can't distinguish cancel from fill. Treat as market maker activity volatility — high flux = active refresh, low flux = about to pull. | ### NOVELS (new combinations) @@ -83,8 +83,10 @@ pub struct IncrementalMicrostructureCalculator { prev_ask_sz_top3: u64, new_orders_top3: u64, // depth renewal counters cancels_top3: u64, - returns_first_half: Vec, // for intra-bar momentum - returns_second_half: Vec, + // Intra-bar momentum: Welford online covariance (O(1) per tick, no allocation) + half1_sum_x: f64, half1_sum_x2: f64, half1_n: u64, + half2_sum_x: f64, half2_sum_x2: f64, half2_n: u64, + cross_sum_xy: f64, // for correlation between halves ofi_prev: f64, // for acceleration ofi_delta_ema: f64, vpin_prev: f64, // for toxicity gradient @@ -100,8 +102,8 @@ pub struct IncrementalMicrostructureCalculator { impl IncrementalMicrostructureCalculator { pub fn new(bar_duration_ns: u64) -> Self; pub fn update(&mut self, snapshot: &Mbp10Snapshot, trade: Option<&Trade>) -> (); - pub fn snapshot(&self) -> MicrostructureFeatures; // [20] f64 - pub fn reset(&mut self); // call at bar boundary + pub fn snapshot(&self) -> MicrostructureFeatures; // [20] f64 — pure read, non-mutating, idempotent + pub fn reset(&mut self); // call at bar boundary — clears all running state } ``` @@ -136,6 +138,21 @@ const OFI_DIM: usize = 20; // was 8 const RECORD_F64_COUNT: usize = 66; // was 54 (42+4+20) ``` +**CRITICAL: OFI_DIM=8 is load-bearing in 6+ files.** All must be updated atomically: + +| File | Location | What to change | +|------|----------|----------------| +| `fxcache.rs:124` | `validate()` | Reads `ofi_dim` from header — will reject old files. Code and data deploy atomically. | +| `fxcache.rs:195,377` | `Vec<[f64; OFI_DIM]>` | Compile-time array generic — all callers cascade. | +| `precompute_features.rs:329,342,434` | `[f64; 8]` literals | Change to `[f64; OFI_DIM]` or use `fxcache::OFI_DIM`. | +| `gpu_experience_collector.rs:747` | `let ofi_dim = if ... { 8 }` | Hardcoded detection. Must use `OFI_DIM` constant. | +| `cuda_pipeline/mod.rs:402,477,516` | `if ofi { 8 } else { 0 }` | Same hardcoded literal. | +| `constructor.rs:52-53,255` | `raw_state_dim = 74` (42+8+16+8) | Must become `42+8+16+OFI_DIM`. Aligned: `(86+7)&!7 = 88`. | +| `hyperopt/adapters/dqn.rs:1437` | Literal `8` | Use `OFI_DIM`. | +| Test files | `[0.0; 8]`, `vec![[0.0; 8]; n]` | All OFI fixtures must use `OFI_DIM`. | + +**Deploy rule:** Code and fxcache data must deploy atomically. New code with OFI_DIM=20 will reject old fxcache (ofi_dim=8 in header). Regenerate fxcache BEFORE deploying new code, or deploy both in the same Argo workflow step. + --- ## Precompute Pipeline @@ -153,7 +170,7 @@ For each minute bar [bar_start, bar_end): The existing OFI calculation path in `precompute_features.rs` already loads MBP-10 snapshots per bar and calls `OFICalculator`. The change: replace `OFICalculator` with `IncrementalMicrostructureCalculator` which computes all 20 features (including the existing 8). -**Precompute time:** 4M bars × ~2000 ticks/bar = 8B ticks × <1μs = ~8 seconds. Negligible. +**Precompute time:** Calculator CPU time is ~8 seconds (8B ticks × <1μs). But disk I/O dominates — 148GB MBP-10 at ~500MB/s = ~5 minutes read time. Total precompute: **~5-10 minutes** depending on I/O. Set Argo workflow timeout accordingly. --- @@ -283,7 +300,8 @@ impl TickProcessor { | Risk | Mitigation | |------|-----------| -| Existing fxcache files incompatible | Regenerate all. Precompute is ~8 seconds for 4M bars. | +| Existing fxcache files incompatible | Regenerate all (~5-10 min). Code and data deploy atomically — new code rejects old fxcache. | +| OFI_DIM=8 hardcoded in 6+ files | All sites enumerated in spec. Must update atomically in single commit. Use shared constant, not literals. | | 12 new features → overfitting on training data | ISV feature gating suppresses irrelevant features. 12 features / 4M bars = negligible risk. | | IncrementalMicrostructureCalculator state drift | Reset at every bar boundary. Running statistics are bounded by definition. | | Databento live feed drops/gaps | Calculator handles missing ticks gracefully (EMA continues with last value). At bar close, features reflect whatever ticks arrived. |