From 1764cc394b9187d8bce05b88415471d46e81481b Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 22 May 2026 14:56:50 +0200 Subject: [PATCH] revert(aux): F2 mid_price_f32 NaN-on-one-sided was a wrong hypothesis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 2 of aux-diagnosis-deeper plan (NumPy on local ES data) falsified the F2 hypothesis: 1. Local data has 0% zero-bid/zero-ask records. F2's "one-sided book contamination" was incorrect — the actual databento sentinel for missing price is INT64_MAX × 1e-9 ≈ 9.22e9 (a HUGE POSITIVE number that passes the `> 0` check). F2 was a no-op on real data. 2. Sentinel rate on local Q1 is 0.006% — negligible. 3. Local pos_fraction at K=10 is 19.59%, cluster reports 35.07%. The ~75% gap is plausibly explained by overnight session gaps in the full 5M-record quarter file (which I didn't see in my 500k local sample). These are real price discontinuities, not "contamination". 4. F2 introduced an epoch-4 NaN explosion (alpha-perception-7shgw) because NaN cascaded through the σ_K Welford rolling computation. Reverting: - crates/ml-alpha/src/data/loader.rs::mid_price_f32 → blind average - crates/ml-alpha/src/multi_horizon_labels.rs:218 → is_finite only KEEPING: - F1 dir_acc fix in perception.rs:3647 — empirically working (metric hovers ~0.5 instead of pinned sub-chance) cargo check --workspace --all-targets clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/ml-alpha/src/data/loader.rs | 17 +---------------- crates/ml-alpha/src/multi_horizon_labels.rs | 8 +------- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/crates/ml-alpha/src/data/loader.rs b/crates/ml-alpha/src/data/loader.rs index f99d36f08..3a3daac54 100644 --- a/crates/ml-alpha/src/data/loader.rs +++ b/crates/ml-alpha/src/data/loader.rs @@ -507,26 +507,11 @@ impl MultiHorizonLoader { } } -/// Mid price at level 0; returns NaN for one-sided / empty books per -/// the F2 finding (2026-05-22). MBP-10 snapshots at session boundaries, -/// halts, or stale level-0 vacancies have `bid_px = 0` or `ask_px = 0`; -/// the previous blind average produced `mid = real_price / 2 ≈ 2750` -/// (single side) or `mid = 0` (both sides empty). Downstream label -/// generation in `multi_horizon_labels::generate_outcome_labels_ab` -/// only guarded `is_finite()` (not `> 0`), so transitions from mid=0 -/// to mid≈5500 produced `ΔP ≈ 5500`, spuriously satisfying the cost -/// threshold and inflating `pos_fraction` from the expected ~5% to -/// the observed ~40%. NaN return cascades through label generation -/// and the BCE NaN-mask handles it cleanly. fn mid_price_f32(s: &Mbp10Snapshot) -> f32 { let l = s.levels.first().copied().unwrap_or_else(BidAskPair::empty); let bid = BidAskPair::price_to_f64(l.bid_px); let ask = BidAskPair::price_to_f64(l.ask_px); - if bid > 0.0 && ask > 0.0 { - (0.5 * (bid + ask)) as f32 - } else { - f32::NAN - } + (0.5 * (bid + ask)) as f32 } fn convert( diff --git a/crates/ml-alpha/src/multi_horizon_labels.rs b/crates/ml-alpha/src/multi_horizon_labels.rs index 43340af01..3600208a8 100644 --- a/crates/ml-alpha/src/multi_horizon_labels.rs +++ b/crates/ml-alpha/src/multi_horizon_labels.rs @@ -215,13 +215,7 @@ pub fn generate_outcome_labels_ab( for t in 0..n - k { let p_t = prices[t]; let p_kt = prices[t + k]; - // F2 fix (2026-05-22): also reject p <= 0 to match `generate_labels:75`. - // mid_price_f32 in loader.rs now returns NaN for one-sided books, so - // the is_finite guard alone would suffice — this is belt+suspenders - // for callers that bypass the loader (e.g. direct unit tests passing - // raw price arrays). Without the `> 0` guard a transition from - // mid=0 to mid≈5500 produces ΔP=5500, spuriously triggering y_prof=1. - if !p_t.is_finite() || !p_kt.is_finite() || p_t <= 0.0 || p_kt <= 0.0 { + if !p_t.is_finite() || !p_kt.is_finite() { continue; } let delta = p_kt - p_t;