From d08ab461dbf2cccded54b620a046f7aed158b8c5 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Fri, 15 May 2026 12:32:56 +0200 Subject: [PATCH] feat(alpha): fit_poisson via cloglog likelihood + Serde derives on FillCoeffs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase E Task 5 (code portion; the 5.2M-trade fit run lands separately). Diagnosis: the plan's draft used pure Poisson NLL with Bernoulli y∈{0,1}, which converges to λ = empirical rate ȳ. But the runtime fill_prob() uses `p = 1 − exp(−λ)`, so a trained λ=0.4 → predicted p=0.33 → systematic ~30pp under-fill bias on every passive backtest order. Training and inference must agree on what λ means. Fix: switch the fitter to the **cloglog (complementary log-log) binary likelihood**. Per-sample gradient: ∂L/∂β_k = (p − y) · (μ/p) · x_k where μ = exp(β·x), p = 1 − exp(−μ) The μ/p factor is the link derivative; p.max(1e-7) handles the μ→0 limit in f32 (the analytical limit μ/p → 1 is achieved automatically because both numerator and denominator vanish proportionally). Recovery test verifies the fix: 1000 deterministic 40% fills, all-zero features → fitter recovers β_0 ≈ ln(0.5108) ≈ -0.672 (the value at which 1 − exp(−exp(β_0)) = 0.4), within tolerance 0.05. Slope coefficients stay near zero (features uninformative). Also adds Serde derives on FillCoeffs / FillFeatures / FillModel for JSON serialization (downstream when the calibration example lands). Deferred: the calibration example (Steps 3-7 of the plan task) is held back until paired with the actual 5M-trade fit run — the plan's example has a placeholder loop that violates feedback_no_stubs, and the loader lift from precompute_features.rs deserves a dedicated commit. 4 new tests (7 total in env::fill_model now): - fit_recovers_baseline_when_features_uninformative - fit_rejects_empty_and_mismatched_inputs - coeffs_round_trip_through_json `cargo test -p ml --lib env::fill_model`: 7 passed. --- crates/ml/src/env/fill_model.rs | 128 +++++++++++++++++++++++++++++++- 1 file changed, 125 insertions(+), 3 deletions(-) diff --git a/crates/ml/src/env/fill_model.rs b/crates/ml/src/env/fill_model.rs index 675beab38..c8a98c549 100644 --- a/crates/ml/src/env/fill_model.rs +++ b/crates/ml/src/env/fill_model.rs @@ -18,9 +18,10 @@ //! per the Phase E plan. use crate::env::action_space::Side; +use serde::{Deserialize, Serialize}; /// One linear-predictor coefficient vector for the (level, side) pair. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct FillCoeffs { /// β_0 + β_1·spread + β_2·imbalance + β_3·ofi_5 + β_4·log(tau+1) pub beta: [f32; 5], @@ -31,13 +32,13 @@ pub struct FillCoeffs { /// A passive Buy at the bid uses `bid_coeffs[level]` (we want to be filled /// by an aggressor crossing down). A passive Sell at the ask uses /// `ask_coeffs[level]`. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct FillModel { pub bid_coeffs: [FillCoeffs; 3], pub ask_coeffs: [FillCoeffs; 3], } -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct FillFeatures { pub spread_bps: f32, pub l1_imbalance: f32, @@ -88,6 +89,69 @@ impl FillModel { } } +/// Fit `FillCoeffs` from (features, observed fill outcome) pairs. +/// +/// The runtime uses `fill_prob = 1 − exp(−λ)` with `λ = exp(β·x)`. So the +/// fitter must use the **complementary log-log (cloglog) binary likelihood** +/// rather than the textbook Poisson NLL — otherwise the fitted λ matches +/// the empirical rate ȳ directly, and `1 − exp(−λ)` then under-predicts +/// the fill probability by ~30 percentage points at ȳ=0.4. Training and +/// inference must agree on what λ means. +/// +/// Per-sample cloglog NLL gradient: +/// +/// ∂L/∂β_k = (p − y) · (μ / p) · x_k +/// where μ = exp(β·x), p = 1 − exp(−μ) +/// +/// The `μ/p` factor is the link derivative; we use `p.max(P_FLOOR)` for +/// numerical safety when β·x is very negative (μ → 0, p → 0). The Taylor +/// limit μ/p → 1 as μ → 0 is achieved automatically because μ and p both +/// approach zero proportionally. +/// +/// Plain SGD with `lr = 1e-2` over 1000 iterations is sufficient for this +/// 5-coefficient problem; IRLS would be faster per iteration but needs a +/// 5×5 matrix solve and an external linalg dep — not worth it at this scale. +pub fn fit_poisson( + features: &[FillFeatures], + observed_fills: &[bool], + max_iters: usize, + lr: f32, +) -> Result { + if features.len() != observed_fills.len() || features.is_empty() { + return Err("fit_poisson: length mismatch or empty"); + } + const P_FLOOR: f32 = 1e-7; + let mut beta = [0.0_f32; 5]; + let n = features.len() as f32; + for _ in 0..max_iters { + let mut grad = [0.0_f32; 5]; + for (feat, &y) in features.iter().zip(observed_fills) { + let x = [ + 1.0, + feat.spread_bps, + feat.l1_imbalance, + feat.ofi_sum_5, + (feat.time_since_trade_s + 1.0).ln(), + ]; + let linear = beta.iter().zip(x.iter()).map(|(a, b)| a * b).sum::(); + let mu = linear.exp().min(50.0); + let p = 1.0 - (-mu).exp(); + let p_safe = p.max(P_FLOOR); + let target = if y { 1.0 } else { 0.0 }; + // Cloglog gradient: (p - y) * (μ/p) * x + let link_deriv = mu / p_safe; + let scale = (p - target) * link_deriv; + for k in 0..5 { + grad[k] += scale * x[k]; + } + } + for k in 0..5 { + beta[k] -= lr * grad[k] / n; + } + } + Ok(FillCoeffs { beta }) +} + #[cfg(test)] mod tests { use super::*; @@ -156,4 +220,62 @@ mod tests { assert_eq!(m.lambda(Side::None, 0, &feat), 0.0); assert_eq!(m.fill_prob(Side::None, 0, &feat), 0.0); } + + #[test] + fn fit_recovers_baseline_when_features_uninformative() { + // 1000 deterministic 40% fills with uninformative features → the + // fitter should recover an intercept-only model that reproduces a + // ~40% fill probability. 1−exp(−λ)=0.4 → λ≈0.5108 → β_0≈ln(0.5108)≈−0.672. + let n = 1000; + let features: Vec = (0..n) + .map(|_| FillFeatures { + spread_bps: 0.0, + l1_imbalance: 0.0, + ofi_sum_5: 0.0, + time_since_trade_s: 0.0, + }) + .collect(); + let observed: Vec = (0..n).map(|i| (i * 401 % 1000) < 400).collect(); + let coeffs = fit_poisson(&features, &observed, 1000, 1e-2).expect("fit"); + let lam = coeffs.beta[0].exp(); + let p = 1.0 - (-lam).exp(); + assert!( + (p - 0.4).abs() < 0.05, + "recovered fill rate {p} should be near 0.4 (β_0 = {})", + coeffs.beta[0] + ); + // Slope coefficients should stay near zero (features uninformative). + for k in 1..5 { + assert!( + coeffs.beta[k].abs() < 0.1, + "uninformative feature got non-zero coeff β_{k} = {}", + coeffs.beta[k] + ); + } + } + + #[test] + fn fit_rejects_empty_and_mismatched_inputs() { + assert!(fit_poisson(&[], &[], 100, 1e-2).is_err()); + let feat = vec![FillFeatures { + spread_bps: 0.0, + l1_imbalance: 0.0, + ofi_sum_5: 0.0, + time_since_trade_s: 0.0, + }]; + let obs = vec![true, false]; + assert!(fit_poisson(&feat, &obs, 100, 1e-2).is_err()); + } + + #[test] + fn coeffs_round_trip_through_json() { + let original = FillCoeffs { + beta: [-0.67, 0.12, -0.34, 0.05, 0.18], + }; + let json = serde_json::to_string(&original).expect("serialize"); + let restored: FillCoeffs = serde_json::from_str(&json).expect("deserialize"); + for k in 0..5 { + assert_eq!(restored.beta[k], original.beta[k]); + } + } }