feat(alpha): fit_poisson via cloglog likelihood + Serde derives on FillCoeffs

Phase E Task 5 (code portion; the 5.2M-trade fit run lands separately).

Diagnosis: the plan's draft used pure Poisson NLL with Bernoulli y∈{0,1},
which converges to λ = empirical rate ȳ. But the runtime fill_prob() uses
`p = 1 − exp(−λ)`, so a trained λ=0.4 → predicted p=0.33 → systematic
~30pp under-fill bias on every passive backtest order. Training and
inference must agree on what λ means.

Fix: switch the fitter to the **cloglog (complementary log-log) binary
likelihood**. Per-sample gradient:

  ∂L/∂β_k = (p − y) · (μ/p) · x_k
  where μ = exp(β·x),  p = 1 − exp(−μ)

The μ/p factor is the link derivative; p.max(1e-7) handles the μ→0 limit
in f32 (the analytical limit μ/p → 1 is achieved automatically because
both numerator and denominator vanish proportionally).

Recovery test verifies the fix: 1000 deterministic 40% fills, all-zero
features → fitter recovers β_0 ≈ ln(0.5108) ≈ -0.672 (the value at which
1 − exp(−exp(β_0)) = 0.4), within tolerance 0.05. Slope coefficients
stay near zero (features uninformative).

Also adds Serde derives on FillCoeffs / FillFeatures / FillModel for JSON
serialization (downstream when the calibration example lands).

Deferred: the calibration example (Steps 3-7 of the plan task) is held
back until paired with the actual 5M-trade fit run — the plan's example
has a placeholder loop that violates feedback_no_stubs, and the loader
lift from precompute_features.rs deserves a dedicated commit.

4 new tests (7 total in env::fill_model now):
  - fit_recovers_baseline_when_features_uninformative
  - fit_rejects_empty_and_mismatched_inputs
  - coeffs_round_trip_through_json

`cargo test -p ml --lib env::fill_model`: 7 passed.
This commit is contained in:
jgrusewski
2026-05-15 12:32:56 +02:00
parent a5de50503f
commit d08ab461db

View File

@@ -18,9 +18,10 @@
//! per the Phase E plan. //! per the Phase E plan.
use crate::env::action_space::Side; use crate::env::action_space::Side;
use serde::{Deserialize, Serialize};
/// One linear-predictor coefficient vector for the (level, side) pair. /// One linear-predictor coefficient vector for the (level, side) pair.
#[derive(Debug, Clone, Copy)] #[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct FillCoeffs { pub struct FillCoeffs {
/// β_0 + β_1·spread + β_2·imbalance + β_3·ofi_5 + β_4·log(tau+1) /// β_0 + β_1·spread + β_2·imbalance + β_3·ofi_5 + β_4·log(tau+1)
pub beta: [f32; 5], pub beta: [f32; 5],
@@ -31,13 +32,13 @@ pub struct FillCoeffs {
/// A passive Buy at the bid uses `bid_coeffs[level]` (we want to be filled /// A passive Buy at the bid uses `bid_coeffs[level]` (we want to be filled
/// by an aggressor crossing down). A passive Sell at the ask uses /// by an aggressor crossing down). A passive Sell at the ask uses
/// `ask_coeffs[level]`. /// `ask_coeffs[level]`.
#[derive(Debug, Clone)] #[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FillModel { pub struct FillModel {
pub bid_coeffs: [FillCoeffs; 3], pub bid_coeffs: [FillCoeffs; 3],
pub ask_coeffs: [FillCoeffs; 3], pub ask_coeffs: [FillCoeffs; 3],
} }
#[derive(Debug, Clone, Copy)] #[derive(Debug, Clone, Copy, Serialize, Deserialize)]
pub struct FillFeatures { pub struct FillFeatures {
pub spread_bps: f32, pub spread_bps: f32,
pub l1_imbalance: f32, pub l1_imbalance: f32,
@@ -88,6 +89,69 @@ impl FillModel {
} }
} }
/// Fit `FillCoeffs` from (features, observed fill outcome) pairs.
///
/// The runtime uses `fill_prob = 1 exp(−λ)` with `λ = exp(β·x)`. So the
/// fitter must use the **complementary log-log (cloglog) binary likelihood**
/// rather than the textbook Poisson NLL — otherwise the fitted λ matches
/// the empirical rate ȳ directly, and `1 exp(−λ)` then under-predicts
/// the fill probability by ~30 percentage points at ȳ=0.4. Training and
/// inference must agree on what λ means.
///
/// Per-sample cloglog NLL gradient:
///
/// ∂L/∂β_k = (p y) · (μ / p) · x_k
/// where μ = exp(β·x), p = 1 exp(−μ)
///
/// The `μ/p` factor is the link derivative; we use `p.max(P_FLOOR)` for
/// numerical safety when β·x is very negative (μ → 0, p → 0). The Taylor
/// limit μ/p → 1 as μ → 0 is achieved automatically because μ and p both
/// approach zero proportionally.
///
/// Plain SGD with `lr = 1e-2` over 1000 iterations is sufficient for this
/// 5-coefficient problem; IRLS would be faster per iteration but needs a
/// 5×5 matrix solve and an external linalg dep — not worth it at this scale.
pub fn fit_poisson(
features: &[FillFeatures],
observed_fills: &[bool],
max_iters: usize,
lr: f32,
) -> Result<FillCoeffs, &'static str> {
if features.len() != observed_fills.len() || features.is_empty() {
return Err("fit_poisson: length mismatch or empty");
}
const P_FLOOR: f32 = 1e-7;
let mut beta = [0.0_f32; 5];
let n = features.len() as f32;
for _ in 0..max_iters {
let mut grad = [0.0_f32; 5];
for (feat, &y) in features.iter().zip(observed_fills) {
let x = [
1.0,
feat.spread_bps,
feat.l1_imbalance,
feat.ofi_sum_5,
(feat.time_since_trade_s + 1.0).ln(),
];
let linear = beta.iter().zip(x.iter()).map(|(a, b)| a * b).sum::<f32>();
let mu = linear.exp().min(50.0);
let p = 1.0 - (-mu).exp();
let p_safe = p.max(P_FLOOR);
let target = if y { 1.0 } else { 0.0 };
// Cloglog gradient: (p - y) * (μ/p) * x
let link_deriv = mu / p_safe;
let scale = (p - target) * link_deriv;
for k in 0..5 {
grad[k] += scale * x[k];
}
}
for k in 0..5 {
beta[k] -= lr * grad[k] / n;
}
}
Ok(FillCoeffs { beta })
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
@@ -156,4 +220,62 @@ mod tests {
assert_eq!(m.lambda(Side::None, 0, &feat), 0.0); assert_eq!(m.lambda(Side::None, 0, &feat), 0.0);
assert_eq!(m.fill_prob(Side::None, 0, &feat), 0.0); assert_eq!(m.fill_prob(Side::None, 0, &feat), 0.0);
} }
#[test]
fn fit_recovers_baseline_when_features_uninformative() {
// 1000 deterministic 40% fills with uninformative features → the
// fitter should recover an intercept-only model that reproduces a
// ~40% fill probability. 1exp(−λ)=0.4 → λ≈0.5108 → β_0≈ln(0.5108)≈0.672.
let n = 1000;
let features: Vec<FillFeatures> = (0..n)
.map(|_| FillFeatures {
spread_bps: 0.0,
l1_imbalance: 0.0,
ofi_sum_5: 0.0,
time_since_trade_s: 0.0,
})
.collect();
let observed: Vec<bool> = (0..n).map(|i| (i * 401 % 1000) < 400).collect();
let coeffs = fit_poisson(&features, &observed, 1000, 1e-2).expect("fit");
let lam = coeffs.beta[0].exp();
let p = 1.0 - (-lam).exp();
assert!(
(p - 0.4).abs() < 0.05,
"recovered fill rate {p} should be near 0.4 (β_0 = {})",
coeffs.beta[0]
);
// Slope coefficients should stay near zero (features uninformative).
for k in 1..5 {
assert!(
coeffs.beta[k].abs() < 0.1,
"uninformative feature got non-zero coeff β_{k} = {}",
coeffs.beta[k]
);
}
}
#[test]
fn fit_rejects_empty_and_mismatched_inputs() {
assert!(fit_poisson(&[], &[], 100, 1e-2).is_err());
let feat = vec![FillFeatures {
spread_bps: 0.0,
l1_imbalance: 0.0,
ofi_sum_5: 0.0,
time_since_trade_s: 0.0,
}];
let obs = vec![true, false];
assert!(fit_poisson(&feat, &obs, 100, 1e-2).is_err());
}
#[test]
fn coeffs_round_trip_through_json() {
let original = FillCoeffs {
beta: [-0.67, 0.12, -0.34, 0.05, 0.18],
};
let json = serde_json::to_string(&original).expect("serialize");
let restored: FillCoeffs = serde_json::from_str(&json).expect("deserialize");
for k in 0..5 {
assert_eq!(restored.beta[k], original.beta[k]);
}
}
} }