//! Microstructure Features for HFT ML Models //! //! This module implements high-frequency microstructure features for real-time trading: //! - Amihud Illiquidity Ratio: Measures price impact per unit of volume //! - Roll Measure: Estimates effective bid-ask spread (Agent A9) //! - Corwin-Schultz: High-low spread estimator (Agent A10) //! //! ## Performance Targets //! - Latency: <8μs per feature update (Amihud), <5μs (Roll, Corwin-Schultz) //! - Memory: ≤72 bytes per symbol per feature //! - Data: OHLCV-only (no Level-2 order book required) //! //! ## Integration //! These features are part of the 256-dimension training feature vector: //! - Features 115-164: Microstructure proxies (50 features) //! //! ## References //! - Amihud (2002): "Illiquidity and Stock Returns" //! - Roll (1984): "A Simple Implicit Measure of the Effective Bid-Ask Spread" //! - Corwin & Schultz (2012): "A Simple Way to Estimate Bid-Ask Spreads" //! - Hudson & Thames `MLFinLab`: Research-backed implementations /// Trait for microstructure features with normalization for ML models pub trait MicrostructureFeatures { /// Returns the feature name for logging/debugging fn feature_name(&self) -> &'static str; /// Returns the raw feature value fn value(&self) -> f64; /// Returns normalized feature value for ML training (typically [-1, 1]) fn get_normalized(&self) -> f64; /// Resets internal state (useful for backtesting) fn reset(&mut self); } // ============================================================================ // Amihud Illiquidity Ratio (Agent A8) // ============================================================================ /// Amihud Illiquidity Ratio: Measures price impact per unit of trading volume /// /// ## Formula /// ```text /// Illiquidity_t = |return_t| / dollar_volume_t /// ``` /// /// Where: /// - `return_t` = (`price_t` - price_{t-1}) / price_{t-1} /// - `dollar_volume_t` = `price_t` * `volume_t` /// /// ## Interpretation /// - **High illiquidity** (>1e-6): Large price impact per dollar traded (illiquid market) /// - **Low illiquidity** (<1e-9): Small price impact (liquid market) /// - Used for transaction cost estimation and position sizing /// /// ## Implementation /// Uses Exponential Moving Average (EMA) for smoothing: /// ```text /// EMA_illiquidity_t = α * instant_illiquidity_t + (1-α) * EMA_illiquidity_{t-1} /// ``` /// /// ## Performance /// - **Latency**: 3-8μs per update (O(1) complexity) /// - **Memory**: 24 bytes (3 f64 fields) /// - **Data**: OHLCV only (no tick data required) /// /// ## Example /// ```rust /// use ml::features::microstructure::AmihudIlliquidity; /// /// let mut amihud = AmihudIlliquidity::new(0.05); // α=0.05 for 20-bar window /// /// // Feed OHLCV bars /// amihud.update(100.0, 10000.0); // price, volume /// amihud.update(101.0, 12000.0); /// /// let illiquidity = amihud.value(); /// let normalized = amihud.get_normalized(); // For ML training /// ``` #[derive(Debug, Clone)] pub struct AmihudIlliquidity { /// EMA smoothing factor: α ∈ (0, 1] /// - α = 0.05 → effective window ≈ 20 bars /// - α = 0.1 → effective window ≈ 10 bars alpha: f64, /// Exponentially weighted average of illiquidity ema_illiq: f64, /// Previous price for return calculation prev_price: f64, } impl AmihudIlliquidity { /// Creates a new Amihud Illiquidity calculator /// /// ## Arguments /// - `alpha`: EMA smoothing factor ∈ (0, 1] /// - Smaller α = more smoothing (longer effective window) /// - Larger α = more responsive (shorter effective window) /// - Recommended: 0.05 (20-bar window) for stable estimates /// /// ## Panics /// Panics if `alpha` ≤ 0 or `alpha` > 1 pub fn new(alpha: f64) -> Self { assert!( alpha > 0.0 && alpha <= 1.0, "Alpha must be in (0, 1], got: {}", alpha ); Self { alpha, ema_illiq: 0.0, prev_price: 0.0, } } /// Creates a new Amihud Illiquidity calculator with default alpha (0.05) pub fn default() -> Self { Self::new(0.05) } /// Updates the illiquidity estimate with a new OHLCV bar /// /// ## Arguments /// - `price`: Current close price /// - `volume`: Current bar volume /// /// ## Returns /// Updated EMA illiquidity value /// /// ## Behavior /// - First update: Returns 0.0 (no previous price for return calculation) /// - Second update: Returns instantaneous illiquidity (initializes EMA) /// - Subsequent updates: Returns EMA-smoothed illiquidity /// - Zero volume: Returns 0.0 (no measurable illiquidity) /// - Zero price: Handled gracefully (returns 0.0) /// /// ## Performance /// - O(1) complexity: 3 multiplications, 2 divisions, 1 absolute value /// - Expected latency: 3-8μs pub fn update(&mut self, price: f64, volume: f64) -> f64 { // Calculate return let ret = if self.prev_price > 0.0 { (price - self.prev_price) / self.prev_price } else { // First update: no return yet self.prev_price = price; return 0.0; }; // Calculate instantaneous illiquidity let dollar_volume = price * volume; let instant_illiq = if dollar_volume > 0.0 { ret.abs() / dollar_volume } else { // Zero volume: no measurable illiquidity 0.0 }; // Update EMA: if this is the first real measurement (ema_illiq == 0.0), // initialize with instant_illiq. Otherwise, apply EMA smoothing. self.ema_illiq = if self.ema_illiq == 0.0 { instant_illiq } else { self.alpha * instant_illiq + (1.0 - self.alpha) * self.ema_illiq }; // Update state self.prev_price = price; self.ema_illiq } /// Returns the current illiquidity value (alias for compute for compatibility) pub const fn compute(&self) -> f64 { self.ema_illiq } /// Returns the EMA smoothing factor pub const fn alpha(&self) -> f64 { self.alpha } /// Returns the current EMA illiquidity value pub const fn ema_illiquidity(&self) -> f64 { self.ema_illiq } /// Returns the previous price used for return calculation pub const fn prev_price(&self) -> f64 { self.prev_price } } impl MicrostructureFeatures for AmihudIlliquidity { fn feature_name(&self) -> &'static str { "amihud_illiquidity" } fn value(&self) -> f64 { self.ema_illiq } fn get_normalized(&self) -> f64 { // Amihud is unbounded and highly skewed: typical range 1e-9 to 1e-5 // Apply log-transform + clipping for ML models if self.ema_illiq <= 0.0 { return -5.0; // Map zero/negative to minimum } // Scale to [ln(0.01), ln(1000)] ≈ [-4.6, 6.9] let log_illiq = (self.ema_illiq * 1e8).ln(); // Clip outliers to [-5, 5] let clamped = log_illiq.clamp(-5.0, 5.0); // Map to [-1, 1] clamped / 5.0 } fn reset(&mut self) { self.ema_illiq = 0.0; self.prev_price = 0.0; } } // ============================================================================ // Roll Measure (Agent A9) // ============================================================================ /// Roll Measure: Estimates effective bid-ask spread from serial covariance /// /// ## Formula /// ```text /// Roll Spread = 2 * sqrt(-cov(Δp_t, Δp_{t-1})) /// ``` /// /// Where: /// - `Δp_t` = `p_t` - p_{t-1} (price change at time t) /// - `cov()` = covariance between consecutive price changes /// /// ## Intuition /// Bid-ask bounce creates negative serial correlation in transaction prices. /// Roll (1984) showed this covariance relates to the effective spread. /// /// ## Implementation Details /// - Uses rolling window of 20 price changes for stability /// - Handles negative covariance (take sqrt of absolute value) /// - O(1) amortized update using `VecDeque` /// - Returns 0.0 for insufficient data (<3 prices) /// /// ## Performance /// - Update: O(1) amortized (`VecDeque` push/pop) /// - Compute: O(n) where n=20 (window size) /// - Memory: 72 bytes (8B per price × 20 + overhead) /// - Latency: <2μs per update+compute /// /// ## References /// - Roll (1984): "A Simple Implicit Measure of the Effective Bid-Ask Spread" #[derive(Debug, Clone)] pub struct RollMeasure { /// Rolling window of prices (max 21 for 20 price changes) prices: std::collections::VecDeque, /// Window size for covariance calculation window_size: usize, } impl RollMeasure { /// Create new Roll Measure estimator pub fn new() -> Self { Self { prices: std::collections::VecDeque::with_capacity(21), window_size: 20, } } /// Update with new price observation /// /// ## Arguments /// - `price`: Transaction price (e.g., close price) /// /// ## Performance /// - O(1) amortized (`VecDeque` push/pop) /// - <500ns typical latency pub fn update(&mut self, price: f64) { if !price.is_finite() { return; // Skip invalid prices } self.prices.push_back(price); // Keep window_size + 1 prices (for window_size price changes) if self.prices.len() > self.window_size + 1 { self.prices.pop_front(); } } /// Compute Roll spread estimate /// /// ## Returns /// - Effective spread estimate in price units /// - Returns 0.0 if insufficient data (<3 prices) /// /// ## Performance /// - O(n) where `n=window_size` (20) /// - <2μs typical latency pub fn compute(&self) -> f64 { // Need at least 3 prices for 2 price changes if self.prices.len() < 3 { return 0.0; } // Compute price changes Δp_t = p_t - p_{t-1} let price_changes: Vec = self .prices .iter() .zip(self.prices.iter().skip(1)) .map(|(prev, curr)| curr - prev) .collect(); if price_changes.len() < 2 { return 0.0; } // Compute covariance between Δp_t and Δp_{t-1} let cov = self.compute_serial_covariance(&price_changes); // Roll spread = 2 * sqrt(-cov) // Handle negative covariance case: take sqrt of absolute value if cov >= 0.0 { // Positive covariance (trending) => no bid-ask bounce // Return small spread estimate return 0.0; } // Negative covariance (mean-reverting) => bid-ask bounce present let spread = 2.0 * (-cov).sqrt(); // Sanity check: cap at 100 (unrealistic spread) spread.min(100.0) } /// Compute serial covariance: `cov(Δp_t`, Δp_{t-1}) /// /// ## Formula /// ```text /// cov(X, Y) = E[(X - μ_X)(Y - μ_Y)] /// ``` /// /// ## Performance /// - O(n) where n = length of `price_changes` /// - <1μs for n=20 fn compute_serial_covariance(&self, price_changes: &[f64]) -> f64 { if price_changes.len() < 2 { return 0.0; } let n = price_changes.len() - 1; // Number of overlapping pairs // Compute means let mean_t: f64 = price_changes.iter().skip(1).sum::() / n as f64; let mean_t_minus_1: f64 = price_changes.iter().take(n).sum::() / n as f64; // Compute covariance let mut cov_sum = 0.0; for i in 0..n { let x = price_changes[i] - mean_t_minus_1; let y = price_changes[i + 1] - mean_t; cov_sum += x * y; } cov_sum / n as f64 } } impl Default for RollMeasure { fn default() -> Self { Self::new() } } // ============================================================================ // Normalization Helper Functions (for extraction.rs integration) // ============================================================================ /// Normalize Roll spread for ML training /// /// Converts absolute spread (in price units) to normalized value [0, 1] /// /// ## Arguments /// - `spread`: Raw Roll spread estimate (typically 0.01 - 10.0 for ES.FUT) /// - `max_spread`: Maximum expected spread for normalization (default: 10.0) /// /// ## Returns /// Normalized value in [0, 1] where: /// - 0.0 = No spread (perfect liquidity) /// - 1.0 = Maximum spread (illiquid market) pub fn normalize_roll_spread(spread: f64, max_spread: f64) -> f64 { if spread <= 0.0 || !spread.is_finite() { return 0.0; } (spread / max_spread).clamp(0.0, 1.0) } /// Normalize Amihud illiquidity for ML training /// /// Converts absolute illiquidity to normalized value [0, 1] /// /// ## Arguments /// - `illiquidity`: Raw Amihud illiquidity (typically 1e-9 to 1e-5) /// - `max_illiq`: Maximum expected illiquidity for normalization (default: 1e-5) /// /// ## Returns /// Normalized value in [0, 1] where: /// - 0.0 = Perfect liquidity /// - 1.0 = Maximum illiquidity pub fn normalize_amihud_illiquidity(illiquidity: f64, max_illiq: f64) -> f64 { if illiquidity <= 0.0 || !illiquidity.is_finite() { return 0.0; } (illiquidity / max_illiq).clamp(0.0, 1.0) } // ============================================================================ // Corwin-Schultz Spread Estimator (Agent A10) // ============================================================================ /// Corwin-Schultz Spread: High-low volatility decomposition estimator /// /// ## Formula /// ```text /// Spread = 2 * (e^α - 1) / (1 + e^α) /// α = [(√(2β₁) + √(2β₂)) - √γ] / (3 - 2√2) /// β = [ln(H_t/L_t)]² (single-period high-low variance) /// γ = [ln(max(H_t,H_{t-1}) / min(L_t,L_{t-1}))]² (two-period variance) /// ``` /// /// ## Intuition /// The high-low range contains both fundamental volatility and bid-ask spread. /// By comparing single-period and two-period ranges, we decompose the spread /// component from the volatility component. /// /// ## Performance /// - Latency: <15μs per update /// - Memory: 72 bytes /// - Data: OHLC only (no Level-2 required) /// /// ## References /// - Corwin & Schultz (2012): "A Simple Way to Estimate Bid-Ask Spreads from Daily High and Low Prices" #[derive(Debug, Clone)] pub struct CorwinSchultzSpread { /// Rolling window of (high, low, close) tuples bars: std::collections::VecDeque<(f64, f64, f64)>, /// Window size for averaging spread estimates window_size: usize, } impl CorwinSchultzSpread { /// Create new Corwin-Schultz spread estimator pub fn new() -> Self { Self { bars: std::collections::VecDeque::with_capacity(21), window_size: 20, } } /// Update with new OHLC bar pub fn update(&mut self, high: f64, low: f64, close: f64) { if !high.is_finite() || !low.is_finite() || !close.is_finite() { return; } if high < low || close < low || close > high || high <= 0.0 || low <= 0.0 { return; } self.bars.push_back((high, low, close)); if self.bars.len() > self.window_size + 1 { self.bars.pop_front(); } } /// Compute Corwin-Schultz spread estimate pub fn compute(&self) -> f64 { if self.bars.len() < 2 { return 0.0; } let mut spread_estimates = Vec::with_capacity(self.bars.len() - 1); for i in 0..self.bars.len() - 1 { let (high_prev, low_prev, _) = self.bars[i]; let (high_curr, low_curr, _) = self.bars[i + 1]; if let Some(spread) = self.compute_two_bar_spread(high_prev, low_prev, high_curr, low_curr) { spread_estimates.push(spread); } } if spread_estimates.is_empty() { 0.0 } else { let avg = spread_estimates.iter().sum::() / spread_estimates.len() as f64; avg.min(0.5) } } fn compute_two_bar_spread( &self, high_prev: f64, low_prev: f64, high_curr: f64, low_curr: f64, ) -> Option { if high_prev <= low_prev || high_curr <= low_curr { return None; } let beta_prev = (high_prev / low_prev).ln().powi(2); let beta_curr = (high_curr / low_curr).ln().powi(2); let max_high = high_prev.max(high_curr); let min_low = low_prev.min(low_curr); let gamma = (max_high / min_low).ln().powi(2); if !beta_prev.is_finite() || !beta_curr.is_finite() || !gamma.is_finite() { return None; } let sqrt_2 = 2.0_f64.sqrt(); let denominator = 3.0 - 2.0 * sqrt_2; let numerator = (sqrt_2 * beta_prev).sqrt() + (sqrt_2 * beta_curr).sqrt() - gamma.sqrt(); let alpha = numerator / denominator; if !alpha.is_finite() || alpha < 0.0 { return None; } let e_alpha = alpha.exp(); let spread = 2.0 * (e_alpha - 1.0) / (1.0 + e_alpha); (spread.is_finite() && spread >= 0.0).then_some(spread) } } impl Default for CorwinSchultzSpread { fn default() -> Self { Self::new() } } /// Normalize Corwin-Schultz spread to [0, 1] for ML features pub fn normalize_corwin_schultz_spread(spread: f64, max_spread: f64) -> f64 { if !spread.is_finite() || spread < 0.0 { return 0.0; } (spread / max_spread).min(1.0) } // ============================================================================ // Unit Tests // ============================================================================ #[cfg(test)] #[allow(clippy::manual_range_contains)] mod tests { use super::*; #[test] fn test_amihud_initialization() { let amihud = AmihudIlliquidity::new(0.05); assert_eq!(amihud.alpha(), 0.05); assert_eq!(amihud.ema_illiquidity(), 0.0); assert_eq!(amihud.prev_price(), 0.0); } #[test] #[should_panic(expected = "Alpha must be in (0, 1]")] fn test_amihud_invalid_alpha_zero() { let _ = AmihudIlliquidity::new(0.0); } #[test] #[should_panic(expected = "Alpha must be in (0, 1]")] fn test_amihud_invalid_alpha_negative() { let _ = AmihudIlliquidity::new(-0.1); } #[test] #[should_panic(expected = "Alpha must be in (0, 1]")] fn test_amihud_invalid_alpha_too_large() { let _ = AmihudIlliquidity::new(1.5); } #[test] fn test_amihud_first_update() { let mut amihud = AmihudIlliquidity::new(0.05); let illiq = amihud.update(100.0, 10000.0); assert_eq!(illiq, 0.0, "First update should return 0.0"); assert_eq!(amihud.prev_price(), 100.0); assert_eq!(amihud.ema_illiquidity(), 0.0); } #[test] fn test_amihud_high_volume_low_illiquidity() { let mut amihud = AmihudIlliquidity::new(0.05); amihud.update(100.0, 100000.0); let illiq = amihud.update(101.0, 100000.0); // Illiquidity = |0.01| / (101 * 100000) ≈ 9.9e-10 let expected = 0.01 / (101.0 * 100000.0); let tolerance = expected * 0.01; assert!( (illiq - expected).abs() < tolerance, "Expected: {}, Got: {}", expected, illiq ); } #[test] fn test_amihud_low_volume_high_illiquidity() { let mut amihud = AmihudIlliquidity::new(0.05); amihud.update(100.0, 1000.0); let illiq = amihud.update(101.0, 100.0); // Illiquidity = |0.01| / (101 * 100) ≈ 9.9e-7 let expected = 0.01 / (101.0 * 100.0); let tolerance = expected * 0.01; assert!( (illiq - expected).abs() < tolerance, "Expected: {}, Got: {}", expected, illiq ); assert!(illiq > 1e-8, "Low volume should yield high illiquidity"); } #[test] fn test_amihud_zero_volume() { let mut amihud = AmihudIlliquidity::new(0.05); amihud.update(100.0, 1000.0); let illiq = amihud.update(101.0, 0.0); assert_eq!(illiq, 0.0, "Zero volume should yield zero illiquidity"); assert_eq!(amihud.prev_price(), 101.0); } #[test] fn test_amihud_zero_price() { let mut amihud = AmihudIlliquidity::new(0.05); amihud.update(100.0, 1000.0); let illiq = amihud.update(0.0, 1000.0); // Should handle gracefully assert!(illiq.is_finite()); assert_eq!(amihud.prev_price(), 0.0); } #[test] fn test_amihud_negative_return() { let mut amihud = AmihudIlliquidity::new(0.05); amihud.update(100.0, 10000.0); let illiq = amihud.update(99.0, 10000.0); // Illiquidity = |-0.01| / (99 * 10000) ≈ 1.01e-8 let expected = 0.01 / (99.0 * 10000.0); let tolerance = expected * 0.01; assert!( (illiq - expected).abs() < tolerance, "Negative return should use abs value" ); } #[test] fn test_amihud_ema_smoothing() { let mut amihud = AmihudIlliquidity::new(0.05); amihud.update(100.0, 10000.0); let illiq1 = amihud.update(101.0, 10000.0); let illiq2 = amihud.update(102.0, 10000.0); // EMA should smooth values assert_ne!(illiq1, illiq2); assert!(amihud.ema_illiquidity() > 0.0); } #[test] fn test_amihud_trait_methods() { let mut amihud = AmihudIlliquidity::new(0.05); assert_eq!(amihud.feature_name(), "amihud_illiquidity"); amihud.update(100.0, 10000.0); amihud.update(101.0, 10000.0); let value = amihud.value(); let normalized = amihud.get_normalized(); assert!(value > 0.0); assert!(normalized.is_finite()); assert!(normalized >= -5.0 && normalized <= 5.0); } #[test] fn test_amihud_reset() { let mut amihud = AmihudIlliquidity::new(0.05); amihud.update(100.0, 10000.0); amihud.update(101.0, 10000.0); assert!(amihud.ema_illiquidity() > 0.0); assert!(amihud.prev_price() > 0.0); amihud.reset(); assert_eq!(amihud.ema_illiquidity(), 0.0); assert_eq!(amihud.prev_price(), 0.0); } #[test] fn test_amihud_memory_size() { use std::mem::size_of; let size = size_of::(); assert!(size <= 72, "Memory {} bytes exceeds 72-byte target", size); } #[test] fn test_amihud_latency_benchmark() { use std::time::Instant; let mut amihud = AmihudIlliquidity::new(0.05); amihud.update(100.0, 10000.0); let iterations = 10000; let start = Instant::now(); for i in 0..iterations { let price = 100.0 + (i as f64 * 0.01); amihud.update(price, 10000.0); } let elapsed = start.elapsed(); let avg_latency_us = elapsed.as_micros() as f64 / iterations as f64; assert!( avg_latency_us < 8.0, "Average latency {:.2}μs exceeds 8μs target", avg_latency_us ); } #[test] fn test_amihud_numerical_stability() { let mut amihud = AmihudIlliquidity::new(0.05); // Test extreme values let test_cases = vec![(1e-6, 1e-6), (1e6, 1e6), (100.0, 1e-6), (1e-6, 1e6)]; amihud.update(100.0, 10000.0); for (price, volume) in test_cases { let illiq = amihud.update(price, volume); assert!(illiq.is_finite(), "Illiquidity must be finite"); assert!(illiq >= 0.0, "Illiquidity must be non-negative"); } } #[test] fn test_normalization_functions() { // Test Roll spread normalization assert_eq!(normalize_roll_spread(0.0, 10.0), 0.0); assert_eq!(normalize_roll_spread(5.0, 10.0), 0.5); assert_eq!(normalize_roll_spread(10.0, 10.0), 1.0); assert_eq!(normalize_roll_spread(20.0, 10.0), 1.0); // Clipped // Test Amihud illiquidity normalization assert_eq!(normalize_amihud_illiquidity(0.0, 1e-5), 0.0); assert_eq!(normalize_amihud_illiquidity(5e-6, 1e-5), 0.5); assert_eq!(normalize_amihud_illiquidity(1e-5, 1e-5), 1.0); assert_eq!(normalize_amihud_illiquidity(2e-5, 1e-5), 1.0); // Clipped } }