foxhunt/crates/ml-features/src/microstructure_features.rs

//! Wave C Microstructure Features for HFT ML Models
//!
//! This module implements 12 microstructure features from `MLFinLab` Chapter 19:
//! - **Spread Estimators** (3): Roll, Corwin-Schultz, High-Low Spread
//! - **Liquidity Metrics** (2): Amihud Illiquidity, Volume-Weighted Spread
//! - **Trade Arrival** (2): Tick Count, Inter-Arrival Time
//! - **Order Flow** (2): Buy/Sell Imbalance, VPIN (not implemented - O(n) complexity)
//! - **Market Impact** (2): Kyle's Lambda (slow-updating), Price Impact
//! - **Efficiency** (1): Variance Ratio
//!
//! ## Integration with Wave A
//! Three features are already implemented in `microstructure.rs` (Wave A):
//! - Roll Measure (Feature 115)
//! - Corwin-Schultz Spread (Feature 116)
//! - Amihud Illiquidity (Feature 117)
//!
//! Wave C adds 9 new features (118-126):
//! - High-Low Spread (118)
//! - Volume-Weighted Spread (119)
//! - Tick Count (120)
//! - Inter-Arrival Time (121)
//! - Buy/Sell Imbalance (122)
//! - Kyle's Lambda (123, slow-updating)
//! - Price Impact (124)
//! - Variance Ratio (125)
//! - Reserved (126)
//!
//! ## Performance Targets
//! - Latency: <200μs for all 12 features per bar (cumulative)
//! - Memory: ≤500 bytes per symbol
//! - Data: OHLCV-only (no Level-2 order book required)
//!
//! ## References
//! - `MLFinLab` Chapter 19: Market Microstructure Features
//! - See `WAVE_C_MICROSTRUCTURE_FEATURE_DESIGN.md` for detailed specifications

use std::collections::VecDeque;

// ============================================================================
// Trait Definition
// ============================================================================

/// Common trait for all microstructure features
pub trait MicrostructureFeature {
    /// Returns the feature name for logging/debugging
    fn feature_name(&self) -> &'static str;

    /// Returns the raw feature value
    fn value(&self) -> f64;

    /// Returns normalized feature value for ML training (typically [-1, 1])
    fn get_normalized(&self) -> f64;

    /// Resets internal state (useful for backtesting)
    fn reset(&mut self);
}

// ============================================================================
// 1. High-Low Spread (Feature 118)
// ============================================================================

/// High-Low Spread: Simple spread estimator from intrabar range
///
/// ## Formula
/// ```text
/// High-Low Spread = (High - Low) / ((High + Low) / 2)
/// ```
///
/// ## Interpretation
/// - Measures intrabar volatility as a proxy for bid-ask spread
/// - Higher values indicate wider spreads (less liquid)
/// - Smoothed with EMA for stability
///
/// ## Performance
/// - Latency: <5μs per update
/// - Memory: 16 bytes (2 f64 fields)
#[derive(Debug, Clone)]
pub struct HighLowSpread {
    /// EMA smoothing factor
    alpha: f64,
    /// Exponentially weighted average of spread
    ema_spread: f64,
}

impl HighLowSpread {
    pub fn new(alpha: f64) -> Self {
        assert!(alpha > 0.0 && alpha <= 1.0, "Alpha must be in (0, 1]");
        Self {
            alpha,
            ema_spread: 0.0,
        }
    }

    pub fn default() -> Self {
        Self::new(0.05) // 20-bar effective window
    }

    /// Update with new OHLC bar
    pub fn update(&mut self, high: f64, low: f64) -> f64 {
        if high <= 0.0 || low <= 0.0 || high < low {
            return self.ema_spread;
        }

        let midpoint = (high + low) / 2.0;
        let instant_spread = (high - low) / midpoint;

        // Initialize EMA on first valid update to avoid slow convergence
        if self.ema_spread == 0.0 {
            self.ema_spread = instant_spread;
        } else {
            self.ema_spread = self.alpha * instant_spread + (1.0 - self.alpha) * self.ema_spread;
        }
        self.ema_spread
    }

    pub const fn compute(&self) -> f64 {
        self.ema_spread
    }
}

impl Default for HighLowSpread {
    fn default() -> Self {
        Self::default()
    }
}

impl MicrostructureFeature for HighLowSpread {
    fn feature_name(&self) -> &'static str {
        "high_low_spread"
    }

    fn value(&self) -> f64 {
        self.ema_spread
    }

    fn get_normalized(&self) -> f64 {
        // High-low spread typically 0.01% - 5.0%
        let clamped = self.ema_spread.clamp(0.0, 0.05);
        (clamped / 0.025) - 1.0 // Map [0, 2.5%] to [-1, 1]
    }

    fn reset(&mut self) {
        self.ema_spread = 0.0;
    }
}

// ============================================================================
// 2. Volume-Weighted Spread (Feature 119)
// ============================================================================

/// Volume-Weighted Spread: Adjusts spread estimate by relative volume
///
/// ## Formula
/// ```text
/// VW_Spread = Spread * (Volume / Avg_Volume)
/// ```
///
/// ## Interpretation
/// - High volume + wide spread = illiquid market under stress
/// - Low volume + wide spread = normal illiquidity
/// - Used for transaction cost estimation
///
/// ## Performance
/// - Latency: <10μs per update
/// - Memory: 32 bytes
#[derive(Debug, Clone)]
pub struct VolumeWeightedSpread {
    alpha: f64,
    ema_volume: f64,
    ema_spread: f64,
}

impl VolumeWeightedSpread {
    pub fn new(alpha: f64) -> Self {
        assert!(alpha > 0.0 && alpha <= 1.0, "Alpha must be in (0, 1]");
        Self {
            alpha,
            ema_volume: 0.0,
            ema_spread: 0.0,
        }
    }

    pub fn default() -> Self {
        Self::new(0.05)
    }

    /// Update with spread and volume
    pub fn update(&mut self, spread: f64, volume: f64) -> f64 {
        if volume <= 0.0 {
            return self.ema_spread;
        }

        // Update average volume
        if self.ema_volume == 0.0 {
            self.ema_volume = volume;
        } else {
            self.ema_volume = self.alpha * volume + (1.0 - self.alpha) * self.ema_volume;
        }

        // Calculate volume-weighted spread
        let volume_ratio = volume / self.ema_volume.max(1.0);
        let vw_spread = spread * volume_ratio;

        self.ema_spread = self.alpha * vw_spread + (1.0 - self.alpha) * self.ema_spread;
        self.ema_spread
    }

    pub const fn compute(&self) -> f64 {
        self.ema_spread
    }
}

impl Default for VolumeWeightedSpread {
    fn default() -> Self {
        Self::default()
    }
}

impl MicrostructureFeature for VolumeWeightedSpread {
    fn feature_name(&self) -> &'static str {
        "volume_weighted_spread"
    }

    fn value(&self) -> f64 {
        self.ema_spread
    }

    fn get_normalized(&self) -> f64 {
        // VW spread typically 0.01% - 10.0% (wider range due to volume weighting)
        let clamped = self.ema_spread.clamp(0.0, 0.10);
        (clamped / 0.05) - 1.0 // Map [0, 5%] to [-1, 1]
    }

    fn reset(&mut self) {
        self.ema_volume = 0.0;
        self.ema_spread = 0.0;
    }
}

// ============================================================================
// 3. Tick Count (Feature 120)
// ============================================================================

/// Tick Count: Number of price changes in rolling window
///
/// ## Formula
/// ```text
/// Tick_Count = Count of bars with non-zero price change
/// ```
///
/// ## Interpretation
/// - High tick count = active trading, good price discovery
/// - Low tick count = stale market, wide spreads
///
/// ## Performance
/// - Latency: <2μs per update
/// - Memory: 24 bytes
#[derive(Debug, Clone)]
pub struct TickCount {
    window_size: usize,
    tick_count: usize,
    prev_price: f64,
    price_changes: VecDeque<bool>,
}

impl TickCount {
    pub fn new(window_size: usize) -> Self {
        Self {
            window_size,
            tick_count: 0,
            prev_price: 0.0,
            price_changes: VecDeque::with_capacity(window_size),
        }
    }

    pub fn default() -> Self {
        Self::new(20) // 20-bar window
    }

    /// Update with new price
    pub fn update(&mut self, price: f64) -> usize {
        if self.prev_price == 0.0 {
            self.prev_price = price;
            return 0;
        }

        let price_changed = (price - self.prev_price).abs() > 1e-9;

        if price_changed {
            self.tick_count += 1;
        }

        self.price_changes.push_back(price_changed);

        if self.price_changes.len() > self.window_size {
            if let Some(old_change) = self.price_changes.pop_front() {
                if old_change {
                    self.tick_count = self.tick_count.saturating_sub(1);
                }
            }
        }

        self.prev_price = price;
        self.tick_count
    }

    pub const fn compute(&self) -> usize {
        self.tick_count
    }
}

impl Default for TickCount {
    fn default() -> Self {
        Self::default()
    }
}

impl MicrostructureFeature for TickCount {
    fn feature_name(&self) -> &'static str {
        "tick_count"
    }

    fn value(&self) -> f64 {
        self.tick_count as f64
    }

    fn get_normalized(&self) -> f64 {
        // Tick count 0-20 (window size)
        let ratio = self.tick_count as f64 / self.window_size as f64;
        2.0 * ratio - 1.0 // Map [0, 1] to [-1, 1]
    }

    fn reset(&mut self) {
        self.tick_count = 0;
        self.prev_price = 0.0;
        self.price_changes.clear();
    }
}

// ============================================================================
// 4. Inter-Arrival Time (Feature 121)
// ============================================================================

/// Inter-Arrival Time: Average time between bars in rolling window
///
/// ## Formula
/// ```text
/// Inter_Arrival = Avg(timestamp`[i]` - timestamp[i-1])
/// ```
///
/// ## Interpretation
/// - Short inter-arrival = high trading activity
/// - Long inter-arrival = low activity, wider spreads
///
/// ## Performance
/// - Latency: <5μs per update
/// - Memory: 160 bytes (20 timestamps)
#[derive(Debug, Clone)]
pub struct InterArrivalTime {
    window_size: usize,
    timestamps: VecDeque<u64>,
}

impl InterArrivalTime {
    pub fn new(window_size: usize) -> Self {
        Self {
            window_size,
            timestamps: VecDeque::with_capacity(window_size),
        }
    }

    pub fn default() -> Self {
        Self::new(20)
    }

    /// Update with new timestamp (nanoseconds)
    pub fn update(&mut self, timestamp_ns: u64) -> f64 {
        self.timestamps.push_back(timestamp_ns);
        if self.timestamps.len() > self.window_size {
            self.timestamps.pop_front();
        }

        self.compute()
    }

    /// Compute average inter-arrival time in seconds
    pub fn compute(&self) -> f64 {
        if self.timestamps.len() < 2 {
            return 0.0;
        }

        let mut total_diff = 0_u64;
        for i in 1..self.timestamps.len() {
            let diff = self.timestamps[i].saturating_sub(self.timestamps[i - 1]);
            total_diff += diff;
        }

        let avg_ns = total_diff as f64 / (self.timestamps.len() - 1) as f64;
        avg_ns / 1_000_000_000.0 // Convert to seconds
    }
}

impl Default for InterArrivalTime {
    fn default() -> Self {
        Self::default()
    }
}

impl MicrostructureFeature for InterArrivalTime {
    fn feature_name(&self) -> &'static str {
        "inter_arrival_time"
    }

    fn value(&self) -> f64 {
        self.compute()
    }

    fn get_normalized(&self) -> f64 {
        // Inter-arrival time: 0.1 - 10 seconds (typical range)
        let log_time = (self.compute() + 0.01).ln();
        let clamped = log_time.clamp(-5.0, 3.0);
        clamped / 4.0 // Map to [-1.25, 0.75], acceptable asymmetry
    }

    fn reset(&mut self) {
        self.timestamps.clear();
    }
}

// ============================================================================
// 5. Buy/Sell Imbalance (Feature 122)
// ============================================================================

/// Buy/Sell Imbalance: Order flow imbalance using tick rule
///
/// ## Formula
/// ```text
/// Imbalance = (Buy_Volume - Sell_Volume) / Total_Volume
/// Trade classified as buy if price_t > price_{t-1}
/// ```
///
/// ## Interpretation
/// - Positive = buying pressure (bullish)
/// - Negative = selling pressure (bearish)
/// - Used for short-term mean reversion signals
///
/// ## Performance
/// - Latency: <3μs per update
/// - Memory: 32 bytes
#[derive(Debug, Clone)]
pub struct BuySellImbalance {
    alpha: f64,
    ema_imbalance: f64,
    prev_price: f64,
}

impl BuySellImbalance {
    pub fn new(alpha: f64) -> Self {
        assert!(alpha > 0.0 && alpha <= 1.0, "Alpha must be in (0, 1]");
        Self {
            alpha,
            ema_imbalance: 0.0,
            prev_price: 0.0,
        }
    }

    pub fn default() -> Self {
        Self::new(0.1) // 10-bar effective window
    }

    /// Update with price and volume (tick rule classification)
    pub fn update(&mut self, price: f64, volume: f64) -> f64 {
        if self.prev_price == 0.0 {
            self.prev_price = price;
            return 0.0;
        }

        if volume <= 0.0 {
            return self.ema_imbalance;
        }

        // Tick rule: positive price change = buy, negative = sell
        let instant_imbalance = if price > self.prev_price {
            1.0
        } else if price < self.prev_price {
            -1.0
        } else {
            0.0 // Zero tick: no classification
        };

        self.ema_imbalance =
            self.alpha * instant_imbalance + (1.0 - self.alpha) * self.ema_imbalance;
        self.prev_price = price;
        self.ema_imbalance
    }

    pub const fn compute(&self) -> f64 {
        self.ema_imbalance
    }
}

impl Default for BuySellImbalance {
    fn default() -> Self {
        Self::default()
    }
}

impl MicrostructureFeature for BuySellImbalance {
    fn feature_name(&self) -> &'static str {
        "buy_sell_imbalance"
    }

    fn value(&self) -> f64 {
        self.ema_imbalance
    }

    fn get_normalized(&self) -> f64 {
        // Already bounded [-1, 1]
        self.ema_imbalance
    }

    fn reset(&mut self) {
        self.ema_imbalance = 0.0;
        self.prev_price = 0.0;
    }
}

// ============================================================================
// 6. Kyle's Lambda (Feature 123) - Slow-Updating Feature
// ============================================================================

/// Kyle's Lambda: Market impact measure from regression
///
/// ## Formula (Incremental OLS)
/// ```text
/// r_t = α + λ * S_t + ε_t
/// S_t = sign(Close - Open) * sqrt(Close * Volume)
/// λ = Cov(r, S) / Var(S)
/// ```
///
/// ## Interpretation
/// - High λ = high price impact (illiquid)
/// - Low λ = low price impact (liquid)
/// - Slow-updating: Recompute every 5 minutes (50+ bars required)
///
/// ## Performance
/// - Latency: 50-100μs when updating, 0μs when cached
/// - Memory: 800 bytes (50-period buffers)
///
/// ## Usage Note
/// ⚠️ Use as slow-updating feature (5-minute intervals), not real-time per-bar
#[derive(Debug, Clone)]
pub struct KyleLambda {
    update_interval_secs: u64,
    last_update_ns: u64,
    cached_lambda: f64,

    // Incremental statistics
    returns: VecDeque<f64>,
    signed_volumes: VecDeque<f64>,
    window_size: usize,
}

impl KyleLambda {
    pub fn new(update_interval_secs: u64, window_size: usize) -> Self {
        Self {
            update_interval_secs,
            last_update_ns: 0,
            cached_lambda: 0.0,
            returns: VecDeque::with_capacity(window_size),
            signed_volumes: VecDeque::with_capacity(window_size),
            window_size,
        }
    }

    pub fn default() -> Self {
        Self::new(300, 50) // 5 minutes, 50 periods
    }

    /// Maybe update lambda (only if interval elapsed)
    pub fn maybe_update(&mut self, timestamp_ns: u64, ret: f64, signed_volume: f64) -> f64 {
        // Add data point
        self.returns.push_back(ret);
        self.signed_volumes.push_back(signed_volume);

        if self.returns.len() > self.window_size {
            self.returns.pop_front();
            self.signed_volumes.pop_front();
        }

        // Check if update needed
        if timestamp_ns - self.last_update_ns >= self.update_interval_secs * 1_000_000_000 {
            self.cached_lambda = self.compute_lambda();
            self.last_update_ns = timestamp_ns;
        }

        self.cached_lambda
    }

    /// Compute Kyle's Lambda via OLS regression
    fn compute_lambda(&self) -> f64 {
        if self.returns.len() < 10 {
            return 0.0; // Insufficient data
        }

        let n = self.returns.len() as f64;

        // Compute means
        let mean_r: f64 = self.returns.iter().sum::<f64>() / n;
        let mean_s: f64 = self.signed_volumes.iter().sum::<f64>() / n;

        // Compute covariance and variance
        let mut cov = 0.0;
        let mut var_s = 0.0;

        for i in 0..self.returns.len() {
            let r_dev = self.returns[i] - mean_r;
            let s_dev = self.signed_volumes[i] - mean_s;
            cov += r_dev * s_dev;
            var_s += s_dev * s_dev;
        }

        if var_s < 1e-12 {
            return 0.0; // No variance, no regression
        }

        cov / var_s
    }

    pub const fn compute(&self) -> f64 {
        self.cached_lambda
    }
}

impl Default for KyleLambda {
    fn default() -> Self {
        Self::default()
    }
}

impl MicrostructureFeature for KyleLambda {
    fn feature_name(&self) -> &'static str {
        "kyles_lambda"
    }

    fn value(&self) -> f64 {
        self.cached_lambda
    }

    fn get_normalized(&self) -> f64 {
        if self.cached_lambda <= 0.0 {
            return -1.0;
        }

        // Kyle's lambda typically 1e-8 to 1e-5
        let log_lambda = (self.cached_lambda * 1e8).ln();
        let clamped = log_lambda.clamp(-5.0, 5.0);
        clamped / 5.0
    }

    fn reset(&mut self) {
        self.last_update_ns = 0;
        self.cached_lambda = 0.0;
        self.returns.clear();
        self.signed_volumes.clear();
    }
}

// ============================================================================
// V10 Block T — Multi-Level Kyle's Lambda (Kolm 2023)
// ============================================================================

/// Multi-level Kyle's λ measuring per-level price-impact sensitivity.
///
/// Per Kolm (2023) — "Deep Order Flow Imbalance" — extends single-level
/// Kyle's λ by running **separate regressions of price-return against OFI
/// at each depth level L**:
///
/// ```text
///   λ_L = cov(return, OFI_L) / var(OFI_L)   for L = 1, 2, 3, 4, 5
/// ```
///
/// Each λ_L quantifies how much price moves per unit of order-flow imbalance
/// observed at depth level L. The vector `[λ_1, …, λ_5]` captures
/// **depth-conditional impact** — markets where most informed flow trades at
/// the inside have large λ_1 and decaying λ_2..λ_5; markets where the
/// information is spread deep have flatter profiles.
///
/// Consumes the per-level OFI vector authored at
/// `OFICalculator::calc_ofi_per_level` (V10 Block F). Update via
/// `maybe_update(timestamp, return, ofi_per_level)`.
pub struct MultiLevelKyleLambda {
    update_interval_secs: u64,
    last_update_ns: u64,
    cached_lambdas: [f64; 5],

    returns: VecDeque<f64>,
    ofis_per_level: [VecDeque<f64>; 5],
    window_size: usize,
}

impl MultiLevelKyleLambda {
    pub fn new(update_interval_secs: u64, window_size: usize) -> Self {
        Self {
            update_interval_secs,
            last_update_ns: 0,
            cached_lambdas: [0.0; 5],
            returns: VecDeque::with_capacity(window_size),
            ofis_per_level: [
                VecDeque::with_capacity(window_size),
                VecDeque::with_capacity(window_size),
                VecDeque::with_capacity(window_size),
                VecDeque::with_capacity(window_size),
                VecDeque::with_capacity(window_size),
            ],
            window_size,
        }
    }

    /// Default: refresh every 5 minutes, 50-point rolling window
    /// (matches the existing scalar `KyleLambda::default()` cadence).
    pub fn with_defaults() -> Self {
        Self::new(300, 50)
    }

    /// Update with a new (return, per-level-OFI) observation.
    ///
    /// `ofi_per_level` should come from
    /// `OFICalculator::calc_ofi_per_level(curr_snap, prev_snap)`.
    /// Lambdas are recomputed only when `update_interval_secs` has elapsed
    /// since the last refresh — between refreshes the cached values are
    /// returned (matches scalar KyleLambda's caching behavior).
    pub fn maybe_update(
        &mut self,
        timestamp_ns: u64,
        ret: f64,
        ofi_per_level: [f64; 5],
    ) -> [f64; 5] {
        self.returns.push_back(ret);
        if self.returns.len() > self.window_size {
            self.returns.pop_front();
        }
        for (i, &ofi) in ofi_per_level.iter().enumerate() {
            self.ofis_per_level[i].push_back(ofi);
            if self.ofis_per_level[i].len() > self.window_size {
                self.ofis_per_level[i].pop_front();
            }
        }

        if timestamp_ns.saturating_sub(self.last_update_ns)
            >= self.update_interval_secs * 1_000_000_000
        {
            for level in 0..5 {
                self.cached_lambdas[level] = self.compute_lambda(level);
            }
            self.last_update_ns = timestamp_ns;
        }
        self.cached_lambdas
    }

    /// Single-level OLS regression `cov(returns, ofis_L) / var(ofis_L)`.
    fn compute_lambda(&self, level: usize) -> f64 {
        if self.returns.len() < 10 {
            return 0.0;
        }
        let ofis = &self.ofis_per_level[level];
        debug_assert_eq!(self.returns.len(), ofis.len());

        let n = self.returns.len() as f64;
        let mean_r: f64 = self.returns.iter().sum::<f64>() / n;
        let mean_o: f64 = ofis.iter().sum::<f64>() / n;

        let mut cov = 0.0;
        let mut var_o = 0.0;
        for i in 0..self.returns.len() {
            let r_dev = self.returns[i] - mean_r;
            let o_dev = ofis[i] - mean_o;
            cov += r_dev * o_dev;
            var_o += o_dev * o_dev;
        }

        if var_o < 1e-12 {
            return 0.0; // No variance, no regression
        }
        cov / var_o
    }

    /// Cached lambda vector (no recompute).
    pub fn lambdas(&self) -> [f64; 5] {
        self.cached_lambdas
    }

    pub fn reset(&mut self) {
        self.last_update_ns = 0;
        self.cached_lambdas = [0.0; 5];
        self.returns.clear();
        for v in self.ofis_per_level.iter_mut() {
            v.clear();
        }
    }
}

impl Default for MultiLevelKyleLambda {
    fn default() -> Self {
        Self::with_defaults()
    }
}

// ============================================================================
// 7. Price Impact (Feature 124)
// ============================================================================

/// Price Impact: Permanent price change after trade
///
/// ## Formula
/// ```text
/// Price_Impact = D_t * (M_{t+τ} - M_t)
/// D_t = Trade direction (+1 buy, -1 sell)
/// M_t = Midpoint (approximated as (High + Low) / 2)
/// τ = 5 bars (forward-looking delay)
/// ```
///
/// ## Interpretation
/// - Positive = price moved with trade (expected impact)
/// - Negative = adverse selection (price moved against trade)
///
/// ## Performance
/// - Latency: <8μs per update
/// - Memory: 160 bytes (5-bar delay buffers)
#[derive(Debug, Clone)]
pub struct PriceImpact {
    alpha: f64,
    ema_impact: f64,
    delay_bars: usize,

    high_buffer: VecDeque<f64>,
    low_buffer: VecDeque<f64>,
    close_buffer: VecDeque<f64>,
}

impl PriceImpact {
    pub fn new(alpha: f64, delay_bars: usize) -> Self {
        assert!(alpha > 0.0 && alpha <= 1.0, "Alpha must be in (0, 1]");
        Self {
            alpha,
            ema_impact: 0.0,
            delay_bars,
            high_buffer: VecDeque::with_capacity(delay_bars + 1),
            low_buffer: VecDeque::with_capacity(delay_bars + 1),
            close_buffer: VecDeque::with_capacity(delay_bars + 1),
        }
    }

    pub fn default() -> Self {
        Self::new(0.05, 5) // 20-bar EMA, 5-bar delay
    }

    /// Update with new OHLC bar
    pub fn update(&mut self, high: f64, low: f64, close: f64) -> f64 {
        let current_midpoint = (high + low) / 2.0;

        self.high_buffer.push_back(high);
        self.low_buffer.push_back(low);
        self.close_buffer.push_back(close);

        // Only compute impact once we have enough bars to establish direction
        if self.close_buffer.len() > self.delay_bars + 1 {
            let old_high = self.high_buffer.pop_front().unwrap_or(high);
            let old_low = self.low_buffer.pop_front().unwrap_or(low);
            let old_close = self.close_buffer.pop_front().unwrap_or(close);

            // Now close_buffer has at least delay_bars+1 elements
            // close_buffer[0] is the close AFTER old_close
            // We need the close BEFORE old_close, which we don't have in the buffer
            // So we need to track it separately or change the approach

            // Alternative: Use the next close in the buffer as reference
            // If old_close < next_close, that's a buy (positive direction)
            let next_close = self.close_buffer.front().copied().unwrap_or(close);
            let direction = (next_close - old_close).signum();

            let old_midpoint = (old_high + old_low) / 2.0;
            let instant_impact = direction * (current_midpoint - old_midpoint);

            self.ema_impact = self.alpha * instant_impact + (1.0 - self.alpha) * self.ema_impact;
        }

        self.ema_impact
    }

    pub const fn compute(&self) -> f64 {
        self.ema_impact
    }
}

impl Default for PriceImpact {
    fn default() -> Self {
        Self::default()
    }
}

impl MicrostructureFeature for PriceImpact {
    fn feature_name(&self) -> &'static str {
        "price_impact"
    }

    fn value(&self) -> f64 {
        self.ema_impact
    }

    fn get_normalized(&self) -> f64 {
        // Price impact typically -2% to +2%
        let clamped = self.ema_impact.clamp(-0.02, 0.02);
        clamped / 0.01 // Map [-1%, 1%] to [-1, 1]
    }

    fn reset(&mut self) {
        self.ema_impact = 0.0;
        self.high_buffer.clear();
        self.low_buffer.clear();
        self.close_buffer.clear();
    }
}

// ============================================================================
// 8. Variance Ratio (Feature 125)
// ============================================================================

/// Variance Ratio: Tests for random walk (market efficiency)
///
/// ## Formula
/// ```text
/// VR(q) = Var(r_t(q)) / (q * Var(r_t))
/// r_t(q) = q-period return
/// r_t = 1-period return
/// ```
///
/// ## Interpretation
/// - VR = 1: Random walk (efficient market)
/// - VR > 1: Positive serial correlation (momentum)
/// - VR < 1: Negative serial correlation (mean reversion)
///
/// ## Performance
/// - Latency: <15μs per update
/// - Memory: 160 bytes (20-bar window)
#[derive(Debug, Clone)]
pub struct VarianceRatio {
    window_size: usize,
    q: usize, // Multi-period lag
    returns: VecDeque<f64>,
}

impl VarianceRatio {
    pub fn new(window_size: usize, q: usize) -> Self {
        assert!(q >= 2, "q must be >= 2");
        assert!(window_size >= q * 2, "Window size must be >= 2*q");
        Self {
            window_size,
            q,
            returns: VecDeque::with_capacity(window_size),
        }
    }

    pub fn default() -> Self {
        Self::new(20, 5) // 20-bar window, 5-period lag
    }

    /// Update with new return
    pub fn update(&mut self, ret: f64) -> f64 {
        self.returns.push_back(ret);
        if self.returns.len() > self.window_size {
            self.returns.pop_front();
        }

        self.compute()
    }

    /// Compute variance ratio
    pub fn compute(&self) -> f64 {
        if self.returns.len() < self.q * 2 {
            return 1.0; // Insufficient data, assume random walk
        }

        // Compute 1-period variance
        let var_1 = self.compute_variance(&self.returns);

        if var_1 < 1e-12 {
            return 1.0; // No variance
        }

        // Compute q-period returns
        let mut returns_q = VecDeque::new();
        for i in 0..self.returns.len() {
            if i + self.q <= self.returns.len() {
                let sum_ret: f64 = self.returns.iter().skip(i).take(self.q).sum();
                returns_q.push_back(sum_ret);
            }
        }

        if returns_q.is_empty() {
            return 1.0;
        }

        let var_q = self.compute_variance(&returns_q);

        // VR(q) = Var(r_q) / (q * Var(r_1))
        let vr = var_q / (self.q as f64 * var_1);

        // Clamp to reasonable range
        vr.clamp(0.1, 3.0)
    }

    fn compute_variance(&self, data: &VecDeque<f64>) -> f64 {
        if data.is_empty() {
            return 0.0;
        }

        let n = data.len() as f64;
        let mean: f64 = data.iter().sum::<f64>() / n;

        let variance: f64 = data.iter().map(|x| (x - mean).powi(2)).sum::<f64>() / n;

        variance
    }
}

impl Default for VarianceRatio {
    fn default() -> Self {
        Self::default()
    }
}

impl MicrostructureFeature for VarianceRatio {
    fn feature_name(&self) -> &'static str {
        "variance_ratio"
    }

    fn value(&self) -> f64 {
        self.compute()
    }

    fn get_normalized(&self) -> f64 {
        // Variance ratio typically 0.5 - 2.0
        // Map to [-1, 1] with VR=1 at center
        let vr = self.compute();
        if vr < 1.0 {
            (vr - 0.5) / 0.5 // Map [0.5, 1.0] to [-1, 0]
        } else {
            (vr - 1.0) / 1.0 // Map [1.0, 2.0] to [0, 1]
        }
    }

    fn reset(&mut self) {
        self.returns.clear();
    }
}

// ============================================================================
// Unit Tests
// ============================================================================

#[cfg(test)]
#[allow(clippy::manual_range_contains)]
mod tests {
    use super::*;

    // High-Low Spread Tests
    #[test]
    fn test_high_low_spread_normal() {
        let mut spread = HighLowSpread::new(0.1);

        // Test with 1% spread - first update initializes EMA directly
        let value = spread.update(101.0, 99.0);
        let expected = (101.0 - 99.0) / ((101.0 + 99.0) / 2.0);
        assert!((value - expected).abs() < 1e-6); // First update: direct initialization

        // Second update should apply EMA smoothing
        let value2 = spread.update(101.0, 99.0);
        let expected2 = 0.1 * expected + 0.9 * expected;
        assert!((value2 - expected2).abs() < 1e-6); // EMA effect
    }

    #[test]
    fn test_high_low_spread_wide() {
        let mut spread = HighLowSpread::new(0.1);

        // 5% spread (wide)
        spread.update(105.0, 95.0);
        assert!(spread.compute() > 0.04);
    }

    // Volume-Weighted Spread Tests
    #[test]
    fn test_volume_weighted_spread() {
        let mut vw_spread = VolumeWeightedSpread::new(0.1);

        vw_spread.update(0.01, 10000.0); // Normal spread, normal volume
        let val1 = vw_spread.compute();

        vw_spread.update(0.01, 50000.0); // Same spread, 5x volume
        let val2 = vw_spread.compute();

        assert!(val2 > val1); // Higher volume should increase VW spread
    }

    // Tick Count Tests
    #[test]
    fn test_tick_count_all_changes() {
        let mut tick_count = TickCount::new(10);

        for i in 0..10 {
            tick_count.update(100.0 + i as f64 * 0.1);
        }

        assert_eq!(tick_count.compute(), 9); // 9 price changes
    }

    #[test]
    fn test_tick_count_no_changes() {
        let mut tick_count = TickCount::new(10);

        for _ in 0..10 {
            tick_count.update(100.0);
        }

        assert_eq!(tick_count.compute(), 0); // No price changes
    }

    // Inter-Arrival Time Tests
    #[test]
    fn test_inter_arrival_time() {
        let mut iat = InterArrivalTime::new(5);

        // 1 second intervals
        for i in 0..5 {
            iat.update(i * 1_000_000_000);
        }

        let avg_time = iat.compute();
        assert!((avg_time - 1.0).abs() < 1e-6); // Should be 1 second
    }

    // Buy/Sell Imbalance Tests
    #[test]
    fn test_buy_sell_imbalance_all_buys() {
        let mut imbalance = BuySellImbalance::new(0.2);

        for i in 0..10 {
            imbalance.update(100.0 + i as f64, 1000.0);
        }

        assert!(imbalance.compute() > 0.5); // Strong buy pressure
    }

    #[test]
    fn test_buy_sell_imbalance_all_sells() {
        let mut imbalance = BuySellImbalance::new(0.2);

        for i in 0..10 {
            imbalance.update(100.0 - i as f64, 1000.0);
        }

        assert!(imbalance.compute() < -0.5); // Strong sell pressure
    }

    // Kyle's Lambda Tests
    #[test]
    fn test_kyles_lambda_insufficient_data() {
        let mut lambda = KyleLambda::new(300, 50);

        // Add only 5 data points
        for i in 0..5 {
            lambda.maybe_update(i * 1_000_000_000, 0.001, 1000.0);
        }

        assert_eq!(lambda.compute(), 0.0); // Should return 0 for insufficient data
    }

    #[test]
    fn test_kyles_lambda_correlation() {
        let mut lambda = KyleLambda::new(0, 50); // Update every call

        // Simulate positive correlation between returns and signed volume
        for i in 0..50 {
            let ret = 0.001 * (i as f64 / 50.0);
            let signed_vol = 1000.0 * (i as f64 / 50.0);
            lambda.maybe_update(i * 1_000_000_000, ret, signed_vol);
        }

        assert!(lambda.compute() > 0.0); // Positive lambda for positive correlation
    }

    // Price Impact Tests
    #[test]
    fn test_price_impact_buy_lifts_price() {
        let mut impact = PriceImpact::new(0.1, 2);

        // Simulate buy (close > prev) and subsequent price increase
        impact.update(100.5, 99.5, 100.0);
        impact.update(101.0, 100.0, 100.5); // Buy
        impact.update(101.5, 100.5, 101.0); // Price lifted
        impact.update(102.0, 101.0, 101.5); // Continued lift

        // After delay, should see positive impact
        assert!(impact.compute() >= 0.0);
    }

    // Variance Ratio Tests
    #[test]
    fn test_variance_ratio_random_walk() {
        let mut vr = VarianceRatio::new(20, 5);

        // Feed random returns (simulating random walk)
        use std::f64::consts::PI;
        for i in 0..20 {
            let ret = (i as f64 * PI).sin() * 0.001;
            vr.update(ret);
        }

        let ratio = vr.compute();
        assert!(ratio > 0.5 && ratio < 2.0); // Should be near 1.0 for random walk
    }

    #[test]
    fn test_variance_ratio_insufficient_data() {
        let vr = VarianceRatio::new(20, 5);
        assert_eq!(vr.compute(), 1.0); // Should default to 1.0
    }

    // Trait Implementation Tests
    #[test]
    fn test_trait_implementations() {
        let features: Vec<Box<dyn MicrostructureFeature>> = vec![
            Box::new(HighLowSpread::default()),
            Box::new(VolumeWeightedSpread::default()),
            Box::new(TickCount::default()),
            Box::new(InterArrivalTime::default()),
            Box::new(BuySellImbalance::default()),
            Box::new(KyleLambda::default()),
            Box::new(PriceImpact::default()),
            Box::new(VarianceRatio::default()),
        ];

        for feature in features {
            assert!(!feature.feature_name().is_empty());
            assert!(feature.get_normalized().is_finite());
        }
    }

    // Normalization Tests
    #[test]
    fn test_normalization_bounds() {
        let mut hl_spread = HighLowSpread::new(0.1);
        hl_spread.update(102.0, 98.0);
        let normalized = hl_spread.get_normalized();
        assert!(normalized >= -1.0 && normalized <= 1.0);

        let mut imbalance = BuySellImbalance::new(0.1);
        imbalance.update(101.0, 1000.0);
        let normalized = imbalance.get_normalized();
        assert!(normalized >= -1.0 && normalized <= 1.0);

        let vr = VarianceRatio::new(20, 5);
        let normalized = vr.get_normalized();
        assert!(normalized >= -1.0 && normalized <= 1.0);
    }

    // Reset Tests
    #[test]
    fn test_reset_all_features() {
        let mut hl_spread = HighLowSpread::new(0.1);
        hl_spread.update(102.0, 98.0);
        hl_spread.reset();
        assert_eq!(hl_spread.value(), 0.0);

        let mut tick_count = TickCount::new(10);
        tick_count.update(100.0);
        tick_count.update(101.0);
        tick_count.reset();
        assert_eq!(tick_count.value(), 0.0);
    }

    // ─────────────────────────────────────────────────────────────────
    // V10 Block T — MultiLevelKyleLambda
    // ─────────────────────────────────────────────────────────────────

    #[test]
    fn test_multilevel_kyle_lambda_cold_start_returns_zeros() {
        let mut k = MultiLevelKyleLambda::new(1, 50);
        // First call: 1 data point < 10 threshold → returns [0; 5]
        let lambdas = k.maybe_update(0, 0.001, [10.0, 5.0, 3.0, 2.0, 1.0]);
        assert_eq!(lambdas, [0.0; 5], "cold start should return zeros");
    }

    #[test]
    fn test_multilevel_kyle_lambda_constant_ofi_zero_variance() {
        let mut k = MultiLevelKyleLambda::new(0, 50); // refresh every call
        // 20 observations of identical OFI → var_o = 0 → λ = 0 (guard branch)
        for i in 0..20 {
            let ts = ((i + 1) as u64) * 2_000_000_000;
            k.maybe_update(ts, 0.001 * (i as f64), [5.0, 3.0, 2.0, 1.0, 0.5]);
        }
        for &v in k.lambdas().iter() {
            assert_eq!(v, 0.0, "constant OFI (zero variance) must yield λ=0");
        }
    }

    #[test]
    fn test_multilevel_kyle_lambda_recovers_linear_l3_signal() {
        // Synthetic data: return = 1.5 × OFI_L3, other levels are
        // high-frequency uncorrelated noise. With 200 samples and noise
        // frequencies well-separated from the signal frequency, residual
        // λ_noise should average to << λ_signal.
        //
        // Honest expectation: λ_L3 → 1.5 exactly; λ_others bounded BELOW
        // |λ_L3| (i.e. clearly distinguishable as noise).
        let mut k = MultiLevelKyleLambda::new(0, 200);
        for i in 0..200 {
            let ts = ((i + 1) as u64) * 1_000_000_000;
            let ofi_l3 = (i as f64).sin() * 100.0;
            let ret = 1.5 * ofi_l3;
            // Use prime-frequency multipliers so noise is far from the
            // signal's unit frequency. Phases also primed to decorrelate.
            let ofi_per = [
                ((i as f64) * 13.0 + 7.0).sin() * 10.0,
                ((i as f64) * 17.0 + 11.0).cos() * 8.0,
                ofi_l3,
                ((i as f64) * 23.0 + 13.0).sin() * 5.0,
                ((i as f64) * 29.0 + 17.0).cos() * 3.0,
            ];
            k.maybe_update(ts, ret, ofi_per);
        }
        let lambdas = k.lambdas();
        // L3 should recover the linear coefficient with tight tolerance.
        assert!(
            (lambdas[2] - 1.5).abs() < 0.01,
            "λ_L3 should recover the linear coefficient 1.5, got {}",
            lambdas[2]
        );
        // Noise levels: |λ| should be CLEARLY below the signal's 1.5.
        // With 200 samples and prime-frequency noise, |λ_noise| < 0.5 typically.
        for level in [0_usize, 1, 3, 4] {
            assert!(
                lambdas[level].abs() < 1.0,
                "λ_L{} should be << λ_signal=1.5 (noise-only), got {}",
                level + 1,
                lambdas[level]
            );
        }
    }

    #[test]
    fn test_multilevel_kyle_lambda_caches_between_refresh_intervals() {
        // Refresh interval = 1 hour: feed enough data to lock cached lambdas,
        // then feed wildly different data within the hour. Cached must not move.
        let mut k = MultiLevelKyleLambda::new(3600, 50);
        for i in 0..15 {
            let ts = ((i + 1) as u64) * 1_000_000_000;
            k.maybe_update(ts, 0.001 * (i as f64), [(i as f64), 0.0, 0.0, 0.0, 0.0]);
        }
        let lambdas_first = k.lambdas();

        // Inject crazy data WITHIN the refresh window
        for i in 15..30 {
            let ts = ((i + 1) as u64) * 1_000_000_000;
            k.maybe_update(ts, 999.0, [999.0, 999.0, 999.0, 999.0, 999.0]);
        }
        let lambdas_after = k.lambdas();
        assert_eq!(
            lambdas_first, lambdas_after,
            "lambdas must remain cached within the refresh interval"
        );
    }
}