foxhunt/crates/trading_engine/src/features/mod.rs

//! Features Module - Core Feature Engineering
//!
//! This module provides the unified feature extraction system that ensures
//! zero training/serving skew across all ML models and trading stages.
//!
//! ## Key Components
//!
//! - `UnifiedFeatureExtractor`: Single source of truth for all feature calculations
//! - Model-specific feature sets: TLOB, MAMBA, DQN, PPO, Liquid, TFT
//! - Data provider integration: Databento (market data) + Benzinga (news/sentiment)
//! - High-performance SIMD optimizations for real-time processing
//!
//! ## Architecture Principles
//!
//! 1. **Single Source of Truth**: All features calculated identically across:
//!    - Training: Historical data processing
//!    - Backtesting: Strategy validation
//!    - Live Trading: Real-time inference
//!
//! 2. **Data Provider Separation**:
//!    - Databento: Market microstructure (trades, quotes, order books)
//!    - Benzinga: News sentiment, analyst ratings, unusual options
//!
//! 3. **Model-Specific Features**:
//!    - TLOB: Order book sequences for transformer analysis
//!    - MAMBA: Long sequences for state space modeling
//!    - DQN: State representation for reinforcement learning
//!    - PPO: Policy-specific features with advantage estimation
//!    - Liquid: Adaptive features for regime detection
//!    - TFT: Multi-horizon sequences with attention inputs
//!
//! ## Usage Example
//!
//! ``rust
//! use core::features::{UnifiedFeatureExtractor, UnifiedConfig};
//!
//! let config = UnifiedConfig::default();
//! let mut extractor = UnifiedFeatureExtractor::new(config);
//!
//! // Extract TLOB features for transformer model
//! let tlob_features = extractor.extract_tlob_features(
//!     &symbol,
//!     &databento_data,
//!     &benzinga_data,
//!     &historical_data
//! ).await?;
//!
//! // Extract DQN features for reinforcement learning
//! let dqn_features = extractor.extract_dqn_features(
//!     &symbol,
//!     &databento_data,
//!     &benzinga_data,
//!     &historical_data,
//!     current_position,
//!     unrealized_pnl
//! ).await?;
//! ``
//!
//! ## Performance Characteristics
//!
//! - **Latency**: Sub-millisecond feature extraction via SIMD
//! - **Throughput**: 10,000+ symbols processed per second
//! - **Memory**: Efficient caching with configurable TTL
//! - **Accuracy**: Identical calculations across all environments

pub mod unified_extractor;

use async_trait::async_trait;

// Types are NOT re-exported - use explicit imports at usage sites:
//   use crate::features::unified_extractor::{...};
use unified_extractor::{
    BaseMarketFeatures, BenzingaNewsData, BenzingaNewsFeatures, DatabentoBuData,
    DatabentoBuFeatures, FeatureError, UnifiedFeatureExtractor,
};

/// Feature extraction result type for ergonomic error handling
pub type FeatureResult<T> = Result<T, FeatureError>;

/// Trait for model-specific feature extraction
#[async_trait]
/// ModelFeatureExtractor
/// 
/// Auto-generated documentation placeholder - enhance with specifics
pub trait ModelFeatureExtractor<F> {
    /// Extract features specific to this model type
    async fn extract_features(
        &mut self,
        extractor: &mut UnifiedFeatureExtractor,
        symbol: &common::types::Symbol,
        databento_data: &DatabentoBuData,
        benzinga_data: &BenzingaNewsData,
        historical_data: &[common::types::MarketTick],
    ) -> FeatureResult<F>;
}

/// Convenience macros for feature extraction
#[macro_export]
macro_rules! extract_features {
    ($extractor:expr, $model:ident, $symbol:expr, $databento:expr, $benzinga:expr, $historical:expr) => {
        $extractor.paste::paste! {
            [<extract_ $model:lower _features>]
        }($symbol, $databento, $benzinga, $historical).await
    };

    ($extractor:expr, $model:ident, $symbol:expr, $databento:expr, $benzinga:expr, $historical:expr, $($extra:expr),+) => {
        $extractor.paste::paste! {
            [<extract_ $model:lower _features>]
        }($symbol, $databento, $benzinga, $historical, $($extra),+).await
    };
}

/// Feature validation utilities
pub mod validation {
    use super::{
        BaseMarketFeatures, BenzingaNewsFeatures, DatabentoBuFeatures, FeatureError, FeatureResult,
    };

    /// Validate feature quality and completeness
    pub fn validate_base_features(features: &BaseMarketFeatures) -> FeatureResult<()> {
        // Check for NaN/Inf values
        if !features.returns_1m.is_finite() {
            return Err(FeatureError::MathematicalError {
                feature: "returns_1m".to_owned(),
                reason: "Non-finite value detected".to_owned(),
            });
        }

        // Validate ranges
        if features.rsi_14 < 0.0 || features.rsi_14 > 100.0 {
            return Err(FeatureError::MathematicalError {
                feature: "rsi_14".to_owned(),
                reason: format!("RSI out of range: {}", features.rsi_14),
            });
        }

        // Check bollinger position is within reasonable bounds
        if features.bollinger_position < -5.0 || features.bollinger_position > 5.0 {
            return Err(FeatureError::MathematicalError {
                feature: "bollinger_position".to_owned(),
                reason: format!(
                    "Bollinger position extreme: {}",
                    features.bollinger_position
                ),
            });
        }

        Ok(())
    }

    /// Validate Databento features
    pub fn validate_databento_features(features: &DatabentoBuFeatures) -> FeatureResult<()> {
        // Spread should be positive
        if features.bid_ask_spread_bps < 0.0 {
            return Err(FeatureError::MathematicalError {
                feature: "bid_ask_spread_bps".to_owned(),
                reason: "Negative spread detected".to_owned(),
            });
        }

        // Order book imbalance should be in [-1, 1]
        if features.order_book_imbalance < -1.0 || features.order_book_imbalance > 1.0 {
            return Err(FeatureError::MathematicalError {
                feature: "order_book_imbalance".to_owned(),
                reason: format!("Imbalance out of range: {}", features.order_book_imbalance),
            });
        }

        // Trade sign should be -1, 0, or 1
        if ![-1, 0, 1].contains(&features.trade_sign) {
            return Err(FeatureError::MathematicalError {
                feature: "trade_sign".to_owned(),
                reason: format!("Invalid trade sign: {}", features.trade_sign),
            });
        }

        Ok(())
    }

    /// Validate Benzinga sentiment features
    pub fn validate_benzinga_features(features: &BenzingaNewsFeatures) -> FeatureResult<()> {
        // Sentiment score should be in [-1, 1]
        if features.sentiment_score < -1.0 || features.sentiment_score > 1.0 {
            return Err(FeatureError::MathematicalError {
                feature: "sentiment_score".to_owned(),
                reason: format!("Sentiment out of range: {}", features.sentiment_score),
            });
        }

        // Confidence should be in [0, 1]
        if features.sentiment_confidence < 0.0 || features.sentiment_confidence > 1.0 {
            return Err(FeatureError::MathematicalError {
                feature: "sentiment_confidence".to_owned(),
                reason: format!("Confidence out of range: {}", features.sentiment_confidence),
            });
        }

        // News velocity should be non-negative
        if features.news_velocity < 0.0 {
            return Err(FeatureError::MathematicalError {
                feature: "news_velocity".to_owned(),
                reason: "Negative news velocity".to_owned(),
            });
        }

        Ok(())
    }
}

/// Performance monitoring for feature extraction
pub mod monitoring {
    use std::collections::HashMap;
    use std::time::{Duration, Instant};

    /// Feature extraction performance metrics
    #[derive(Debug, Clone)]
    pub struct FeatureMetrics {
        /// Extraction Time
        pub extraction_time: Duration,
        /// Feature Count
        pub feature_count: usize,
        /// Cache Hits
        pub cache_hits: usize,
        /// Cache Misses
        pub cache_misses: usize,
        /// Validation Time
        pub validation_time: Duration,
    }

    /// Performance monitor for feature extraction
    #[derive(Debug)]
    pub struct FeatureMonitor {
        metrics: HashMap<String, Vec<FeatureMetrics>>,
        start_times: HashMap<String, Instant>,
    }

    impl FeatureMonitor {
        pub fn new() -> Self {
            Self {
                metrics: HashMap::new(),
                start_times: HashMap::new(),
            }
        }

        /// Start timing a feature extraction operation
        pub fn start_timing(&mut self, operation: &str) {
            self.start_times
                .insert(operation.to_owned(), Instant::now());
        }

        /// End timing and record metrics
        pub fn end_timing(
            &mut self,
            operation: &str,
            feature_count: usize,
            cache_hits: usize,
            cache_misses: usize,
        ) {
            if let Some(start_time) = self.start_times.remove(operation) {
                let extraction_time = start_time.elapsed();
                let metrics = FeatureMetrics {
                    extraction_time,
                    feature_count,
                    cache_hits,
                    cache_misses,
                    validation_time: Duration::from_nanos(0), // Set by validation
                };

                self.metrics
                    .entry(operation.to_owned())
                    .or_default()
                    .push(metrics);
            }
        }

        /// Get average extraction time for an operation
        pub fn average_extraction_time(&self, operation: &str) -> Option<Duration> {
            self.metrics.get(operation).and_then(|metrics| {
                if metrics.is_empty() {
                    return None;
                }

                let total: Duration = metrics.iter().map(|m| m.extraction_time).sum();
                Some(total / metrics.len() as u32)
            })
        }

        /// Get cache hit rate for an operation
        pub fn cache_hit_rate(&self, operation: &str) -> Option<f64> {
            self.metrics.get(operation).and_then(|metrics| {
                if metrics.is_empty() {
                    return None;
                }

                let total_hits: usize = metrics.iter().map(|m| m.cache_hits).sum();
                let total_requests: usize =
                    metrics.iter().map(|m| m.cache_hits + m.cache_misses).sum();

                if total_requests == 0 {
    // None variant
                    None
                } else {
    // Some variant
                    Some(total_hits as f64 / total_requests as f64)
                }
            })
        }
    }

    impl Default for FeatureMonitor {
        fn default() -> Self {
            Self::new()
        }
    }
}

/// Testing utilities for feature validation
#[cfg(test)]
pub mod test_utils {
    use chrono::Utc;
    use common::types::{Exchange, HftTimestamp, MarketTick, TickType};
    use common::{Price, Quantity, Symbol, TradeEvent};
    use rust_decimal_macros::dec;
    use super::unified_extractor::{
        BenzingaNewsData, DatabentoBuData, NewsArticle, SentimentScore,
    };

    /// Create mock Databento data for testing
    pub fn create_mock_databento_data() -> DatabentoBuData {
        let bid_price = Price::from_f64(100.50).unwrap_or(Price::ZERO);
        let ask_price = Price::from_f64(100.51).unwrap_or(Price::ZERO);
        let bid_qty = Quantity::from_f64(1000.0).unwrap_or(Quantity::ZERO);
        let ask_qty = Quantity::from_f64(800.0).unwrap_or(Quantity::ZERO);

        DatabentoBuData {
            order_book: vec![
                (bid_price, bid_qty),
                (ask_price, ask_qty),
            ],
            trades: vec![TradeEvent::new(
                "AAPL".to_string(),
                dec!(100.505),
                dec!(100),
                Utc::now(),
            ).with_trade_id("T123")],
            quotes: vec![],
            timestamp: Utc::now(),
        }
    }

    /// Create mock Benzinga data for testing
    pub fn create_mock_benzinga_data() -> BenzingaNewsData {
        BenzingaNewsData {
            articles: vec![NewsArticle {
                title: "Apple Reports Strong Q4 Earnings".to_string(),
                content: "Apple exceeded expectations...".to_string(),
                source: "Reuters".to_string(),
                timestamp: Utc::now(),
                symbols: vec![Symbol::new("AAPL".to_string())],
                category: "earnings".to_string(),
                importance: 0.8,
            }],
            sentiment_scores: vec![SentimentScore {
                symbol: Symbol::new("AAPL".to_string()),
                score: 0.6,
                confidence: 0.9,
                timestamp: Utc::now(),
            }],
            analyst_ratings: vec![],
            unusual_options: vec![],
            timestamp: Utc::now(),
        }
    }

    /// Create mock historical market data
    #[allow(clippy::unwrap_used)]
    pub fn create_mock_historical_data() -> Vec<MarketTick> {
        let base_price = 100.0_f64;
        let mut data = Vec::new();

        for i in 0..1000_u64 {
            let price_change = (i as f64 / 100.0).sin() * 0.01;
            let price = Price::from_f64(base_price + price_change).unwrap_or(Price::ZERO);
            let size = Quantity::from_f64(1000.0 + (i % 500) as f64).unwrap_or(Quantity::ZERO);

            data.push(MarketTick {
                symbol: Symbol::new("AAPL".to_string()),
                price,
                size,
                timestamp: HftTimestamp::now().unwrap(),
                tick_type: TickType::Trade,
                exchange: Exchange::NASDAQ,
                sequence_number: i,
            });
        }

        data
    }
}