foxhunt/data/src/providers/benzinga/mod.rs

//! # Benzinga Provider Module
//!
//! This module provides comprehensive integration with Benzinga Pro API for financial
//! news, sentiment analysis, analyst ratings, and unusual options activity.
//!
//! ## Components
//!
//! - **Streaming Provider**: Real-time WebSocket streaming for live data feeds
//! - **Historical Provider**: REST API access for historical news and events
//! - **Production Providers**: Enhanced versions with advanced features
//! - **ML Integration**: Feature extraction for machine learning models
//! - **HFT Integration**: Complete orchestration layer for high-frequency trading
//!
//! ## Architecture
//!
//! The Benzinga integration follows a multi-tier provider pattern:
//! - `BenzingaStreamingProvider`: Basic WebSocket streaming implementation
//! - `ProductionBenzingaProvider`: Production-grade with rate limiting, deduplication, circuit breakers
//! - `BenzingaHistoricalProvider`: Basic REST API access
//! - `ProductionBenzingaHistoricalProvider`: Production-grade with caching, retry logic, bulk operations
//! - `BenzingaMLExtractor`: ML feature extraction and time series preparation
//! - `BenzingaHFTIntegration`: Complete orchestration layer with trading signal generation
//!
//! ## Usage
//!
//! ### Production Real-time Streaming
//!
//! ```rust,no_run
//! use data::providers::benzinga::{ProductionBenzingaProvider, ProductionBenzingaConfig};
//! use data::providers::traits::RealTimeProvider;
//! use common::Symbol;
//!
//! # async fn example() -> anyhow::Result<()> {
//! let config = ProductionBenzingaConfig {
//!     api_key: "your-benzinga-api-key".to_string(),
//!     enable_news: true,
//!     enable_sentiment: true,
//!     enable_ratings: true,
//!     enable_options: true,
//!     rate_limit_per_second: 100,
//!     enable_ml_integration: true,
//!     ..Default::default()
//! };
//!
//! let mut provider = ProductionBenzingaProvider::new(config)?;
//! provider.connect().await?;
//! provider.subscribe(vec![Symbol::from("AAPL"), Symbol::from("SPY")]).await?;
//!
//! let mut stream = provider.stream().await?;
//! while let Some(event) = stream.next().await {
//!     match event {
//!         MarketDataEvent::NewsAlert(news) => {
//!             println!("News: {} - Impact: {:?}", news.headline, news.impact_score);
//!         }
//!         MarketDataEvent::SentimentUpdate(sentiment) => {
//!             println!("Sentiment for {}: {:.3}", sentiment.symbol, sentiment.sentiment_score);
//!         }
//!         MarketDataEvent::AnalystRating(rating) => {
//!             println!("Rating: {} {} -> {}", rating.symbol, rating.action, rating.current_rating);
//!         }
//!         MarketDataEvent::UnusualOptions(options) => {
//!             println!("Options: {} {:?} Vol: {}", options.symbol, options.activity_type, options.volume);
//!         }
//!         _ => {}
//!     }
//! }
//! # Ok(())
//! # }
//! ```
//!
//! ### Production Historical Data
//!
//! ```rust,no_run
//! use data::providers::benzinga::{ProductionBenzingaHistoricalProvider, ProductionBenzingaHistoricalConfig};
//! use chrono::{Utc, Duration};
//!
//! # async fn example() -> anyhow::Result<()> {
//! let config = ProductionBenzingaHistoricalConfig {
//!     api_key: "your-benzinga-api-key".to_string(),
//!     enable_caching: true,
//!     enable_bulk_download: true,
//!     rate_limit_per_second: 10,
//!     ..Default::default()
//! };
//!
//! let provider = ProductionBenzingaHistoricalProvider::new(config)?;
//! let symbols = ["AAPL", "SPY"];
//! let end = Utc::now();
//! let start = end - Duration::days(7);
//!
//! // Get all events (news, ratings, earnings, options) in parallel
//! let events = provider.get_all_events(Some(&symbols), start, end).await?;
//! println!("Retrieved {} historical events", events.len());
//!
//! // Get specific event types
//! let news = provider.get_news_events(Some(&symbols), start, end).await?;
//! let ratings = provider.get_rating_events(Some(&symbols), start, end).await?;
//! let options = provider.get_options_events(Some(&symbols), start, end).await?;
//!
//! # Ok(())
//! # }
//! ```
//!
//! ### ML Feature Extraction
//!
//! ```rust,no_run
//! use data::providers::benzinga::{BenzingaMLExtractor, BenzingaMLConfig};
//! use data::providers::common::MarketDataEvent;
//! use chrono::Utc;
//! use common::Symbol;
//!
//! # async fn example() -> anyhow::Result<()> {
//! let config = BenzingaMLConfig {
//!     feature_window_minutes: 60,
//!     enable_nlp_features: true,
//!     enable_sentiment_indicators: true,
//!     normalization_method: data::providers::benzinga::NormalizationMethod::ZScore,
//!     ..Default::default()
//! };
//!
//! let mut extractor = BenzingaMLExtractor::new(config);
//!
//! // Process real-time events
//! let event = MarketDataEvent::NewsAlert(/* news event */);
//! extractor.process_event(&event).await?;
//!
//! // Extract features for ML models
//! let symbol = Symbol::from("AAPL");
//! let features = extractor.extract_features(&symbol, Utc::now()).await?;
//!
//! println!("Feature vector dimension: {}", extractor.get_feature_dimension());
//! println!("Feature names: {:?}", extractor.get_feature_names());
//!
//! // Batch feature extraction
//! let symbols = vec![Symbol::from("AAPL"), Symbol::from("SPY")];
//! let batch_features = extractor.extract_features_batch(&symbols, Utc::now()).await?;
//!
//! # Ok(())
//! # }
//! ```
//!
//! ### HFT Integration (Complete System)
//!
//! ```rust,no_run
//! use data::providers::benzinga::{BenzingaHFTIntegration, BenzingaIntegrationConfig, TradingSignal, TradingSignalType};
//! use config::ConfigManager;
//! use common::Symbol;
//! use std::sync::Arc;
//!
//! # async fn example() -> anyhow::Result<()> {
//! let config = BenzingaIntegrationConfig {
//!     enable_streaming: true,
//!     enable_historical: true,
//!     enable_ml_integration: true,
//!     symbols: vec![Symbol::from("AAPL"), Symbol::from("SPY")],
//!     signal_config: SignalConfig {
//!         news_impact_threshold: 0.7,
//!         sentiment_momentum_threshold: 0.5,
//!         analyst_rating_enabled: true,
//!         options_flow_threshold: 1000,
//!     },
//!     ..Default::default()
//! };
//!
//! // Create comprehensive HFT integration
//! let mut integration = BenzingaHFTIntegration::new(config).await?;
//! integration.start().await?;
//!
//! // Process trading signals in real-time
//! while let Some(signal) = integration.next_signal().await {
//!     match signal.signal_type {
//!         TradingSignalType::NewsImpact => {
//!             println!("News Impact: {} - Strength: {:.3}", signal.symbol, signal.strength);
//!             // Route to trading engine...
//!         }
//!         TradingSignalType::SentimentShift => {
//!             println!("Sentiment Shift: {} - Direction: {}", signal.symbol,
//!                     if signal.strength > 0.0 { "Bullish" } else { "Bearish" });
//!         }
//!         TradingSignalType::AnalystAction => {
//!             println!("Analyst Action: {} - Confidence: {:.3}", signal.symbol, signal.confidence);
//!         }
//!         TradingSignalType::OptionsFlow => {
//!             println!("Options Flow: {} - Activity: {:.0}", signal.symbol, signal.strength);
//!         }
//!     }
//! }
//!
//! integration.stop().await?;
//! # Ok(())
//! # }
//! ```
//!
//! ## Event Types
//!
//! The Benzinga providers emit the following `MarketDataEvent` types:
//!
//! - `NewsAlert`: Breaking financial news with impact scoring and smart categorization
//! - `SentimentUpdate`: AI-powered sentiment analysis scores with technical indicators
//! - `AnalystRating`: Analyst upgrades, downgrades, and price targets with consensus tracking
//! - `UnusualOptions`: Unusual options activity detection with sentiment analysis
//! - `ConnectionStatus`: Provider connection state changes
//! - `Error`: Provider error notifications with recovery information
//!
//! ## Production Features
//!
//! ### Streaming Provider
//! - Advanced rate limiting with token bucket algorithm
//! - Message deduplication using SHA-256 hashing
//! - Circuit breakers for fault tolerance
//! - Smart categorization with ML-enhanced classification
//! - Batch processing for efficiency
//! - Comprehensive metrics and monitoring
//!
//! ### Historical Provider
//! - Redis and in-memory caching with TTL
//! - Retry logic with exponential backoff
//! - Bulk data download capabilities
//! - Data quality validation and filtering
//! - Concurrent API requests with semaphore control
//! - Comprehensive event coverage (news, earnings, ratings, options, calendar)
//!
//! ### ML Integration
//! - 50+ engineered features for temporal ML models
//! - Real-time feature extraction for TFT and Liquid Networks
//! - Technical indicators applied to sentiment data
//! - NLP features with keyword and topic analysis
//! - Multiple normalization methods (Z-score, Min-Max, Robust)
//! - Batch processing and caching for performance
//!
//! ### HFT Integration
//! - Complete orchestration layer for high-frequency trading
//! - Real-time trading signal generation from news/sentiment events
//! - ML model integration with feature queues for TFT and Liquid Networks
//! - Event-driven architecture optimized for sub-millisecond latency
//! - Automated symbol monitoring and signal routing
//! - Performance metrics and latency monitoring
//! - Signal strength calibration and confidence scoring
//!
//! ## Configuration
//!
//! All providers require a Benzinga Pro API key. Set the `BENZINGA_API_KEY`
//! environment variable or provide it directly in the configuration.
//!
//! Optional Redis caching can be enabled by setting `REDIS_URL` environment variable.
//!
//! ## Rate Limits
//!
//! Benzinga Pro has rate limits that vary by subscription tier:
//! - Basic: 5 requests/second
//! - Professional: 20 requests/second  
//! - Enterprise: 100+ requests/second
//!
//! The providers implement automatic rate limiting and respect API quotas.

// Import required types using canonical paths
// Import types for factory methods
use crate::providers::benzinga::production_streaming::{ProductionBenzingaProvider, ProductionBenzingaConfig};
use crate::providers::benzinga::production_historical::{ProductionBenzingaHistoricalProvider, ProductionBenzingaHistoricalConfig};
// Note: BenzingaConfig, BenzingaHistoricalProvider, BenzingaStreamingConfig, BenzingaStreamingProvider
// are re-exported below for external consumption

// Re-export the streaming provider
pub mod streaming;

// Re-export the historical provider
pub mod historical;

// Production-grade providers
pub mod production_historical;
pub mod production_streaming;

// ML integration module
pub mod ml_integration;

// HFT integration orchestration
pub mod integration;

// Convenience re-exports for common types that are frequently used

// Re-export core types from common module
pub use crate::providers::common::{
    NewsEvent, NewsEventType, SentimentEvent, SentimentPeriod, AnalystRatingEvent, RatingAction,
    UnusualOptionsEvent, OptionsContract, OptionsType, OptionsSentiment, UnusualOptionsType,
};

// Re-export benzinga-specific types from historical module
pub use self::historical::{
    BenzingaChannel, BenzingaNewsArticle, BenzingaRating, BenzingaTag, BenzingaEarnings,
    BenzingaEconomicEvent,
};

// Re-export the main config and provider types for external consumption
pub use crate::providers::benzinga::historical::{BenzingaConfig, BenzingaHistoricalProvider};
pub use crate::providers::benzinga::streaming::{BenzingaStreamingConfig, BenzingaStreamingProvider};

// Production provider re-exports
// DO NOT RE-EXPORT - Use explicit imports at usage sites
// pub use crate::providers::benzinga::production_historical::{
//     ProductionBenzingaHistoricalConfig, ProductionBenzingaHistoricalProvider,
// };

// ML integration re-exports
// DO NOT RE-EXPORT - Use explicit imports at usage sites
// pub use crate::providers::benzinga::ml_integration::{
//     BenzingaFeatureVector, BenzingaMLConfig, BenzingaMLExtractor, NormalizationMethod,
// };

// HFT integration re-exports
// DO NOT RE-EXPORT - Use explicit imports at usage sites
// pub use crate::providers::benzinga::integration::{
//     BenzingaHFTIntegration, MLModelIntegration, SignalConfig,
//     TradingSignal,
// };

/// Benzinga provider factory for creating provider instances
pub struct BenzingaProviderFactory;

impl BenzingaProviderFactory {
    /// Create a new production streaming provider with the given configuration
    pub fn create_production_streaming_provider(
        config: ProductionBenzingaConfig,
    ) -> crate::error::Result<ProductionBenzingaProvider> {
        ProductionBenzingaProvider::new(config)
    }

    /// Create a new production historical provider with the given configuration
    pub fn create_production_historical_provider(
        config: ProductionBenzingaHistoricalConfig,
    ) -> crate::error::Result<ProductionBenzingaHistoricalProvider> {
        ProductionBenzingaHistoricalProvider::new(config)
    }

    /// Create ML feature extractor
    pub fn create_ml_extractor(config: ml_integration::BenzingaMLConfig) -> ml_integration::BenzingaMLExtractor {
        ml_integration::BenzingaMLExtractor::new(config)
    }

    /// Create a basic streaming provider with the given configuration
    pub fn create_streaming_provider(
        config: BenzingaStreamingConfig,
    ) -> crate::error::Result<BenzingaStreamingProvider> {
        BenzingaStreamingProvider::new(config)
    }

    /// Create a basic historical provider with the given configuration
    pub fn create_historical_provider(
        config: BenzingaConfig,
    ) -> crate::error::Result<BenzingaHistoricalProvider> {
        BenzingaHistoricalProvider::new(config)
    }

    /// Create a production streaming provider from environment variables
    pub fn create_production_streaming_from_env() -> crate::error::Result<ProductionBenzingaProvider>
    {
        let config = ProductionBenzingaConfig::default();
        Self::create_production_streaming_provider(config)
    }

    /// Create a production historical provider from environment variables
    pub fn create_production_historical_from_env(
    ) -> crate::error::Result<ProductionBenzingaHistoricalProvider> {
        let config = ProductionBenzingaHistoricalConfig::default();
        Self::create_production_historical_provider(config)
    }

    /// Create ML extractor from environment
    pub fn create_ml_extractor_from_env() -> ml_integration::BenzingaMLExtractor {
        let config = ml_integration::BenzingaMLConfig::default();
        Self::create_ml_extractor(config)
    }

    /// Create HFT integration instance
    pub async fn create_hft_integration(
        _config: BenzingaStreamingConfig,
    ) -> crate::error::Result<integration::BenzingaHFTIntegration> {
        // Create a default config manager for now - this needs proper implementation
        let default_config = config::manager::ServiceConfig {
            name: "benzinga_service".to_string(),
            environment: "development".to_string(),
            version: "1.0.0".to_string(),
            settings: serde_json::json!({}),
        };
        let config_manager = config::manager::ConfigManager::new(default_config);
        integration::BenzingaHFTIntegration::new(config_manager).await
    }

    /// Create HFT integration from environment variables
    pub async fn create_hft_integration_from_env() -> crate::error::Result<integration::BenzingaHFTIntegration> {
        let config = BenzingaStreamingConfig::default();
        Self::create_hft_integration(config).await
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_factory_creation_with_api_key() {
        let streaming_config = ProductionBenzingaConfig {
            api_key: "test-key".to_string(),
            ..Default::default()
        };

        let result =
            BenzingaProviderFactory::create_production_streaming_provider(streaming_config);
        assert!(result.is_ok());

        let historical_config = ProductionBenzingaHistoricalConfig {
            api_key: "test-key".to_string(),
            ..Default::default()
        };

        let result =
            BenzingaProviderFactory::create_production_historical_provider(historical_config);
        assert!(result.is_ok());
    }

    #[test]
    fn test_factory_creation_without_api_key() {
        let streaming_config = ProductionBenzingaConfig {
            api_key: "".to_string(),
            ..Default::default()
        };

        let result =
            BenzingaProviderFactory::create_production_streaming_provider(streaming_config);
        assert!(result.is_err());
    }

    #[test]
    fn test_ml_extractor_creation() {
        let config = BenzingaMLConfig::default();
        let extractor = BenzingaProviderFactory::create_ml_extractor(config);

        assert!(extractor.get_feature_dimension() > 0);
        assert!(!extractor.get_feature_names().is_empty());
    }

    #[test]
    fn test_factory_from_env() {
        // These will use default values from environment variables
        let streaming_result = BenzingaProviderFactory::create_production_streaming_from_env();
        let historical_result = BenzingaProviderFactory::create_production_historical_from_env();
        let ml_extractor = BenzingaProviderFactory::create_ml_extractor_from_env();

        // May fail due to missing API key in test environment, but should not panic
        // In production with proper API key, these would succeed
        assert!(streaming_result.is_err() || streaming_result.is_ok());
        assert!(historical_result.is_err() || historical_result.is_ok());
        assert!(ml_extractor.get_feature_dimension() > 0);
    }

    #[tokio::test]
    async fn test_hft_integration_creation() {
        use common::Symbol;

        let config = BenzingaStreamingConfig {
            api_key: "test-key".to_string(),
            enable_news: true,
            enable_sentiment: true,
            ..Default::default()
        };

        let result = BenzingaProviderFactory::create_hft_integration(config).await;
        // May fail due to missing API key or other dependencies in test environment
        assert!(result.is_err() || result.is_ok());
    }
}