//! Unified Historical Data Loader for ML Training
//!
//! Modern data pipeline for training financial ML models using dual data providers:
//! - Databento for high-quality market data
//! - Benzinga for news sentiment data
//!
//! This replaces the old Polygon-based pipeline with a more robust architecture
//! that ensures training and serving feature extraction are identical.

use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant};

use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use tokio::sync::RwLock;
use tracing::{debug, info};

use crate::features::extraction::{extract_ml_features, OHLCVBar};
use crate::features::unified::{FeatureExtractionConfig, UnifiedFeatureExtractor};
use crate::safety::MLSafetyManager;
use crate::{MLError, MLResult};
use common::types::{Price, Symbol, Volume};
// Using placeholder OrderLevel type
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct OrderLevel {
    pub price: f64,
    pub quantity: f64,
}

/// Configuration for the unified data loader
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct UnifiedDataLoaderConfig {
    /// `Databento` API configuration
    pub databento_config: DatabentoConfig,
    /// Benzinga API configuration  
    pub benzinga_config: BenzingaConfig,
    /// Data processing settings
    pub processing: DataProcessingConfig,
    /// Training data settings
    pub training: TrainingDataConfig,
    /// Feature extraction settings
    pub feature_extraction: FeatureExtractionSettings,
}

/// `Databento` historical data provider configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DatabentoConfig {
    /// API key for `Databento`
    pub api_key: String,
    /// API endpoint
    pub endpoint: String,
    /// Dataset to use (e.g., "XNAS.ITCH")
    pub dataset: String,
    /// Symbols to subscribe to
    pub symbols: Vec<String>,
    /// Data types to request
    pub data_types: Vec<String>,
    /// Request timeout in seconds
    pub timeout_seconds: u64,
    /// Rate limit (requests per second)
    pub rate_limit: u32,
}

/// Benzinga news provider configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BenzingaConfig {
    /// API key for Benzinga
    pub api_key: String,
    /// API endpoint
    pub endpoint: String,
    /// News channels to monitor
    pub channels: Vec<String>,
    /// Symbols to get news for
    pub symbols: Vec<String>,
    /// Request timeout in seconds
    pub timeout_seconds: u64,
    /// Rate limit (requests per second)
    pub rate_limit: u32,
}

/// Data processing configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DataProcessingConfig {
    /// Time window for aggregating data (seconds)
    pub window_size_seconds: u64,
    /// Overlap between windows (seconds)
    pub window_overlap_seconds: u64,
    /// Maximum data age to include (hours)
    pub max_age_hours: u64,
    /// Enable data quality filtering
    pub enable_quality_filtering: bool,
    /// Minimum data completeness ratio (0.0 to 1.0)
    pub min_completeness_ratio: f64,
}

/// Training data configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TrainingDataConfig {
    /// Training/validation split ratio
    pub train_val_split: f64,
    /// Sequence length for time series models
    pub sequence_length: usize,
    /// Prediction horizon (number of steps ahead)
    pub prediction_horizon: usize,
    /// Batch size for training
    pub batch_size: usize,
    /// Maximum samples per symbol
    pub max_samples_per_symbol: Option<usize>,
    /// Enable data augmentation
    pub enable_augmentation: bool,
}

/// Feature extraction settings
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FeatureExtractionSettings {
    /// Use the same UnifiedFeatureExtractor as trading
    pub use_unified_extractor: bool,
    /// Feature normalization method
    pub normalization: String,
    /// Feature selection method
    pub feature_selection: Option<String>,
    /// Dimensionality reduction method
    pub dimensionality_reduction: Option<String>,
}

/// Market data container for training
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MarketDataContainer {
    /// Symbol this data is for
    pub symbol: Symbol,
    /// Timestamp of the data point
    pub timestamp: DateTime<Utc>,
    /// Price data
    pub price_data: PriceData,
    /// Volume data
    pub volume_data: VolumeData,
    /// Order book data (if available)
    pub orderbook_data: Option<OrderBookData>,
    /// News sentiment data
    pub news_sentiment: Option<NewsSentimentData>,
    /// Metadata
    pub metadata: HashMap<String, String>,
}

/// Price data structure
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PriceData {
    pub open: Price,
    pub high: Price,
    pub low: Price,
    pub close: Price,
    pub bid: Option<Price>,
    pub ask: Option<Price>,
    pub mid: Option<Price>,
    pub spread: Option<Price>,
    pub vwap: Option<Price>,
}

/// Volume data structure
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VolumeData {
    pub volume: Volume,
    pub bid_volume: Option<Volume>,
    pub ask_volume: Option<Volume>,
    pub trade_count: Option<u64>,
    pub dollar_volume: Option<f64>,
}

/// Order book data structure
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OrderBookData {
    pub bid_levels: Vec<OrderLevel>,
    pub ask_levels: Vec<OrderLevel>,
    pub timestamp: DateTime<Utc>,
    pub sequence: Option<u64>,
}

/// News sentiment data structure
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NewsSentimentData {
    pub sentiment_score: f64,
    pub sentiment_label: String,
    pub confidence: f64,
    pub news_count: u32,
    pub topics: Vec<String>,
    pub timestamp: DateTime<Utc>,
}

/// Training dataset structure
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TrainingDataset {
    /// Training samples with features
    pub training_samples: Vec<TrainingSample>,
    /// Validation samples
    pub validation_samples: Vec<TrainingSample>,
    /// Feature metadata
    pub feature_metadata: FeatureMetadata,
    /// Dataset statistics
    pub statistics: DatasetStatistics,
}

/// Individual training sample
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TrainingSample {
    /// Input features (51-dim from extract_ml_features, or 2-dim fallback)
    pub features: Vec<f64>,
    /// Target values for supervised learning
    pub targets: Vec<f64>,
    /// Sequence timestamp
    pub timestamp: DateTime<Utc>,
    /// Symbol identifier
    pub symbol: Symbol,
    /// Sample weight (for weighted training)
    pub weight: f64,
    /// Metadata
    pub metadata: HashMap<String, String>,
}

/// Feature metadata
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FeatureMetadata {
    pub feature_names: Vec<String>,
    pub feature_types: Vec<String>,
    pub feature_statistics: HashMap<String, FeatureStats>,
    pub normalization_params: HashMap<String, NormalizationParams>,
}

/// Feature statistics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FeatureStats {
    pub mean: f64,
    pub std: f64,
    pub min: f64,
    pub max: f64,
    pub percentiles: HashMap<String, f64>,
}

/// Normalization parameters
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NormalizationParams {
    pub method: String,
    pub params: HashMap<String, f64>,
}

/// Dataset statistics
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DatasetStatistics {
    pub total_samples: usize,
    pub training_samples: usize,
    pub validation_samples: usize,
    pub symbols: Vec<String>,
    pub time_range: (DateTime<Utc>, DateTime<Utc>),
    pub data_quality_score: f64,
    pub completeness_ratio: f64,
}

/// Unified data loader using `Databento` and Benzinga
#[derive(Debug)]
pub struct UnifiedDataLoader {
    config: UnifiedDataLoaderConfig,
    /// Unified feature extractor for consistent training/serving feature vectors
    feature_extractor: Option<UnifiedFeatureExtractor>,
    safety_manager: Arc<MLSafetyManager>,
    databento_provider: DatabentoHistoricalProvider,
    benzinga_provider: BenzingaHistoricalProvider,
    cache: Arc<RwLock<HashMap<String, CachedData>>>,
}

/// Cached data structure
#[derive(Debug, Clone)]
struct CachedData {
    data: Vec<MarketDataContainer>,
    timestamp: Instant,
    ttl_seconds: u64,
}

/// `Databento` historical data provider
#[derive(Debug)]
pub struct DatabentoHistoricalProvider {
    config: DatabentoConfig,
    client: reqwest::Client,
}

/// Benzinga historical data provider  
#[derive(Debug)]
pub struct BenzingaHistoricalProvider {
    config: BenzingaConfig,
    client: reqwest::Client,
}

impl Default for UnifiedDataLoaderConfig {
    fn default() -> Self {
        // Default test symbols for configuration
        let default_symbols = vec![
            "TEST_SYM_1".to_owned(),
            "TEST_SYM_2".to_owned(),
            "TEST_SYM_3".to_owned(),
        ];

        Self {
            databento_config: DatabentoConfig {
                api_key: std::env::var("DATABENTO_API_KEY")
                    .unwrap_or_else(|_| "DATABENTO_API_KEY_REQUIRED".to_owned()),
                endpoint: "https://hist.databento.com/v2".to_owned(),
                dataset: "XNAS.ITCH".to_owned(),
                symbols: default_symbols.clone(),
                data_types: vec![
                    "trades".to_owned(),
                    "quotes".to_owned(),
                    "mbp-10".to_owned(),
                ],
                timeout_seconds: 30,
                rate_limit: 10,
            },
            benzinga_config: BenzingaConfig {
                api_key: std::env::var("BENZINGA_API_KEY")
                    .unwrap_or_else(|_| "BENZINGA_API_KEY_REQUIRED".to_owned()),
                endpoint: "https://api.benzinga.com/api/v2".to_owned(),
                channels: vec!["news".to_owned(), "ratings".to_owned()],
                symbols: default_symbols,
                timeout_seconds: 30,
                rate_limit: 5,
            },
            processing: DataProcessingConfig {
                window_size_seconds: 60,
                window_overlap_seconds: 30,
                max_age_hours: 24,
                enable_quality_filtering: true,
                min_completeness_ratio: 0.8,
            },
            training: TrainingDataConfig {
                train_val_split: 0.8,
                sequence_length: 120,
                prediction_horizon: 1,
                batch_size: 32,
                max_samples_per_symbol: Some(10000),
                enable_augmentation: false,
            },
            feature_extraction: FeatureExtractionSettings {
                use_unified_extractor: true,
                normalization: "zscore".to_owned(),
                feature_selection: None,
                dimensionality_reduction: None,
            },
        }
    }
}

impl UnifiedDataLoader {
    /// Create new unified data loader
    pub fn new(config: UnifiedDataLoaderConfig) -> MLResult<Self> {
        let safety_manager = Arc::new(MLSafetyManager::new(
            crate::safety::MLSafetyConfig::default(),
        ));

        let databento_provider = DatabentoHistoricalProvider::new(config.databento_config.clone())?;
        let benzinga_provider = BenzingaHistoricalProvider::new(config.benzinga_config.clone())?;

        let feature_extractor =
            config.feature_extraction.use_unified_extractor.then(|| {
                UnifiedFeatureExtractor::new(
                    FeatureExtractionConfig::default(),
                    Arc::clone(&safety_manager),
                )
            });

        Ok(Self {
            config,
            feature_extractor,
            safety_manager,
            databento_provider,
            benzinga_provider,
            cache: Arc::new(RwLock::new(HashMap::new())),
        })
    }

    /// Load training data for given symbols and time range
    pub async fn load_training_data(
        &self,
        symbols: &[Symbol],
        start_time: DateTime<Utc>,
        end_time: DateTime<Utc>,
    ) -> MLResult<TrainingDataset> {
        info!(
            "Loading training data for {} symbols from {} to {}",
            symbols.len(),
            start_time,
            end_time
        );

        let start_load = Instant::now();

        // Load market data from Databento
        let market_data_futures = symbols.iter().map(|symbol| {
            self.databento_provider
                .load_historical_data(symbol.clone(), start_time, end_time)
        });

        // Load news data from Benzinga
        let news_data_futures = symbols.iter().map(|symbol| {
            self.benzinga_provider
                .load_news_data(symbol.clone(), start_time, end_time)
        });

        let market_data_results = futures::future::join_all(market_data_futures).await;
        let news_data_results = futures::future::join_all(news_data_futures).await;

        // Process and merge data
        let mut all_containers = Vec::new();

        for (i, symbol) in symbols.into_iter().enumerate() {
            let market_data = market_data_results[i].as_ref().map_err(|e| {
                MLError::TrainingError(format!("Failed to load market data for {}: {}", symbol, e))
            })?;

            let news_data = news_data_results[i].as_ref().map_err(|e| {
                MLError::TrainingError(format!("Failed to load news data for {}: {}", symbol, e))
            })?;

            // Merge market and news data by timestamp
            let merged_data = self
                .merge_market_and_news_data(market_data, news_data)
                .await?;
            all_containers.extend(merged_data);
        }

        info!(
            "Loaded {} data containers in {:?}",
            all_containers.len(),
            start_load.elapsed()
        );

        // Convert to training samples using UnifiedFeatureExtractor
        let training_samples = self.create_training_samples(all_containers).await?;

        // Split into training and validation sets
        let split_idx =
            (training_samples.len() as f64 * self.config.training.train_val_split) as usize;
        let (training_samples, validation_samples) = training_samples.split_at(split_idx);

        // Generate metadata and statistics
        let feature_metadata = self.generate_feature_metadata(training_samples)?;
        let statistics = self.calculate_dataset_statistics(
            training_samples,
            validation_samples,
            symbols,
            start_time,
            end_time,
        )?;

        info!(
            "Created training dataset with {} training samples, {} validation samples",
            training_samples.len(),
            validation_samples.len()
        );

        Ok(TrainingDataset {
            training_samples: training_samples.to_vec(), // cpu-side split
            validation_samples: validation_samples.to_vec(), // cpu-side split
            feature_metadata,
            statistics,
        })
    }

    /// Merge market data and news data by timestamp
    async fn merge_market_and_news_data(
        &self,
        market_data: &[MarketDataContainer],
        news_data: &[NewsSentimentData],
    ) -> MLResult<Vec<MarketDataContainer>> {
        let mut merged_data = Vec::new();

        for market_point in market_data {
            let mut container = market_point.clone();

            // Find the most recent news sentiment for this timestamp
            let relevant_news = news_data
                .iter()
                .filter(|news| news.timestamp <= container.timestamp)
                .max_by_key(|news| news.timestamp);

            if let Some(news) = relevant_news {
                container.news_sentiment = Some(news.clone());
            }

            merged_data.push(container);
        }

        Ok(merged_data)
    }

    /// Create training samples using the full 51-dimension feature extractor
    async fn create_training_samples(
        &self,
        containers: Vec<MarketDataContainer>,
    ) -> MLResult<Vec<TrainingSample>> {
        if containers.is_empty() {
            return Ok(Vec::new());
        }

        // Convert MarketDataContainers to OHLCVBars for feature extraction
        let bars: Vec<OHLCVBar> = containers
            .iter()
            .map(|c| OHLCVBar {
                timestamp: c.timestamp,
                open: c.price_data.open.as_f64(),
                high: c.price_data.high.as_f64(),
                low: c.price_data.low.as_f64(),
                close: c.price_data.close.as_f64(),
                volume: c.volume_data.volume.as_f64(),
            })
            .collect();

        // Use full feature extractor when available, otherwise fall back to basic features
        if self.feature_extractor.is_some() {
            match extract_ml_features(&bars) {
                Ok(feature_vectors) => {
                    // extract_ml_features skips a warmup period, so feature_vectors.len()
                    // may be less than containers.len(). Align from the end so each
                    // feature vector corresponds to the correct container.
                    let offset = containers.len().saturating_sub(feature_vectors.len());
                    let mut samples = Vec::with_capacity(feature_vectors.len());

                    for (fv, container) in
                        feature_vectors.iter().zip(containers.get(offset..).unwrap_or(&[]))
                    {
                        let features: Vec<f64> = fv.to_vec(); // cpu-side feature copy
                        let targets = self.create_target_values(container)?;

                        samples.push(TrainingSample {
                            features,
                            targets,
                            timestamp: container.timestamp,
                            symbol: container.symbol.clone(),
                            weight: 1.0,
                            metadata: container.metadata.clone(),
                        });
                    }

                    info!(
                        "Created {} training samples with 51-dim features ({} bars skipped for warmup)",
                        samples.len(),
                        offset,
                    );
                    return Ok(samples);
                }
                Err(e) => {
                    debug!(
                        "Full feature extraction failed ({}), falling back to basic features",
                        e
                    );
                }
            }
        }

        // Fallback: basic 2-feature extraction (close price, volume)
        let mut samples = Vec::with_capacity(containers.len());
        for container in &containers {
            let features = vec![
                container.price_data.close.as_f64(),
                container.volume_data.volume.as_f64(),
            ];
            let targets = self.create_target_values(container)?;

            samples.push(TrainingSample {
                features,
                targets,
                timestamp: container.timestamp,
                symbol: container.symbol.clone(),
                weight: 1.0,
                metadata: container.metadata.clone(),
            });
        }

        info!(
            "Created {} training samples with basic 2-dim features (fallback)",
            samples.len(),
        );
        Ok(samples)
    }

    /// Create target values for supervised learning
    fn create_target_values(&self, _container: &MarketDataContainer) -> MLResult<Vec<f64>> {
        // Example target: price direction (1.0 for up, -1.0 for down, 0.0 for sideways)
        // In a real implementation, this would look at future price movements
        Ok(vec![0.0]) // Placeholder
    }

    /// Generate feature metadata
    fn generate_feature_metadata(&self, samples: &[TrainingSample]) -> MLResult<FeatureMetadata> {
        let dim = samples.first().map_or(0, |s| s.features.len());
        let feature_names = if self.feature_extractor.is_some() && dim == 51 {
            // 51-dimension feature names matching extraction.rs ordering
            vec![
                "open", "high", "low", "close", "volume",
                "returns", "log_returns", "high_low_range", "close_open_range",
                "ema_5", "ema_10", "ema_20", "ema_50",
                "rsi_14", "macd_line", "macd_signal", "macd_histogram",
                "bb_upper", "bb_middle", "bb_lower", "bb_width", "bb_pctb",
                "atr_14",
                "higher_high", "lower_low", "inside_bar",
                "vol_sma_20", "vol_ratio", "vol_momentum",
                "hour_sin", "hour_cos", "dow_sin", "dow_cos",
                "std_20", "skewness_20", "kurtosis_20", "z_score",
                "roll_measure", "amihud_illiq", "corwin_schultz",
                "regime_vol", "regime_trend", "regime_mean_rev",
                "ofi_1", "ofi_2", "ofi_3", "ofi_4",
                "ofi_5", "ofi_6", "ofi_7", "ofi_8",
            ]
            .into_iter()
            .map(String::from)
            .collect()
        } else {
            vec!["close".to_owned(), "volume".to_owned()]
        };

        let feature_types = vec!["continuous".to_owned(); feature_names.len()];

        Ok(FeatureMetadata {
            feature_names,
            feature_types,
            feature_statistics: HashMap::new(),
            normalization_params: HashMap::new(),
        })
    }

    /// Calculate dataset statistics
    fn calculate_dataset_statistics(
        &self,
        training_samples: &[TrainingSample],
        validation_samples: &[TrainingSample],
        symbols: &[Symbol],
        start_time: DateTime<Utc>,
        end_time: DateTime<Utc>,
    ) -> MLResult<DatasetStatistics> {
        Ok(DatasetStatistics {
            total_samples: training_samples.len() + validation_samples.len(),
            training_samples: training_samples.len(),
            validation_samples: validation_samples.len(),
            symbols: symbols.iter().map(|s| s.to_string()).collect(),
            time_range: (start_time, end_time),
            data_quality_score: 0.95, // Placeholder
            completeness_ratio: 0.98, // Placeholder
        })
    }
}

impl DatabentoHistoricalProvider {
    /// Create new `Databento` provider
    pub fn new(config: DatabentoConfig) -> MLResult<Self> {
        let client = reqwest::Client::builder()
            .timeout(Duration::from_secs(config.timeout_seconds))
            .build()
            .map_err(|e| MLError::ConfigError(format!("Failed to create HTTP client: {}", e)))?;

        Ok(Self { config, client })
    }

    /// Load historical market data from `Databento`
    pub async fn load_historical_data(
        &self,
        symbol: Symbol,
        start_time: DateTime<Utc>,
        end_time: DateTime<Utc>,
    ) -> MLResult<Vec<MarketDataContainer>> {
        debug!(
            "Loading Databento data for {} from {} to {}",
            symbol, start_time, end_time
        );

        // Placeholder implementation - in reality this would make API calls to Databento
        // and parse the response into MarketDataContainer structures

        Ok(vec![])
    }
}

impl BenzingaHistoricalProvider {
    /// Create new Benzinga provider
    pub fn new(config: BenzingaConfig) -> MLResult<Self> {
        let client = reqwest::Client::builder()
            .timeout(Duration::from_secs(config.timeout_seconds))
            .build()
            .map_err(|e| MLError::ConfigError(format!("Failed to create HTTP client: {}", e)))?;

        Ok(Self { config, client })
    }

    /// Load historical news sentiment data from Benzinga
    pub async fn load_news_data(
        &self,
        symbol: Symbol,
        start_time: DateTime<Utc>,
        end_time: DateTime<Utc>,
    ) -> MLResult<Vec<NewsSentimentData>> {
        debug!(
            "Loading Benzinga news for {} from {} to {}",
            symbol, start_time, end_time
        );

        // Placeholder implementation - in reality this would make API calls to Benzinga
        // and parse the response into NewsSentimentData structures

        Ok(vec![])
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_unified_data_loader_config_default() {
        let config = UnifiedDataLoaderConfig::default();
        assert!(!config.databento_config.api_key.is_empty());
        assert!(!config.benzinga_config.api_key.is_empty());
        assert!(config.feature_extraction.use_unified_extractor);
    }

    #[test]
    fn test_training_sample_creation() {
        // Note: UnifiedFinancialFeatures doesn't have a Default impl due to its complexity
        // This test validates the TrainingSample structure expectations
        let test_symbol = "TEST_SYMBOL";

        // Basic validation of TrainingSample structure
        assert!(!test_symbol.is_empty());

        // Would create sample like:
        // let sample = TrainingSample {
        //     features: UnifiedFinancialFeatures { /* complex initialization */ },
        //     targets: vec![1.0],
        //     timestamp: Utc::now(),
        //     symbol: Symbol::from(test_symbol),
        //     weight: 1.0,
        //     metadata: HashMap::new(),
        // };
    }

    #[tokio::test]
    async fn test_data_loader_creation() {
        let config = UnifiedDataLoaderConfig::default();
        let loader = UnifiedDataLoader::new(config);
        assert!(loader.is_ok());
    }
}
