//! Test Data Generation for Integration Testing
//! 
//! Generates synthetic market data, model artifacts, and test scenarios
//! for comprehensive ML training and trading pipeline testing.

use anyhow::Result;
use chrono::{DateTime, Utc, Duration};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use rand::prelude::*;

/// Test data generator for creating realistic market data and ML artifacts
pub struct TestDataGenerator {
    rng: ThreadRng,
    symbols: Vec<String>,
    data_cache: HashMap<String, Vec<MarketTick>>,
}

impl TestDataGenerator {
    pub fn new() -> Self {
        Self {
            rng: thread_rng(),
            symbols: vec![
                "AAPL".to_string(),
                "MSFT".to_string(), 
                "GOOGL".to_string(),
                "TSLA".to_string(),
                "NVDA".to_string(),
                "SPY".to_string(),
                "QQQ".to_string(),
            ],
            data_cache: HashMap::new(),
        }
    }

    /// Generate comprehensive test data for all scenarios
    pub async fn generate_synthetic_data(&mut self) -> Result<()> {
        // Generate market data for all symbols
        for symbol in &self.symbols.clone() {
            let ticks = self.generate_market_data(symbol, 10000).await?;
            self.data_cache.insert(symbol.clone(), ticks);
        }

        // Generate model training datasets
        self.generate_training_datasets().await?;

        // Generate model artifacts
        self.generate_model_artifacts().await?;

        Ok(())
    }

    /// Generate realistic market tick data with various market conditions
    pub async fn generate_market_data(&mut self, symbol: &str, count: usize) -> Result<Vec<MarketTick>> {
        let mut ticks = Vec::with_capacity(count);
        let start_time = Utc::now() - Duration::hours(24);
        let mut current_time = start_time;
        
        // Initialize realistic starting price based on symbol
        let mut current_price = match symbol {
            "AAPL" => 150.0,
            "MSFT" => 300.0,
            "GOOGL" => 2500.0,
            "TSLA" => 200.0,
            "NVDA" => 400.0,
            "SPY" => 450.0,
            "QQQ" => 350.0,
            _ => 100.0,
        };

        let base_volume = match symbol {
            "SPY" | "QQQ" => 50_000_000,
            "AAPL" | "MSFT" => 30_000_000,
            _ => 10_000_000,
        };

        for i in 0..count {
            // Generate price movement with realistic volatility
            let volatility = self.calculate_volatility(symbol, i);
            let price_change = self.rng.gen_range(-volatility..volatility);
            current_price = (current_price + price_change).max(0.01);

            // Generate volume with realistic patterns
            let volume_multiplier = self.generate_volume_pattern(i, count);
            let volume = (base_volume as f64 * volume_multiplier) as u64;

            // Add market microstructure noise
            let bid_ask_spread = self.calculate_spread(symbol, current_price);
            let bid = current_price - bid_ask_spread / 2.0;
            let ask = current_price + bid_ask_spread / 2.0;

            let tick = MarketTick {
                symbol: symbol.to_string(),
                timestamp: current_time,
                price: current_price,
                volume,
                bid,
                ask,
                market_condition: self.determine_market_condition(i, count),
            };

            ticks.push(tick);
            current_time = current_time + Duration::milliseconds(100);
        }

        Ok(ticks)
    }

    /// Generate training datasets with features and labels
    async fn generate_training_datasets(&mut self) -> Result<()> {
        for symbol in &self.symbols.clone() {
            let dataset = self.create_training_dataset(symbol).await?;
            self.save_training_dataset(symbol, &dataset).await?;
        }
        Ok(())
    }

    /// Create training dataset with features and labels for ML models
    async fn create_training_dataset(&mut self, symbol: &str) -> Result<TrainingDataset> {
        let ticks = self.data_cache.get(symbol).ok_or_else(|| {
            anyhow::anyhow!("No market data available for symbol: {}", symbol)
        })?;

        let mut features = Vec::new();
        let mut labels = Vec::new();

        // Generate features using sliding window
        let window_size = 50;
        for i in window_size..ticks.len() {
            let window = &ticks[i-window_size..i];
            let feature_vector = self.extract_features(window);
            let label = self.generate_label(&ticks[i-1], &ticks[i]);
            
            features.push(feature_vector);
            labels.push(label);
        }

        Ok(TrainingDataset {
            symbol: symbol.to_string(),
            features,
            labels,
            metadata: DatasetMetadata {
                created_at: Utc::now(),
                window_size,
                feature_count: features.first().map(|f| f.len()).unwrap_or(0),
                sample_count: features.len(),
            },
        })
    }

    /// Extract features from market data window
    fn extract_features(&self, window: &[MarketTick]) -> Vec<f64> {
        let mut features = Vec::new();

        // Price-based features
        let prices: Vec<f64> = window.iter().map(|t| t.price).collect();
        features.extend(self.calculate_price_features(&prices));

        // Volume-based features
        let volumes: Vec<f64> = window.iter().map(|t| t.volume as f64).collect();
        features.extend(self.calculate_volume_features(&volumes));

        // Technical indicators
        features.extend(self.calculate_technical_indicators(&prices, &volumes));

        // Microstructure features
        features.extend(self.calculate_microstructure_features(window));

        features
    }

    /// Calculate price-based features
    fn calculate_price_features(&self, prices: &[f64]) -> Vec<f64> {
        let mut features = Vec::new();
        
        if prices.len() < 2 {
            return vec![0.0; 10]; // Return zeros if insufficient data
        }

        // Returns
        let returns: Vec<f64> = prices.windows(2)
            .map(|w| (w[1] / w[0] - 1.0))
            .collect();

        // Statistical measures
        features.push(returns.iter().sum::<f64>() / returns.len() as f64); // Mean return
        features.push(self.calculate_std(&returns)); // Volatility
        features.push(self.calculate_skewness(&returns)); // Skewness
        features.push(self.calculate_kurtosis(&returns)); // Kurtosis

        // Price levels
        features.push(prices.last().unwrap() / prices.iter().sum::<f64>() * prices.len() as f64); // Price relative to mean
        features.push(prices.last().unwrap() / prices.iter().fold(f64::NEG_INFINITY, |a, &b| a.max(b))); // Price relative to max
        features.push(prices.last().unwrap() / prices.iter().fold(f64::INFINITY, |a, &b| a.min(b))); // Price relative to min

        // Momentum indicators
        if prices.len() >= 10 {
            let recent_avg = prices[prices.len()-10..].iter().sum::<f64>() / 10.0;
            let older_avg = prices[0..10].iter().sum::<f64>() / 10.0;
            features.push(recent_avg / older_avg - 1.0); // Momentum
        } else {
            features.push(0.0);
        }

        // Trend indicators
        features.push(self.calculate_linear_trend(&prices)); // Linear trend slope
        features.push(if prices.last() > prices.first() { 1.0 } else { -1.0 }); // Overall direction

        features
    }

    /// Calculate volume-based features
    fn calculate_volume_features(&self, volumes: &[f64]) -> Vec<f64> {
        let mut features = Vec::new();

        if volumes.is_empty() {
            return vec![0.0; 5];
        }

        let mean_volume = volumes.iter().sum::<f64>() / volumes.len() as f64;
        let current_volume = volumes.last().unwrap();

        features.push(current_volume / mean_volume); // Volume relative to average
        features.push(self.calculate_std(volumes)); // Volume volatility
        features.push(current_volume / volumes.iter().fold(f64::NEG_INFINITY, |a, &b| a.max(b))); // Volume relative to max
        features.push(self.calculate_volume_momentum(volumes)); // Volume momentum
        features.push(self.calculate_volume_trend(volumes)); // Volume trend

        features
    }

    /// Calculate technical indicators
    fn calculate_technical_indicators(&self, prices: &[f64], volumes: &[f64]) -> Vec<f64> {
        let mut features = Vec::new();

        // RSI (simplified)
        features.push(self.calculate_rsi(prices));

        // MACD (simplified)
        let (macd, signal) = self.calculate_macd(prices);
        features.push(macd);
        features.push(signal);
        features.push(macd - signal); // MACD histogram

        // Bollinger Bands
        let (upper, lower, middle) = self.calculate_bollinger_bands(prices);
        features.push((prices.last().unwrap() - middle) / (upper - lower)); // Position within bands

        // Volume-weighted average price
        features.push(self.calculate_vwap(prices, volumes));

        features
    }

    /// Calculate microstructure features
    fn calculate_microstructure_features(&self, window: &[MarketTick]) -> Vec<f64> {
        let mut features = Vec::new();

        if window.is_empty() {
            return vec![0.0; 5];
        }

        // Bid-ask spread statistics
        let spreads: Vec<f64> = window.iter().map(|t| t.ask - t.bid).collect();
        features.push(spreads.iter().sum::<f64>() / spreads.len() as f64); // Average spread
        features.push(self.calculate_std(&spreads)); // Spread volatility

        // Market impact estimation
        features.push(self.estimate_market_impact(window));

        // Order flow imbalance (simplified)
        features.push(self.calculate_order_flow_imbalance(window));

        // Price improvement opportunity
        features.push(self.calculate_price_improvement(window));

        features
    }

    /// Generate label for supervised learning
    fn generate_label(&self, current: &MarketTick, next: &MarketTick) -> f64 {
        let return_threshold = 0.001; // 0.1% threshold
        let price_return = (next.price / current.price) - 1.0;

        if price_return > return_threshold {
            1.0 // Buy signal
        } else if price_return < -return_threshold {
            -1.0 // Sell signal
        } else {
            0.0 // Hold signal
        }
    }

    /// Generate model artifacts for deployment testing
    async fn generate_model_artifacts(&mut self) -> Result<()> {
        let model_types = vec!["DQN", "PPO", "MAMBA", "TFT", "ENSEMBLE"];
        
        for model_type in model_types {
            for symbol in &self.symbols.clone() {
                let artifact = self.create_model_artifact(model_type, symbol).await?;
                self.save_model_artifact(&artifact).await?;
            }
        }

        Ok(())
    }

    /// Create a model artifact for testing
    async fn create_model_artifact(&self, model_type: &str, symbol: &str) -> Result<ModelArtifact> {
        Ok(ModelArtifact {
            model_id: format!("{}_{}_test_v1.0", model_type.to_lowercase(), symbol.to_lowercase()),
            model_type: model_type.to_string(),
            symbol: symbol.to_string(),
            version: "1.0.0".to_string(),
            created_at: Utc::now(),
            hyperparameters: self.generate_hyperparameters(model_type),
            performance_metrics: self.generate_mock_performance(),
            model_path: format!("/tmp/test_models/{}_{}.pkl", model_type.to_lowercase(), symbol.to_lowercase()),
            metadata: TestModelMetrics {
                training_samples: 10000,
                validation_accuracy: 0.75 + self.rng.gen::<f64>() * 0.2,
                feature_count: 50,
                training_duration_minutes: 30 + self.rng.gen_range(0..120),
            },
        })
    }

    /// Save training dataset to file
    async fn save_training_dataset(&self, symbol: &str, dataset: &TrainingDataset) -> Result<()> {
        let path = format!("/tmp/test_data/training_dataset_{}.json", symbol.to_lowercase());
        tokio::fs::create_dir_all("/tmp/test_data").await?;
        
        let content = serde_json::to_string_pretty(dataset)?;
        tokio::fs::write(&path, content).await?;
        
        Ok(())
    }

    /// Save model artifact
    async fn save_model_artifact(&self, artifact: &ModelArtifact) -> Result<()> {
        let path = format!("/tmp/test_models/{}.json", artifact.model_id);
        tokio::fs::create_dir_all("/tmp/test_models").await?;
        
        let content = serde_json::to_string_pretty(artifact)?;
        tokio::fs::write(&path, content).await?;
        
        Ok(())
    }

    // Helper functions for feature calculation
    fn calculate_volatility(&mut self, symbol: &str, index: usize) -> f64 {
        let base_volatility = match symbol {
            "TSLA" => 0.05,
            "NVDA" => 0.04,
            "GOOGL" => 0.03,
            _ => 0.02,
        };
        
        // Add time-of-day and market condition effects
        let time_factor = 1.0 + 0.5 * (index as f64 / 100.0).sin();
        base_volatility * time_factor * self.rng.gen_range(0.5..1.5)
    }

    fn generate_volume_pattern(&mut self, index: usize, total: usize) -> f64 {
        // U-shaped volume pattern (high at open/close, low at midday)
        let time_factor = ((index as f64 / total as f64) * 2.0 * std::f64::consts::PI).cos().abs();
        0.3 + 0.7 * time_factor + 0.2 * self.rng.gen::<f64>()
    }

    fn calculate_spread(&self, symbol: &str, price: f64) -> f64 {
        let base_spread_bps = match symbol {
            "SPY" | "QQQ" => 1.0,
            "AAPL" | "MSFT" => 2.0,
            _ => 5.0,
        };
        
        price * base_spread_bps / 10000.0
    }

    fn determine_market_condition(&self, index: usize, total: usize) -> MarketCondition {
        let phase = index as f64 / total as f64;
        match phase {
            p if p < 0.3 => MarketCondition::Trending,
            p if p < 0.7 => MarketCondition::Ranging,
            _ => MarketCondition::Volatile,
        }
    }

    // Statistical helper functions
    fn calculate_std(&self, values: &[f64]) -> f64 {
        if values.len() < 2 {
            return 0.0;
        }
        
        let mean = values.iter().sum::<f64>() / values.len() as f64;
        let variance = values.iter()
            .map(|x| (x - mean).powi(2))
            .sum::<f64>() / (values.len() - 1) as f64;
        variance.sqrt()
    }

    fn calculate_skewness(&self, values: &[f64]) -> f64 {
        if values.len() < 3 {
            return 0.0;
        }
        
        let mean = values.iter().sum::<f64>() / values.len() as f64;
        let std = self.calculate_std(values);
        
        if std == 0.0 {
            return 0.0;
        }
        
        let skew = values.iter()
            .map(|x| ((x - mean) / std).powi(3))
            .sum::<f64>() / values.len() as f64;
        skew
    }

    fn calculate_kurtosis(&self, values: &[f64]) -> f64 {
        if values.len() < 4 {
            return 0.0;
        }
        
        let mean = values.iter().sum::<f64>() / values.len() as f64;
        let std = self.calculate_std(values);
        
        if std == 0.0 {
            return 0.0;
        }
        
        let kurt = values.iter()
            .map(|x| ((x - mean) / std).powi(4))
            .sum::<f64>() / values.len() as f64;
        kurt - 3.0 // Excess kurtosis
    }

    fn calculate_linear_trend(&self, values: &[f64]) -> f64 {
        if values.len() < 2 {
            return 0.0;
        }
        
        let n = values.len() as f64;
        let x_mean = (n - 1.0) / 2.0;
        let y_mean = values.iter().sum::<f64>() / n;
        
        let numerator: f64 = values.iter().enumerate()
            .map(|(i, &y)| (i as f64 - x_mean) * (y - y_mean))
            .sum();
        
        let denominator: f64 = (0..values.len())
            .map(|i| (i as f64 - x_mean).powi(2))
            .sum();
        
        if denominator == 0.0 {
            0.0
        } else {
            numerator / denominator
        }
    }

    // Technical indicator implementations (simplified)
    fn calculate_rsi(&self, prices: &[f64]) -> f64 {
        if prices.len() < 14 {
            return 50.0; // Neutral RSI
        }
        
        let changes: Vec<f64> = prices.windows(2)
            .map(|w| w[1] - w[0])
            .collect();
        
        let gains: f64 = changes.iter().filter(|&&x| x > 0.0).sum();
        let losses: f64 = changes.iter().filter(|&&x| x < 0.0).map(|x| -x).sum();
        
        if losses == 0.0 {
            100.0
        } else {
            let rs = gains / losses;
            100.0 - (100.0 / (1.0 + rs))
        }
    }

    fn calculate_macd(&self, prices: &[f64]) -> (f64, f64) {
        if prices.len() < 26 {
            return (0.0, 0.0);
        }
        
        // Simplified MACD calculation
        let ema12 = self.calculate_ema(prices, 12);
        let ema26 = self.calculate_ema(prices, 26);
        let macd = ema12 - ema26;
        let signal = self.calculate_ema(&vec![macd], 9);
        
        (macd, signal)
    }

    fn calculate_ema(&self, values: &[f64], period: usize) -> f64 {
        if values.is_empty() || period == 0 {
            return 0.0;
        }
        
        let alpha = 2.0 / (period as f64 + 1.0);
        let mut ema = values[0];
        
        for &value in &values.skip(1) {
            ema = alpha * value + (1.0 - alpha) * ema;
        }
        
        ema
    }

    fn calculate_bollinger_bands(&self, prices: &[f64]) -> (f64, f64, f64) {
        if prices.len() < 20 {
            let price = prices.last().unwrap_or(&100.0);
            return (*price * 1.02, *price * 0.98, *price);
        }
        
        let period = 20.min(prices.len());
        let recent_prices = &prices[prices.len()-period..];
        
        let mean = recent_prices.iter().sum::<f64>() / period as f64;
        let std = self.calculate_std(recent_prices);
        
        (mean + 2.0 * std, mean - 2.0 * std, mean)
    }

    fn calculate_vwap(&self, prices: &[f64], volumes: &[f64]) -> f64 {
        if prices.len() != volumes.len() || prices.is_empty() {
            return prices.last().unwrap_or(&100.0).clone();
        }
        
        let total_value: f64 = prices.into_iter().zip(volumes.into_iter())
            .map(|(p, v)| p * v)
            .sum();
        let total_volume: f64 = volumes.iter().sum();
        
        if total_volume == 0.0 {
            prices.last().unwrap().clone()
        } else {
            total_value / total_volume
        }
    }

    fn calculate_volume_momentum(&self, volumes: &[f64]) -> f64 {
        if volumes.len() < 10 {
            return 0.0;
        }
        
        let recent = &volumes[volumes.len()-5..];
        let older = &volumes[volumes.len()-10..volumes.len()-5];
        
        let recent_avg = recent.iter().sum::<f64>() / recent.len() as f64;
        let older_avg = older.iter().sum::<f64>() / older.len() as f64;
        
        if older_avg == 0.0 {
            0.0
        } else {
            (recent_avg / older_avg) - 1.0
        }
    }

    fn calculate_volume_trend(&self, volumes: &[f64]) -> f64 {
        self.calculate_linear_trend(volumes)
    }

    fn estimate_market_impact(&self, window: &[MarketTick]) -> f64 {
        // Simplified market impact estimation
        if window.len() < 2 {
            return 0.0;
        }
        
        let volume_factor = window.last().unwrap().volume as f64 / 1_000_000.0;
        let volatility = self.calculate_std(&window.iter().map(|t| t.price).collect::<Vec<_>>());
        
        volume_factor * volatility * 0.1 // Simplified impact model
    }

    fn calculate_order_flow_imbalance(&self, window: &[MarketTick]) -> f64 {
        // Simplified order flow imbalance
        if window.is_empty() {
            return 0.0;
        }
        
        let mut imbalance = 0.0;
        for tick in window {
            let mid = (tick.bid + tick.ask) / 2.0;
            if tick.price > mid {
                imbalance += 1.0; // Buy-side trade
            } else if tick.price < mid {
                imbalance -= 1.0; // Sell-side trade
            }
        }
        
        imbalance / window.len() as f64
    }

    fn calculate_price_improvement(&self, window: &[MarketTick]) -> f64 {
        if window.is_empty() {
            return 0.0;
        }
        
        // Calculate potential price improvement based on spread
        let avg_spread = window.iter()
            .map(|t| t.ask - t.bid)
            .sum::<f64>() / window.len() as f64;
        
        avg_spread / window.last().unwrap().price
    }

    fn generate_hyperparameters(&mut self, model_type: &str) -> HashMap<String, String> {
        let mut params = HashMap::new();
        
        match model_type {
            "DQN" => {
                params.insert("learning_rate".to_string(), "0.001".to_string());
                params.insert("epsilon".to_string(), "0.1".to_string());
                params.insert("buffer_size".to_string(), "100000".to_string());
                params.insert("batch_size".to_string(), "32".to_string());
            },
            "PPO" => {
                params.insert("learning_rate".to_string(), "0.0003".to_string());
                params.insert("clip_range".to_string(), "0.2".to_string());
                params.insert("n_epochs".to_string(), "10".to_string());
            },
            "MAMBA" => {
                params.insert("d_model".to_string(), "512".to_string());
                params.insert("n_layers".to_string(), "8".to_string());
                params.insert("d_state".to_string(), "16".to_string());
            },
            "TFT" => {
                params.insert("hidden_size".to_string(), "256".to_string());
                params.insert("num_heads".to_string(), "8".to_string());
                params.insert("dropout".to_string(), "0.1".to_string());
            },
            _ => {
                params.insert("learning_rate".to_string(), "0.001".to_string());
            }
        }
        
        params
    }

    fn generate_mock_performance(&mut self) -> HashMap<String, f64> {
        let mut metrics = HashMap::new();
        
        metrics.insert("accuracy".to_string(), 0.6 + self.rng.gen::<f64>() * 0.3);
        metrics.insert("precision".to_string(), 0.6 + self.rng.gen::<f64>() * 0.3);
        metrics.insert("recall".to_string(), 0.6 + self.rng.gen::<f64>() * 0.3);
        metrics.insert("f1_score".to_string(), 0.6 + self.rng.gen::<f64>() * 0.3);
        metrics.insert("sharpe_ratio".to_string(), 1.0 + self.rng.gen::<f64>() * 1.5);
        
        metrics
    }
}

/// Market tick data structure
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MarketTick {
    pub symbol: String,
    pub timestamp: DateTime<Utc>,
    pub price: f64,
    pub volume: u64,
    pub bid: f64,
    pub ask: f64,
    pub market_condition: MarketCondition,
}

/// Market condition enumeration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum MarketCondition {
    Trending,
    Ranging,
    Volatile,
}

/// Training dataset structure
#[derive(Debug, Serialize, Deserialize)]
pub struct TrainingDataset {
    pub symbol: String,
    pub features: Vec<Vec<f64>>,
    pub labels: Vec<f64>,
    pub metadata: DatasetMetadata,
}

/// Dataset metadata
#[derive(Debug, Serialize, Deserialize)]
pub struct DatasetMetadata {
    pub created_at: DateTime<Utc>,
    pub window_size: usize,
    pub feature_count: usize,
    pub sample_count: usize,
}

/// Model artifact for testing deployment
#[derive(Debug, Serialize, Deserialize)]
pub struct ModelArtifact {
    pub model_id: String,
    pub model_type: String,
    pub symbol: String,
    pub version: String,
    pub created_at: DateTime<Utc>,
    pub hyperparameters: HashMap<String, String>,
    pub performance_metrics: HashMap<String, f64>,
    pub model_path: String,
    pub metadata: TestModelMetrics,
}

/// Model metadata
#[derive(Debug, Serialize, Deserialize)]
pub struct TestModelMetrics {
    pub training_samples: usize,
    pub validation_accuracy: f64,
    pub feature_count: usize,
    pub training_duration_minutes: u32,
}