foxhunt/services/backtesting_service/src/ml_strategy_engine.rs

//! ML-powered strategy execution engine for backtesting
//!
//! This module integrates the shared ML strategy from common crate to ensure
//! ONE SINGLE SYSTEM across trading and backtesting services.

use anyhow::Result;
use chrono::{DateTime, Datelike, Timelike, Utc};
use std::collections::HashMap;
use std::sync::Arc;
use tracing::{debug, info};
use serde::{Deserialize, Serialize};
use rust_decimal::{Decimal, prelude::ToPrimitive};

use config::structures::BacktestingStrategyConfig;
use crate::storage::StorageManager;
use crate::strategy_engine::{MarketData, BacktestTrade, TradeSide, TradeSignal, StrategyExecutor, Portfolio};

// Import shared ML strategy (ONE SINGLE SYSTEM)
use common::ml_strategy::{SharedMLStrategy, MLPrediction as CommonMLPrediction};

/// ML model prediction result for backtesting
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MLPrediction {
    /// Model identifier
    pub model_id: String,
    /// Prediction value (0.0-1.0)
    pub prediction_value: f64,
    /// Confidence score (0.0-1.0)
    pub confidence: f64,
    /// Features used for prediction
    pub features: Vec<f64>,
    /// Prediction timestamp
    pub timestamp: DateTime<Utc>,
    /// Inference latency in microseconds
    pub inference_latency_us: u64,
}

/// ML model performance tracking for backtesting
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct MLModelPerformance {
    /// Model identifier
    pub model_id: String,
    /// Total predictions made
    pub total_predictions: u64,
    /// Correct predictions (when outcome is known)
    pub correct_predictions: u64,
    /// Average inference latency
    pub avg_latency_us: f64,
    /// Average confidence score
    pub avg_confidence: f64,
    /// Model accuracy percentage
    pub accuracy_percentage: f64,
    /// Returns generated when following this model
    pub returns: Vec<f64>,
    /// Sharpe ratio for this model
    pub sharpe_ratio: f64,
    /// Maximum drawdown when following this model
    pub max_drawdown: f64,
}

/// ML feature extractor for market data
#[derive(Debug)]
pub struct MLFeatureExtractor {
    /// Lookback window for features
    pub lookback_periods: usize,
    /// Price history buffer
    price_history: Vec<f64>,
    /// Volume history buffer
    volume_history: Vec<f64>,
}

impl MLFeatureExtractor {
    /// Create new feature extractor
    pub fn new(lookback_periods: usize) -> Self {
        Self {
            lookback_periods,
            price_history: Vec::with_capacity(lookback_periods + 1),
            volume_history: Vec::with_capacity(lookback_periods + 1),
        }
    }

    /// Extract features from market data
    pub fn extract_features(&mut self, market_data: &MarketData) -> Vec<f64> {
        // Update price and volume history
        self.price_history.push(market_data.close.to_f64().unwrap_or(0.0));
        self.volume_history.push(market_data.volume.to_f64().unwrap_or(0.0));

        // Keep only the required lookback periods
        if self.price_history.len() > self.lookback_periods {
            self.price_history.remove(0);
        }
        if self.volume_history.len() > self.lookback_periods {
            self.volume_history.remove(0);
        }

        // Extract technical features
        let mut features = Vec::new();

        if self.price_history.len() >= 2 {
            // Price momentum (returns)
            let current_price = self.price_history.last().copied().unwrap_or(0.0);
            let prev_price = self.price_history.get(self.price_history.len() - 2).copied().unwrap_or(current_price);
            let price_return = if prev_price != 0.0 {
                (current_price - prev_price) / prev_price
            } else {
                0.0
            };
            features.push(price_return);

            // Short-term moving average
            if self.price_history.len() >= 5 {
                let short_ma: f64 = self.price_history.iter().rev().take(5).sum::<f64>() / 5.0;
                let ma_ratio = if short_ma != 0.0 { current_price / short_ma - 1.0 } else { 0.0 };
                features.push(ma_ratio);
            } else {
                features.push(0.0);
            }

            // Price volatility (rolling standard deviation)
            if self.price_history.len() >= 10 {
                let recent_returns: Vec<f64> = self.price_history
                    .windows(2)
                    .rev()
                    .take(9)
                    .map(|w| (w[1] - w[0]) / w[0])
                    .collect();
                
                let mean_return = recent_returns.iter().sum::<f64>() / recent_returns.len() as f64;
                let variance = recent_returns.iter()
                    .map(|&r| (r - mean_return).powi(2))
                    .sum::<f64>() / recent_returns.len() as f64;
                let volatility = variance.sqrt();
                features.push(volatility);
            } else {
                features.push(0.0);
            }
        } else {
            features.extend_from_slice(&[0.0, 0.0, 0.0]);
        }

        // Volume features
        if self.volume_history.len() >= 2 {
            let current_volume = self.volume_history.last().copied().unwrap_or(0.0);
            let prev_volume = self.volume_history.get(self.volume_history.len() - 2).copied().unwrap_or(current_volume);
            let volume_ratio = if prev_volume != 0.0 {
                current_volume / prev_volume - 1.0
            } else {
                0.0
            };
            features.push(volume_ratio);

            // Volume moving average
            if self.volume_history.len() >= 5 {
                let volume_ma = self.volume_history.iter().rev().take(5).sum::<f64>() / 5.0;
                let volume_ma_ratio = if volume_ma != 0.0 { current_volume / volume_ma - 1.0 } else { 0.0 };
                features.push(volume_ma_ratio);
            } else {
                features.push(0.0);
            }
        } else {
            features.extend_from_slice(&[0.0, 0.0]);
        }

        // Add time-based features
        let hour = market_data.timestamp.hour() as f64 / 24.0; // Normalized hour
        let day_of_week = market_data.timestamp.weekday().num_days_from_monday() as f64 / 6.0; // Normalized day
        features.push(hour);
        features.push(day_of_week);

        // Normalize all features to [-1, 1] range using tanh
        features.iter().map(|&f| f.tanh()).collect()
    }
}

/// ML-powered strategy for backtesting (uses shared ML strategy - ONE SINGLE SYSTEM)
pub struct MLPoweredStrategy {
    /// Strategy name
    name: String,
    /// Shared ML strategy (ONE SINGLE SYSTEM)
    strategy: Arc<SharedMLStrategy>,
    /// Feature extractor (kept for backward compatibility with local types)
    #[allow(dead_code)]
    feature_extractor: MLFeatureExtractor,
    /// Model performance tracking (local copy for backward compatibility)
    model_performance: HashMap<String, MLModelPerformance>,
    /// Current position size based on confidence
    confidence_based_sizing: bool,
    /// Minimum confidence threshold for trades
    min_confidence_threshold: f64,
}

// NOTE: Old model simulator implementations removed.
// We now use SharedMLStrategy from common crate (ONE SINGLE SYSTEM).
// This eliminates code duplication and ensures consistent ML predictions
// across trading and backtesting services.

impl std::fmt::Debug for MLPoweredStrategy {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("MLPoweredStrategy")
            .field("name", &self.name)
            .field("confidence_based_sizing", &self.confidence_based_sizing)
            .field("min_confidence_threshold", &self.min_confidence_threshold)
            .field("model_performance_count", &self.model_performance.len())
            .finish()
    }
}

impl MLPoweredStrategy {
    /// Create new ML-powered strategy (uses shared strategy - ONE SINGLE SYSTEM)
    pub fn new(name: String, lookback_periods: usize) -> Self {
        // Use shared ML strategy (ONE SINGLE SYSTEM)
        let min_confidence_threshold = 0.6;
        let strategy = Arc::new(SharedMLStrategy::new(lookback_periods, min_confidence_threshold));

        Self {
            name,
            strategy,
            feature_extractor: MLFeatureExtractor::new(lookback_periods),
            model_performance: HashMap::new(),
            confidence_based_sizing: true,
            min_confidence_threshold,
        }
    }

    /// Get ensemble prediction from all models (delegates to shared strategy)
    pub async fn get_ensemble_prediction(&mut self, market_data: &MarketData) -> Result<Vec<MLPrediction>> {
        // Use shared ML strategy (ONE SINGLE SYSTEM)
        let price = market_data.close.to_f64().unwrap_or(0.0);
        let volume = market_data.volume.to_f64().unwrap_or(0.0);
        let timestamp = market_data.timestamp;

        // Get predictions from shared strategy
        let common_predictions = self.strategy.get_ensemble_prediction(price, volume, timestamp).await?;

        // Convert to local type for backward compatibility
        let predictions = common_predictions.iter().map(|p| MLPrediction {
            model_id: p.model_id.clone(),
            prediction_value: p.prediction_value,
            confidence: p.confidence,
            features: p.features.clone(),
            timestamp: p.timestamp,
            inference_latency_us: p.inference_latency_us,
        }).collect();

        Ok(predictions)
    }

    /// Calculate weighted ensemble prediction
    pub fn calculate_ensemble_vote(&self, predictions: &[MLPrediction]) -> Option<(f64, f64)> {
        if predictions.is_empty() {
            return None;
        }

        let total_confidence: f64 = predictions.iter().map(|p| p.confidence).sum();
        if total_confidence == 0.0 {
            return None;
        }

        // Weighted average by confidence
        let weighted_prediction: f64 = predictions.iter()
            .map(|p| p.prediction_value * p.confidence)
            .sum::<f64>() / total_confidence;

        let average_confidence: f64 = predictions.iter().map(|p| p.confidence).sum::<f64>() / predictions.len() as f64;

        Some((weighted_prediction, average_confidence))
    }

    /// Validate predictions against actual market outcomes (delegates to shared strategy)
    pub async fn validate_predictions(&mut self, predictions: &[MLPrediction], actual_return: f64) {
        // Convert to common predictions
        let common_predictions: Vec<CommonMLPrediction> = predictions.iter().map(|p| CommonMLPrediction {
            model_id: p.model_id.clone(),
            prediction_value: p.prediction_value,
            confidence: p.confidence,
            features: p.features.clone(),
            timestamp: p.timestamp,
            inference_latency_us: p.inference_latency_us,
        }).collect();

        // Delegate to shared strategy (ONE SINGLE SYSTEM)
        self.strategy.validate_predictions(&common_predictions, actual_return).await;

        // Update local performance tracking for backward compatibility
        let shared_performance = self.strategy.get_performance_summary().await;
        for (model_id, perf) in shared_performance {
            self.model_performance.insert(model_id.clone(), MLModelPerformance {
                model_id: model_id.clone(),
                total_predictions: perf.total_predictions,
                correct_predictions: perf.correct_predictions,
                avg_latency_us: perf.avg_latency_us,
                avg_confidence: perf.avg_confidence,
                accuracy_percentage: perf.accuracy_percentage,
                returns: perf.returns,
                sharpe_ratio: perf.sharpe_ratio,
                max_drawdown: perf.max_drawdown,
            });
        }
    }

    /// Get performance summary for all models
    pub fn get_performance_summary(&self) -> HashMap<String, MLModelPerformance> {
        self.model_performance.clone()
    }
}

impl StrategyExecutor for MLPoweredStrategy {
    fn execute(
        &self,
        market_data: &MarketData,
        _portfolio: &Portfolio,
        parameters: &HashMap<String, String>,
    ) -> Result<Vec<TradeSignal>> {
        // This is a bit tricky because we need mutable access to call predict
        // In a real implementation, you'd want to redesign this to avoid the issue
        // For now, we'll create a simplified version that doesn't update the feature extractor
        
        let mut signals = Vec::new();
        
        // Extract basic features without updating history (simplified for demo)
        let price = market_data.close.to_f64().unwrap_or(0.0);
        let volume = market_data.volume.to_f64().unwrap_or(0.0);
        
        // Create simplified features
        let features = vec![
            (price - 100.0) / 100.0, // Normalized price change from baseline
            (volume - 1000.0) / 1000.0, // Normalized volume
            0.0, 0.0, 0.0, 0.0, 0.0 // Placeholder features
        ];
        
        // Simple prediction using DQN-like logic
        let weights = vec![0.1, -0.05, 0.2, 0.15, -0.1, 0.08, 0.03];
        let linear_output: f64 = features.iter()
            .zip(weights.iter())
            .map(|(f, w)| f * w)
            .sum();
        
        let prediction_value = 1.0 / (1.0 + (-linear_output).exp());
        let confidence = 0.5 + (prediction_value - 0.5).abs() * 0.8;
        
        // Get minimum confidence from parameters
        let min_confidence = parameters.get("min_confidence")
            .and_then(|s| s.parse::<f64>().ok())
            .unwrap_or(self.min_confidence_threshold);
        
        // Generate signal if confidence is high enough
        if confidence >= min_confidence {
            let quantity = if self.confidence_based_sizing {
                // Size position based on confidence
                Decimal::try_from(confidence * 1000.0).unwrap_or(Decimal::from(100))
            } else {
                Decimal::from(100)
            };
            
            if prediction_value > 0.6 {
                signals.push(TradeSignal {
                    symbol: market_data.symbol.clone(),
                    side: TradeSide::Buy,
                    quantity,
                    strength: Decimal::try_from(confidence)
                        .unwrap_or_else(|_| Decimal::try_from(0.5)
                            .unwrap_or(Decimal::ONE / Decimal::from(2))),
                    reason: format!("ML prediction: {:.3} (confidence: {:.3})", prediction_value, confidence),
                    features: None,
                    news_events: None,
                });
            } else if prediction_value < 0.4 {
                signals.push(TradeSignal {
                    symbol: market_data.symbol.clone(),
                    side: TradeSide::Sell,
                    quantity,
                    strength: Decimal::try_from(confidence)
                        .unwrap_or_else(|_| Decimal::try_from(0.5)
                            .unwrap_or(Decimal::ONE / Decimal::from(2))),
                    reason: format!("ML prediction: {:.3} (confidence: {:.3})", prediction_value, confidence),
                    features: None,
                    news_events: None,
                });
            }
        }
        
        Ok(signals)
    }

    fn name(&self) -> &str {
        &self.name
    }
}

/// ML Strategy Engine with model performance tracking
pub struct MLStrategyEngine {
    /// Base strategy engine
    base_engine: crate::strategy_engine::StrategyEngine,
    /// ML-powered strategies
    ml_strategies: HashMap<String, MLPoweredStrategy>,
    /// Model performance tracking across backtests
    global_model_performance: HashMap<String, MLModelPerformance>,
}

impl MLStrategyEngine {
    /// Create new ML strategy engine
    pub async fn new(
        config: &BacktestingStrategyConfig,
        storage_manager: Arc<StorageManager>,
    ) -> Result<Self> {
        // Create repositories from storage manager
        let repositories = Arc::new(crate::repository_impl::create_repositories(storage_manager).await?);
        let base_engine = crate::strategy_engine::StrategyEngine::new(config, repositories).await?;
        
        let mut ml_strategies = HashMap::new();
        
        // Add ML-powered strategies
        ml_strategies.insert(
            "ml_momentum".to_string(),
            MLPoweredStrategy::new("ml_momentum".to_string(), 20)
        );
        
        ml_strategies.insert(
            "ml_ensemble".to_string(), 
            MLPoweredStrategy::new("ml_ensemble".to_string(), 50)
        );
        
        Ok(Self {
            base_engine,
            ml_strategies,
            global_model_performance: HashMap::new(),
        })
    }

    /// Execute backtest with ML model validation
    pub async fn execute_ml_backtest(
        &mut self,
        context: &crate::service::BacktestContext,
    ) -> Result<(Vec<BacktestTrade>, HashMap<String, MLModelPerformance>)> {
        info!("Executing ML-powered backtest {} for strategy {}", context.id, context.strategy_name);

        // Check if this is an ML strategy
        let is_ml_strategy = self.ml_strategies.contains_key(&context.strategy_name);

        if is_ml_strategy {
            // Execute ML-powered backtest with model validation
            self.execute_ml_strategy_backtest(context).await
        } else {
            // Fall back to base strategy engine
            let trades = self.base_engine.execute_backtest(context).await?;
            Ok((trades, HashMap::new()))
        }
    }

    /// Execute backtest for ML strategy with model performance tracking
    async fn execute_ml_strategy_backtest(
        &mut self,
        context: &crate::service::BacktestContext,
    ) -> Result<(Vec<BacktestTrade>, HashMap<String, MLModelPerformance>)> {
        // Load market data for the backtest period
        let market_data = self.base_engine
            .load_market_data(
                &context.symbols,
                context.started_at,
                context.completed_at.unwrap_or(chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0)),
            )
            .await?;

        let trades = Vec::new();
        let mut previous_price = None;
        let total_data_points = market_data.len();

        // Get ML strategy reference
        let ml_strategy = self.ml_strategies.get_mut(&context.strategy_name)
            .ok_or_else(|| anyhow::anyhow!("ML strategy {} not found", context.strategy_name))?;

        // Process each data point with ML predictions
        for (i, data_point) in market_data.into_iter().enumerate() {
            // Get ML predictions (async call to shared strategy)
            let predictions = ml_strategy.get_ensemble_prediction(&data_point).await?;

            // Calculate ensemble vote
            if let Some((ensemble_prediction, ensemble_confidence)) = ml_strategy.calculate_ensemble_vote(&predictions) {
                debug!("Ensemble prediction: {:.3} (confidence: {:.3})", ensemble_prediction, ensemble_confidence);

                // Validate predictions against future returns if we have next price
                if let Some(prev_price) = previous_price {
                    let current_price = data_point.close.to_f64().unwrap_or(prev_price);
                    let actual_return = (current_price - prev_price) / prev_price;
                    ml_strategy.validate_predictions(&predictions, actual_return).await;
                }
            }

            previous_price = Some(data_point.close.to_f64().unwrap_or(0.0));

            // Generate and execute trades using base strategy logic
            // (This would integrate with the existing strategy execution logic)
            if i % 100 == 0 {
                let progress = (i as f64 / total_data_points as f64) * 100.0;
                debug!("ML backtest progress: {:.1}%", progress);
            }
        }

        // Get final model performance
        let model_performance = ml_strategy.get_performance_summary();

        // Update global performance tracking
        for (model_id, perf) in &model_performance {
            self.global_model_performance.insert(model_id.clone(), perf.clone());
        }

        info!("ML backtest completed with {} trades and {} model evaluations", 
            trades.len(), model_performance.len());
        
        Ok((trades, model_performance))
    }

    /// Get model performance across all backtests
    pub fn get_global_model_performance(&self) -> &HashMap<String, MLModelPerformance> {
        &self.global_model_performance
    }

    /// Generate model performance report
    pub fn generate_performance_report(&self) -> String {
        let mut report = String::new();
        report.push_str("=== ML Model Performance Report ===\n\n");

        for (model_id, performance) in &self.global_model_performance {
            report.push_str(&format!("Model: {}\n", model_id));
            report.push_str(&format!("  Total Predictions: {}\n", performance.total_predictions));
            report.push_str(&format!("  Accuracy: {:.2}%\n", performance.accuracy_percentage));
            report.push_str(&format!("  Average Confidence: {:.3}\n", performance.avg_confidence));
            report.push_str(&format!("  Average Latency: {:.1}μs\n", performance.avg_latency_us));
            if performance.sharpe_ratio != 0.0 {
                report.push_str(&format!("  Sharpe Ratio: {:.3}\n", performance.sharpe_ratio));
            }
            if performance.max_drawdown != 0.0 {
                report.push_str(&format!("  Max Drawdown: {:.2}%\n", performance.max_drawdown * 100.0));
            }
            report.push_str("\n");
        }

        report
    }
}