foxhunt/ml/src/risk/advanced_risk_engine.rs

//! Advanced Risk Management Engine
//!
//! Production-ready risk management system implementing 2025 best practices for HFT systems.
//! Features real-time VaR calculation, stress testing, position monitoring, and regulatory compliance.

use std::collections::HashMap;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, RwLock, Mutex};

use chrono::{DateTime, Utc, Duration};
use crossbeam::queue::ArrayQueue;
use ndarray::{Array1, Array2};
use rand::Rng;
use rand_distr::{Distribution, StandardNormal};
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use tokio::sync::mpsc;
use core::types::prelude::*;

use crate::{MLResult, MLError};
// use crate::safe_operations; // DISABLED - module not found

/// Monte Carlo simulation for VaR calculation
pub fn simulate_random_shock(volatility: f64) -> f64 {
    let mut rng = rand::thread_rng();
    let z: f64 = rng.sample(StandardNormal);
    z * volatility
}

/// Stress testing engine with parallel execution
#[derive(Debug)]
/// StressTestEngine component.
pub struct StressTestEngine {
    scenarios: Vec<StressScenario>,
    thread_pool: rayon::ThreadPool,
    results_queue: Arc<ArrayQueue<StressTestResult>>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
/// StressScenario component.
pub struct StressScenario {
    pub name: String,
    pub description: String,
    pub shock_magnitude: f64,
    pub affected_assets: Vec<AssetId>,
    pub correlation_shock: Option<f64>,
    pub volatility_multiplier: f64,
    pub duration_days: u32,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
/// StressTestResult component.
pub struct StressTestResult {
    pub scenario_name: String,
    pub portfolio_pnl: f64,
    pub max_drawdown: f64,
    pub var_breach_probability: f64,
    pub liquidity_shortfall: f64,
    pub recovery_time_days: u32,
    pub passed: bool,
    pub confidence_level: f64,
    pub timestamp: DateTime<Utc>,
}

impl StressTestEngine {
    /// Create new stress testing engine
    pub fn new(scenarios: Vec<StressScenario>) -> Self {
        let thread_pool = rayon::ThreadPoolBuilder::new()
            .num_threads(num_cpus::get())
            .build()
            .map_err(|e| anyhow!("Failed to create thread pool: {:?}", e))?;
        
        let results_queue = Arc::new(ArrayQueue::new(1000));
        
        Self {
            scenarios,
            thread_pool,
            results_queue,
        }
    }

    /// Run all stress scenarios in parallel
    pub fn run_stress_tests(
        &self,
        portfolio: &HashMap<AssetId, f64>,
        var_engine: &RealTimeVarEngine,
    ) -> Vec<StressTestResult> {
        let results: Vec<_> = self.scenarios
            .par_iter()
            .map(|scenario| self.execute_scenario(scenario, portfolio, var_engine))
            .collect();
        
        results.into_iter().filter_map(Result::ok).collect()
    }

    fn execute_scenario(
        &self,
        scenario: &StressScenario,
        portfolio: &HashMap<AssetId, f64>,
        var_engine: &RealTimeVarEngine,
    ) -> Result<StressTestResult, AdvancedRiskError> {
        // Apply scenario shocks to portfolio
        let mut stressed_portfolio = portfolio.clone();
        
        for asset_id in &scenario.affected_assets {
            if let Some(position) = stressed_portfolio.get_mut(asset_id) {
                *position *= 1.0 + scenario.shock_magnitude;
            }
        }
        
        // Calculate stressed VaR
        let stressed_var = var_engine.calculate_portfolio_var(&stressed_portfolio, 10000)?;
        
        // Calculate portfolio P&L under stress
        let portfolio_pnl = self.calculate_stressed_pnl(portfolio, scenario);
        
        // Determine if scenario passed
        let max_acceptable_loss = portfolio.values().sum::<f64>() * 0.05; // 5% max loss
        let passed = portfolio_pnl.abs() <= max_acceptable_loss;
        
        Ok(StressTestResult {
            scenario_name: scenario.name.clone(),
            portfolio_pnl,
            max_drawdown: portfolio_pnl.min(0.0),
            var_breach_probability: if stressed_var.var_95 > 0.0 { 0.05 } else { 0.0 },
            liquidity_shortfall: 0.0, // Would calculate based on liquidation costs
            recovery_time_days: if passed { 0 } else { scenario.duration_days },
            passed,
            confidence_level: 0.95,
            timestamp: Utc::now().timestamp_nanos_opt().unwrap_or(0) as u64,
        })
    }

    fn calculate_stressed_pnl(
        &self,
        portfolio: &HashMap<AssetId, f64>,
        scenario: &StressScenario,
    ) -> f64 {
        let mut total_pnl = 0.0;
        
        for (asset_id, position) in portfolio {
            if scenario.affected_assets.contains(asset_id) {
                total_pnl += position * scenario.shock_magnitude;
            }
        }
        
        total_pnl
    }
}

/// Position monitoring system with hierarchical limits
#[derive(Debug)]
/// PositionMonitor component.
pub struct PositionMonitor {
    account_limits: Arc<RwLock<HashMap<AccountId, AccountLimits>>>,
    strategy_limits: Arc<RwLock<HashMap<StrategyId, StrategyLimits>>>,
    instrument_limits: Arc<RwLock<HashMap<AssetId, InstrumentLimits>>>,
    current_positions: Arc<RwLock<HashMap<AssetId, f64>>>,
    circuit_breaker: Arc<AtomicBool>,
    limit_breach_sender: mpsc::Sender<LimitBreach>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
/// AccountLimits component.
pub struct AccountLimits {
    pub max_gross_exposure: f64,
    pub max_net_exposure: f64,
    pub max_var_95: f64,
    pub max_concentration: f64,
    pub max_leverage: f64,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
/// StrategyLimits component.
pub struct StrategyLimits {
    pub max_position_size: f64,
    pub max_daily_pnl_loss: f64,
    pub max_drawdown: f64,
    pub enabled: bool,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
/// InstrumentLimits component.
pub struct InstrumentLimits {
    pub max_position: f64,
    pub max_order_size: f64,
    pub min_price: f64,
    pub max_price: f64,
    pub trading_enabled: bool,
}

#[derive(Debug, Clone)]
/// LimitBreach component.
pub struct LimitBreach {
    pub limit_type: String,
    pub current_value: f64,
    pub limit_value: f64,
    pub asset_id: Option<AssetId>,
    pub timestamp: DateTime<Utc>,
}

impl PositionMonitor {
    /// Create new position monitoring system
    pub fn new() -> (Self, mpsc::Receiver<LimitBreach>) {
        let (sender, receiver) = mpsc::channel(1000);
        
        (Self {
            account_limits: Arc::new(RwLock::new(HashMap::new())),
            strategy_limits: Arc::new(RwLock::new(HashMap::new())),
            instrument_limits: Arc::new(RwLock::new(HashMap::new())),
            current_positions: Arc::new(RwLock::new(HashMap::new())),
            circuit_breaker: Arc::new(AtomicBool::new(false)),
            limit_breach_sender,
        }, receiver)
    }

    /// Check if order passes all risk limits
    pub async fn check_order_limits(
        &self,
        asset_id: AssetId,
        order_size: f64,
        account_id: AccountId,
        strategy_id: StrategyId,
    ) -> Result<(), AdvancedRiskError> {
        // Check circuit breaker
        if self.circuit_breaker.load(Ordering::Relaxed) {
            return Err(AdvancedRiskError::CircuitBreakerError {
                reason: "Global circuit breaker activated".to_string(),
            });
        }

        // Check instrument limits
        self.check_instrument_limits(asset_id, order_size).await?;
        
        // Check strategy limits
        self.check_strategy_limits(strategy_id, asset_id, order_size).await?;
        
        // Check account limits
        self.check_account_limits(account_id, asset_id, order_size).await?;
        
        Ok(())
    }

    async fn check_instrument_limits(
        &self,
        asset_id: AssetId,
        order_size: f64,
    ) -> Result<(), AdvancedRiskError> {
        let limits = self.instrument_limits.read()?;
        let positions = self.current_positions.read()?;
        
        if let Some(limit) = limits.get(&asset_id) {
            if !limit.trading_enabled {
                return Err(AdvancedRiskError::PositionLimitError {
                    limit_type: "Trading disabled".to_string(),
                });
            }
            
            if order_size.abs() > limit.max_order_size {
                return Err(AdvancedRiskError::PositionLimitError {
                    limit_type: "Order size limit exceeded".to_string(),
                });
            }
            
            let current_position = positions.get(&asset_id).copied().unwrap_or(0.0);
            let projected_position = current_position + order_size;
            
            if projected_position.abs() > limit.max_position {
                let _ = self.limit_breach_sender.try_send(LimitBreach {
                    limit_type: "Position limit breach".to_string(),
                    current_value: projected_position.abs(),
                    limit_value: limit.max_position,
                    asset_id: Some(asset_id),
                    timestamp: Utc::now().timestamp_nanos_opt().unwrap_or(0) as u64,
                });
                
                return Err(AdvancedRiskError::PositionLimitError {
                    limit_type: "Position limit exceeded".to_string(),
                });
            }
        }
        
        Ok(())
    }

    async fn check_strategy_limits(
        &self,
        strategy_id: StrategyId,
        asset_id: AssetId,
        order_size: f64,
    ) -> Result<(), AdvancedRiskError> {
        let limits = self.strategy_limits.read()?;
        
        if let Some(limit) = limits.get(&strategy_id) {
            if !limit.enabled {
                return Err(AdvancedRiskError::PositionLimitError {
                    limit_type: "Strategy disabled".to_string(),
                });
            }
            
            if order_size.abs() > limit.max_position_size {
                return Err(AdvancedRiskError::PositionLimitError {
                    limit_type: "Strategy position size exceeded".to_string(),
                });
            }
        }
        
        Ok(())
    }

    async fn check_account_limits(
        &self,
        account_id: AccountId,
        asset_id: AssetId,
        order_size: f64,
    ) -> Result<(), AdvancedRiskError> {
        let limits = self.account_limits.read()?;
        let positions = self.current_positions.read()?;
        
        if let Some(limit) = limits.get(&account_id) {
            // Calculate projected exposure
            let current_gross_exposure: f64 = positions.values().map(|p| p.abs()).sum();
            let projected_gross_exposure = current_gross_exposure + order_size.abs();
            
            if projected_gross_exposure > limit.max_gross_exposure {
                return Err(AdvancedRiskError::PositionLimitError {
                    limit_type: "Account gross exposure exceeded".to_string(),
                });
            }
        }
        
        Ok(())
    }

    /// Trigger emergency circuit breaker
    pub fn activate_circuit_breaker(&self, reason: String) {
        self.circuit_breaker.store(true, Ordering::Relaxed);
        
        let _ = self.limit_breach_sender.try_send(LimitBreach {
            limit_type: "Circuit breaker activated".to_string(),
            current_value: 1.0,
            limit_value: 0.0,
            asset_id: None,
            timestamp: Utc::now(),
        });
    }

    /// Deactivate circuit breaker
    pub fn deactivate_circuit_breaker(&self) {
        self.circuit_breaker.store(false, Ordering::Relaxed);
    }
}

/// Portfolio optimization engine with dynamic correlation analysis
#[derive(Debug)]
/// PortfolioOptimizer component.
pub struct PortfolioOptimizer {
    correlation_estimator: OnlineCorrelationEstimator,
    expected_returns: Arc<RwLock<HashMap<AssetId, f64>>>,
    risk_aversion: f64,
}

#[derive(Debug)]
/// OnlineCorrelationEstimizer component.
pub struct OnlineCorrelationEstimizer {
    correlation_matrix: Arc<RwLock<Array2<f64>>>,
    means: Arc<RwLock<HashMap<AssetId, f64>>>,
    n_observations: Arc<Mutex<u64>>,
    decay_factor: f64,
}

impl OnlineCorrelationEstimator {
    pub fn new(decay_factor: f64) -> Self {
        Self {
            correlation_matrix: Arc::new(RwLock::new(Array2::zeros((0, 0)))),
            means: Arc::new(RwLock::new(HashMap::new())),
            n_observations: Arc::new(Mutex::new(0)),
            decay_factor,
        }
    }
}

#[derive(Debug, Clone)]
/// OptimizationResult component.
pub struct OptimizationResult {
    pub optimal_weights: HashMap<AssetId, f64>,
    pub expected_return: f64,
    pub expected_risk: f64,
    pub sharpe_ratio: f64,
    pub timestamp: DateTime<Utc>,
}

impl PortfolioOptimizer {
    /// Create new portfolio optimizer
    pub fn new(risk_aversion: f64) -> Self {
        Self {
            correlation_estimator: OnlineCorrelationEstimizer::new(0.94),
            expected_returns: Arc::new(RwLock::new(HashMap::new())),
            risk_aversion,
        }
    }

    /// Optimize portfolio using mean-variance optimization
    pub fn optimize_portfolio(
        &self,
        current_positions: &HashMap<AssetId, f64>,
        target_return: Option<f64>,
    ) -> Result<OptimizationResult, AdvancedRiskError> {
        let returns = self.expected_returns.read()?;
        let correlations = self.correlation_estimator.correlation_matrix.read()?;
        
        // Simple mean-variance optimization (simplified)
        let mut optimal_weights = HashMap::new();
        let num_assets = current_positions.len();
        let equal_weight = 1.0 / num_assets as f64;
        
        for asset_id in current_positions.keys() {
            optimal_weights.insert(*asset_id, equal_weight);
        }
        
        // Calculate expected portfolio return and risk
        let expected_return = self.calculate_portfolio_return(&optimal_weights, &returns);
        let expected_risk = self.calculate_portfolio_risk(&optimal_weights, &correlations);
        
        let sharpe_ratio = if expected_risk > 0.0 {
            expected_return / expected_risk
        } else {
            0.0
        };
        
        Ok(OptimizationResult {
            optimal_weights,
            expected_return,
            expected_risk,
            sharpe_ratio,
            timestamp: Utc::now(),
        })
    }

    fn calculate_portfolio_return(
        &self,
        weights: &HashMap<AssetId, f64>,
        returns: &HashMap<AssetId, f64>,
    ) -> f64 {
        weights.iter()
            .map(|(asset_id, weight)| weight * returns.get(asset_id).copied().unwrap_or(0.0))
            .sum()
    }

    fn calculate_portfolio_risk(
        &self,
        weights: &HashMap<AssetId, f64>,
        correlations: &Array2<f64>,
    ) -> f64 {
        // Simplified risk calculation
        // In practice, this would involve full covariance matrix multiplication
        weights.values().map(|w| w.powi(2)).sum::<f64>().sqrt()
    }
}

impl OnlineCorrelationEstimator {
    /// Create new online correlation estimator
    pub fn new(decay_factor: f64) -> Self {
        Self {
            correlation_matrix: Arc::new(RwLock::new(Array2::zeros((100, 100)))),
            means: Arc::new(RwLock::new(HashMap::new())),
            n_observations: Arc::new(Mutex::new(0)),
            decay_factor,
        }
    }

    /// Update correlations with new return data
    pub fn update_correlations(&self, returns: &HashMap<AssetId, f64>) {
        let mut means = self.means.write()?;
        let mut n_obs = self.n_observations.lock()?;
        *n_obs += 1;
        
        // Update running means using EWMA
        for (asset_id, return_value) in returns {
            let current_mean = means.get(asset_id).copied().unwrap_or(0.0);
            let updated_mean = self.decay_factor * current_mean + (1.0 - self.decay_factor) * return_value;
            means.insert(*asset_id, updated_mean);
        }
    }
}

/// Regulatory compliance engine
#[derive(Debug)]
/// ComplianceEngine component.
pub struct ComplianceEngine {
    position_limits: HashMap<String, f64>,
    reporting_buffer: Arc<Mutex<Vec<ComplianceEvent>>>,
    violation_count: Arc<AtomicU64>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
/// ComplianceEvent component.
pub struct ComplianceEvent {
    pub event_type: String,
    pub description: String,
    pub severity: ComplianceSeverity,
    pub asset_id: Option<AssetId>,
    pub value: f64,
    pub timestamp: DateTime<Utc>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
/// ComplianceSeverity component.
pub enum ComplianceSeverity {
    Info,
    Warning,
    Critical,
    Breach,
}

impl ComplianceEngine {
    /// Create new compliance engine
    pub fn new() -> Self {
        Self {
            position_limits: HashMap::new(),
            reporting_buffer: Arc::new(Mutex::new(Vec::new())),
            violation_count: Arc::new(AtomicU64::new(0)),
        }
    }

    /// Check order for regulatory compliance
    pub fn check_compliance(
        &self,
        asset_id: AssetId,
        order_size: f64,
        current_positions: &HashMap<AssetId, f64>,
    ) -> Result<(), AdvancedRiskError> {
        // Example: Large trader reporting threshold
        if order_size.abs() > 1_000_000.0 {
            self.log_compliance_event(ComplianceEvent {
                event_type: "Large Trade".to_string(),
                description: format!("Large order size: {}", order_size),
                severity: ComplianceSeverity::Info,
                asset_id: Some(asset_id),
                value: order_size,
                timestamp: Utc::now(),
            });
        }

        // Example: Position concentration check
        let total_exposure: f64 = current_positions.values().map(|p| p.abs()).sum();
        let current_position = current_positions.get(&asset_id).copied().unwrap_or(0.0);
        let concentration = (current_position + order_size).abs() / total_exposure;
        
        if concentration > 0.1 {  // 10% concentration limit
            self.violation_count.fetch_add(1, Ordering::Relaxed);
            return Err(AdvancedRiskError::ComplianceError {
                rule: "Position concentration limit exceeded".to_string(),
            });
        }

        Ok(())
    }

    fn log_compliance_event(&self, event: ComplianceEvent) {
        if let Ok(mut buffer) = self.reporting_buffer.lock() {
            buffer.push(event);
            
            // Maintain buffer size
            if buffer.len() > 10000 {
                buffer.drain(0..1000);
            }
        }
    }

    /// Get compliance report
    pub fn generate_compliance_report(&self) -> Vec<ComplianceEvent> {
        self.reporting_buffer.lock()
            .map(|buffer| buffer.clone())
            .unwrap_or_default()
    }
}

/// Main advanced risk management system
#[derive(Debug)]
/// AdvancedRiskManagementSystem component.
pub struct AdvancedRiskManagementSystem {
    var_engine: RealTimeVarEngine,
    stress_engine: StressTestEngine,
    position_monitor: PositionMonitor,
    portfolio_optimizer: PortfolioOptimizer,
    compliance_engine: ComplianceEngine,
    config: AdvancedRiskConfig,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
/// AdvancedRiskConfig component.
pub struct AdvancedRiskConfig {
    pub var_confidence_levels: Vec<f64>,
    pub stress_scenarios_enabled: bool,
    pub position_monitoring_enabled: bool,
    pub portfolio_optimization_enabled: bool,
    pub compliance_checking_enabled: bool,
    pub update_frequency_ms: u64,
    pub max_portfolio_var: f64,
    pub max_concentration: f64,
}

impl AdvancedRiskManagementSystem {
    /// Create new advanced risk management system
    pub fn new(config: AdvancedRiskConfig) -> (Self, mpsc::Receiver<LimitBreach>) {
        let var_engine = RealTimeVarEngine::new(config.var_confidence_levels.clone(), 252);
        
        // Default stress scenarios
        let stress_scenarios = vec![
            StressScenario {
                name: "Market Crash".to_string(),
                description: "20% market decline".to_string(),
                shock_magnitude: -0.20,
                affected_assets: vec![], // Would be populated with relevant assets
                correlation_shock: Some(0.8), // High correlation during crisis
                volatility_multiplier: 2.0,
                duration_days: 5,
            },
            StressScenario {
                name: "Liquidity Crisis".to_string(),
                description: "Severe liquidity constraints".to_string(),
                shock_magnitude: -0.10,
                affected_assets: vec![],
                correlation_shock: None,
                volatility_multiplier: 1.5,
                duration_days: 10,
            },
        ];
        
        let stress_engine = StressTestEngine::new(stress_scenarios);
        let (position_monitor, limit_receiver) = PositionMonitor::new();
        let portfolio_optimizer = PortfolioOptimizer::new(1.0); // Moderate risk aversion
        let compliance_engine = ComplianceEngine::new();
        
        (Self {
            var_engine,
            stress_engine,
            position_monitor,
            portfolio_optimizer,
            compliance_engine,
            config,
        }, limit_receiver)
    }

    /// Comprehensive risk check for new order
    pub async fn check_order_risk(
        &self,
        asset_id: AssetId,
        order_size: f64,
        account_id: AccountId,
        strategy_id: StrategyId,
        current_positions: &HashMap<AssetId, f64>,
    ) -> Result<OrderRiskAssessment, AdvancedRiskError> {
        let start_time = std::time::Instant::now();
        
        // 1. Position limit checks
        if self.config.position_monitoring_enabled {
            self.position_monitor
                .check_order_limits(asset_id, order_size, account_id, strategy_id)
                .await?;
        }
        
        // 2. Compliance checks
        if self.config.compliance_checking_enabled {
            self.compliance_engine
                .check_compliance(asset_id, order_size, current_positions)?;
        }
        
        // 3. VaR impact assessment
        let var_impact = if current_positions.contains_key(&asset_id) {
            self.var_engine
                .calculate_incremental_var(
                    asset_id,
                    order_size,
                    self.config.max_portfolio_var,
                )?
        } else {
            0.0
        };
        
        // 4. Portfolio optimization impact (optional, for advisory)
        let optimization_advice = if self.config.portfolio_optimization_enabled {
            Some(self.portfolio_optimizer.optimize_portfolio(current_positions, None)?)
        } else {
            None
        };
        
        let processing_time = start_time.elapsed();
        
        Ok(OrderRiskAssessment {
            approved: true,
            var_impact,
            optimization_advice,
            processing_time_nanos: processing_time.as_nanos() as u64,
            timestamp: Utc::now(),
        })
    }

    /// Run comprehensive stress testing
    pub fn run_stress_tests(
        &self,
        current_positions: &HashMap<AssetId, f64>,
    ) -> Vec<StressTestResult> {
        if self.config.stress_scenarios_enabled {
            self.stress_engine.run_stress_tests(current_positions, &self.var_engine)
        } else {
            vec![]
        }
    }

    /// Update risk models with new market data
    pub fn update_risk_models(&self, market_data: &HashMap<AssetId, f64>) {
        self.var_engine.update_volatility_estimates(market_data);
        self.portfolio_optimizer.correlation_estimator.update_correlations(market_data);
    }
}

#[derive(Debug, Clone)]
/// OrderRiskAssessment component.
pub struct OrderRiskAssessment {
    pub approved: bool,
    pub var_impact: f64,
    pub optimization_advice: Option<OptimizationResult>,
    pub processing_time_nanos: u64,
    pub timestamp: DateTime<Utc>,
}