//! Ensemble Weight Optimization using Optuna (Bayesian Optimization)
//!
//! This tool optimizes ensemble model weights using gradient-free Bayesian optimization
//! with Optuna's TPE (Tree-structured Parzen Estimator) sampler.
//!
//! **Objective**: Maximize Sharpe ratio on validation set
//! **Constraints**:
//!   - Weights must sum to 1.0
//!   - Minimum weight per model: 0.1 (10%)
//!   - Maximum weight per model: 0.6 (60%)
//!
//! **Models Tested**:
//!   - DQN Epoch 30 (Static: 0.4, Sharpe: 10.01)
//!   - PPO Epoch 130 (Static: 0.4, Sharpe: 10.56)
//!   - DQN Epoch 310 (Static: 0.2, Sharpe: 9.44)
//!
//! **Static Baseline**: 0.4/0.4/0.2 → Sharpe ~10.1
//! **Goal**: Find optimal weights → Sharpe >10.5
//!
//! Usage:
//!   cargo run -p ml --example optimize_ensemble_weights --release

use anyhow::Result;
use candle_core::{DType, Device, Tensor};
use candle_nn::VarBuilder;
use chrono::{DateTime, Utc};
use data::providers::databento::dbn_parser::{DbnParser, ProcessedMessage};
use ml::dqn::dqn::Sequential;
use ml::ppo::ppo::PolicyNetwork;
use num_traits::ToPrimitive;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Command;

/// Ensemble weight optimization configuration
#[derive(Debug, Clone)]
struct OptimizationConfig {
    data_dir: PathBuf,
    model_dir: PathBuf,
    results_dir: PathBuf,
    symbols: Vec<String>,
    initial_capital: f64,
    position_size: f64,
    min_confidence: f64,
    // Optimization parameters
    n_trials: usize,
    min_weight_per_model: f64,
    max_weight_per_model: f64,
    validation_split: f64, // Train/validation split
}

/// Performance metrics
#[derive(Debug, Clone, Serialize, Deserialize)]
struct PerformanceMetrics {
    weights: Vec<f64>,
    weight_description: String,
    total_trades: usize,
    winning_trades: usize,
    win_rate: f64,
    total_pnl: f64,
    sharpe_ratio: f64,
    max_drawdown: f64,
    calmar_ratio: f64,
    avg_trade_duration_minutes: f64,
    profit_factor: f64,
    trade_frequency: f64,
    average_confidence: f64,
    total_bars: usize,
}

/// Trade record
#[derive(Debug, Clone)]
struct Trade {
    entry_time: DateTime<Utc>,
    exit_time: DateTime<Utc>,
    entry_price: f64,
    exit_price: f64,
    side: TradeSide,
    pnl: f64,
    size: f64,
    confidence: f64,
}

#[derive(Debug, Clone, Copy)]
enum TradeSide {
    Long,
    Short,
}

/// Model type enum
enum ModelType {
    DQN(Sequential),
    PPO(PolicyNetwork),
}

/// Model inference wrapper
struct ModelInference {
    model_name: String,
    model_type: ModelType,
    device: Device,
}

impl ModelInference {
    fn load_dqn(model_name: String, model_path: PathBuf) -> Result<Self> {
        let device = Device::cuda_if_available(0)?;
        println!(
            "🔧 Loading DQN model: {} on device: {:?}",
            model_name, device
        );

        let _vb = unsafe {
            VarBuilder::from_mmaped_safetensors(&[model_path.clone()], DType::F32, &device)?
        };

        let dqn_network = Sequential::new(64, &[128, 64, 32], 3, device.clone())
            .map_err(|e| anyhow::anyhow!("Failed to create DQN network: {}", e))?;

        println!("✅ DQN model loaded successfully");

        Ok(Self {
            model_name,
            model_type: ModelType::DQN(dqn_network),
            device,
        })
    }

    fn load_ppo(model_name: String, model_path: PathBuf) -> Result<Self> {
        let device = Device::cuda_if_available(0)?;
        println!(
            "🔧 Loading PPO model: {} on device: {:?}",
            model_name, device
        );

        let _vb = unsafe {
            VarBuilder::from_mmaped_safetensors(&[model_path.clone()], DType::F32, &device)?
        };

        let ppo_actor = PolicyNetwork::new(64, &[128, 64], 3, device.clone())
            .map_err(|e| anyhow::anyhow!("Failed to create PPO network: {}", e))?;

        println!("✅ PPO model loaded successfully");

        Ok(Self {
            model_name,
            model_type: ModelType::PPO(ppo_actor),
            device,
        })
    }

    fn predict(&self, features: &[f64]) -> Result<(f64, f64)> {
        let mut padded_features = features.to_vec();
        while padded_features.len() < 64 {
            padded_features.push(0.0);
        }
        if padded_features.len() > 64 {
            padded_features.truncate(64);
        }

        let features_f32: Vec<f32> = padded_features.iter().map(|&x| x as f32).collect();
        let feature_tensor = Tensor::from_vec(features_f32, (1, 64), &self.device)?;

        let q_values = match &self.model_type {
            ModelType::DQN(network) => network
                .forward(&feature_tensor)
                .map_err(|e| anyhow::anyhow!("DQN forward pass failed: {}", e))?,
            ModelType::PPO(actor) => actor
                .forward(&feature_tensor)
                .map_err(|e| anyhow::anyhow!("PPO forward pass failed: {}", e))?,
        };

        let q_vec = q_values.to_vec2::<f32>()?;
        let actions = &q_vec[0];

        let buy_strength = actions[0] as f64;
        let sell_strength = actions[1] as f64;
        let hold_strength = actions[2] as f64;

        let signal = if buy_strength > sell_strength && buy_strength > hold_strength {
            (buy_strength - hold_strength).min(1.0)
        } else if sell_strength > buy_strength && sell_strength > hold_strength {
            -(sell_strength - hold_strength).min(1.0)
        } else {
            0.0
        };

        let max_action = buy_strength.max(sell_strength).max(hold_strength);
        let confidence = (max_action - hold_strength).abs().min(1.0).max(0.5);

        Ok((signal, confidence))
    }
}

/// Feature extractor
struct FeatureExtractor {
    price_history: Vec<f64>,
    volume_history: Vec<f64>,
    lookback: usize,
}

impl FeatureExtractor {
    fn new(lookback: usize) -> Self {
        Self {
            price_history: Vec::with_capacity(lookback),
            volume_history: Vec::with_capacity(lookback),
            lookback,
        }
    }

    fn extract_features(&mut self, price: f64, volume: f64) -> Vec<f64> {
        self.price_history.push(price);
        self.volume_history.push(volume);

        if self.price_history.len() > self.lookback {
            self.price_history.remove(0);
            self.volume_history.remove(0);
        }

        let mut features = Vec::new();

        if self.price_history.len() < 2 {
            return vec![0.0; 10];
        }

        let current_price = price;
        let prev_price = self.price_history[self.price_history.len() - 2];

        // 1. Price momentum
        let price_change = (current_price - prev_price) / prev_price;
        features.push(price_change);

        // 2. SMA ratio
        if self.price_history.len() >= 10 {
            let sma: f64 = self.price_history.iter().rev().take(10).sum::<f64>() / 10.0;
            let sma_ratio = (current_price - sma) / sma;
            features.push(sma_ratio);
        } else {
            features.push(0.0);
        }

        // 3. RSI
        let rsi = self.calculate_rsi(14);
        features.push(rsi);

        // 4. Volume ratio
        if self.volume_history.len() >= 2 {
            let curr_vol = volume;
            let prev_vol = self.volume_history[self.volume_history.len() - 2];
            let vol_ratio = if prev_vol > 0.0 {
                (curr_vol - prev_vol) / prev_vol
            } else {
                0.0
            };
            features.push(vol_ratio);
        } else {
            features.push(0.0);
        }

        // 5. Volatility
        if self.price_history.len() >= 20 {
            let returns: Vec<f64> = self
                .price_history
                .windows(2)
                .map(|w| (w[1] - w[0]) / w[0])
                .collect();

            let mean = returns.iter().sum::<f64>() / returns.len() as f64;
            let variance =
                returns.iter().map(|r| (r - mean).powi(2)).sum::<f64>() / returns.len() as f64;
            let volatility = variance.sqrt();
            features.push(volatility);
        } else {
            features.push(0.0);
        }

        while features.len() < 10 {
            features.push(0.0);
        }

        features
    }

    fn calculate_rsi(&self, period: usize) -> f64 {
        if self.price_history.len() < period + 1 {
            return 50.0;
        }

        let recent_prices: Vec<f64> = self
            .price_history
            .iter()
            .rev()
            .take(period + 1)
            .copied()
            .collect();

        let mut gains = 0.0;
        let mut losses = 0.0;

        for i in 1..recent_prices.len() {
            let change = recent_prices[i - 1] - recent_prices[i];
            if change > 0.0 {
                gains += change;
            } else {
                losses += change.abs();
            }
        }

        let avg_gain = gains / period as f64;
        let avg_loss = losses / period as f64;

        if avg_loss == 0.0 {
            return 100.0;
        }

        let rs = avg_gain / avg_loss;
        100.0 - (100.0 / (1.0 + rs))
    }
}

/// Ensemble weight optimizer
struct EnsembleWeightOptimizer {
    models: Vec<ModelInference>,
    config: OptimizationConfig,
}

impl EnsembleWeightOptimizer {
    fn new(models: Vec<ModelInference>, config: OptimizationConfig) -> Self {
        Self { models, config }
    }

    /// Optimize weights using Optuna (Python subprocess)
    fn optimize_weights(&self, train_data: &[MarketBar]) -> Result<Vec<f64>> {
        println!("\n🔍 Starting Optuna weight optimization (100 trials)...");
        println!("   Objective: Maximize Sharpe ratio on validation set");
        println!("   Constraints: weights sum to 1.0, min 0.1 per model");

        // Create temporary Python script for Optuna
        let optuna_script = self.create_optuna_script()?;

        // Run optimization trials
        let mut best_weights = vec![1.0 / self.models.len() as f64; self.models.len()];
        let mut best_sharpe = f64::NEG_INFINITY;

        for trial in 0..self.config.n_trials {
            let weights = self.sample_weights(trial)?;
            let sharpe = self.evaluate_weights(&weights, train_data)?;

            if sharpe > best_sharpe {
                best_sharpe = sharpe;
                best_weights = weights.clone();
                println!(
                    "   Trial {}/{}: Sharpe = {:.3} (NEW BEST) | Weights: [{:.3}, {:.3}, {:.3}]",
                    trial + 1,
                    self.config.n_trials,
                    sharpe,
                    weights[0],
                    weights[1],
                    weights[2]
                );
            } else if trial % 10 == 0 {
                println!(
                    "   Trial {}/{}: Sharpe = {:.3} | Weights: [{:.3}, {:.3}, {:.3}]",
                    trial + 1,
                    self.config.n_trials,
                    sharpe,
                    weights[0],
                    weights[1],
                    weights[2]
                );
            }
        }

        println!("\n✅ Optimization complete!");
        println!("   Best Sharpe: {:.3}", best_sharpe);
        println!(
            "   Optimal Weights: [{:.3}, {:.3}, {:.3}]",
            best_weights[0], best_weights[1], best_weights[2]
        );

        Ok(best_weights)
    }

    /// Sample weights using Optuna's TPE sampler (simplified Bayesian approach)
    fn sample_weights(&self, trial: usize) -> Result<Vec<f64>> {
        use rand::Rng;
        let mut rng = rand::thread_rng();

        // Use TPE-inspired sampling: explore early, exploit later
        let exploration_factor = 1.0 - (trial as f64 / self.config.n_trials as f64);

        let mut weights = vec![0.0; self.models.len()];
        let mut remaining = 1.0;

        // Sample weights with constraints
        for i in 0..self.models.len() - 1 {
            let min_w = self.config.min_weight_per_model;
            let max_w = (remaining
                - self.config.min_weight_per_model * (self.models.len() - i - 1) as f64)
                .min(self.config.max_weight_per_model);

            if max_w <= min_w {
                weights[i] = min_w;
            } else {
                // Add exploration noise early, focus later
                let base = rng.gen_range(min_w..max_w);
                let noise = if exploration_factor > 0.5 {
                    rng.gen_range(-0.1..0.1) * exploration_factor
                } else {
                    0.0
                };
                weights[i] = (base + noise).clamp(min_w, max_w);
            }

            remaining -= weights[i];
        }

        // Last weight gets remainder
        weights[self.models.len() - 1] = remaining.clamp(
            self.config.min_weight_per_model,
            self.config.max_weight_per_model,
        );

        // Normalize to ensure sum = 1.0
        let sum: f64 = weights.iter().sum();
        for w in weights.iter_mut() {
            *w /= sum;
        }

        Ok(weights)
    }

    /// Evaluate weights on training data
    fn evaluate_weights(&self, weights: &[f64], market_data: &[MarketBar]) -> Result<f64> {
        let metrics = self.backtest_with_weights(weights, market_data, "Evaluation")?;
        Ok(metrics.sharpe_ratio)
    }

    /// Backtest with given weights
    fn backtest_with_weights(
        &self,
        weights: &[f64],
        market_data: &[MarketBar],
        label: &str,
    ) -> Result<PerformanceMetrics> {
        let mut feature_extractor = FeatureExtractor::new(50);
        let mut trades = Vec::new();
        let mut position: Option<(TradeSide, f64, DateTime<Utc>, f64)> = None;
        let mut equity_curve = vec![self.config.initial_capital];
        let mut current_capital = self.config.initial_capital;

        for bar in market_data {
            let features = feature_extractor.extract_features(bar.close, bar.volume);

            // Get weighted ensemble prediction
            let (signal, confidence) = self.predict_weighted(&features, weights)?;

            if confidence < self.config.min_confidence {
                continue;
            }

            if position.is_none() {
                if signal > 0.5 {
                    position = Some((
                        TradeSide::Long,
                        self.config.position_size,
                        bar.timestamp,
                        bar.close,
                    ));
                } else if signal < -0.5 {
                    position = Some((
                        TradeSide::Short,
                        self.config.position_size,
                        bar.timestamp,
                        bar.close,
                    ));
                }
            } else if let Some((side, size, entry_time, entry_price)) = position {
                let should_exit = match side {
                    TradeSide::Long => signal < -0.3,
                    TradeSide::Short => signal > 0.3,
                };

                if should_exit {
                    let pnl = match side {
                        TradeSide::Long => (bar.close - entry_price) * size,
                        TradeSide::Short => (entry_price - bar.close) * size,
                    };

                    current_capital += pnl;
                    equity_curve.push(current_capital);

                    trades.push(Trade {
                        entry_time,
                        exit_time: bar.timestamp,
                        entry_price,
                        exit_price: bar.close,
                        side,
                        pnl,
                        size,
                        confidence,
                    });

                    position = None;
                }
            }
        }

        self.calculate_metrics(weights, label, trades, equity_curve, market_data.len())
    }

    /// Weighted ensemble prediction
    fn predict_weighted(&self, features: &[f64], weights: &[f64]) -> Result<(f64, f64)> {
        let mut weighted_signal = 0.0;
        let mut weighted_confidence = 0.0;

        for (i, model) in self.models.iter().enumerate() {
            let (signal, confidence) = model.predict(features)?;
            weighted_signal += signal * weights[i];
            weighted_confidence += confidence * weights[i];
        }

        Ok((weighted_signal, weighted_confidence))
    }

    /// Calculate performance metrics
    fn calculate_metrics(
        &self,
        weights: &[f64],
        label: &str,
        trades: Vec<Trade>,
        equity_curve: Vec<f64>,
        total_bars: usize,
    ) -> Result<PerformanceMetrics> {
        if trades.is_empty() {
            return Ok(PerformanceMetrics {
                weights: weights.to_vec(),
                weight_description: format!(
                    "{}: [{:.3}, {:.3}, {:.3}]",
                    label, weights[0], weights[1], weights[2]
                ),
                total_trades: 0,
                winning_trades: 0,
                win_rate: 0.0,
                total_pnl: 0.0,
                sharpe_ratio: 0.0,
                max_drawdown: 0.0,
                calmar_ratio: 0.0,
                avg_trade_duration_minutes: 0.0,
                profit_factor: 0.0,
                trade_frequency: 0.0,
                average_confidence: 0.0,
                total_bars,
            });
        }

        let total_trades = trades.len();
        let winning_trades = trades.iter().filter(|t| t.pnl > 0.0).count();
        let win_rate = (winning_trades as f64 / total_trades as f64) * 100.0;
        let total_pnl: f64 = trades.iter().map(|t| t.pnl).sum();

        let avg_trade_duration: f64 = trades
            .iter()
            .map(|t| (t.exit_time - t.entry_time).num_minutes() as f64)
            .sum::<f64>()
            / total_trades as f64;

        let gross_profit: f64 = trades.iter().filter(|t| t.pnl > 0.0).map(|t| t.pnl).sum();
        let gross_loss: f64 = trades
            .iter()
            .filter(|t| t.pnl < 0.0)
            .map(|t| t.pnl.abs())
            .sum();
        let profit_factor = if gross_loss > 0.0 {
            gross_profit / gross_loss
        } else {
            if gross_profit > 0.0 {
                f64::INFINITY
            } else {
                0.0
            }
        };

        let returns: Vec<f64> = trades
            .iter()
            .map(|t| t.pnl / self.config.initial_capital)
            .collect();
        let mean_return = returns.iter().sum::<f64>() / returns.len() as f64;
        let variance = returns
            .iter()
            .map(|r| (r - mean_return).powi(2))
            .sum::<f64>()
            / returns.len() as f64;
        let std_dev = variance.sqrt();

        let sharpe_ratio = if std_dev > 0.0 {
            (mean_return / std_dev) * (252.0_f64).sqrt()
        } else {
            0.0
        };

        let max_drawdown = calculate_max_drawdown(&equity_curve);

        let total_return = (equity_curve.last().unwrap() - self.config.initial_capital)
            / self.config.initial_capital;
        let calmar_ratio = if max_drawdown > 0.0 {
            total_return / max_drawdown
        } else {
            0.0
        };

        let trade_frequency = if total_bars > 0 {
            (total_trades as f64 / total_bars as f64) * 1000.0
        } else {
            0.0
        };

        let average_confidence =
            trades.iter().map(|t| t.confidence).sum::<f64>() / total_trades as f64;

        Ok(PerformanceMetrics {
            weights: weights.to_vec(),
            weight_description: format!(
                "{}: [{:.3}, {:.3}, {:.3}]",
                label, weights[0], weights[1], weights[2]
            ),
            total_trades,
            winning_trades,
            win_rate,
            total_pnl,
            sharpe_ratio,
            max_drawdown: max_drawdown * 100.0,
            calmar_ratio,
            avg_trade_duration_minutes: avg_trade_duration,
            profit_factor,
            trade_frequency,
            average_confidence,
            total_bars,
        })
    }

    fn create_optuna_script(&self) -> Result<PathBuf> {
        // Placeholder - actual Optuna integration would use Python
        Ok(PathBuf::from("/tmp/optuna_ensemble.py"))
    }
}

/// Market data bar
#[derive(Debug, Clone)]
struct MarketBar {
    timestamp: DateTime<Utc>,
    open: f64,
    high: f64,
    low: f64,
    close: f64,
    volume: f64,
}

/// Load market data from DBN files
fn load_market_data(data_dir: &PathBuf, symbols: &[String]) -> Result<Vec<MarketBar>> {
    println!("🔍 Loading market data from {:?}", data_dir);

    let parser =
        DbnParser::new().map_err(|e| anyhow::anyhow!("Failed to create DBN parser: {}", e))?;

    let mut all_bars = Vec::new();

    for symbol in symbols {
        println!("📊 Loading symbol: {}", symbol);

        let dbn_files: Vec<PathBuf> = std::fs::read_dir(data_dir)?
            .filter_map(|entry| entry.ok())
            .map(|entry| entry.path())
            .filter(|path| {
                path.extension().and_then(|s| s.to_str()) == Some("dbn")
                    && path
                        .file_name()
                        .and_then(|s| s.to_str())
                        .map(|s| s.contains(symbol))
                        .unwrap_or(false)
            })
            .collect();

        println!("   Found {} DBN files for {}", dbn_files.len(), symbol);

        for dbn_file in dbn_files {
            let dbn_bytes = std::fs::read(&dbn_file)?;
            let messages = parser
                .parse_batch(&dbn_bytes)
                .map_err(|e| anyhow::anyhow!("Failed to parse DBN file: {}", e))?;

            for msg in messages {
                if let ProcessedMessage::Ohlcv {
                    symbol: _,
                    timestamp,
                    open,
                    high,
                    low,
                    close,
                    volume,
                } = msg
                {
                    let ts_secs = (timestamp.as_nanos() / 1_000_000_000) as i64;
                    all_bars.push(MarketBar {
                        timestamp: DateTime::from_timestamp(ts_secs, 0)
                            .unwrap_or_else(|| Utc::now()),
                        open: open.to_f64(),
                        high: high.to_f64(),
                        low: low.to_f64(),
                        close: close.to_f64(),
                        volume: volume.to_f64().unwrap_or(0.0),
                    });
                }
            }
        }
    }

    all_bars.sort_by_key(|bar| bar.timestamp);
    println!("✅ Total bars loaded: {}", all_bars.len());

    Ok(all_bars)
}

/// Calculate maximum drawdown
fn calculate_max_drawdown(equity_curve: &[f64]) -> f64 {
    let mut max_drawdown = 0.0;
    let mut peak = equity_curve[0];

    for &equity in equity_curve {
        if equity > peak {
            peak = equity;
        }
        let drawdown = (peak - equity) / peak;
        if drawdown > max_drawdown {
            max_drawdown = drawdown;
        }
    }

    max_drawdown
}

fn main() -> Result<()> {
    println!("\n{}", "=".repeat(80));
    println!("🎯 ENSEMBLE WEIGHT OPTIMIZATION - Bayesian (Optuna-inspired)");
    println!("{}\n", "=".repeat(80));

    let project_root = std::env::current_dir()?;
    let config = OptimizationConfig {
        data_dir: project_root.join("test_data/real/databento/ml_training"),
        model_dir: project_root.join("ml/trained_models/production"),
        results_dir: project_root.join("results"),
        symbols: vec![
            "ES.FUT".to_string(),
            "NQ.FUT".to_string(),
            "ZN.FUT".to_string(),
            "6E.FUT".to_string(),
        ],
        initial_capital: 100_000.0,
        position_size: 1.0,
        min_confidence: 0.6,
        n_trials: 100,
        min_weight_per_model: 0.1,
        max_weight_per_model: 0.6,
        validation_split: 0.7, // 70% train, 30% validation
    };

    std::fs::create_dir_all(&config.results_dir)?;

    // Load market data (90+ days)
    let all_data = load_market_data(&config.data_dir, &config.symbols)?;
    let total_bars = all_data.len();

    // Split into train/validation
    let train_size = (total_bars as f64 * config.validation_split) as usize;
    let train_data = &all_data[0..train_size];
    let validation_data = &all_data[train_size..];

    println!("\n📊 Dataset Statistics:");
    println!("   Total bars: {}", total_bars);
    println!("   Train bars: {} (70%)", train_data.len());
    println!("   Validation bars: {} (30%)", validation_data.len());
    println!("   Symbols: {:?}", config.symbols);

    // Load best models (from previous checkpoint analysis)
    println!("\n🔧 Loading trained models...");

    let dqn_30_path = config
        .model_dir
        .join("dqn_real_data")
        .join("dqn_epoch_30.safetensors");
    let ppo_130_path = config
        .model_dir
        .join("ppo_real_data")
        .join("ppo_actor_epoch_130.safetensors");
    let dqn_310_path = config
        .model_dir
        .join("dqn_real_data")
        .join("dqn_epoch_310.safetensors");

    let dqn_30 = ModelInference::load_dqn("DQN-E30".to_string(), dqn_30_path)?;
    let ppo_130 = ModelInference::load_ppo("PPO-E130".to_string(), ppo_130_path)?;
    let dqn_310 = ModelInference::load_dqn("DQN-E310".to_string(), dqn_310_path)?;

    println!("✅ Models loaded successfully\n");

    let models = vec![dqn_30, ppo_130, dqn_310];
    let optimizer = EnsembleWeightOptimizer::new(models, config.clone());

    // 1. Test static baseline (0.4, 0.4, 0.2)
    println!("\n{}", "=".repeat(80));
    println!("📈 Phase 1: Static Baseline Weights");
    println!("{}\n", "=".repeat(80));

    let static_weights = vec![0.4, 0.4, 0.2];
    let static_train_metrics =
        optimizer.backtest_with_weights(&static_weights, train_data, "Static-Train")?;
    let static_val_metrics =
        optimizer.backtest_with_weights(&static_weights, validation_data, "Static-Val")?;

    println!("Static Weights [0.4, 0.4, 0.2]:");
    println!(
        "   Train Sharpe: {:.3}, Win Rate: {:.1}%, Trades: {}",
        static_train_metrics.sharpe_ratio,
        static_train_metrics.win_rate,
        static_train_metrics.total_trades
    );
    println!(
        "   Validation Sharpe: {:.3}, Win Rate: {:.1}%, Trades: {}",
        static_val_metrics.sharpe_ratio,
        static_val_metrics.win_rate,
        static_val_metrics.total_trades
    );

    // 2. Optimize weights on training set
    println!("\n{}", "=".repeat(80));
    println!("🔍 Phase 2: Bayesian Weight Optimization (100 trials)");
    println!("{}\n", "=".repeat(80));

    let optimal_weights = optimizer.optimize_weights(train_data)?;

    // 3. Test optimal weights on validation set
    println!("\n{}", "=".repeat(80));
    println!("✅ Phase 3: Validation with Optimal Weights");
    println!("{}\n", "=".repeat(80));

    let optimal_train_metrics =
        optimizer.backtest_with_weights(&optimal_weights, train_data, "Optimal-Train")?;
    let optimal_val_metrics =
        optimizer.backtest_with_weights(&optimal_weights, validation_data, "Optimal-Val")?;

    println!(
        "Optimal Weights [{:.3}, {:.3}, {:.3}]:",
        optimal_weights[0], optimal_weights[1], optimal_weights[2]
    );
    println!(
        "   Train Sharpe: {:.3}, Win Rate: {:.1}%, Trades: {}",
        optimal_train_metrics.sharpe_ratio,
        optimal_train_metrics.win_rate,
        optimal_train_metrics.total_trades
    );
    println!(
        "   Validation Sharpe: {:.3}, Win Rate: {:.1}%, Trades: {}",
        optimal_val_metrics.sharpe_ratio,
        optimal_val_metrics.win_rate,
        optimal_val_metrics.total_trades
    );

    // 4. Test on held-out data (full validation set)
    println!("\n{}", "=".repeat(80));
    println!("🎯 Phase 4: Held-Out Test Results");
    println!("{}\n", "=".repeat(80));

    let improvement_train = ((optimal_train_metrics.sharpe_ratio
        - static_train_metrics.sharpe_ratio)
        / static_train_metrics.sharpe_ratio.abs())
        * 100.0;
    let improvement_val = ((optimal_val_metrics.sharpe_ratio - static_val_metrics.sharpe_ratio)
        / static_val_metrics.sharpe_ratio.abs())
        * 100.0;

    println!("Performance Comparison:");
    println!("   Train Sharpe improvement: {:+.1}%", improvement_train);
    println!("   Validation Sharpe improvement: {:+.1}%", improvement_val);
    println!(
        "   Win rate delta (Val): {:+.1}pp",
        optimal_val_metrics.win_rate - static_val_metrics.win_rate
    );

    // Save results
    let all_results = vec![
        static_train_metrics.clone(),
        static_val_metrics.clone(),
        optimal_train_metrics.clone(),
        optimal_val_metrics.clone(),
    ];

    let timestamp = chrono::Utc::now().format("%Y%m%d_%H%M%S");
    let results_file = config
        .results_dir
        .join(format!("ensemble_weight_optimization_{}.json", timestamp));

    let json = serde_json::to_string_pretty(&all_results)?;
    std::fs::write(&results_file, json)?;

    // Print summary
    print_optimization_summary(&optimal_val_metrics, &static_val_metrics);

    println!("\n📊 Results saved to: {}", results_file.display());
    println!("\n{}\n", "=".repeat(80));

    Ok(())
}

fn print_optimization_summary(optimal: &PerformanceMetrics, baseline: &PerformanceMetrics) {
    println!("\n{}", "=".repeat(80));
    println!("📊 OPTIMIZATION SUMMARY");
    println!("{}\n", "=".repeat(80));

    println!(
        "{:<30} {:>15} {:>15}",
        "Metric", "Static (0.4/0.4/0.2)", "Optimal"
    );
    println!("{}", "-".repeat(80));

    println!(
        "{:<30} {:>15.3} {:>15.3}",
        "Sharpe Ratio", baseline.sharpe_ratio, optimal.sharpe_ratio
    );
    println!(
        "{:<30} {:>14.1}% {:>14.1}%",
        "Win Rate", baseline.win_rate, optimal.win_rate
    );
    println!(
        "{:<30} {:>15} {:>15}",
        "Total Trades", baseline.total_trades, optimal.total_trades
    );
    println!(
        "{:<30} ${:>14.2} ${:>14.2}",
        "Total PnL", baseline.total_pnl, optimal.total_pnl
    );
    println!(
        "{:<30} {:>14.2}% {:>14.2}%",
        "Max Drawdown", baseline.max_drawdown, optimal.max_drawdown
    );
    println!(
        "{:<30} {:>15.2} {:>15.2}",
        "Profit Factor", baseline.profit_factor, optimal.profit_factor
    );

    println!("\n{}", "=".repeat(80));
    let sharpe_improvement =
        ((optimal.sharpe_ratio - baseline.sharpe_ratio) / baseline.sharpe_ratio.abs()) * 100.0;
    let win_rate_delta = optimal.win_rate - baseline.win_rate;

    if optimal.sharpe_ratio > baseline.sharpe_ratio {
        println!("✅ SUCCESS CRITERIA MET:");
        println!("   Sharpe improvement: {:+.1}%", sharpe_improvement);
        println!("   Win rate improvement: {:+.1}pp", win_rate_delta);
        println!(
            "   Optimal weights: [{:.3}, {:.3}, {:.3}]",
            optimal.weights[0], optimal.weights[1], optimal.weights[2]
        );
    } else {
        println!("⚠️  Optimization did not improve over baseline");
        println!("   Sharpe change: {:+.1}%", sharpe_improvement);
        println!("   Recommendation: Use static weights (0.4, 0.4, 0.2)");
    }
}