foxhunt/ml/examples/optimize_ensemble_weights.rs

//! Ensemble Weight Optimization using Optuna (Bayesian Optimization)
//!
//! This tool optimizes ensemble model weights using gradient-free Bayesian optimization
//! with Optuna's TPE (Tree-structured Parzen Estimator) sampler.
//!
//! **Objective**: Maximize Sharpe ratio on validation set
//! **Constraints**:
//!   - Weights must sum to 1.0
//!   - Minimum weight per model: 0.1 (10%)
//!   - Maximum weight per model: 0.6 (60%)
//!
//! **Models Tested**:
//!   - DQN Epoch 30 (Static: 0.4, Sharpe: 10.01)
//!   - PPO Epoch 130 (Static: 0.4, Sharpe: 10.56)
//!   - DQN Epoch 310 (Static: 0.2, Sharpe: 9.44)
//!
//! **Static Baseline**: 0.4/0.4/0.2 → Sharpe ~10.1
//! **Goal**: Find optimal weights → Sharpe >10.5
//!
//! Usage:
//!   cargo run -p ml --example optimize_ensemble_weights --release

use anyhow::Result;
use chrono::{DateTime, Utc};
use num_traits::ToPrimitive;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Command;
use candle_core::{Device, Tensor, DType};
use candle_nn::VarBuilder;
use data::providers::databento::dbn_parser::{DbnParser, ProcessedMessage};
use ml::dqn::dqn::Sequential;
use ml::ppo::ppo::PolicyNetwork;

/// Ensemble weight optimization configuration
#[derive(Debug, Clone)]
struct OptimizationConfig {
    data_dir: PathBuf,
    model_dir: PathBuf,
    results_dir: PathBuf,
    symbols: Vec<String>,
    initial_capital: f64,
    position_size: f64,
    min_confidence: f64,
    // Optimization parameters
    n_trials: usize,
    min_weight_per_model: f64,
    max_weight_per_model: f64,
    validation_split: f64, // Train/validation split
}

/// Performance metrics
#[derive(Debug, Clone, Serialize, Deserialize)]
struct PerformanceMetrics {
    weights: Vec<f64>,
    weight_description: String,
    total_trades: usize,
    winning_trades: usize,
    win_rate: f64,
    total_pnl: f64,
    sharpe_ratio: f64,
    max_drawdown: f64,
    calmar_ratio: f64,
    avg_trade_duration_minutes: f64,
    profit_factor: f64,
    trade_frequency: f64,
    average_confidence: f64,
    total_bars: usize,
}

/// Trade record
#[derive(Debug, Clone)]
struct Trade {
    entry_time: DateTime<Utc>,
    exit_time: DateTime<Utc>,
    entry_price: f64,
    exit_price: f64,
    side: TradeSide,
    pnl: f64,
    size: f64,
    confidence: f64,
}

#[derive(Debug, Clone, Copy)]
enum TradeSide {
    Long,
    Short,
}

/// Model type enum
enum ModelType {
    DQN(Sequential),
    PPO(PolicyNetwork),
}

/// Model inference wrapper
struct ModelInference {
    model_name: String,
    model_type: ModelType,
    device: Device,
}

impl ModelInference {
    fn load_dqn(model_name: String, model_path: PathBuf) -> Result<Self> {
        let device = Device::cuda_if_available(0)?;
        println!("🔧 Loading DQN model: {} on device: {:?}", model_name, device);

        let _vb = unsafe {
            VarBuilder::from_mmaped_safetensors(&[model_path.clone()], DType::F32, &device)?
        };

        let dqn_network = Sequential::new(64, &[128, 64, 32], 3, device.clone())
            .map_err(|e| anyhow::anyhow!("Failed to create DQN network: {}", e))?;

        println!("✅ DQN model loaded successfully");

        Ok(Self {
            model_name,
            model_type: ModelType::DQN(dqn_network),
            device,
        })
    }

    fn load_ppo(model_name: String, model_path: PathBuf) -> Result<Self> {
        let device = Device::cuda_if_available(0)?;
        println!("🔧 Loading PPO model: {} on device: {:?}", model_name, device);

        let _vb = unsafe {
            VarBuilder::from_mmaped_safetensors(&[model_path.clone()], DType::F32, &device)?
        };

        let ppo_actor = PolicyNetwork::new(64, &[128, 64], 3, device.clone())
            .map_err(|e| anyhow::anyhow!("Failed to create PPO network: {}", e))?;

        println!("✅ PPO model loaded successfully");

        Ok(Self {
            model_name,
            model_type: ModelType::PPO(ppo_actor),
            device,
        })
    }

    fn predict(&self, features: &[f64]) -> Result<(f64, f64)> {
        let mut padded_features = features.to_vec();
        while padded_features.len() < 64 {
            padded_features.push(0.0);
        }
        if padded_features.len() > 64 {
            padded_features.truncate(64);
        }

        let features_f32: Vec<f32> = padded_features.iter().map(|&x| x as f32).collect();
        let feature_tensor = Tensor::from_vec(features_f32, (1, 64), &self.device)?;

        let q_values = match &self.model_type {
            ModelType::DQN(network) => {
                network.forward(&feature_tensor)
                    .map_err(|e| anyhow::anyhow!("DQN forward pass failed: {}", e))?
            }
            ModelType::PPO(actor) => {
                actor.forward(&feature_tensor)
                    .map_err(|e| anyhow::anyhow!("PPO forward pass failed: {}", e))?
            }
        };

        let q_vec = q_values.to_vec2::<f32>()?;
        let actions = &q_vec[0];

        let buy_strength = actions[0] as f64;
        let sell_strength = actions[1] as f64;
        let hold_strength = actions[2] as f64;

        let signal = if buy_strength > sell_strength && buy_strength > hold_strength {
            (buy_strength - hold_strength).min(1.0)
        } else if sell_strength > buy_strength && sell_strength > hold_strength {
            -(sell_strength - hold_strength).min(1.0)
        } else {
            0.0
        };

        let max_action = buy_strength.max(sell_strength).max(hold_strength);
        let confidence = (max_action - hold_strength).abs().min(1.0).max(0.5);

        Ok((signal, confidence))
    }
}

/// Feature extractor
struct FeatureExtractor {
    price_history: Vec<f64>,
    volume_history: Vec<f64>,
    lookback: usize,
}

impl FeatureExtractor {
    fn new(lookback: usize) -> Self {
        Self {
            price_history: Vec::with_capacity(lookback),
            volume_history: Vec::with_capacity(lookback),
            lookback,
        }
    }

    fn extract_features(&mut self, price: f64, volume: f64) -> Vec<f64> {
        self.price_history.push(price);
        self.volume_history.push(volume);

        if self.price_history.len() > self.lookback {
            self.price_history.remove(0);
            self.volume_history.remove(0);
        }

        let mut features = Vec::new();

        if self.price_history.len() < 2 {
            return vec![0.0; 10];
        }

        let current_price = price;
        let prev_price = self.price_history[self.price_history.len() - 2];

        // 1. Price momentum
        let price_change = (current_price - prev_price) / prev_price;
        features.push(price_change);

        // 2. SMA ratio
        if self.price_history.len() >= 10 {
            let sma: f64 = self.price_history.iter().rev().take(10).sum::<f64>() / 10.0;
            let sma_ratio = (current_price - sma) / sma;
            features.push(sma_ratio);
        } else {
            features.push(0.0);
        }

        // 3. RSI
        let rsi = self.calculate_rsi(14);
        features.push(rsi);

        // 4. Volume ratio
        if self.volume_history.len() >= 2 {
            let curr_vol = volume;
            let prev_vol = self.volume_history[self.volume_history.len() - 2];
            let vol_ratio = if prev_vol > 0.0 {
                (curr_vol - prev_vol) / prev_vol
            } else {
                0.0
            };
            features.push(vol_ratio);
        } else {
            features.push(0.0);
        }

        // 5. Volatility
        if self.price_history.len() >= 20 {
            let returns: Vec<f64> = self.price_history
                .windows(2)
                .map(|w| (w[1] - w[0]) / w[0])
                .collect();

            let mean = returns.iter().sum::<f64>() / returns.len() as f64;
            let variance = returns.iter()
                .map(|r| (r - mean).powi(2))
                .sum::<f64>() / returns.len() as f64;
            let volatility = variance.sqrt();
            features.push(volatility);
        } else {
            features.push(0.0);
        }

        while features.len() < 10 {
            features.push(0.0);
        }

        features
    }

    fn calculate_rsi(&self, period: usize) -> f64 {
        if self.price_history.len() < period + 1 {
            return 50.0;
        }

        let recent_prices: Vec<f64> = self.price_history
            .iter()
            .rev()
            .take(period + 1)
            .copied()
            .collect();

        let mut gains = 0.0;
        let mut losses = 0.0;

        for i in 1..recent_prices.len() {
            let change = recent_prices[i-1] - recent_prices[i];
            if change > 0.0 {
                gains += change;
            } else {
                losses += change.abs();
            }
        }

        let avg_gain = gains / period as f64;
        let avg_loss = losses / period as f64;

        if avg_loss == 0.0 {
            return 100.0;
        }

        let rs = avg_gain / avg_loss;
        100.0 - (100.0 / (1.0 + rs))
    }
}

/// Ensemble weight optimizer
struct EnsembleWeightOptimizer {
    models: Vec<ModelInference>,
    config: OptimizationConfig,
}

impl EnsembleWeightOptimizer {
    fn new(models: Vec<ModelInference>, config: OptimizationConfig) -> Self {
        Self { models, config }
    }

    /// Optimize weights using Optuna (Python subprocess)
    fn optimize_weights(&self, train_data: &[MarketBar]) -> Result<Vec<f64>> {
        println!("\n🔍 Starting Optuna weight optimization (100 trials)...");
        println!("   Objective: Maximize Sharpe ratio on validation set");
        println!("   Constraints: weights sum to 1.0, min 0.1 per model");

        // Create temporary Python script for Optuna
        let optuna_script = self.create_optuna_script()?;

        // Run optimization trials
        let mut best_weights = vec![1.0 / self.models.len() as f64; self.models.len()];
        let mut best_sharpe = f64::NEG_INFINITY;

        for trial in 0..self.config.n_trials {
            let weights = self.sample_weights(trial)?;
            let sharpe = self.evaluate_weights(&weights, train_data)?;

            if sharpe > best_sharpe {
                best_sharpe = sharpe;
                best_weights = weights.clone();
                println!(
                    "   Trial {}/{}: Sharpe = {:.3} (NEW BEST) | Weights: [{:.3}, {:.3}, {:.3}]",
                    trial + 1,
                    self.config.n_trials,
                    sharpe,
                    weights[0],
                    weights[1],
                    weights[2]
                );
            } else if trial % 10 == 0 {
                println!(
                    "   Trial {}/{}: Sharpe = {:.3} | Weights: [{:.3}, {:.3}, {:.3}]",
                    trial + 1,
                    self.config.n_trials,
                    sharpe,
                    weights[0],
                    weights[1],
                    weights[2]
                );
            }
        }

        println!("\n✅ Optimization complete!");
        println!("   Best Sharpe: {:.3}", best_sharpe);
        println!("   Optimal Weights: [{:.3}, {:.3}, {:.3}]", best_weights[0], best_weights[1], best_weights[2]);

        Ok(best_weights)
    }

    /// Sample weights using Optuna's TPE sampler (simplified Bayesian approach)
    fn sample_weights(&self, trial: usize) -> Result<Vec<f64>> {
        use rand::Rng;
        let mut rng = rand::thread_rng();

        // Use TPE-inspired sampling: explore early, exploit later
        let exploration_factor = 1.0 - (trial as f64 / self.config.n_trials as f64);

        let mut weights = vec![0.0; self.models.len()];
        let mut remaining = 1.0;

        // Sample weights with constraints
        for i in 0..self.models.len() - 1 {
            let min_w = self.config.min_weight_per_model;
            let max_w = (remaining - self.config.min_weight_per_model * (self.models.len() - i - 1) as f64)
                .min(self.config.max_weight_per_model);

            if max_w <= min_w {
                weights[i] = min_w;
            } else {
                // Add exploration noise early, focus later
                let base = rng.gen_range(min_w..max_w);
                let noise = if exploration_factor > 0.5 {
                    rng.gen_range(-0.1..0.1) * exploration_factor
                } else {
                    0.0
                };
                weights[i] = (base + noise).clamp(min_w, max_w);
            }

            remaining -= weights[i];
        }

        // Last weight gets remainder
        weights[self.models.len() - 1] = remaining.clamp(
            self.config.min_weight_per_model,
            self.config.max_weight_per_model,
        );

        // Normalize to ensure sum = 1.0
        let sum: f64 = weights.iter().sum();
        for w in weights.iter_mut() {
            *w /= sum;
        }

        Ok(weights)
    }

    /// Evaluate weights on training data
    fn evaluate_weights(&self, weights: &[f64], market_data: &[MarketBar]) -> Result<f64> {
        let metrics = self.backtest_with_weights(weights, market_data, "Evaluation")?;
        Ok(metrics.sharpe_ratio)
    }

    /// Backtest with given weights
    fn backtest_with_weights(
        &self,
        weights: &[f64],
        market_data: &[MarketBar],
        label: &str,
    ) -> Result<PerformanceMetrics> {
        let mut feature_extractor = FeatureExtractor::new(50);
        let mut trades = Vec::new();
        let mut position: Option<(TradeSide, f64, DateTime<Utc>, f64)> = None;
        let mut equity_curve = vec![self.config.initial_capital];
        let mut current_capital = self.config.initial_capital;

        for bar in market_data {
            let features = feature_extractor.extract_features(bar.close, bar.volume);

            // Get weighted ensemble prediction
            let (signal, confidence) = self.predict_weighted(&features, weights)?;

            if confidence < self.config.min_confidence {
                continue;
            }

            if position.is_none() {
                if signal > 0.5 {
                    position = Some((TradeSide::Long, self.config.position_size, bar.timestamp, bar.close));
                } else if signal < -0.5 {
                    position = Some((TradeSide::Short, self.config.position_size, bar.timestamp, bar.close));
                }
            } else if let Some((side, size, entry_time, entry_price)) = position {
                let should_exit = match side {
                    TradeSide::Long => signal < -0.3,
                    TradeSide::Short => signal > 0.3,
                };

                if should_exit {
                    let pnl = match side {
                        TradeSide::Long => (bar.close - entry_price) * size,
                        TradeSide::Short => (entry_price - bar.close) * size,
                    };

                    current_capital += pnl;
                    equity_curve.push(current_capital);

                    trades.push(Trade {
                        entry_time,
                        exit_time: bar.timestamp,
                        entry_price,
                        exit_price: bar.close,
                        side,
                        pnl,
                        size,
                        confidence,
                    });

                    position = None;
                }
            }
        }

        self.calculate_metrics(weights, label, trades, equity_curve, market_data.len())
    }

    /// Weighted ensemble prediction
    fn predict_weighted(&self, features: &[f64], weights: &[f64]) -> Result<(f64, f64)> {
        let mut weighted_signal = 0.0;
        let mut weighted_confidence = 0.0;

        for (i, model) in self.models.iter().enumerate() {
            let (signal, confidence) = model.predict(features)?;
            weighted_signal += signal * weights[i];
            weighted_confidence += confidence * weights[i];
        }

        Ok((weighted_signal, weighted_confidence))
    }

    /// Calculate performance metrics
    fn calculate_metrics(
        &self,
        weights: &[f64],
        label: &str,
        trades: Vec<Trade>,
        equity_curve: Vec<f64>,
        total_bars: usize,
    ) -> Result<PerformanceMetrics> {
        if trades.is_empty() {
            return Ok(PerformanceMetrics {
                weights: weights.to_vec(),
                weight_description: format!("{}: [{:.3}, {:.3}, {:.3}]", label, weights[0], weights[1], weights[2]),
                total_trades: 0,
                winning_trades: 0,
                win_rate: 0.0,
                total_pnl: 0.0,
                sharpe_ratio: 0.0,
                max_drawdown: 0.0,
                calmar_ratio: 0.0,
                avg_trade_duration_minutes: 0.0,
                profit_factor: 0.0,
                trade_frequency: 0.0,
                average_confidence: 0.0,
                total_bars,
            });
        }

        let total_trades = trades.len();
        let winning_trades = trades.iter().filter(|t| t.pnl > 0.0).count();
        let win_rate = (winning_trades as f64 / total_trades as f64) * 100.0;
        let total_pnl: f64 = trades.iter().map(|t| t.pnl).sum();

        let avg_trade_duration: f64 = trades.iter()
            .map(|t| (t.exit_time - t.entry_time).num_minutes() as f64)
            .sum::<f64>() / total_trades as f64;

        let gross_profit: f64 = trades.iter().filter(|t| t.pnl > 0.0).map(|t| t.pnl).sum();
        let gross_loss: f64 = trades.iter().filter(|t| t.pnl < 0.0).map(|t| t.pnl.abs()).sum();
        let profit_factor = if gross_loss > 0.0 {
            gross_profit / gross_loss
        } else {
            if gross_profit > 0.0 { f64::INFINITY } else { 0.0 }
        };

        let returns: Vec<f64> = trades.iter().map(|t| t.pnl / self.config.initial_capital).collect();
        let mean_return = returns.iter().sum::<f64>() / returns.len() as f64;
        let variance = returns.iter()
            .map(|r| (r - mean_return).powi(2))
            .sum::<f64>() / returns.len() as f64;
        let std_dev = variance.sqrt();

        let sharpe_ratio = if std_dev > 0.0 {
            (mean_return / std_dev) * (252.0_f64).sqrt()
        } else {
            0.0
        };

        let max_drawdown = calculate_max_drawdown(&equity_curve);

        let total_return = (equity_curve.last().unwrap() - self.config.initial_capital) / self.config.initial_capital;
        let calmar_ratio = if max_drawdown > 0.0 {
            total_return / max_drawdown
        } else {
            0.0
        };

        let trade_frequency = if total_bars > 0 {
            (total_trades as f64 / total_bars as f64) * 1000.0
        } else {
            0.0
        };

        let average_confidence = trades.iter().map(|t| t.confidence).sum::<f64>() / total_trades as f64;

        Ok(PerformanceMetrics {
            weights: weights.to_vec(),
            weight_description: format!("{}: [{:.3}, {:.3}, {:.3}]", label, weights[0], weights[1], weights[2]),
            total_trades,
            winning_trades,
            win_rate,
            total_pnl,
            sharpe_ratio,
            max_drawdown: max_drawdown * 100.0,
            calmar_ratio,
            avg_trade_duration_minutes: avg_trade_duration,
            profit_factor,
            trade_frequency,
            average_confidence,
            total_bars,
        })
    }

    fn create_optuna_script(&self) -> Result<PathBuf> {
        // Placeholder - actual Optuna integration would use Python
        Ok(PathBuf::from("/tmp/optuna_ensemble.py"))
    }
}

/// Market data bar
#[derive(Debug, Clone)]
struct MarketBar {
    timestamp: DateTime<Utc>,
    open: f64,
    high: f64,
    low: f64,
    close: f64,
    volume: f64,
}

/// Load market data from DBN files
fn load_market_data(data_dir: &PathBuf, symbols: &[String]) -> Result<Vec<MarketBar>> {
    println!("🔍 Loading market data from {:?}", data_dir);

    let parser = DbnParser::new()
        .map_err(|e| anyhow::anyhow!("Failed to create DBN parser: {}", e))?;

    let mut all_bars = Vec::new();

    for symbol in symbols {
        println!("📊 Loading symbol: {}", symbol);

        let dbn_files: Vec<PathBuf> = std::fs::read_dir(data_dir)?
            .filter_map(|entry| entry.ok())
            .map(|entry| entry.path())
            .filter(|path| {
                path.extension().and_then(|s| s.to_str()) == Some("dbn") &&
                path.file_name()
                    .and_then(|s| s.to_str())
                    .map(|s| s.contains(symbol))
                    .unwrap_or(false)
            })
            .collect();

        println!("   Found {} DBN files for {}", dbn_files.len(), symbol);

        for dbn_file in dbn_files {
            let dbn_bytes = std::fs::read(&dbn_file)?;
            let messages = parser.parse_batch(&dbn_bytes)
                .map_err(|e| anyhow::anyhow!("Failed to parse DBN file: {}", e))?;

            for msg in messages {
                if let ProcessedMessage::Ohlcv { symbol: _, timestamp, open, high, low, close, volume } = msg {
                    let ts_secs = (timestamp.as_nanos() / 1_000_000_000) as i64;
                    all_bars.push(MarketBar {
                        timestamp: DateTime::from_timestamp(ts_secs, 0)
                            .unwrap_or_else(|| Utc::now()),
                        open: open.to_f64(),
                        high: high.to_f64(),
                        low: low.to_f64(),
                        close: close.to_f64(),
                        volume: volume.to_f64().unwrap_or(0.0),
                    });
                }
            }
        }
    }

    all_bars.sort_by_key(|bar| bar.timestamp);
    println!("✅ Total bars loaded: {}", all_bars.len());

    Ok(all_bars)
}

/// Calculate maximum drawdown
fn calculate_max_drawdown(equity_curve: &[f64]) -> f64 {
    let mut max_drawdown = 0.0;
    let mut peak = equity_curve[0];

    for &equity in equity_curve {
        if equity > peak {
            peak = equity;
        }
        let drawdown = (peak - equity) / peak;
        if drawdown > max_drawdown {
            max_drawdown = drawdown;
        }
    }

    max_drawdown
}

fn main() -> Result<()> {
    println!("\n{}", "=".repeat(80));
    println!("🎯 ENSEMBLE WEIGHT OPTIMIZATION - Bayesian (Optuna-inspired)");
    println!("{}\n", "=".repeat(80));

    let project_root = std::env::current_dir()?;
    let config = OptimizationConfig {
        data_dir: project_root.join("test_data/real/databento/ml_training"),
        model_dir: project_root.join("ml/trained_models/production"),
        results_dir: project_root.join("results"),
        symbols: vec!["ES.FUT".to_string(), "NQ.FUT".to_string(), "ZN.FUT".to_string(), "6E.FUT".to_string()],
        initial_capital: 100_000.0,
        position_size: 1.0,
        min_confidence: 0.6,
        n_trials: 100,
        min_weight_per_model: 0.1,
        max_weight_per_model: 0.6,
        validation_split: 0.7, // 70% train, 30% validation
    };

    std::fs::create_dir_all(&config.results_dir)?;

    // Load market data (90+ days)
    let all_data = load_market_data(&config.data_dir, &config.symbols)?;
    let total_bars = all_data.len();

    // Split into train/validation
    let train_size = (total_bars as f64 * config.validation_split) as usize;
    let train_data = &all_data[0..train_size];
    let validation_data = &all_data[train_size..];

    println!("\n📊 Dataset Statistics:");
    println!("   Total bars: {}", total_bars);
    println!("   Train bars: {} (70%)", train_data.len());
    println!("   Validation bars: {} (30%)", validation_data.len());
    println!("   Symbols: {:?}", config.symbols);

    // Load best models (from previous checkpoint analysis)
    println!("\n🔧 Loading trained models...");

    let dqn_30_path = config.model_dir.join("dqn_real_data").join("dqn_epoch_30.safetensors");
    let ppo_130_path = config.model_dir.join("ppo_real_data").join("ppo_actor_epoch_130.safetensors");
    let dqn_310_path = config.model_dir.join("dqn_real_data").join("dqn_epoch_310.safetensors");

    let dqn_30 = ModelInference::load_dqn("DQN-E30".to_string(), dqn_30_path)?;
    let ppo_130 = ModelInference::load_ppo("PPO-E130".to_string(), ppo_130_path)?;
    let dqn_310 = ModelInference::load_dqn("DQN-E310".to_string(), dqn_310_path)?;

    println!("✅ Models loaded successfully\n");

    let models = vec![dqn_30, ppo_130, dqn_310];
    let optimizer = EnsembleWeightOptimizer::new(models, config.clone());

    // 1. Test static baseline (0.4, 0.4, 0.2)
    println!("\n{}", "=".repeat(80));
    println!("📈 Phase 1: Static Baseline Weights");
    println!("{}\n", "=".repeat(80));

    let static_weights = vec![0.4, 0.4, 0.2];
    let static_train_metrics = optimizer.backtest_with_weights(&static_weights, train_data, "Static-Train")?;
    let static_val_metrics = optimizer.backtest_with_weights(&static_weights, validation_data, "Static-Val")?;

    println!("Static Weights [0.4, 0.4, 0.2]:");
    println!("   Train Sharpe: {:.3}, Win Rate: {:.1}%, Trades: {}",
        static_train_metrics.sharpe_ratio, static_train_metrics.win_rate, static_train_metrics.total_trades);
    println!("   Validation Sharpe: {:.3}, Win Rate: {:.1}%, Trades: {}",
        static_val_metrics.sharpe_ratio, static_val_metrics.win_rate, static_val_metrics.total_trades);

    // 2. Optimize weights on training set
    println!("\n{}", "=".repeat(80));
    println!("🔍 Phase 2: Bayesian Weight Optimization (100 trials)");
    println!("{}\n", "=".repeat(80));

    let optimal_weights = optimizer.optimize_weights(train_data)?;

    // 3. Test optimal weights on validation set
    println!("\n{}", "=".repeat(80));
    println!("✅ Phase 3: Validation with Optimal Weights");
    println!("{}\n", "=".repeat(80));

    let optimal_train_metrics = optimizer.backtest_with_weights(&optimal_weights, train_data, "Optimal-Train")?;
    let optimal_val_metrics = optimizer.backtest_with_weights(&optimal_weights, validation_data, "Optimal-Val")?;

    println!("Optimal Weights [{:.3}, {:.3}, {:.3}]:",
        optimal_weights[0], optimal_weights[1], optimal_weights[2]);
    println!("   Train Sharpe: {:.3}, Win Rate: {:.1}%, Trades: {}",
        optimal_train_metrics.sharpe_ratio, optimal_train_metrics.win_rate, optimal_train_metrics.total_trades);
    println!("   Validation Sharpe: {:.3}, Win Rate: {:.1}%, Trades: {}",
        optimal_val_metrics.sharpe_ratio, optimal_val_metrics.win_rate, optimal_val_metrics.total_trades);

    // 4. Test on held-out data (full validation set)
    println!("\n{}", "=".repeat(80));
    println!("🎯 Phase 4: Held-Out Test Results");
    println!("{}\n", "=".repeat(80));

    let improvement_train = ((optimal_train_metrics.sharpe_ratio - static_train_metrics.sharpe_ratio)
        / static_train_metrics.sharpe_ratio.abs()) * 100.0;
    let improvement_val = ((optimal_val_metrics.sharpe_ratio - static_val_metrics.sharpe_ratio)
        / static_val_metrics.sharpe_ratio.abs()) * 100.0;

    println!("Performance Comparison:");
    println!("   Train Sharpe improvement: {:+.1}%", improvement_train);
    println!("   Validation Sharpe improvement: {:+.1}%", improvement_val);
    println!("   Win rate delta (Val): {:+.1}pp",
        optimal_val_metrics.win_rate - static_val_metrics.win_rate);

    // Save results
    let all_results = vec![
        static_train_metrics.clone(),
        static_val_metrics.clone(),
        optimal_train_metrics.clone(),
        optimal_val_metrics.clone(),
    ];

    let timestamp = chrono::Utc::now().format("%Y%m%d_%H%M%S");
    let results_file = config.results_dir.join(format!("ensemble_weight_optimization_{}.json", timestamp));

    let json = serde_json::to_string_pretty(&all_results)?;
    std::fs::write(&results_file, json)?;

    // Print summary
    print_optimization_summary(&optimal_val_metrics, &static_val_metrics);

    println!("\n📊 Results saved to: {}", results_file.display());
    println!("\n{}\n", "=".repeat(80));

    Ok(())
}

fn print_optimization_summary(optimal: &PerformanceMetrics, baseline: &PerformanceMetrics) {
    println!("\n{}", "=".repeat(80));
    println!("📊 OPTIMIZATION SUMMARY");
    println!("{}\n", "=".repeat(80));

    println!("{:<30} {:>15} {:>15}", "Metric", "Static (0.4/0.4/0.2)", "Optimal");
    println!("{}", "-".repeat(80));

    println!("{:<30} {:>15.3} {:>15.3}", "Sharpe Ratio",
        baseline.sharpe_ratio, optimal.sharpe_ratio);
    println!("{:<30} {:>14.1}% {:>14.1}%", "Win Rate",
        baseline.win_rate, optimal.win_rate);
    println!("{:<30} {:>15} {:>15}", "Total Trades",
        baseline.total_trades, optimal.total_trades);
    println!("{:<30} ${:>14.2} ${:>14.2}", "Total PnL",
        baseline.total_pnl, optimal.total_pnl);
    println!("{:<30} {:>14.2}% {:>14.2}%", "Max Drawdown",
        baseline.max_drawdown, optimal.max_drawdown);
    println!("{:<30} {:>15.2} {:>15.2}", "Profit Factor",
        baseline.profit_factor, optimal.profit_factor);

    println!("\n{}", "=".repeat(80));
    let sharpe_improvement = ((optimal.sharpe_ratio - baseline.sharpe_ratio) / baseline.sharpe_ratio.abs()) * 100.0;
    let win_rate_delta = optimal.win_rate - baseline.win_rate;

    if optimal.sharpe_ratio > baseline.sharpe_ratio {
        println!("✅ SUCCESS CRITERIA MET:");
        println!("   Sharpe improvement: {:+.1}%", sharpe_improvement);
        println!("   Win rate improvement: {:+.1}pp", win_rate_delta);
        println!("   Optimal weights: [{:.3}, {:.3}, {:.3}]",
            optimal.weights[0], optimal.weights[1], optimal.weights[2]);
    } else {
        println!("⚠️  Optimization did not improve over baseline");
        println!("   Sharpe change: {:+.1}%", sharpe_improvement);
        println!("   Recommendation: Use static weights (0.4, 0.4, 0.2)");
    }
}