foxhunt/backtesting/examples/feature_comparison_backtest.rs

//! Feature Comparison Backtest: 26-Feature System vs 18-Feature Baseline
//!
//! Agent A19: Comprehensive performance analysis comparing enhanced 26-feature
//! ML system against 18-feature baseline across ES.FUT, NQ.FUT, ZN.FUT.
//!
//! Metrics:
//! - Win rate (target: 46-51% vs baseline 41.81%)
//! - Sharpe ratio (target: 0.5-1.0 vs baseline -6.5192)
//! - Max drawdown (target: <14%)
//! - Total PnL (baseline: -55.90)
//! - Statistical significance (t-tests)
//! - Feature importance analysis

use anyhow::Result;
use backtesting::{
    BacktestConfig, BacktestEngine, ReplayConfig, Strategy, StrategyConfig, StrategyContext,
    StrategyResult, TradingSignal,
};
use chrono::{DateTime, Duration, Utc};
use common::ml_strategy::{MLStrategy, SimpleDQNAdapter};
use common::{Order, Position, Price, Quantity, Symbol};
use rust_decimal::Decimal;
use rust_decimal_macros::dec;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::path::PathBuf;
use trading_engine::types::events::MarketEvent;

/// Feature set configuration for A/B testing
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum FeatureSet {
    Baseline18,  // Original 18 features (pre-Wave 19)
    Enhanced26,  // New 26 features (post-Wave 19, Agents A1-A7)
}

/// ML-based trading strategy with configurable feature set
struct MLTradingStrategy {
    feature_set: FeatureSet,
    ml_strategy: MLStrategy,
    dqn_adapter: SimpleDQNAdapter,
    initial_capital: Decimal,
    trades_executed: usize,
    winning_trades: usize,
    total_pnl: Decimal,
    peak_value: Decimal,
    max_drawdown: Decimal,
    returns_history: Vec<Decimal>,
    trades_history: Vec<TradeRecord>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
struct TradeRecord {
    timestamp: DateTime<Utc>,
    symbol: Symbol,
    side: String,
    quantity: Decimal,
    entry_price: Decimal,
    exit_price: Option<Decimal>,
    pnl: Option<Decimal>,
    is_winner: Option<bool>,
}

impl MLTradingStrategy {
    fn new(feature_set: FeatureSet) -> Self {
        let ml_strategy = MLStrategy::new(200); // 200 bar lookback
        let dqn_adapter = SimpleDQNAdapter::new_with_26_features().unwrap();

        Self {
            feature_set,
            ml_strategy,
            dqn_adapter,
            initial_capital: Decimal::ZERO,
            trades_executed: 0,
            winning_trades: 0,
            total_pnl: Decimal::ZERO,
            peak_value: Decimal::ZERO,
            max_drawdown: Decimal::ZERO,
            returns_history: Vec::new(),
            trades_history: Vec::new(),
        }
    }

    /// Extract features based on configured feature set
    fn extract_features(&self, market_event: &MarketEvent) -> Result<Vec<f32>> {
        // Get 26-feature vector from ML strategy
        let full_features = self.ml_strategy.extract_features(market_event)?;

        match self.feature_set {
            FeatureSet::Enhanced26 => {
                // Use all 26 features
                Ok(full_features)
            }
            FeatureSet::Baseline18 => {
                // Use only first 18 features (pre-Wave 19 baseline)
                // This simulates the original system before ADX, Stochastic, CCI, etc. were added
                Ok(full_features[..18].to_vec())
            }
        }
    }

    fn calculate_sharpe_ratio(&self) -> Decimal {
        if self.returns_history.len() < 2 {
            return Decimal::ZERO;
        }

        let n = Decimal::from(self.returns_history.len());
        let mean_return = self.returns_history.iter().sum::<Decimal>() / n;

        let variance = self.returns_history.iter()
            .map(|r| {
                let diff = *r - mean_return;
                diff * diff
            })
            .sum::<Decimal>() / (n - Decimal::ONE);

        let std_dev = Decimal::try_from(
            variance.to_f64().unwrap_or(0.0).sqrt()
        ).unwrap_or(Decimal::ZERO);

        if std_dev > Decimal::ZERO {
            // Annualized Sharpe (assuming 252 trading days)
            let annualization_factor = Decimal::try_from(252.0_f64.sqrt()).unwrap_or(dec!(15.87));
            mean_return * annualization_factor / std_dev
        } else {
            Decimal::ZERO
        }
    }
}

#[async_trait::async_trait(?Send)]
impl Strategy for MLTradingStrategy {
    fn name(&self) -> &str {
        match self.feature_set {
            FeatureSet::Baseline18 => "ML_Strategy_18_Features_Baseline",
            FeatureSet::Enhanced26 => "ML_Strategy_26_Features_Enhanced",
        }
    }

    async fn initialize(&mut self, initial_capital: Decimal, _config: StrategyConfig) -> Result<()> {
        self.initial_capital = initial_capital;
        self.peak_value = initial_capital;
        println!(
            "Initialized {} with capital: {}",
            self.name(),
            initial_capital
        );
        Ok(())
    }

    async fn on_market_event(
        &mut self,
        event: &MarketEvent,
        context: &StrategyContext,
    ) -> Result<Vec<TradingSignal>> {
        let mut signals = Vec::new();

        if let MarketEvent::Trade { symbol, price, .. } = event {
            // Extract features based on configured feature set
            let features = match self.extract_features(event) {
                Ok(f) => f,
                Err(e) => {
                    eprintln!("Feature extraction error: {}", e);
                    return Ok(signals);
                }
            };

            // Get ML prediction using appropriate adapter
            let action = match self.feature_set {
                FeatureSet::Enhanced26 => {
                    self.dqn_adapter.predict(&features)?
                }
                FeatureSet::Baseline18 => {
                    // For 18-feature baseline, we need a compatible adapter
                    // Using SimpleDQN's linear combination approach
                    let score: f32 = features.iter()
                        .take(18)
                        .enumerate()
                        .map(|(i, &f)| {
                            // Simplified weights for baseline (first 18 features)
                            let weight = match i {
                                0..=4 => 0.05,   // OHLCV features
                                5 => 0.12,       // RSI
                                6..=7 => 0.08,   // EMA
                                8..=10 => 0.10,  // MACD
                                11..=13 => 0.16, // Bollinger Bands
                                14..=17 => 0.08, // Other indicators
                                _ => 0.0,
                            };
                            f * weight
                        })
                        .sum();

                    // Sigmoid activation
                    let sigmoid = 1.0 / (1.0 + (-score).exp());

                    if sigmoid > 0.6 {
                        common::ml_strategy::TradingAction::Buy
                    } else if sigmoid < 0.4 {
                        common::ml_strategy::TradingAction::Sell
                    } else {
                        common::ml_strategy::TradingAction::Hold
                    }
                }
            };

            // Generate trading signals based on ML prediction
            let position_size = context.account_balance * dec!(0.02); // 2% position sizing
            let price_decimal: Decimal = (*price).into();
            let quantity = (position_size / price_decimal).round_dp(0);

            use backtesting::SignalType;

            match action {
                common::ml_strategy::TradingAction::Buy => {
                    signals.push(TradingSignal {
                        symbol: symbol.clone(),
                        signal_type: SignalType::Buy,
                        quantity: Quantity::from_f64(quantity.to_f64().unwrap_or(0.0))
                            .unwrap_or(Quantity::ZERO),
                        target_price: Some(*price),
                        stop_loss: None,
                        take_profit: None,
                        confidence: dec!(0.75),
                        metadata: {
                            let mut m = HashMap::new();
                            m.insert("feature_set".to_string(), serde_json::json!(format!("{:?}", self.feature_set)));
                            m.insert("feature_count".to_string(), serde_json::json!(features.len()));
                            m
                        },
                    });
                }
                common::ml_strategy::TradingAction::Sell => {
                    signals.push(TradingSignal {
                        symbol: symbol.clone(),
                        signal_type: SignalType::Sell,
                        quantity: Quantity::from_f64(quantity.to_f64().unwrap_or(0.0))
                            .unwrap_or(Quantity::ZERO),
                        target_price: Some(*price),
                        stop_loss: None,
                        take_profit: None,
                        confidence: dec!(0.75),
                        metadata: {
                            let mut m = HashMap::new();
                            m.insert("feature_set".to_string(), serde_json::json!(format!("{:?}", self.feature_set)));
                            m.insert("feature_count".to_string(), serde_json::json!(features.len()));
                            m
                        },
                    });
                }
                common::ml_strategy::TradingAction::Hold => {
                    // No signal
                }
            }
        }

        Ok(signals)
    }

    async fn on_order_update(&mut self, order: &Order, _context: &StrategyContext) -> Result<()> {
        if order.status == common::OrderStatus::Filled {
            self.trades_executed += 1;
            println!(
                "[{}] Trade #{}: {} {} @ {}",
                self.name(),
                self.trades_executed,
                order.side,
                order.quantity,
                order.average_price.unwrap_or(order.price.unwrap_or(Price::ZERO))
            );
        }
        Ok(())
    }

    async fn on_position_update(
        &mut self,
        _position: &Position,
        context: &StrategyContext,
    ) -> Result<()> {
        let current_value = context.account_balance;

        // Update peak value and drawdown
        if current_value > self.peak_value {
            self.peak_value = current_value;
        }

        let current_drawdown = (self.peak_value - current_value) / self.peak_value;
        if current_drawdown > self.max_drawdown {
            self.max_drawdown = current_drawdown;
        }

        // Calculate period return
        if self.initial_capital > Decimal::ZERO {
            let period_return = (current_value - self.initial_capital) / self.initial_capital;
            self.returns_history.push(period_return);
        }

        self.total_pnl = current_value - self.initial_capital;

        Ok(())
    }

    async fn finalize(&mut self, context: &StrategyContext) -> Result<StrategyResult> {
        let final_value = context.account_balance;
        let total_return = if self.initial_capital > Decimal::ZERO {
            (final_value - self.initial_capital) / self.initial_capital
        } else {
            Decimal::ZERO
        };

        let win_rate = if self.trades_executed > 0 {
            Decimal::from(self.winning_trades) / Decimal::from(self.trades_executed)
        } else {
            Decimal::ZERO
        };

        let sharpe_ratio = self.calculate_sharpe_ratio();

        println!("\n=== {} Final Results ===", self.name());
        println!("Total Trades: {}", self.trades_executed);
        println!("Win Rate: {:.2}%", win_rate * dec!(100));
        println!("Total Return: {:.2}%", total_return * dec!(100));
        println!("Sharpe Ratio: {:.4}", sharpe_ratio);
        println!("Max Drawdown: {:.2}%", self.max_drawdown * dec!(100));
        println!("Final PnL: {:.2}", self.total_pnl);
        println!("Feature Count: {}", match self.feature_set {
            FeatureSet::Baseline18 => 18,
            FeatureSet::Enhanced26 => 26,
        });

        Ok(StrategyResult {
            strategy_name: self.name().to_string(),
            total_return,
            annualized_return: total_return, // Simplified
            max_drawdown: self.max_drawdown,
            sharpe_ratio,
            total_trades: self.trades_executed as u64,
            win_rate,
            avg_trade_return: if self.trades_executed > 0 {
                self.total_pnl / Decimal::from(self.trades_executed)
            } else {
                Decimal::ZERO
            },
            final_value,
            trades: vec![],
            performance_timeline: vec![],
        })
    }

    async fn get_state(&self) -> Result<serde_json::Value> {
        Ok(serde_json::json!({
            "name": self.name(),
            "feature_set": format!("{:?}", self.feature_set),
            "trades_executed": self.trades_executed,
            "winning_trades": self.winning_trades,
            "total_pnl": self.total_pnl,
            "max_drawdown": self.max_drawdown,
            "sharpe_ratio": self.calculate_sharpe_ratio(),
        }))
    }
}

/// Run backtest comparison for a single symbol
async fn run_symbol_backtest(
    symbol: &str,
    dbn_file_path: PathBuf,
    feature_set: FeatureSet,
) -> Result<StrategyResult> {
    println!("\n{'=':=<80}");
    println!("Running {} backtest on {}",
        match feature_set {
            FeatureSet::Baseline18 => "18-FEATURE BASELINE",
            FeatureSet::Enhanced26 => "26-FEATURE ENHANCED",
        },
        symbol
    );
    println!("{'=':=<80}\n");

    let config = BacktestConfig {
        initial_capital: dec!(100000), // $100k starting capital
        replay_config: ReplayConfig {
            start_time: Utc::now() - Duration::days(30),
            end_time: Utc::now(),
            tick_by_tick: false,
            speed_multiplier: 1.0,
            symbols: vec![Symbol(symbol.to_string())],
        },
        strategy_config: StrategyConfig {
            max_position_size: dec!(50000),
            risk_per_trade: dec!(0.02), // 2% risk
            max_open_positions: 3,
            stop_loss_pct: Some(dec!(0.05)), // 5% stop loss
            take_profit_pct: Some(dec!(0.10)), // 10% take profit
            position_sizing_enabled: true,
            commission_rate: dec!(0.0002), // 0.02% commission
            slippage_factor: dec!(0.0001), // 0.01% slippage
            parameters: HashMap::new(),
        },
        risk_free_rate: dec!(0.02), // 2% annual risk-free rate
        enable_logging: true,
        snapshot_interval: 3600,
        max_memory_usage: 1024 * 1024 * 1024,
    };

    let mut engine = BacktestEngine::new(config).await?;

    let strategy = Box::new(MLTradingStrategy::new(feature_set));
    engine.set_strategy(strategy).await?;

    let result = engine.run().await?;

    Ok(result.strategy_result)
}

/// Calculate t-test for statistical significance
fn calculate_t_test(
    baseline_metrics: &[StrategyResult],
    enhanced_metrics: &[StrategyResult],
) -> (Decimal, Decimal) {
    // Calculate means
    let baseline_sharpe_mean = baseline_metrics.iter()
        .map(|r| r.sharpe_ratio)
        .sum::<Decimal>() / Decimal::from(baseline_metrics.len());

    let enhanced_sharpe_mean = enhanced_metrics.iter()
        .map(|r| r.sharpe_ratio)
        .sum::<Decimal>() / Decimal::from(enhanced_metrics.len());

    // Calculate standard deviations
    let baseline_variance = baseline_metrics.iter()
        .map(|r| {
            let diff = r.sharpe_ratio - baseline_sharpe_mean;
            diff * diff
        })
        .sum::<Decimal>() / Decimal::from(baseline_metrics.len());

    let enhanced_variance = enhanced_metrics.iter()
        .map(|r| {
            let diff = r.sharpe_ratio - enhanced_sharpe_mean;
            diff * diff
        })
        .sum::<Decimal>() / Decimal::from(enhanced_metrics.len());

    let pooled_std = Decimal::try_from(
        ((baseline_variance + enhanced_variance) / dec!(2)).to_f64().unwrap_or(0.0).sqrt()
    ).unwrap_or(dec!(0.0001));

    let n = Decimal::from(baseline_metrics.len());
    let t_stat = (enhanced_sharpe_mean - baseline_sharpe_mean) /
        (pooled_std * Decimal::try_from((2.0 / n.to_f64().unwrap_or(1.0)).sqrt()).unwrap_or(Decimal::ONE));

    // Simple p-value approximation (2-tailed)
    let p_value = if t_stat.abs() > dec!(2.0) {
        dec!(0.05) // Significant
    } else {
        dec!(0.15) // Not significant
    };

    (t_stat, p_value)
}

#[tokio::main]
async fn main() -> Result<()> {
    println!("\n{'#':=<80}");
    println!("# Agent A19: Feature Comparison Backtest");
    println!("# 26-Feature Enhanced System vs 18-Feature Baseline");
    println!("{'#':=<80}\n");

    let test_data_dir = PathBuf::from("/home/jgrusewski/Work/foxhunt/test_data/real/databento");

    let symbols = vec![
        ("ES.FUT", test_data_dir.join("ml_training/ES.FUT_ohlcv-1m_2024-03-25.dbn")),
        ("NQ.FUT", test_data_dir.join("NQ.FUT_ohlcv-1m_2024-01-02.dbn")),
        ("ZN.FUT", test_data_dir.join("ml_training/ZN.FUT_ohlcv-1m_2024-04-17.dbn")),
    ];

    let mut baseline_results = Vec::new();
    let mut enhanced_results = Vec::new();

    for (symbol, dbn_path) in &symbols {
        // Run baseline (18 features)
        match run_symbol_backtest(symbol, dbn_path.clone(), FeatureSet::Baseline18).await {
            Ok(result) => baseline_results.push(result),
            Err(e) => eprintln!("Baseline backtest failed for {}: {}", symbol, e),
        }

        // Run enhanced (26 features)
        match run_symbol_backtest(symbol, dbn_path.clone(), FeatureSet::Enhanced26).await {
            Ok(result) => enhanced_results.push(result),
            Err(e) => eprintln!("Enhanced backtest failed for {}: {}", symbol, e),
        }
    }

    // Statistical analysis
    println!("\n{'#':=<80}");
    println!("# STATISTICAL SIGNIFICANCE ANALYSIS");
    println!("{'#':=<80}\n");

    if !baseline_results.is_empty() && !enhanced_results.is_empty() {
        let (t_stat, p_value) = calculate_t_test(&baseline_results, &enhanced_results);

        println!("T-statistic: {:.4}", t_stat);
        println!("P-value: {:.4}", p_value);
        println!("Significance: {}", if p_value < dec!(0.05) {
            "SIGNIFICANT (p < 0.05) ✓"
        } else {
            "NOT SIGNIFICANT (p >= 0.05)"
        });
    }

    // Summary comparison table
    println!("\n{'#':=<80}");
    println!("# PERFORMANCE COMPARISON SUMMARY");
    println!("{'#':=<80}\n");
    println!("{:<20} | {:>15} | {:>15} | {:>15}", "Metric", "18-Feature", "26-Feature", "Improvement");
    println!("{:-<70}", "");

    if !baseline_results.is_empty() && !enhanced_results.is_empty() {
        let baseline_avg_sharpe = baseline_results.iter().map(|r| r.sharpe_ratio).sum::<Decimal>()
            / Decimal::from(baseline_results.len());
        let enhanced_avg_sharpe = enhanced_results.iter().map(|r| r.sharpe_ratio).sum::<Decimal>()
            / Decimal::from(enhanced_results.len());

        let baseline_avg_wr = baseline_results.iter().map(|r| r.win_rate).sum::<Decimal>()
            / Decimal::from(baseline_results.len());
        let enhanced_avg_wr = enhanced_results.iter().map(|r| r.win_rate).sum::<Decimal>()
            / Decimal::from(enhanced_results.len());

        let baseline_avg_dd = baseline_results.iter().map(|r| r.max_drawdown).sum::<Decimal>()
            / Decimal::from(baseline_results.len());
        let enhanced_avg_dd = enhanced_results.iter().map(|r| r.max_drawdown).sum::<Decimal>()
            / Decimal::from(enhanced_results.len());

        println!("{:<20} | {:>15.4} | {:>15.4} | {:>+14.2}%",
            "Sharpe Ratio", baseline_avg_sharpe, enhanced_avg_sharpe,
            ((enhanced_avg_sharpe - baseline_avg_sharpe) / baseline_avg_sharpe.abs().max(dec!(0.01))) * dec!(100)
        );
        println!("{:<20} | {:>14.2}% | {:>14.2}% | {:>+14.2}%",
            "Win Rate", baseline_avg_wr * dec!(100), enhanced_avg_wr * dec!(100),
            ((enhanced_avg_wr - baseline_avg_wr) / baseline_avg_wr.max(dec!(0.01))) * dec!(100)
        );
        println!("{:<20} | {:>14.2}% | {:>14.2}% | {:>+14.2}%",
            "Max Drawdown", baseline_avg_dd * dec!(100), enhanced_avg_dd * dec!(100),
            ((baseline_avg_dd - enhanced_avg_dd) / baseline_avg_dd.max(dec!(0.01))) * dec!(100)
        );
    }

    println!("\n{'#':=<80}");
    println!("# Feature Comparison Backtest Complete");
    println!("{'#':=<80}\n");

    Ok(())
}