foxhunt/analyze_backtest_results.py

#!/usr/bin/env python3
"""
Deep Analysis of Comprehensive Backtest Results
Analyzes 100 checkpoint backtest data for trading insights
"""

import json
import statistics
from datetime import datetime
from collections import defaultdict

# Load the backtest results
with open('results/comprehensive_backtest_results_20251014_143309.json', 'r') as f:
    results = json.load(f)

# Categorization functions
def categorize_by_performance(result):
    """Categorize models as High/Medium/Low performers"""
    sharpe = result['sharpe_ratio']
    win_rate = result['win_rate']
    pnl = result['total_pnl']
    
    # High performer: Sharpe > 5, Win Rate > 55%, PnL > 50
    if sharpe > 5 and win_rate > 55 and pnl > 50:
        return 'High'
    # Low performer: Sharpe < 0 or Win Rate < 35% or PnL < -50
    elif sharpe < 0 or win_rate < 35 or pnl < -50:
        return 'Low'
    else:
        return 'Medium'

def analyze_by_model_type():
    """Analyze performance by model type (DQN vs PPO)"""
    dqn_results = [r for r in results if r['model_type'] == 'DQN']
    ppo_results = [r for r in results if r['model_type'] == 'PPO']
    
    print("\n" + "="*80)
    print("MODEL TYPE ANALYSIS")
    print("="*80)
    
    for model_type, model_results in [('DQN', dqn_results), ('PPO', ppo_results)]:
        # Filter out models with no trades
        active_models = [r for r in model_results if r['total_trades'] > 0]
        
        if not active_models:
            print(f"\n{model_type}: No active models")
            continue
        
        avg_sharpe = statistics.mean([r['sharpe_ratio'] for r in active_models])
        avg_win_rate = statistics.mean([r['win_rate'] for r in active_models])
        avg_pnl = statistics.mean([r['total_pnl'] for r in active_models])
        avg_trades = statistics.mean([r['total_trades'] for r in active_models])
        
        profitable_models = len([r for r in active_models if r['total_pnl'] > 0])
        
        print(f"\n{model_type} Models:")
        print(f"  Total Models: {len(model_results)} ({len(active_models)} active)")
        print(f"  Profitable Models: {profitable_models}/{len(active_models)} ({profitable_models/len(active_models)*100:.1f}%)")
        print(f"  Average Sharpe Ratio: {avg_sharpe:.2f}")
        print(f"  Average Win Rate: {avg_win_rate:.2f}%")
        print(f"  Average PnL: ${avg_pnl:.2f}")
        print(f"  Average Total Trades: {avg_trades:.0f}")

def analyze_by_epoch():
    """Analyze how performance changes with training epochs"""
    print("\n" + "="*80)
    print("EPOCH PROGRESSION ANALYSIS")
    print("="*80)
    
    for model_type in ['DQN', 'PPO']:
        model_results = [r for r in results if r['model_type'] == model_type]
        
        # Group by epoch ranges
        epoch_ranges = {
            'Early (10-100)': [r for r in model_results if 10 <= r['epoch'] <= 100 and r['total_trades'] > 0],
            'Mid (110-300)': [r for r in model_results if 110 <= r['epoch'] <= 300 and r['total_trades'] > 0],
            'Late (310-500)': [r for r in model_results if 310 <= r['epoch'] <= 500 and r['total_trades'] > 0]
        }
        
        print(f"\n{model_type} Epoch Progression:")
        for range_name, range_results in epoch_ranges.items():
            if not range_results:
                continue
            
            avg_sharpe = statistics.mean([r['sharpe_ratio'] for r in range_results])
            avg_win_rate = statistics.mean([r['win_rate'] for r in range_results])
            avg_pnl = statistics.mean([r['total_pnl'] for r in range_results])
            profitable = len([r for r in range_results if r['total_pnl'] > 0])
            
            print(f"  {range_name}: {len(range_results)} models, {profitable} profitable")
            print(f"    Avg Sharpe: {avg_sharpe:.2f}, Win Rate: {avg_win_rate:.1f}%, PnL: ${avg_pnl:.2f}")

def analyze_trade_characteristics():
    """Analyze trade patterns and characteristics"""
    print("\n" + "="*80)
    print("TRADE CHARACTERISTICS ANALYSIS")
    print("="*80)
    
    active_models = [r for r in results if r['total_trades'] > 10]
    
    # Trade frequency analysis
    high_freq = [r for r in active_models if r['trade_frequency'] > 50]
    low_freq = [r for r in active_models if r['trade_frequency'] < 20]
    
    print(f"\nTrade Frequency Impact:")
    print(f"  High Frequency (>50 trades/day): {len(high_freq)} models")
    if high_freq:
        avg_pnl_high = statistics.mean([r['total_pnl'] for r in high_freq])
        avg_sharpe_high = statistics.mean([r['sharpe_ratio'] for r in high_freq])
        profitable_high = len([r for r in high_freq if r['total_pnl'] > 0])
        print(f"    Profitable: {profitable_high}/{len(high_freq)} ({profitable_high/len(high_freq)*100:.1f}%)")
        print(f"    Avg PnL: ${avg_pnl_high:.2f}, Avg Sharpe: {avg_sharpe_high:.2f}")
    
    print(f"  Low Frequency (<20 trades/day): {len(low_freq)} models")
    if low_freq:
        avg_pnl_low = statistics.mean([r['total_pnl'] for r in low_freq])
        avg_sharpe_low = statistics.mean([r['sharpe_ratio'] for r in low_freq])
        profitable_low = len([r for r in low_freq if r['total_pnl'] > 0])
        print(f"    Profitable: {profitable_low}/{len(low_freq)} ({profitable_low/len(low_freq)*100:.1f}%)")
        print(f"    Avg PnL: ${avg_pnl_low:.2f}, Avg Sharpe: {avg_sharpe_low:.2f}")
    
    # Hold time analysis
    short_hold = [r for r in active_models if r['avg_trade_duration'] < 20]
    long_hold = [r for r in active_models if r['avg_trade_duration'] > 60]
    
    print(f"\nAverage Hold Time Impact:")
    print(f"  Short Hold (<20 bars): {len(short_hold)} models")
    if short_hold:
        avg_pnl_short = statistics.mean([r['total_pnl'] for r in short_hold])
        avg_win_rate_short = statistics.mean([r['win_rate'] for r in short_hold])
        profitable_short = len([r for r in short_hold if r['total_pnl'] > 0])
        print(f"    Profitable: {profitable_short}/{len(short_hold)} ({profitable_short/len(short_hold)*100:.1f}%)")
        print(f"    Avg PnL: ${avg_pnl_short:.2f}, Win Rate: {avg_win_rate_short:.1f}%")
    
    print(f"  Long Hold (>60 bars): {len(long_hold)} models")
    if long_hold:
        avg_pnl_long = statistics.mean([r['total_pnl'] for r in long_hold])
        avg_win_rate_long = statistics.mean([r['win_rate'] for r in long_hold])
        profitable_long = len([r for r in long_hold if r['total_pnl'] > 0])
        print(f"    Profitable: {profitable_long}/{len(long_hold)} ({profitable_long/len(long_hold)*100:.1f}%)")
        print(f"    Avg PnL: ${avg_pnl_long:.2f}, Win Rate: {avg_win_rate_long:.1f}%")
    
    # Win rate distribution
    print(f"\nWin Rate Distribution:")
    win_rate_bins = {
        '<30%': [r for r in active_models if r['win_rate'] < 30],
        '30-45%': [r for r in active_models if 30 <= r['win_rate'] < 45],
        '45-55%': [r for r in active_models if 45 <= r['win_rate'] < 55],
        '55-65%': [r for r in active_models if 55 <= r['win_rate'] < 65],
        '>65%': [r for r in active_models if r['win_rate'] >= 65]
    }
    
    for bin_name, bin_results in win_rate_bins.items():
        if not bin_results:
            continue
        avg_pnl = statistics.mean([r['total_pnl'] for r in bin_results])
        profitable = len([r for r in bin_results if r['total_pnl'] > 0])
        print(f"  {bin_name}: {len(bin_results)} models, {profitable} profitable, Avg PnL: ${avg_pnl:.2f}")

def find_top_performers():
    """Identify top performing models"""
    print("\n" + "="*80)
    print("TOP PERFORMERS")
    print("="*80)
    
    # Filter active models with significant trades
    active_models = [r for r in results if r['total_trades'] > 50]
    
    # Sort by different metrics
    by_sharpe = sorted(active_models, key=lambda x: x['sharpe_ratio'], reverse=True)[:10]
    by_pnl = sorted(active_models, key=lambda x: x['total_pnl'], reverse=True)[:10]
    by_profit_factor = sorted([r for r in active_models if r['profit_factor'] and r['profit_factor'] > 0], 
                              key=lambda x: x['profit_factor'], reverse=True)[:10]
    
    print("\nTop 10 by Sharpe Ratio (>50 trades):")
    for i, model in enumerate(by_sharpe, 1):
        print(f"  {i}. {model['model_name']}: Sharpe {model['sharpe_ratio']:.2f}, "
              f"Win Rate {model['win_rate']:.1f}%, PnL ${model['total_pnl']:.2f}, "
              f"Trades {model['total_trades']}")
    
    print("\nTop 10 by Total PnL (>50 trades):")
    for i, model in enumerate(by_pnl, 1):
        print(f"  {i}. {model['model_name']}: PnL ${model['total_pnl']:.2f}, "
              f"Sharpe {model['sharpe_ratio']:.2f}, Win Rate {model['win_rate']:.1f}%, "
              f"Trades {model['total_trades']}")
    
    print("\nTop 10 by Profit Factor (>50 trades):")
    for i, model in enumerate(by_profit_factor, 1):
        print(f"  {i}. {model['model_name']}: Profit Factor {model['profit_factor']:.2f}, "
              f"PnL ${model['total_pnl']:.2f}, Win Rate {model['win_rate']:.1f}%")

def analyze_risk_metrics():
    """Analyze risk-adjusted performance"""
    print("\n" + "="*80)
    print("RISK-ADJUSTED PERFORMANCE ANALYSIS")
    print("="*80)
    
    active_models = [r for r in results if r['total_trades'] > 10 and r['max_drawdown'] > 0]
    
    # Sort by Calmar ratio (return/max drawdown)
    positive_calmar = [r for r in active_models if r['calmar_ratio'] > 0]
    by_calmar = sorted(positive_calmar, key=lambda x: x['calmar_ratio'], reverse=True)[:10]
    
    print("\nTop 10 by Calmar Ratio (return/max drawdown):")
    for i, model in enumerate(by_calmar, 1):
        print(f"  {i}. {model['model_name']}: Calmar {model['calmar_ratio']:.2f}, "
              f"Max DD {model['max_drawdown']*100:.4f}%, PnL ${model['total_pnl']:.2f}")
    
    # Analyze drawdown patterns
    small_dd = [r for r in active_models if r['max_drawdown'] < 0.001]  # <0.1%
    large_dd = [r for r in active_models if r['max_drawdown'] > 0.05]    # >5%
    
    print(f"\nDrawdown Distribution:")
    print(f"  Small Drawdown (<0.1%): {len(small_dd)} models")
    if small_dd:
        avg_pnl_small = statistics.mean([r['total_pnl'] for r in small_dd])
        profitable_small = len([r for r in small_dd if r['total_pnl'] > 0])
        print(f"    Profitable: {profitable_small}/{len(small_dd)} ({profitable_small/len(small_dd)*100:.1f}%)")
        print(f"    Avg PnL: ${avg_pnl_small:.2f}")
    
    print(f"  Large Drawdown (>5%): {len(large_dd)} models")
    if large_dd:
        avg_pnl_large = statistics.mean([r['total_pnl'] for r in large_dd])
        profitable_large = len([r for r in large_dd if r['total_pnl'] > 0])
        print(f"    Profitable: {profitable_large}/{len(large_dd)} ({profitable_large/len(large_dd)*100:.1f}%)")
        print(f"    Avg PnL: ${avg_pnl_large:.2f}")

def generate_actionable_insights():
    """Generate actionable insights for production"""
    print("\n" + "="*80)
    print("ACTIONABLE INSIGHTS FOR PRODUCTION")
    print("="*80)
    
    active_models = [r for r in results if r['total_trades'] > 10]
    
    insights = []
    
    # Insight 1: Optimal epoch range
    dqn_profitable = [r for r in active_models if r['model_type'] == 'DQN' and r['total_pnl'] > 50]
    ppo_profitable = [r for r in active_models if r['model_type'] == 'PPO' and r['total_pnl'] > 50]
    
    if dqn_profitable:
        dqn_epochs = [r['epoch'] for r in dqn_profitable]
        insights.append(f"1. DQN Optimal Epochs: {min(dqn_epochs)}-{max(dqn_epochs)} "
                       f"(found {len(dqn_profitable)} profitable models)")
    
    if ppo_profitable:
        ppo_epochs = [r['epoch'] for r in ppo_profitable]
        insights.append(f"2. PPO Optimal Epochs: {min(ppo_epochs)}-{max(ppo_epochs)} "
                       f"(found {len(ppo_profitable)} profitable models)")
    
    # Insight 2: Trade frequency sweet spot
    high_sharpe = [r for r in active_models if r['sharpe_ratio'] > 5]
    if high_sharpe:
        avg_freq = statistics.mean([r['trade_frequency'] for r in high_sharpe])
        avg_hold = statistics.mean([r['avg_trade_duration'] for r in high_sharpe])
        insights.append(f"3. High Sharpe Models (>5): Trade frequency ~{avg_freq:.1f} trades/day, "
                       f"Hold time ~{avg_hold:.1f} bars")
    
    # Insight 3: Win rate vs profit factor correlation
    high_win_rate = [r for r in active_models if r['win_rate'] > 55]
    if high_win_rate:
        profitable_high_wr = len([r for r in high_win_rate if r['total_pnl'] > 0])
        insights.append(f"4. Win Rate >55%: {profitable_high_wr}/{len(high_win_rate)} models profitable "
                       f"({profitable_high_wr/len(high_win_rate)*100:.1f}%)")
    
    # Insight 4: Risk control
    low_dd_profitable = [r for r in active_models if r['max_drawdown'] < 0.01 and r['total_pnl'] > 20]
    insights.append(f"5. Low Drawdown Winners (<1% DD, >$20 PnL): {len(low_dd_profitable)} models - "
                   f"excellent risk control")
    
    # Insight 5: Model type recommendation
    dqn_active = [r for r in active_models if r['model_type'] == 'DQN']
    ppo_active = [r for r in active_models if r['model_type'] == 'PPO']
    
    dqn_profit_rate = len([r for r in dqn_active if r['total_pnl'] > 0]) / len(dqn_active) if dqn_active else 0
    ppo_profit_rate = len([r for r in ppo_active if r['total_pnl'] > 0]) / len(ppo_active) if ppo_active else 0
    
    better_model = 'DQN' if dqn_profit_rate > ppo_profit_rate else 'PPO'
    insights.append(f"6. Model Type Preference: {better_model} ({max(dqn_profit_rate, ppo_profit_rate)*100:.1f}% "
                   f"profitability vs {min(dqn_profit_rate, ppo_profit_rate)*100:.1f}%)")
    
    # Insight 7: Trading frequency
    high_freq_models = [r for r in active_models if r['trade_frequency'] > 50]
    high_freq_profitable = len([r for r in high_freq_models if r['total_pnl'] > 0])
    insights.append(f"7. High Frequency Trading (>50/day): {high_freq_profitable}/{len(high_freq_models)} "
                   f"profitable - {'recommended' if high_freq_profitable/len(high_freq_models) > 0.5 else 'not recommended'}")
    
    # Insight 8: Extreme performers
    extreme_sharpe = [r for r in active_models if r['sharpe_ratio'] > 8]
    insights.append(f"8. Extreme Sharpe Ratio (>8): {len(extreme_sharpe)} models - potential overfitting "
                   f"or exceptional performance")
    
    # Insight 9: Consistency
    consistent = [r for r in active_models if r['win_rate'] > 50 and r['profit_factor'] and 
                  r['profit_factor'] > 2 and r['calmar_ratio'] > 5]
    insights.append(f"9. Consistent Performers (50%+ WR, PF>2, Calmar>5): {len(consistent)} models - "
                   f"best candidates for production")
    
    # Insight 10: No-trade models
    no_trade = len([r for r in results if r['total_trades'] == 0])
    insights.append(f"10. No-Trade Models: {no_trade}/100 - indicates poor training or overfitting")
    
    # Insight 11: Trade count vs performance
    optimal_trade_count = [r for r in active_models if 100 <= r['total_trades'] <= 500 and r['total_pnl'] > 0]
    insights.append(f"11. Optimal Trade Count (100-500): {len(optimal_trade_count)} profitable models - "
                   f"good balance of activity and selectivity")
    
    # Insight 12: Hold time patterns
    short_hold_profitable = [r for r in active_models if r['avg_trade_duration'] < 20 and r['total_pnl'] > 20]
    insights.append(f"12. Short-Term Trading (<20 bars): {len(short_hold_profitable)} highly profitable models - "
                   f"scalping strategy viable")
    
    for insight in insights:
        print(f"\n{insight}")

# Run all analyses
if __name__ == '__main__':
    print("\n" + "="*80)
    print("COMPREHENSIVE BACKTEST ANALYSIS")
    print("100 Checkpoint Models (DQN + PPO)")
    print("Date Range: 2025-07-16 to 2025-10-14")
    print("="*80)
    
    analyze_by_model_type()
    analyze_by_epoch()
    analyze_trade_characteristics()
    find_top_performers()
    analyze_risk_metrics()
    generate_actionable_insights()
    
    print("\n" + "="*80)
    print("ANALYSIS COMPLETE")
    print("="*80 + "\n")