foxhunt/scripts/analyze_checkpoints_simple.py

#!/usr/bin/env python3
"""
Simple Checkpoint Comparison Analysis (no matplotlib dependency)
Analyzes backtest results for all DQN and PPO checkpoints
"""

import json
import sys
from pathlib import Path
from collections import defaultdict

def load_results(results_file):
    """Load backtest results from JSON"""
    with open(results_file, 'r') as f:
        return json.load(f)

def filter_valid_results(results):
    """Filter out checkpoints with no trades or invalid metrics"""
    return [r for r in results if r['total_trades'] > 0]

def create_comparison_analysis(results):
    """Analyze and compare DQN vs PPO checkpoints"""

    # Separate DQN and PPO
    dqn_results = [r for r in results if r['model_type'] == 'DQN']
    ppo_results = [r for r in results if r['model_type'] == 'PPO']

    print("\n" + "="*100)
    print("📊 CHECKPOINT BACKTESTING ANALYSIS - COMPLETE RESULTS")
    print("="*100)

    print(f"\n✅ Total Checkpoints Tested: {len(results)}")
    print(f"   - DQN: {len(dqn_results)} checkpoints")
    print(f"   - PPO: {len(ppo_results)} checkpoints")

    # Top 10 DQN
    print("\n" + "="*100)
    print("🔵 TOP 10 DQN CHECKPOINTS (Ranked by Sharpe Ratio)")
    print("="*100)

    dqn_sorted = sorted(dqn_results, key=lambda x: x['sharpe_ratio'], reverse=True)[:10]

    print(f"{'Rank':<6} {'Epoch':<8} {'Sharpe':<10} {'Win Rate':<12} {'Trades':<8} {'PnL':<14} {'Drawdown':<12} {'Freq':<10}")
    print("-"*100)

    for rank, r in enumerate(dqn_sorted, 1):
        print(f"{rank:<6} {r['epoch']:<8} {r['sharpe_ratio']:<10.3f} {r['win_rate']:<11.1f}% {r['total_trades']:<8} ${r['total_pnl']:<13.2f} {r['max_drawdown']*100:<11.2f}% {r['trade_frequency']:<10.1f}")

    # Top 10 PPO
    print("\n" + "="*100)
    print("🟢 TOP 10 PPO CHECKPOINTS (Ranked by Sharpe Ratio)")
    print("="*100)

    ppo_sorted = sorted(ppo_results, key=lambda x: x['sharpe_ratio'], reverse=True)[:10]

    print(f"{'Rank':<6} {'Epoch':<8} {'Sharpe':<10} {'Win Rate':<12} {'Trades':<8} {'PnL':<14} {'Drawdown':<12} {'Freq':<10}")
    print("-"*100)

    for rank, r in enumerate(ppo_sorted, 1):
        print(f"{rank:<6} {r['epoch']:<8} {r['sharpe_ratio']:<10.3f} {r['win_rate']:<11.1f}% {r['total_trades']:<8} ${r['total_pnl']:<13.2f} {r['max_drawdown']*100:<11.2f}% {r['trade_frequency']:<10.1f}")

    # Statistical summary
    print("\n" + "="*100)
    print("📈 STATISTICAL SUMMARY")
    print("="*100)

    dqn_sharpe = [r['sharpe_ratio'] for r in dqn_results]
    dqn_win_rate = [r['win_rate'] for r in dqn_results]
    dqn_trades = [r['total_trades'] for r in dqn_results]
    dqn_pnl = [r['total_pnl'] for r in dqn_results]

    ppo_sharpe = [r['sharpe_ratio'] for r in ppo_results]
    ppo_win_rate = [r['win_rate'] for r in ppo_results]
    ppo_trades = [r['total_trades'] for r in ppo_results]
    ppo_pnl = [r['total_pnl'] for r in ppo_results]

    print(f"\n{'Metric':<30} {'DQN':>20} {'PPO':>20} {'Winner':>20}")
    print("-"*100)

    # Calculate stats
    metrics = [
        ('Checkpoints Tested', len(dqn_results), len(ppo_results)),
        ('Avg Sharpe Ratio', sum(dqn_sharpe)/len(dqn_sharpe), sum(ppo_sharpe)/len(ppo_sharpe)),
        ('Max Sharpe Ratio', max(dqn_sharpe), max(ppo_sharpe)),
        ('Min Sharpe Ratio', min(dqn_sharpe), min(ppo_sharpe)),
        ('Avg Win Rate (%)', sum(dqn_win_rate)/len(dqn_win_rate), sum(ppo_win_rate)/len(ppo_win_rate)),
        ('Avg Total Trades', sum(dqn_trades)/len(dqn_trades), sum(ppo_trades)/len(ppo_trades)),
        ('Max Total Trades', max(dqn_trades), max(ppo_trades)),
        ('Avg PnL ($)', sum(dqn_pnl)/len(dqn_pnl), sum(ppo_pnl)/len(ppo_pnl)),
        ('Best PnL ($)', max(dqn_pnl), max(ppo_pnl)),
        ('Worst PnL ($)', min(dqn_pnl), min(ppo_pnl)),
    ]

    for name, dqn_val, ppo_val in metrics:
        if name == 'Checkpoints Tested':
            winner = 'DQN' if dqn_val > ppo_val else 'PPO' if ppo_val > dqn_val else 'Tie'
            print(f"{name:<30} {int(dqn_val):>20} {int(ppo_val):>20} {winner:>20}")
        else:
            winner = 'DQN' if dqn_val > ppo_val else 'PPO' if ppo_val > dqn_val else 'Tie'
            print(f"{name:<30} {dqn_val:>20.3f} {ppo_val:>20.3f} {winner:>20}")

    # Best overall checkpoints
    print("\n" + "="*100)
    print("🏆 BEST CHECKPOINTS (Highest Sharpe Ratio)")
    print("="*100)

    best_dqn = max(dqn_results, key=lambda x: x['sharpe_ratio'])
    best_ppo = max(ppo_results, key=lambda x: x['sharpe_ratio'])

    print(f"\n🔵 Best DQN: Epoch {best_dqn['epoch']}")
    print(f"   Sharpe Ratio: {best_dqn['sharpe_ratio']:.3f}")
    print(f"   Win Rate: {best_dqn['win_rate']:.1f}%")
    print(f"   Total Trades: {best_dqn['total_trades']}")
    print(f"   Total PnL: ${best_dqn['total_pnl']:.2f}")
    print(f"   Max Drawdown: {best_dqn['max_drawdown']:.2%}")
    print(f"   Trade Frequency: {best_dqn['trade_frequency']:.1f} trades/1000 bars")

    print(f"\n🟢 Best PPO: Epoch {best_ppo['epoch']}")
    print(f"   Sharpe Ratio: {best_ppo['sharpe_ratio']:.3f}")
    print(f"   Win Rate: {best_ppo['win_rate']:.1f}%")
    print(f"   Total Trades: {best_ppo['total_trades']}")
    print(f"   Total PnL: ${best_ppo['total_pnl']:.2f}")
    print(f"   Max Drawdown: {best_ppo['max_drawdown']:.2%}")
    print(f"   Trade Frequency: {best_ppo['trade_frequency']:.1f} trades/1000 bars")

    # Training phase analysis
    print("\n" + "="*100)
    print("📊 TRAINING PHASE ANALYSIS")
    print("="*100)

    # DQN Early vs Late
    dqn_early = [r for r in dqn_results if r['epoch'] <= 200]
    dqn_late = [r for r in dqn_results if r['epoch'] > 200]

    print(f"\n🔵 DQN Performance by Training Phase")
    print(f"\nEarly Epochs (≤200): {len(dqn_early)} checkpoints")
    if dqn_early:
        print(f"   Avg Sharpe: {sum(r['sharpe_ratio'] for r in dqn_early)/len(dqn_early):.3f}")
        print(f"   Avg Trades: {sum(r['total_trades'] for r in dqn_early)/len(dqn_early):.1f}")
        print(f"   Avg Win Rate: {sum(r['win_rate'] for r in dqn_early)/len(dqn_early):.1f}%")

    print(f"\nLate Epochs (>200): {len(dqn_late)} checkpoints")
    if dqn_late:
        print(f"   Avg Sharpe: {sum(r['sharpe_ratio'] for r in dqn_late)/len(dqn_late):.3f}")
        print(f"   Avg Trades: {sum(r['total_trades'] for r in dqn_late)/len(dqn_late):.1f}")
        print(f"   Avg Win Rate: {sum(r['win_rate'] for r in dqn_late)/len(dqn_late):.1f}%")

    # PPO Early vs Late
    ppo_early = [r for r in ppo_results if r['epoch'] <= 200]
    ppo_late = [r for r in ppo_results if r['epoch'] > 200]

    print(f"\n🟢 PPO Performance by Training Phase")
    print(f"\nEarly Epochs (≤200): {len(ppo_early)} checkpoints")
    if ppo_early:
        print(f"   Avg Sharpe: {sum(r['sharpe_ratio'] for r in ppo_early)/len(ppo_early):.3f}")
        print(f"   Avg Trades: {sum(r['total_trades'] for r in ppo_early)/len(ppo_early):.1f}")
        print(f"   Avg Win Rate: {sum(r['win_rate'] for r in ppo_early)/len(ppo_early):.1f}%")

    print(f"\nLate Epochs (>200): {len(ppo_late)} checkpoints")
    if ppo_late:
        print(f"   Avg Sharpe: {sum(r['sharpe_ratio'] for r in ppo_late)/len(ppo_late):.3f}")
        print(f"   Avg Trades: {sum(r['total_trades'] for r in ppo_late)/len(ppo_late):.1f}")
        print(f"   Avg Win Rate: {sum(r['win_rate'] for r in ppo_late)/len(ppo_late):.1f}%")

    # Key findings
    print("\n" + "="*100)
    print("🔍 KEY FINDINGS")
    print("="*100)

    print("\n1. **Hypothesis Validation: Early Epochs (10-100) vs Late Epochs (400-500)**")

    # Compare early vs very late
    dqn_very_early = [r for r in dqn_results if 10 <= r['epoch'] <= 100]
    dqn_very_late = [r for r in dqn_results if 400 <= r['epoch'] <= 500]

    if dqn_very_early and dqn_very_late:
        early_sharpe = sum(r['sharpe_ratio'] for r in dqn_very_early) / len(dqn_very_early)
        late_sharpe = sum(r['sharpe_ratio'] for r in dqn_very_late) / len(dqn_very_late)
        early_trades = sum(r['total_trades'] for r in dqn_very_early) / len(dqn_very_early)
        late_trades = sum(r['total_trades'] for r in dqn_very_late) / len(dqn_very_late)

        print(f"\n   DQN Early (10-100):")
        print(f"   - Avg Sharpe: {early_sharpe:.3f}")
        print(f"   - Avg Trades: {early_trades:.1f}")

        print(f"\n   DQN Late (400-500):")
        print(f"   - Avg Sharpe: {late_sharpe:.3f}")
        print(f"   - Avg Trades: {late_trades:.1f}")

        if late_sharpe > early_sharpe:
            print(f"\n   ✅ HYPOTHESIS CONFIRMED: Late epochs have {late_sharpe/early_sharpe:.2f}x better Sharpe ratio")
        else:
            print(f"\n   ❌ HYPOTHESIS REJECTED: Early epochs have better Sharpe ratio")

    print("\n2. **Optimal Training Duration**")

    # Find best epoch ranges
    dqn_by_range = defaultdict(list)
    for r in dqn_results:
        epoch_range = (r['epoch'] // 100) * 100
        dqn_by_range[epoch_range].append(r)

    print(f"\n   DQN Best Performance by Epoch Range:")
    for epoch_range in sorted(dqn_by_range.keys()):
        checkpoints = dqn_by_range[epoch_range]
        avg_sharpe = sum(r['sharpe_ratio'] for r in checkpoints) / len(checkpoints)
        best = max(checkpoints, key=lambda x: x['sharpe_ratio'])
        print(f"   Epochs {epoch_range}-{epoch_range+99}: Avg Sharpe {avg_sharpe:.3f}, Best: Epoch {best['epoch']} (Sharpe {best['sharpe_ratio']:.3f})")

    ppo_by_range = defaultdict(list)
    for r in ppo_results:
        epoch_range = (r['epoch'] // 100) * 100
        ppo_by_range[epoch_range].append(r)

    print(f"\n   PPO Best Performance by Epoch Range:")
    for epoch_range in sorted(ppo_by_range.keys()):
        checkpoints = ppo_by_range[epoch_range]
        avg_sharpe = sum(r['sharpe_ratio'] for r in checkpoints) / len(checkpoints)
        best = max(checkpoints, key=lambda x: x['sharpe_ratio'])
        print(f"   Epochs {epoch_range}-{epoch_range+99}: Avg Sharpe {avg_sharpe:.3f}, Best: Epoch {best['epoch']} (Sharpe {best['sharpe_ratio']:.3f})")

    print("\n3. **DQN vs PPO Comparison**")

    overall_best = max(dqn_results + ppo_results, key=lambda x: x['sharpe_ratio'])
    print(f"\n   🏆 Overall Winner: {overall_best['model_type']} Epoch {overall_best['epoch']}")
    print(f"      Sharpe Ratio: {overall_best['sharpe_ratio']:.3f}")
    print(f"      PnL: ${overall_best['total_pnl']:.2f}")

    return dqn_results, ppo_results

def create_markdown_report(dqn_results, ppo_results, output_dir):
    """Create comprehensive markdown report"""

    dqn_sorted = sorted(dqn_results, key=lambda x: x['sharpe_ratio'], reverse=True)[:10]
    ppo_sorted = sorted(ppo_results, key=lambda x: x['sharpe_ratio'], reverse=True)[:10]

    report = []
    report.append("# Checkpoint Backtesting Results")
    report.append(f"\n**Date**: 2025-10-14")
    report.append(f"**Total Checkpoints Tested**: {len(dqn_results) + len(ppo_results)}")
    report.append(f"**Data**: 6E.FUT (Euro FX Futures), 7,223 bars, 4 days")
    report.append("\n---\n")

    # Executive Summary
    best_dqn = max(dqn_results, key=lambda x: x['sharpe_ratio'])
    best_ppo = max(ppo_results, key=lambda x: x['sharpe_ratio'])

    report.append("## Executive Summary")
    report.append(f"\n### DQN Performance")
    report.append(f"- **Best Checkpoint**: Epoch {best_dqn['epoch']}")
    report.append(f"- **Best Sharpe Ratio**: {best_dqn['sharpe_ratio']:.3f}")
    report.append(f"- **Best PnL**: ${best_dqn['total_pnl']:.2f}")
    report.append(f"- **Win Rate**: {best_dqn['win_rate']:.1f}%")

    report.append(f"\n### PPO Performance")
    report.append(f"- **Best Checkpoint**: Epoch {best_ppo['epoch']}")
    report.append(f"- **Best Sharpe Ratio**: {best_ppo['sharpe_ratio']:.3f}")
    report.append(f"- **Best PnL**: ${best_ppo['total_pnl']:.2f}")
    report.append(f"- **Win Rate**: {best_ppo['win_rate']:.1f}%")

    # Top 10 DQN
    report.append("\n---\n")
    report.append("## Top 10 DQN Checkpoints")
    report.append("\n| Rank | Epoch | Sharpe | Win Rate | Trades | PnL | Drawdown | Trade Freq |")
    report.append("|------|-------|--------|----------|--------|-----|----------|------------|")

    for rank, r in enumerate(dqn_sorted, 1):
        report.append(f"| {rank} | {r['epoch']} | {r['sharpe_ratio']:.3f} | {r['win_rate']:.1f}% | {r['total_trades']} | ${r['total_pnl']:.2f} | {r['max_drawdown']:.2%} | {r['trade_frequency']:.1f} |")

    # Top 10 PPO
    report.append("\n---\n")
    report.append("## Top 10 PPO Checkpoints")
    report.append("\n| Rank | Epoch | Sharpe | Win Rate | Trades | PnL | Drawdown | Trade Freq |")
    report.append("|------|-------|--------|----------|--------|-----|----------|------------|")

    for rank, r in enumerate(ppo_sorted, 1):
        report.append(f"| {rank} | {r['epoch']} | {r['sharpe_ratio']:.3f} | {r['win_rate']:.1f}% | {r['total_trades']} | ${r['total_pnl']:.2f} | {r['max_drawdown']:.2%} | {r['trade_frequency']:.1f} |")

    # Production Recommendations
    report.append("\n---\n")
    report.append("## Production Deployment Recommendations")
    report.append(f"\n### Primary Recommendation: **{best_dqn['model_type']} Epoch {best_dqn['epoch']}**")
    report.append(f"- Sharpe Ratio: {best_dqn['sharpe_ratio']:.3f}")
    report.append(f"- Win Rate: {best_dqn['win_rate']:.1f}%")
    report.append(f"- Total PnL: ${best_dqn['total_pnl']:.2f}")
    report.append(f"- Max Drawdown: {best_dqn['max_drawdown']:.2%}")

    report.append(f"\n### Alternative: **{best_ppo['model_type']} Epoch {best_ppo['epoch']}**")
    report.append(f"- Sharpe Ratio: {best_ppo['sharpe_ratio']:.3f}")
    report.append(f"- Win Rate: {best_ppo['win_rate']:.1f}%")
    report.append(f"- Total PnL: ${best_ppo['total_pnl']:.2f}")
    report.append(f"- Max Drawdown: {best_ppo['max_drawdown']:.2%}")

    # Save report
    report_file = output_dir / 'CHECKPOINT_BACKTEST_REPORT.md'
    with open(report_file, 'w') as f:
        f.write('\n'.join(report))

    print(f"\n📄 Markdown report saved to: {report_file}")

def main():
    if len(sys.argv) < 2:
        print("Usage: python analyze_checkpoints_simple.py <results_json_file>")
        sys.exit(1)

    results_file = Path(sys.argv[1])
    if not results_file.exists():
        print(f"Error: Results file not found: {results_file}")
        sys.exit(1)

    # Create output directory
    output_dir = Path(__file__).parent.parent / 'results'
    output_dir.mkdir(exist_ok=True)

    # Load results
    print(f"\n📖 Loading results from: {results_file}")
    results = load_results(results_file)

    # Filter valid results
    valid_results = filter_valid_results(results)
    print(f"✅ Found {len(valid_results)} checkpoints with valid trades")

    # Create comprehensive analysis
    dqn_results, ppo_results = create_comparison_analysis(valid_results)

    # Create markdown report
    create_markdown_report(dqn_results, ppo_results, output_dir)

    print("\n✅ Analysis complete!")

if __name__ == '__main__':
    main()