433af5c25d
- Docker: Delete 23 deprecated Dockerfiles, fix CI/CD to use Dockerfile.foxhunt-build - Config: Remove 36 .env files, keep 4 essential, delete config/environments/ - Docs: Archive 614 Wave D files to docs/archive/wave_d/, 95% reduction in root - Scripts: Delete 56 deprecated scripts, keep 58 production-critical (49% reduction) - Python: Organize 37 scripts into scripts/python/ subdirectories, delete ml/python/ - Build: Remove 1GB artifacts, delete old venvs, clean Python cache from git - Migrations: Delete deprecated directory (4,432 lines), remove duplicate database/migrations/ - Infrastructure: Delete deployment/ (61 files), docs/scripts/ (8 files) Total impact: ~2,500 files cleaned, 750MB+ space freed, zero production impact All deleted scripts backed up to archives. runpod/ and tests/runpod/ preserved. data_acquisition_service retained per user request.
335 lines
16 KiB
Python
335 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Deep Analysis of Comprehensive Backtest Results
|
|
Analyzes 100 checkpoint backtest data for trading insights
|
|
"""
|
|
|
|
import json
|
|
import statistics
|
|
from datetime import datetime
|
|
from collections import defaultdict
|
|
|
|
# Load the backtest results
|
|
with open('results/comprehensive_backtest_results_20251014_143309.json', 'r') as f:
|
|
results = json.load(f)
|
|
|
|
# Categorization functions
|
|
def categorize_by_performance(result):
|
|
"""Categorize models as High/Medium/Low performers"""
|
|
sharpe = result['sharpe_ratio']
|
|
win_rate = result['win_rate']
|
|
pnl = result['total_pnl']
|
|
|
|
# High performer: Sharpe > 5, Win Rate > 55%, PnL > 50
|
|
if sharpe > 5 and win_rate > 55 and pnl > 50:
|
|
return 'High'
|
|
# Low performer: Sharpe < 0 or Win Rate < 35% or PnL < -50
|
|
elif sharpe < 0 or win_rate < 35 or pnl < -50:
|
|
return 'Low'
|
|
else:
|
|
return 'Medium'
|
|
|
|
def analyze_by_model_type():
|
|
"""Analyze performance by model type (DQN vs PPO)"""
|
|
dqn_results = [r for r in results if r['model_type'] == 'DQN']
|
|
ppo_results = [r for r in results if r['model_type'] == 'PPO']
|
|
|
|
print("\n" + "="*80)
|
|
print("MODEL TYPE ANALYSIS")
|
|
print("="*80)
|
|
|
|
for model_type, model_results in [('DQN', dqn_results), ('PPO', ppo_results)]:
|
|
# Filter out models with no trades
|
|
active_models = [r for r in model_results if r['total_trades'] > 0]
|
|
|
|
if not active_models:
|
|
print(f"\n{model_type}: No active models")
|
|
continue
|
|
|
|
avg_sharpe = statistics.mean([r['sharpe_ratio'] for r in active_models])
|
|
avg_win_rate = statistics.mean([r['win_rate'] for r in active_models])
|
|
avg_pnl = statistics.mean([r['total_pnl'] for r in active_models])
|
|
avg_trades = statistics.mean([r['total_trades'] for r in active_models])
|
|
|
|
profitable_models = len([r for r in active_models if r['total_pnl'] > 0])
|
|
|
|
print(f"\n{model_type} Models:")
|
|
print(f" Total Models: {len(model_results)} ({len(active_models)} active)")
|
|
print(f" Profitable Models: {profitable_models}/{len(active_models)} ({profitable_models/len(active_models)*100:.1f}%)")
|
|
print(f" Average Sharpe Ratio: {avg_sharpe:.2f}")
|
|
print(f" Average Win Rate: {avg_win_rate:.2f}%")
|
|
print(f" Average PnL: ${avg_pnl:.2f}")
|
|
print(f" Average Total Trades: {avg_trades:.0f}")
|
|
|
|
def analyze_by_epoch():
|
|
"""Analyze how performance changes with training epochs"""
|
|
print("\n" + "="*80)
|
|
print("EPOCH PROGRESSION ANALYSIS")
|
|
print("="*80)
|
|
|
|
for model_type in ['DQN', 'PPO']:
|
|
model_results = [r for r in results if r['model_type'] == model_type]
|
|
|
|
# Group by epoch ranges
|
|
epoch_ranges = {
|
|
'Early (10-100)': [r for r in model_results if 10 <= r['epoch'] <= 100 and r['total_trades'] > 0],
|
|
'Mid (110-300)': [r for r in model_results if 110 <= r['epoch'] <= 300 and r['total_trades'] > 0],
|
|
'Late (310-500)': [r for r in model_results if 310 <= r['epoch'] <= 500 and r['total_trades'] > 0]
|
|
}
|
|
|
|
print(f"\n{model_type} Epoch Progression:")
|
|
for range_name, range_results in epoch_ranges.items():
|
|
if not range_results:
|
|
continue
|
|
|
|
avg_sharpe = statistics.mean([r['sharpe_ratio'] for r in range_results])
|
|
avg_win_rate = statistics.mean([r['win_rate'] for r in range_results])
|
|
avg_pnl = statistics.mean([r['total_pnl'] for r in range_results])
|
|
profitable = len([r for r in range_results if r['total_pnl'] > 0])
|
|
|
|
print(f" {range_name}: {len(range_results)} models, {profitable} profitable")
|
|
print(f" Avg Sharpe: {avg_sharpe:.2f}, Win Rate: {avg_win_rate:.1f}%, PnL: ${avg_pnl:.2f}")
|
|
|
|
def analyze_trade_characteristics():
|
|
"""Analyze trade patterns and characteristics"""
|
|
print("\n" + "="*80)
|
|
print("TRADE CHARACTERISTICS ANALYSIS")
|
|
print("="*80)
|
|
|
|
active_models = [r for r in results if r['total_trades'] > 10]
|
|
|
|
# Trade frequency analysis
|
|
high_freq = [r for r in active_models if r['trade_frequency'] > 50]
|
|
low_freq = [r for r in active_models if r['trade_frequency'] < 20]
|
|
|
|
print(f"\nTrade Frequency Impact:")
|
|
print(f" High Frequency (>50 trades/day): {len(high_freq)} models")
|
|
if high_freq:
|
|
avg_pnl_high = statistics.mean([r['total_pnl'] for r in high_freq])
|
|
avg_sharpe_high = statistics.mean([r['sharpe_ratio'] for r in high_freq])
|
|
profitable_high = len([r for r in high_freq if r['total_pnl'] > 0])
|
|
print(f" Profitable: {profitable_high}/{len(high_freq)} ({profitable_high/len(high_freq)*100:.1f}%)")
|
|
print(f" Avg PnL: ${avg_pnl_high:.2f}, Avg Sharpe: {avg_sharpe_high:.2f}")
|
|
|
|
print(f" Low Frequency (<20 trades/day): {len(low_freq)} models")
|
|
if low_freq:
|
|
avg_pnl_low = statistics.mean([r['total_pnl'] for r in low_freq])
|
|
avg_sharpe_low = statistics.mean([r['sharpe_ratio'] for r in low_freq])
|
|
profitable_low = len([r for r in low_freq if r['total_pnl'] > 0])
|
|
print(f" Profitable: {profitable_low}/{len(low_freq)} ({profitable_low/len(low_freq)*100:.1f}%)")
|
|
print(f" Avg PnL: ${avg_pnl_low:.2f}, Avg Sharpe: {avg_sharpe_low:.2f}")
|
|
|
|
# Hold time analysis
|
|
short_hold = [r for r in active_models if r['avg_trade_duration'] < 20]
|
|
long_hold = [r for r in active_models if r['avg_trade_duration'] > 60]
|
|
|
|
print(f"\nAverage Hold Time Impact:")
|
|
print(f" Short Hold (<20 bars): {len(short_hold)} models")
|
|
if short_hold:
|
|
avg_pnl_short = statistics.mean([r['total_pnl'] for r in short_hold])
|
|
avg_win_rate_short = statistics.mean([r['win_rate'] for r in short_hold])
|
|
profitable_short = len([r for r in short_hold if r['total_pnl'] > 0])
|
|
print(f" Profitable: {profitable_short}/{len(short_hold)} ({profitable_short/len(short_hold)*100:.1f}%)")
|
|
print(f" Avg PnL: ${avg_pnl_short:.2f}, Win Rate: {avg_win_rate_short:.1f}%")
|
|
|
|
print(f" Long Hold (>60 bars): {len(long_hold)} models")
|
|
if long_hold:
|
|
avg_pnl_long = statistics.mean([r['total_pnl'] for r in long_hold])
|
|
avg_win_rate_long = statistics.mean([r['win_rate'] for r in long_hold])
|
|
profitable_long = len([r for r in long_hold if r['total_pnl'] > 0])
|
|
print(f" Profitable: {profitable_long}/{len(long_hold)} ({profitable_long/len(long_hold)*100:.1f}%)")
|
|
print(f" Avg PnL: ${avg_pnl_long:.2f}, Win Rate: {avg_win_rate_long:.1f}%")
|
|
|
|
# Win rate distribution
|
|
print(f"\nWin Rate Distribution:")
|
|
win_rate_bins = {
|
|
'<30%': [r for r in active_models if r['win_rate'] < 30],
|
|
'30-45%': [r for r in active_models if 30 <= r['win_rate'] < 45],
|
|
'45-55%': [r for r in active_models if 45 <= r['win_rate'] < 55],
|
|
'55-65%': [r for r in active_models if 55 <= r['win_rate'] < 65],
|
|
'>65%': [r for r in active_models if r['win_rate'] >= 65]
|
|
}
|
|
|
|
for bin_name, bin_results in win_rate_bins.items():
|
|
if not bin_results:
|
|
continue
|
|
avg_pnl = statistics.mean([r['total_pnl'] for r in bin_results])
|
|
profitable = len([r for r in bin_results if r['total_pnl'] > 0])
|
|
print(f" {bin_name}: {len(bin_results)} models, {profitable} profitable, Avg PnL: ${avg_pnl:.2f}")
|
|
|
|
def find_top_performers():
|
|
"""Identify top performing models"""
|
|
print("\n" + "="*80)
|
|
print("TOP PERFORMERS")
|
|
print("="*80)
|
|
|
|
# Filter active models with significant trades
|
|
active_models = [r for r in results if r['total_trades'] > 50]
|
|
|
|
# Sort by different metrics
|
|
by_sharpe = sorted(active_models, key=lambda x: x['sharpe_ratio'], reverse=True)[:10]
|
|
by_pnl = sorted(active_models, key=lambda x: x['total_pnl'], reverse=True)[:10]
|
|
by_profit_factor = sorted([r for r in active_models if r['profit_factor'] and r['profit_factor'] > 0],
|
|
key=lambda x: x['profit_factor'], reverse=True)[:10]
|
|
|
|
print("\nTop 10 by Sharpe Ratio (>50 trades):")
|
|
for i, model in enumerate(by_sharpe, 1):
|
|
print(f" {i}. {model['model_name']}: Sharpe {model['sharpe_ratio']:.2f}, "
|
|
f"Win Rate {model['win_rate']:.1f}%, PnL ${model['total_pnl']:.2f}, "
|
|
f"Trades {model['total_trades']}")
|
|
|
|
print("\nTop 10 by Total PnL (>50 trades):")
|
|
for i, model in enumerate(by_pnl, 1):
|
|
print(f" {i}. {model['model_name']}: PnL ${model['total_pnl']:.2f}, "
|
|
f"Sharpe {model['sharpe_ratio']:.2f}, Win Rate {model['win_rate']:.1f}%, "
|
|
f"Trades {model['total_trades']}")
|
|
|
|
print("\nTop 10 by Profit Factor (>50 trades):")
|
|
for i, model in enumerate(by_profit_factor, 1):
|
|
print(f" {i}. {model['model_name']}: Profit Factor {model['profit_factor']:.2f}, "
|
|
f"PnL ${model['total_pnl']:.2f}, Win Rate {model['win_rate']:.1f}%")
|
|
|
|
def analyze_risk_metrics():
|
|
"""Analyze risk-adjusted performance"""
|
|
print("\n" + "="*80)
|
|
print("RISK-ADJUSTED PERFORMANCE ANALYSIS")
|
|
print("="*80)
|
|
|
|
active_models = [r for r in results if r['total_trades'] > 10 and r['max_drawdown'] > 0]
|
|
|
|
# Sort by Calmar ratio (return/max drawdown)
|
|
positive_calmar = [r for r in active_models if r['calmar_ratio'] > 0]
|
|
by_calmar = sorted(positive_calmar, key=lambda x: x['calmar_ratio'], reverse=True)[:10]
|
|
|
|
print("\nTop 10 by Calmar Ratio (return/max drawdown):")
|
|
for i, model in enumerate(by_calmar, 1):
|
|
print(f" {i}. {model['model_name']}: Calmar {model['calmar_ratio']:.2f}, "
|
|
f"Max DD {model['max_drawdown']*100:.4f}%, PnL ${model['total_pnl']:.2f}")
|
|
|
|
# Analyze drawdown patterns
|
|
small_dd = [r for r in active_models if r['max_drawdown'] < 0.001] # <0.1%
|
|
large_dd = [r for r in active_models if r['max_drawdown'] > 0.05] # >5%
|
|
|
|
print(f"\nDrawdown Distribution:")
|
|
print(f" Small Drawdown (<0.1%): {len(small_dd)} models")
|
|
if small_dd:
|
|
avg_pnl_small = statistics.mean([r['total_pnl'] for r in small_dd])
|
|
profitable_small = len([r for r in small_dd if r['total_pnl'] > 0])
|
|
print(f" Profitable: {profitable_small}/{len(small_dd)} ({profitable_small/len(small_dd)*100:.1f}%)")
|
|
print(f" Avg PnL: ${avg_pnl_small:.2f}")
|
|
|
|
print(f" Large Drawdown (>5%): {len(large_dd)} models")
|
|
if large_dd:
|
|
avg_pnl_large = statistics.mean([r['total_pnl'] for r in large_dd])
|
|
profitable_large = len([r for r in large_dd if r['total_pnl'] > 0])
|
|
print(f" Profitable: {profitable_large}/{len(large_dd)} ({profitable_large/len(large_dd)*100:.1f}%)")
|
|
print(f" Avg PnL: ${avg_pnl_large:.2f}")
|
|
|
|
def generate_actionable_insights():
|
|
"""Generate actionable insights for production"""
|
|
print("\n" + "="*80)
|
|
print("ACTIONABLE INSIGHTS FOR PRODUCTION")
|
|
print("="*80)
|
|
|
|
active_models = [r for r in results if r['total_trades'] > 10]
|
|
|
|
insights = []
|
|
|
|
# Insight 1: Optimal epoch range
|
|
dqn_profitable = [r for r in active_models if r['model_type'] == 'DQN' and r['total_pnl'] > 50]
|
|
ppo_profitable = [r for r in active_models if r['model_type'] == 'PPO' and r['total_pnl'] > 50]
|
|
|
|
if dqn_profitable:
|
|
dqn_epochs = [r['epoch'] for r in dqn_profitable]
|
|
insights.append(f"1. DQN Optimal Epochs: {min(dqn_epochs)}-{max(dqn_epochs)} "
|
|
f"(found {len(dqn_profitable)} profitable models)")
|
|
|
|
if ppo_profitable:
|
|
ppo_epochs = [r['epoch'] for r in ppo_profitable]
|
|
insights.append(f"2. PPO Optimal Epochs: {min(ppo_epochs)}-{max(ppo_epochs)} "
|
|
f"(found {len(ppo_profitable)} profitable models)")
|
|
|
|
# Insight 2: Trade frequency sweet spot
|
|
high_sharpe = [r for r in active_models if r['sharpe_ratio'] > 5]
|
|
if high_sharpe:
|
|
avg_freq = statistics.mean([r['trade_frequency'] for r in high_sharpe])
|
|
avg_hold = statistics.mean([r['avg_trade_duration'] for r in high_sharpe])
|
|
insights.append(f"3. High Sharpe Models (>5): Trade frequency ~{avg_freq:.1f} trades/day, "
|
|
f"Hold time ~{avg_hold:.1f} bars")
|
|
|
|
# Insight 3: Win rate vs profit factor correlation
|
|
high_win_rate = [r for r in active_models if r['win_rate'] > 55]
|
|
if high_win_rate:
|
|
profitable_high_wr = len([r for r in high_win_rate if r['total_pnl'] > 0])
|
|
insights.append(f"4. Win Rate >55%: {profitable_high_wr}/{len(high_win_rate)} models profitable "
|
|
f"({profitable_high_wr/len(high_win_rate)*100:.1f}%)")
|
|
|
|
# Insight 4: Risk control
|
|
low_dd_profitable = [r for r in active_models if r['max_drawdown'] < 0.01 and r['total_pnl'] > 20]
|
|
insights.append(f"5. Low Drawdown Winners (<1% DD, >$20 PnL): {len(low_dd_profitable)} models - "
|
|
f"excellent risk control")
|
|
|
|
# Insight 5: Model type recommendation
|
|
dqn_active = [r for r in active_models if r['model_type'] == 'DQN']
|
|
ppo_active = [r for r in active_models if r['model_type'] == 'PPO']
|
|
|
|
dqn_profit_rate = len([r for r in dqn_active if r['total_pnl'] > 0]) / len(dqn_active) if dqn_active else 0
|
|
ppo_profit_rate = len([r for r in ppo_active if r['total_pnl'] > 0]) / len(ppo_active) if ppo_active else 0
|
|
|
|
better_model = 'DQN' if dqn_profit_rate > ppo_profit_rate else 'PPO'
|
|
insights.append(f"6. Model Type Preference: {better_model} ({max(dqn_profit_rate, ppo_profit_rate)*100:.1f}% "
|
|
f"profitability vs {min(dqn_profit_rate, ppo_profit_rate)*100:.1f}%)")
|
|
|
|
# Insight 7: Trading frequency
|
|
high_freq_models = [r for r in active_models if r['trade_frequency'] > 50]
|
|
high_freq_profitable = len([r for r in high_freq_models if r['total_pnl'] > 0])
|
|
insights.append(f"7. High Frequency Trading (>50/day): {high_freq_profitable}/{len(high_freq_models)} "
|
|
f"profitable - {'recommended' if high_freq_profitable/len(high_freq_models) > 0.5 else 'not recommended'}")
|
|
|
|
# Insight 8: Extreme performers
|
|
extreme_sharpe = [r for r in active_models if r['sharpe_ratio'] > 8]
|
|
insights.append(f"8. Extreme Sharpe Ratio (>8): {len(extreme_sharpe)} models - potential overfitting "
|
|
f"or exceptional performance")
|
|
|
|
# Insight 9: Consistency
|
|
consistent = [r for r in active_models if r['win_rate'] > 50 and r['profit_factor'] and
|
|
r['profit_factor'] > 2 and r['calmar_ratio'] > 5]
|
|
insights.append(f"9. Consistent Performers (50%+ WR, PF>2, Calmar>5): {len(consistent)} models - "
|
|
f"best candidates for production")
|
|
|
|
# Insight 10: No-trade models
|
|
no_trade = len([r for r in results if r['total_trades'] == 0])
|
|
insights.append(f"10. No-Trade Models: {no_trade}/100 - indicates poor training or overfitting")
|
|
|
|
# Insight 11: Trade count vs performance
|
|
optimal_trade_count = [r for r in active_models if 100 <= r['total_trades'] <= 500 and r['total_pnl'] > 0]
|
|
insights.append(f"11. Optimal Trade Count (100-500): {len(optimal_trade_count)} profitable models - "
|
|
f"good balance of activity and selectivity")
|
|
|
|
# Insight 12: Hold time patterns
|
|
short_hold_profitable = [r for r in active_models if r['avg_trade_duration'] < 20 and r['total_pnl'] > 20]
|
|
insights.append(f"12. Short-Term Trading (<20 bars): {len(short_hold_profitable)} highly profitable models - "
|
|
f"scalping strategy viable")
|
|
|
|
for insight in insights:
|
|
print(f"\n{insight}")
|
|
|
|
# Run all analyses
|
|
if __name__ == '__main__':
|
|
print("\n" + "="*80)
|
|
print("COMPREHENSIVE BACKTEST ANALYSIS")
|
|
print("100 Checkpoint Models (DQN + PPO)")
|
|
print("Date Range: 2025-07-16 to 2025-10-14")
|
|
print("="*80)
|
|
|
|
analyze_by_model_type()
|
|
analyze_by_epoch()
|
|
analyze_trade_characteristics()
|
|
find_top_performers()
|
|
analyze_risk_metrics()
|
|
generate_actionable_insights()
|
|
|
|
print("\n" + "="*80)
|
|
print("ANALYSIS COMPLETE")
|
|
print("="*80 + "\n")
|