chore: Major codebase cleanup - remove deprecated files and organize structure
- Docker: Delete 23 deprecated Dockerfiles, fix CI/CD to use Dockerfile.foxhunt-build - Config: Remove 36 .env files, keep 4 essential, delete config/environments/ - Docs: Archive 614 Wave D files to docs/archive/wave_d/, 95% reduction in root - Scripts: Delete 56 deprecated scripts, keep 58 production-critical (49% reduction) - Python: Organize 37 scripts into scripts/python/ subdirectories, delete ml/python/ - Build: Remove 1GB artifacts, delete old venvs, clean Python cache from git - Migrations: Delete deprecated directory (4,432 lines), remove duplicate database/migrations/ - Infrastructure: Delete deployment/ (61 files), docs/scripts/ (8 files) Total impact: ~2,500 files cleaned, 750MB+ space freed, zero production impact All deleted scripts backed up to archives. runpod/ and tests/runpod/ preserved. data_acquisition_service retained per user request.
This commit is contained in:
jgrusewski
334
scripts/python/analysis/analyze_backtest_results.py
Normal file
334
scripts/python/analysis/analyze_backtest_results.py
Normal file
@@ -0,0 +1,334 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Deep Analysis of Comprehensive Backtest Results
|
||||
Analyzes 100 checkpoint backtest data for trading insights
|
||||
"""
|
||||
|
||||
import json
|
||||
import statistics
|
||||
from datetime import datetime
|
||||
from collections import defaultdict
|
||||
|
||||
# Load the backtest results
|
||||
with open('results/comprehensive_backtest_results_20251014_143309.json', 'r') as f:
|
||||
results = json.load(f)
|
||||
|
||||
# Categorization functions
|
||||
def categorize_by_performance(result):
|
||||
"""Categorize models as High/Medium/Low performers"""
|
||||
sharpe = result['sharpe_ratio']
|
||||
win_rate = result['win_rate']
|
||||
pnl = result['total_pnl']
|
||||
|
||||
# High performer: Sharpe > 5, Win Rate > 55%, PnL > 50
|
||||
if sharpe > 5 and win_rate > 55 and pnl > 50:
|
||||
return 'High'
|
||||
# Low performer: Sharpe < 0 or Win Rate < 35% or PnL < -50
|
||||
elif sharpe < 0 or win_rate < 35 or pnl < -50:
|
||||
return 'Low'
|
||||
else:
|
||||
return 'Medium'
|
||||
|
||||
def analyze_by_model_type():
|
||||
"""Analyze performance by model type (DQN vs PPO)"""
|
||||
dqn_results = [r for r in results if r['model_type'] == 'DQN']
|
||||
ppo_results = [r for r in results if r['model_type'] == 'PPO']
|
||||
|
||||
print("\n" + "="*80)
|
||||
print("MODEL TYPE ANALYSIS")
|
||||
print("="*80)
|
||||
|
||||
for model_type, model_results in [('DQN', dqn_results), ('PPO', ppo_results)]:
|
||||
# Filter out models with no trades
|
||||
active_models = [r for r in model_results if r['total_trades'] > 0]
|
||||
|
||||
if not active_models:
|
||||
print(f"\n{model_type}: No active models")
|
||||
continue
|
||||
|
||||
avg_sharpe = statistics.mean([r['sharpe_ratio'] for r in active_models])
|
||||
avg_win_rate = statistics.mean([r['win_rate'] for r in active_models])
|
||||
avg_pnl = statistics.mean([r['total_pnl'] for r in active_models])
|
||||
avg_trades = statistics.mean([r['total_trades'] for r in active_models])
|
||||
|
||||
profitable_models = len([r for r in active_models if r['total_pnl'] > 0])
|
||||
|
||||
print(f"\n{model_type} Models:")
|
||||
print(f" Total Models: {len(model_results)} ({len(active_models)} active)")
|
||||
print(f" Profitable Models: {profitable_models}/{len(active_models)} ({profitable_models/len(active_models)*100:.1f}%)")
|
||||
print(f" Average Sharpe Ratio: {avg_sharpe:.2f}")
|
||||
print(f" Average Win Rate: {avg_win_rate:.2f}%")
|
||||
print(f" Average PnL: ${avg_pnl:.2f}")
|
||||
print(f" Average Total Trades: {avg_trades:.0f}")
|
||||
|
||||
def analyze_by_epoch():
|
||||
"""Analyze how performance changes with training epochs"""
|
||||
print("\n" + "="*80)
|
||||
print("EPOCH PROGRESSION ANALYSIS")
|
||||
print("="*80)
|
||||
|
||||
for model_type in ['DQN', 'PPO']:
|
||||
model_results = [r for r in results if r['model_type'] == model_type]
|
||||
|
||||
# Group by epoch ranges
|
||||
epoch_ranges = {
|
||||
'Early (10-100)': [r for r in model_results if 10 <= r['epoch'] <= 100 and r['total_trades'] > 0],
|
||||
'Mid (110-300)': [r for r in model_results if 110 <= r['epoch'] <= 300 and r['total_trades'] > 0],
|
||||
'Late (310-500)': [r for r in model_results if 310 <= r['epoch'] <= 500 and r['total_trades'] > 0]
|
||||
}
|
||||
|
||||
print(f"\n{model_type} Epoch Progression:")
|
||||
for range_name, range_results in epoch_ranges.items():
|
||||
if not range_results:
|
||||
continue
|
||||
|
||||
avg_sharpe = statistics.mean([r['sharpe_ratio'] for r in range_results])
|
||||
avg_win_rate = statistics.mean([r['win_rate'] for r in range_results])
|
||||
avg_pnl = statistics.mean([r['total_pnl'] for r in range_results])
|
||||
profitable = len([r for r in range_results if r['total_pnl'] > 0])
|
||||
|
||||
print(f" {range_name}: {len(range_results)} models, {profitable} profitable")
|
||||
print(f" Avg Sharpe: {avg_sharpe:.2f}, Win Rate: {avg_win_rate:.1f}%, PnL: ${avg_pnl:.2f}")
|
||||
|
||||
def analyze_trade_characteristics():
|
||||
"""Analyze trade patterns and characteristics"""
|
||||
print("\n" + "="*80)
|
||||
print("TRADE CHARACTERISTICS ANALYSIS")
|
||||
print("="*80)
|
||||
|
||||
active_models = [r for r in results if r['total_trades'] > 10]
|
||||
|
||||
# Trade frequency analysis
|
||||
high_freq = [r for r in active_models if r['trade_frequency'] > 50]
|
||||
low_freq = [r for r in active_models if r['trade_frequency'] < 20]
|
||||
|
||||
print(f"\nTrade Frequency Impact:")
|
||||
print(f" High Frequency (>50 trades/day): {len(high_freq)} models")
|
||||
if high_freq:
|
||||
avg_pnl_high = statistics.mean([r['total_pnl'] for r in high_freq])
|
||||
avg_sharpe_high = statistics.mean([r['sharpe_ratio'] for r in high_freq])
|
||||
profitable_high = len([r for r in high_freq if r['total_pnl'] > 0])
|
||||
print(f" Profitable: {profitable_high}/{len(high_freq)} ({profitable_high/len(high_freq)*100:.1f}%)")
|
||||
print(f" Avg PnL: ${avg_pnl_high:.2f}, Avg Sharpe: {avg_sharpe_high:.2f}")
|
||||
|
||||
print(f" Low Frequency (<20 trades/day): {len(low_freq)} models")
|
||||
if low_freq:
|
||||
avg_pnl_low = statistics.mean([r['total_pnl'] for r in low_freq])
|
||||
avg_sharpe_low = statistics.mean([r['sharpe_ratio'] for r in low_freq])
|
||||
profitable_low = len([r for r in low_freq if r['total_pnl'] > 0])
|
||||
print(f" Profitable: {profitable_low}/{len(low_freq)} ({profitable_low/len(low_freq)*100:.1f}%)")
|
||||
print(f" Avg PnL: ${avg_pnl_low:.2f}, Avg Sharpe: {avg_sharpe_low:.2f}")
|
||||
|
||||
# Hold time analysis
|
||||
short_hold = [r for r in active_models if r['avg_trade_duration'] < 20]
|
||||
long_hold = [r for r in active_models if r['avg_trade_duration'] > 60]
|
||||
|
||||
print(f"\nAverage Hold Time Impact:")
|
||||
print(f" Short Hold (<20 bars): {len(short_hold)} models")
|
||||
if short_hold:
|
||||
avg_pnl_short = statistics.mean([r['total_pnl'] for r in short_hold])
|
||||
avg_win_rate_short = statistics.mean([r['win_rate'] for r in short_hold])
|
||||
profitable_short = len([r for r in short_hold if r['total_pnl'] > 0])
|
||||
print(f" Profitable: {profitable_short}/{len(short_hold)} ({profitable_short/len(short_hold)*100:.1f}%)")
|
||||
print(f" Avg PnL: ${avg_pnl_short:.2f}, Win Rate: {avg_win_rate_short:.1f}%")
|
||||
|
||||
print(f" Long Hold (>60 bars): {len(long_hold)} models")
|
||||
if long_hold:
|
||||
avg_pnl_long = statistics.mean([r['total_pnl'] for r in long_hold])
|
||||
avg_win_rate_long = statistics.mean([r['win_rate'] for r in long_hold])
|
||||
profitable_long = len([r for r in long_hold if r['total_pnl'] > 0])
|
||||
print(f" Profitable: {profitable_long}/{len(long_hold)} ({profitable_long/len(long_hold)*100:.1f}%)")
|
||||
print(f" Avg PnL: ${avg_pnl_long:.2f}, Win Rate: {avg_win_rate_long:.1f}%")
|
||||
|
||||
# Win rate distribution
|
||||
print(f"\nWin Rate Distribution:")
|
||||
win_rate_bins = {
|
||||
'<30%': [r for r in active_models if r['win_rate'] < 30],
|
||||
'30-45%': [r for r in active_models if 30 <= r['win_rate'] < 45],
|
||||
'45-55%': [r for r in active_models if 45 <= r['win_rate'] < 55],
|
||||
'55-65%': [r for r in active_models if 55 <= r['win_rate'] < 65],
|
||||
'>65%': [r for r in active_models if r['win_rate'] >= 65]
|
||||
}
|
||||
|
||||
for bin_name, bin_results in win_rate_bins.items():
|
||||
if not bin_results:
|
||||
continue
|
||||
avg_pnl = statistics.mean([r['total_pnl'] for r in bin_results])
|
||||
profitable = len([r for r in bin_results if r['total_pnl'] > 0])
|
||||
print(f" {bin_name}: {len(bin_results)} models, {profitable} profitable, Avg PnL: ${avg_pnl:.2f}")
|
||||
|
||||
def find_top_performers():
|
||||
"""Identify top performing models"""
|
||||
print("\n" + "="*80)
|
||||
print("TOP PERFORMERS")
|
||||
print("="*80)
|
||||
|
||||
# Filter active models with significant trades
|
||||
active_models = [r for r in results if r['total_trades'] > 50]
|
||||
|
||||
# Sort by different metrics
|
||||
by_sharpe = sorted(active_models, key=lambda x: x['sharpe_ratio'], reverse=True)[:10]
|
||||
by_pnl = sorted(active_models, key=lambda x: x['total_pnl'], reverse=True)[:10]
|
||||
by_profit_factor = sorted([r for r in active_models if r['profit_factor'] and r['profit_factor'] > 0],
|
||||
key=lambda x: x['profit_factor'], reverse=True)[:10]
|
||||
|
||||
print("\nTop 10 by Sharpe Ratio (>50 trades):")
|
||||
for i, model in enumerate(by_sharpe, 1):
|
||||
print(f" {i}. {model['model_name']}: Sharpe {model['sharpe_ratio']:.2f}, "
|
||||
f"Win Rate {model['win_rate']:.1f}%, PnL ${model['total_pnl']:.2f}, "
|
||||
f"Trades {model['total_trades']}")
|
||||
|
||||
print("\nTop 10 by Total PnL (>50 trades):")
|
||||
for i, model in enumerate(by_pnl, 1):
|
||||
print(f" {i}. {model['model_name']}: PnL ${model['total_pnl']:.2f}, "
|
||||
f"Sharpe {model['sharpe_ratio']:.2f}, Win Rate {model['win_rate']:.1f}%, "
|
||||
f"Trades {model['total_trades']}")
|
||||
|
||||
print("\nTop 10 by Profit Factor (>50 trades):")
|
||||
for i, model in enumerate(by_profit_factor, 1):
|
||||
print(f" {i}. {model['model_name']}: Profit Factor {model['profit_factor']:.2f}, "
|
||||
f"PnL ${model['total_pnl']:.2f}, Win Rate {model['win_rate']:.1f}%")
|
||||
|
||||
def analyze_risk_metrics():
|
||||
"""Analyze risk-adjusted performance"""
|
||||
print("\n" + "="*80)
|
||||
print("RISK-ADJUSTED PERFORMANCE ANALYSIS")
|
||||
print("="*80)
|
||||
|
||||
active_models = [r for r in results if r['total_trades'] > 10 and r['max_drawdown'] > 0]
|
||||
|
||||
# Sort by Calmar ratio (return/max drawdown)
|
||||
positive_calmar = [r for r in active_models if r['calmar_ratio'] > 0]
|
||||
by_calmar = sorted(positive_calmar, key=lambda x: x['calmar_ratio'], reverse=True)[:10]
|
||||
|
||||
print("\nTop 10 by Calmar Ratio (return/max drawdown):")
|
||||
for i, model in enumerate(by_calmar, 1):
|
||||
print(f" {i}. {model['model_name']}: Calmar {model['calmar_ratio']:.2f}, "
|
||||
f"Max DD {model['max_drawdown']*100:.4f}%, PnL ${model['total_pnl']:.2f}")
|
||||
|
||||
# Analyze drawdown patterns
|
||||
small_dd = [r for r in active_models if r['max_drawdown'] < 0.001] # <0.1%
|
||||
large_dd = [r for r in active_models if r['max_drawdown'] > 0.05] # >5%
|
||||
|
||||
print(f"\nDrawdown Distribution:")
|
||||
print(f" Small Drawdown (<0.1%): {len(small_dd)} models")
|
||||
if small_dd:
|
||||
avg_pnl_small = statistics.mean([r['total_pnl'] for r in small_dd])
|
||||
profitable_small = len([r for r in small_dd if r['total_pnl'] > 0])
|
||||
print(f" Profitable: {profitable_small}/{len(small_dd)} ({profitable_small/len(small_dd)*100:.1f}%)")
|
||||
print(f" Avg PnL: ${avg_pnl_small:.2f}")
|
||||
|
||||
print(f" Large Drawdown (>5%): {len(large_dd)} models")
|
||||
if large_dd:
|
||||
avg_pnl_large = statistics.mean([r['total_pnl'] for r in large_dd])
|
||||
profitable_large = len([r for r in large_dd if r['total_pnl'] > 0])
|
||||
print(f" Profitable: {profitable_large}/{len(large_dd)} ({profitable_large/len(large_dd)*100:.1f}%)")
|
||||
print(f" Avg PnL: ${avg_pnl_large:.2f}")
|
||||
|
||||
def generate_actionable_insights():
|
||||
"""Generate actionable insights for production"""
|
||||
print("\n" + "="*80)
|
||||
print("ACTIONABLE INSIGHTS FOR PRODUCTION")
|
||||
print("="*80)
|
||||
|
||||
active_models = [r for r in results if r['total_trades'] > 10]
|
||||
|
||||
insights = []
|
||||
|
||||
# Insight 1: Optimal epoch range
|
||||
dqn_profitable = [r for r in active_models if r['model_type'] == 'DQN' and r['total_pnl'] > 50]
|
||||
ppo_profitable = [r for r in active_models if r['model_type'] == 'PPO' and r['total_pnl'] > 50]
|
||||
|
||||
if dqn_profitable:
|
||||
dqn_epochs = [r['epoch'] for r in dqn_profitable]
|
||||
insights.append(f"1. DQN Optimal Epochs: {min(dqn_epochs)}-{max(dqn_epochs)} "
|
||||
f"(found {len(dqn_profitable)} profitable models)")
|
||||
|
||||
if ppo_profitable:
|
||||
ppo_epochs = [r['epoch'] for r in ppo_profitable]
|
||||
insights.append(f"2. PPO Optimal Epochs: {min(ppo_epochs)}-{max(ppo_epochs)} "
|
||||
f"(found {len(ppo_profitable)} profitable models)")
|
||||
|
||||
# Insight 2: Trade frequency sweet spot
|
||||
high_sharpe = [r for r in active_models if r['sharpe_ratio'] > 5]
|
||||
if high_sharpe:
|
||||
avg_freq = statistics.mean([r['trade_frequency'] for r in high_sharpe])
|
||||
avg_hold = statistics.mean([r['avg_trade_duration'] for r in high_sharpe])
|
||||
insights.append(f"3. High Sharpe Models (>5): Trade frequency ~{avg_freq:.1f} trades/day, "
|
||||
f"Hold time ~{avg_hold:.1f} bars")
|
||||
|
||||
# Insight 3: Win rate vs profit factor correlation
|
||||
high_win_rate = [r for r in active_models if r['win_rate'] > 55]
|
||||
if high_win_rate:
|
||||
profitable_high_wr = len([r for r in high_win_rate if r['total_pnl'] > 0])
|
||||
insights.append(f"4. Win Rate >55%: {profitable_high_wr}/{len(high_win_rate)} models profitable "
|
||||
f"({profitable_high_wr/len(high_win_rate)*100:.1f}%)")
|
||||
|
||||
# Insight 4: Risk control
|
||||
low_dd_profitable = [r for r in active_models if r['max_drawdown'] < 0.01 and r['total_pnl'] > 20]
|
||||
insights.append(f"5. Low Drawdown Winners (<1% DD, >$20 PnL): {len(low_dd_profitable)} models - "
|
||||
f"excellent risk control")
|
||||
|
||||
# Insight 5: Model type recommendation
|
||||
dqn_active = [r for r in active_models if r['model_type'] == 'DQN']
|
||||
ppo_active = [r for r in active_models if r['model_type'] == 'PPO']
|
||||
|
||||
dqn_profit_rate = len([r for r in dqn_active if r['total_pnl'] > 0]) / len(dqn_active) if dqn_active else 0
|
||||
ppo_profit_rate = len([r for r in ppo_active if r['total_pnl'] > 0]) / len(ppo_active) if ppo_active else 0
|
||||
|
||||
better_model = 'DQN' if dqn_profit_rate > ppo_profit_rate else 'PPO'
|
||||
insights.append(f"6. Model Type Preference: {better_model} ({max(dqn_profit_rate, ppo_profit_rate)*100:.1f}% "
|
||||
f"profitability vs {min(dqn_profit_rate, ppo_profit_rate)*100:.1f}%)")
|
||||
|
||||
# Insight 7: Trading frequency
|
||||
high_freq_models = [r for r in active_models if r['trade_frequency'] > 50]
|
||||
high_freq_profitable = len([r for r in high_freq_models if r['total_pnl'] > 0])
|
||||
insights.append(f"7. High Frequency Trading (>50/day): {high_freq_profitable}/{len(high_freq_models)} "
|
||||
f"profitable - {'recommended' if high_freq_profitable/len(high_freq_models) > 0.5 else 'not recommended'}")
|
||||
|
||||
# Insight 8: Extreme performers
|
||||
extreme_sharpe = [r for r in active_models if r['sharpe_ratio'] > 8]
|
||||
insights.append(f"8. Extreme Sharpe Ratio (>8): {len(extreme_sharpe)} models - potential overfitting "
|
||||
f"or exceptional performance")
|
||||
|
||||
# Insight 9: Consistency
|
||||
consistent = [r for r in active_models if r['win_rate'] > 50 and r['profit_factor'] and
|
||||
r['profit_factor'] > 2 and r['calmar_ratio'] > 5]
|
||||
insights.append(f"9. Consistent Performers (50%+ WR, PF>2, Calmar>5): {len(consistent)} models - "
|
||||
f"best candidates for production")
|
||||
|
||||
# Insight 10: No-trade models
|
||||
no_trade = len([r for r in results if r['total_trades'] == 0])
|
||||
insights.append(f"10. No-Trade Models: {no_trade}/100 - indicates poor training or overfitting")
|
||||
|
||||
# Insight 11: Trade count vs performance
|
||||
optimal_trade_count = [r for r in active_models if 100 <= r['total_trades'] <= 500 and r['total_pnl'] > 0]
|
||||
insights.append(f"11. Optimal Trade Count (100-500): {len(optimal_trade_count)} profitable models - "
|
||||
f"good balance of activity and selectivity")
|
||||
|
||||
# Insight 12: Hold time patterns
|
||||
short_hold_profitable = [r for r in active_models if r['avg_trade_duration'] < 20 and r['total_pnl'] > 20]
|
||||
insights.append(f"12. Short-Term Trading (<20 bars): {len(short_hold_profitable)} highly profitable models - "
|
||||
f"scalping strategy viable")
|
||||
|
||||
for insight in insights:
|
||||
print(f"\n{insight}")
|
||||
|
||||
# Run all analyses
|
||||
if __name__ == '__main__':
|
||||
print("\n" + "="*80)
|
||||
print("COMPREHENSIVE BACKTEST ANALYSIS")
|
||||
print("100 Checkpoint Models (DQN + PPO)")
|
||||
print("Date Range: 2025-07-16 to 2025-10-14")
|
||||
print("="*80)
|
||||
|
||||
analyze_by_model_type()
|
||||
analyze_by_epoch()
|
||||
analyze_trade_characteristics()
|
||||
find_top_performers()
|
||||
analyze_risk_metrics()
|
||||
generate_actionable_insights()
|
||||
|
||||
print("\n" + "="*80)
|
||||
print("ANALYSIS COMPLETE")
|
||||
print("="*80 + "\n")
|
||||
145
scripts/python/analysis/analyze_clippy_warnings.py
Normal file
145
scripts/python/analysis/analyze_clippy_warnings.py
Normal file
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Analyze clippy warnings from output file."""
|
||||
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
def analyze_clippy_output(filepath):
|
||||
"""Analyze clippy output and categorize warnings."""
|
||||
|
||||
warning_types = defaultdict(list)
|
||||
warning_counts = defaultdict(int)
|
||||
file_warnings = defaultdict(list)
|
||||
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
lines = f.readlines()
|
||||
|
||||
current_warning = None
|
||||
current_file = None
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
# Extract warning type and location
|
||||
if line.strip().startswith('warning:') and '-->' in (lines[i+1] if i+1 < len(lines) else ''):
|
||||
warning_text = line.strip()
|
||||
location = lines[i+1].strip()
|
||||
|
||||
# Extract file path
|
||||
match = re.search(r'--> (.+):(\d+):(\d+)', location)
|
||||
if match:
|
||||
file_path = match.group(1)
|
||||
line_num = match.group(2)
|
||||
|
||||
# Extract warning type from help text or note
|
||||
warning_type = "unknown"
|
||||
for j in range(i+1, min(i+10, len(lines))):
|
||||
if 'clippy::' in lines[j]:
|
||||
clippy_match = re.search(r'clippy::(\w+)', lines[j])
|
||||
if clippy_match:
|
||||
warning_type = clippy_match.group(1)
|
||||
break
|
||||
|
||||
warning_counts[warning_type] += 1
|
||||
file_warnings[file_path].append({
|
||||
'type': warning_type,
|
||||
'line': line_num,
|
||||
'text': warning_text.replace('warning: ', '')
|
||||
})
|
||||
|
||||
return warning_counts, file_warnings
|
||||
|
||||
def categorize_severity(warning_type):
|
||||
"""Categorize warning by severity."""
|
||||
|
||||
critical = {
|
||||
'correctness', 'suspicious', 'panic', 'unwrap_used',
|
||||
'expect_used', 'indexing_slicing', 'panic_in_result_fn'
|
||||
}
|
||||
|
||||
high = {
|
||||
'perf', 'performance', 'nursery', 'cargo',
|
||||
'missing_errors_doc', 'missing_panics_doc', 'must_use_candidate'
|
||||
}
|
||||
|
||||
medium = {
|
||||
'complexity', 'style', 'pedantic', 'redundant_clone',
|
||||
'needless_pass_by_value', 'unnecessary_wraps'
|
||||
}
|
||||
|
||||
low = {
|
||||
'restriction', 'needless_raw_string_hashes', 'doc_markdown',
|
||||
'missing_const_for_fn', 'default_numeric_fallback'
|
||||
}
|
||||
|
||||
if warning_type in critical:
|
||||
return 'CRITICAL'
|
||||
elif warning_type in high:
|
||||
return 'HIGH'
|
||||
elif warning_type in medium:
|
||||
return 'MEDIUM'
|
||||
elif warning_type in low:
|
||||
return 'LOW'
|
||||
else:
|
||||
return 'UNKNOWN'
|
||||
|
||||
def main():
|
||||
filepath = Path('/home/jgrusewski/Work/foxhunt/clippy_full_output.txt')
|
||||
|
||||
warning_counts, file_warnings = analyze_clippy_output(filepath)
|
||||
|
||||
# Categorize by severity
|
||||
severity_counts = defaultdict(int)
|
||||
severity_types = defaultdict(list)
|
||||
|
||||
for warning_type, count in warning_counts.items():
|
||||
severity = categorize_severity(warning_type)
|
||||
severity_counts[severity] += count
|
||||
severity_types[severity].append((warning_type, count))
|
||||
|
||||
print("=" * 80)
|
||||
print("CLIPPY WARNING ANALYSIS")
|
||||
print("=" * 80)
|
||||
print()
|
||||
|
||||
# Summary by severity
|
||||
print("SEVERITY SUMMARY:")
|
||||
print("-" * 80)
|
||||
for severity in ['CRITICAL', 'HIGH', 'MEDIUM', 'LOW', 'UNKNOWN']:
|
||||
if severity in severity_counts:
|
||||
print(f"{severity:10s}: {severity_counts[severity]:5d} warnings")
|
||||
print()
|
||||
|
||||
# Top warnings by type
|
||||
print("TOP WARNING TYPES:")
|
||||
print("-" * 80)
|
||||
sorted_warnings = sorted(warning_counts.items(), key=lambda x: x[1], reverse=True)
|
||||
for warning_type, count in sorted_warnings[:20]:
|
||||
severity = categorize_severity(warning_type)
|
||||
print(f" [{severity:8s}] {warning_type:40s}: {count:5d}")
|
||||
print()
|
||||
|
||||
# Files with most warnings
|
||||
print("FILES WITH MOST WARNINGS:")
|
||||
print("-" * 80)
|
||||
sorted_files = sorted(file_warnings.items(), key=lambda x: len(x[1]), reverse=True)
|
||||
for filepath, warnings in sorted_files[:15]:
|
||||
print(f" {filepath:60s}: {len(warnings):5d} warnings")
|
||||
print()
|
||||
|
||||
# Critical warnings detail
|
||||
print("CRITICAL WARNINGS DETAIL:")
|
||||
print("-" * 80)
|
||||
for warning_type, count in sorted_warnings:
|
||||
if categorize_severity(warning_type) == 'CRITICAL':
|
||||
print(f"\n{warning_type.upper()} ({count} occurrences):")
|
||||
# Find examples
|
||||
examples = []
|
||||
for filepath, warnings in file_warnings.items():
|
||||
for w in warnings:
|
||||
if w['type'] == warning_type and len(examples) < 3:
|
||||
examples.append(f" - {filepath}:{w['line']} - {w['text']}")
|
||||
for ex in examples:
|
||||
print(ex)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
32
scripts/python/analysis/analyze_failures.py
Normal file
32
scripts/python/analysis/analyze_failures.py
Normal file
@@ -0,0 +1,32 @@
|
||||
import re
|
||||
|
||||
with open('test_output.log', 'r') as f:
|
||||
lines = f.readlines()
|
||||
|
||||
current_package = None
|
||||
failures = {}
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
# Match package name
|
||||
if 'Running' in line and ('tests/' in line or 'src/lib.rs' in line or 'src/main.rs' in line):
|
||||
match = re.search(r'Running.*\(target/debug/deps/([^-]+)', line)
|
||||
if match:
|
||||
current_package = match.group(1)
|
||||
|
||||
# Match test failures
|
||||
if 'test result: FAILED' in line:
|
||||
match = re.search(r'(\d+) passed; (\d+) failed', line)
|
||||
if match and current_package:
|
||||
passed = int(match.group(1))
|
||||
failed = int(match.group(2))
|
||||
if current_package not in failures:
|
||||
failures[current_package] = {'passed': 0, 'failed': 0}
|
||||
failures[current_package]['passed'] += passed
|
||||
failures[current_package]['failed'] += failed
|
||||
|
||||
print("PACKAGES WITH TEST FAILURES:")
|
||||
print("=" * 80)
|
||||
for pkg, stats in sorted(failures.items()):
|
||||
total = stats['passed'] + stats['failed']
|
||||
rate = stats['passed'] / total * 100 if total > 0 else 0
|
||||
print(f"{pkg:40s} {stats['passed']:3d}/{total:3d} passed ({rate:5.1f}%)")
|
||||
95
scripts/python/analysis/analyze_gc_data.py
Normal file
95
scripts/python/analysis/analyze_gc_data.py
Normal file
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Analyze the downloaded Gold Futures data.
|
||||
"""
|
||||
|
||||
import databento as db
|
||||
import pandas as pd
|
||||
|
||||
def main():
|
||||
filepath = "test_data/real/databento/GC_continuous_ohlcv-1m_2024-01-02_to_2024-01-31.dbn"
|
||||
|
||||
print("=" * 80)
|
||||
print("GOLD FUTURES DATA ANALYSIS")
|
||||
print("=" * 80)
|
||||
print(f"File: {filepath}")
|
||||
print()
|
||||
|
||||
# Load data
|
||||
store = db.DBNStore.from_file(filepath)
|
||||
df = store.to_df()
|
||||
|
||||
print(f"Total records: {len(df):,}")
|
||||
print(f"Date range: {df.index[0]} to {df.index[-1]}")
|
||||
print(f"Trading days covered: {(df.index[-1] - df.index[0]).days + 1}")
|
||||
print()
|
||||
|
||||
# Check data completeness by day
|
||||
print("Records per day:")
|
||||
df['date'] = df.index.date
|
||||
records_per_day = df.groupby('date').size()
|
||||
print(records_per_day)
|
||||
print()
|
||||
|
||||
print(f"Average records per day: {records_per_day.mean():.0f}")
|
||||
print(f"Min records per day: {records_per_day.min()}")
|
||||
print(f"Max records per day: {records_per_day.max()}")
|
||||
print()
|
||||
|
||||
# Trading hours analysis
|
||||
df['hour'] = df.index.hour
|
||||
print("Records per hour (UTC):")
|
||||
records_per_hour = df.groupby('hour').size().sort_index()
|
||||
for hour, count in records_per_hour.items():
|
||||
print(f" {hour:02d}:00 - {count:,} bars")
|
||||
print()
|
||||
|
||||
# Price analysis
|
||||
print("Price Statistics:")
|
||||
print(f" Open: ${df['open'].min():.2f} - ${df['open'].max():.2f}")
|
||||
print(f" Close: ${df['close'].min():.2f} - ${df['close'].max():.2f}")
|
||||
print(f" Range: ${df['low'].min():.2f} - ${df['high'].max():.2f}")
|
||||
print()
|
||||
|
||||
# Volatility
|
||||
df['returns'] = df['close'].pct_change()
|
||||
print(f"Max single-bar return: {df['returns'].max()*100:.2f}%")
|
||||
print(f"Min single-bar return: {df['returns'].min()*100:.2f}%")
|
||||
print(f"Std dev of returns: {df['returns'].std()*100:.3f}%")
|
||||
print()
|
||||
|
||||
# Volume analysis
|
||||
print("Volume Statistics:")
|
||||
print(f" Average: {df['volume'].mean():.0f}")
|
||||
print(f" Median: {df['volume'].median():.0f}")
|
||||
print(f" Max: {df['volume'].max():.0f}")
|
||||
print(f" Zero volume bars: {(df['volume'] == 0).sum()} ({(df['volume'] == 0).sum()/len(df)*100:.1f}%)")
|
||||
print()
|
||||
|
||||
# Data quality checks
|
||||
print("Data Quality Checks:")
|
||||
print(f" Missing values: {df.isnull().sum().sum()}")
|
||||
print(f" Duplicate timestamps: {df.index.duplicated().sum()}")
|
||||
print(f" High > Low: {(df['high'] >= df['low']).sum()} / {len(df)} ✅" if (df['high'] >= df['low']).all() else " ⚠️ High > Low violations detected")
|
||||
print(f" OHLC consistency: {((df['high'] >= df['open']) & (df['high'] >= df['close']) & (df['low'] <= df['open']) & (df['low'] <= df['close'])).sum()} / {len(df)}")
|
||||
print()
|
||||
|
||||
# Symbol info
|
||||
print("Metadata:")
|
||||
print(f" Dataset: {store.dataset}")
|
||||
print(f" Schema: {store.schema}")
|
||||
print(f" Symbols: {store.symbols}")
|
||||
print()
|
||||
|
||||
print("=" * 80)
|
||||
print("CONCLUSION")
|
||||
print("=" * 80)
|
||||
print(f"✅ Downloaded {len(df):,} 1-minute OHLCV bars for Gold Futures (GC)")
|
||||
print(f"✅ Cost: $0.00 (free data)")
|
||||
print(f"✅ Data quality: Good (no price spikes >20%, consistent OHLC)")
|
||||
print(f"ℹ️ Trading hours: Gold futures trade primarily during CME hours")
|
||||
print(f"ℹ️ ~26 bars/day suggests limited trading hours coverage in this dataset")
|
||||
print("=" * 80)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
237
scripts/python/analysis/analyze_price_action.py
Normal file
237
scripts/python/analysis/analyze_price_action.py
Normal file
@@ -0,0 +1,237 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Detailed price action analysis for ES futures data.
|
||||
Shows actual price movements to understand market regimes.
|
||||
"""
|
||||
|
||||
import databento as db
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
|
||||
FILES = [
|
||||
{
|
||||
"path": "test_data/real/databento/ES.FUT_ohlcv-1m_2024-01-02.dbn",
|
||||
"date": "2024-01-02",
|
||||
"label": "Baseline"
|
||||
},
|
||||
{
|
||||
"path": "test_data/real/databento/ESH4_ohlcv-1m_2024-01-03.dbn",
|
||||
"date": "2024-01-03",
|
||||
"label": "Trending"
|
||||
},
|
||||
{
|
||||
"path": "test_data/real/databento/ESH4_ohlcv-1m_2024-01-04.dbn",
|
||||
"date": "2024-01-04",
|
||||
"label": "Ranging"
|
||||
},
|
||||
{
|
||||
"path": "test_data/real/databento/ESH4_ohlcv-1m_2024-01-05.dbn",
|
||||
"date": "2024-01-05",
|
||||
"label": "Volatile"
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def analyze_price_action(df, date, label):
|
||||
"""Analyze price action in detail."""
|
||||
print(f"\n{'=' * 80}")
|
||||
print(f"{date} ({label})")
|
||||
print('=' * 80)
|
||||
|
||||
# Basic stats
|
||||
prices = df['close']
|
||||
open_price = df['open'].iloc[0]
|
||||
close_price = df['close'].iloc[-1]
|
||||
high_price = df['high'].max()
|
||||
low_price = df['low'].min()
|
||||
|
||||
# Remove obvious outliers for better analysis
|
||||
# (e.g., the $36.05 in 2024-01-02)
|
||||
price_median = prices.median()
|
||||
price_std = prices.std()
|
||||
# Keep prices within 5 std devs of median
|
||||
mask = np.abs(prices - price_median) <= (5 * price_std)
|
||||
clean_prices = prices[mask]
|
||||
|
||||
if len(clean_prices) < len(prices) * 0.5:
|
||||
print(f"⚠️ WARNING: {len(prices) - len(clean_prices)} outliers detected and filtered")
|
||||
print(f" Original range: ${prices.min():.2f} - ${prices.max():.2f}")
|
||||
print(f" Clean range: ${clean_prices.min():.2f} - ${clean_prices.max():.2f}")
|
||||
|
||||
# Recompute with clean data
|
||||
open_price = clean_prices.iloc[0] if len(clean_prices) > 0 else open_price
|
||||
close_price = clean_prices.iloc[-1] if len(clean_prices) > 0 else close_price
|
||||
high_price = clean_prices.max()
|
||||
low_price = clean_prices.min()
|
||||
|
||||
# Price movement
|
||||
net_change = close_price - open_price
|
||||
net_change_pct = (net_change / open_price) * 100
|
||||
total_range = high_price - low_price
|
||||
range_pct = (total_range / open_price) * 100
|
||||
|
||||
print(f"\n📊 Price Statistics:")
|
||||
print(f" Open: ${open_price:.2f}")
|
||||
print(f" Close: ${close_price:.2f}")
|
||||
print(f" High: ${high_price:.2f}")
|
||||
print(f" Low: ${low_price:.2f}")
|
||||
print(f" Net change: ${net_change:+.2f} ({net_change_pct:+.2f}%)")
|
||||
print(f" Range: ${total_range:.2f} ({range_pct:.2f}%)")
|
||||
|
||||
# Returns analysis
|
||||
returns = prices.pct_change().dropna()
|
||||
|
||||
print(f"\n📈 Returns Analysis:")
|
||||
print(f" Mean return: {returns.mean() * 100:.4f}%")
|
||||
print(f" Std dev: {returns.std() * 100:.4f}%")
|
||||
print(f" Skewness: {returns.skew():.4f}")
|
||||
print(f" Max drawdown: {(returns.cumsum().min() * 100):.2f}%")
|
||||
print(f" Max runup: {(returns.cumsum().max() * 100):.2f}%")
|
||||
|
||||
# Direction analysis
|
||||
up_bars = (df['close'] > df['open']).sum()
|
||||
down_bars = (df['close'] < df['open']).sum()
|
||||
doji_bars = (df['close'] == df['open']).sum()
|
||||
|
||||
print(f"\n📊 Bar Direction:")
|
||||
print(f" Up bars: {up_bars:,} ({up_bars/len(df)*100:.1f}%)")
|
||||
print(f" Down bars: {down_bars:,} ({down_bars/len(df)*100:.1f}%)")
|
||||
print(f" Doji bars: {doji_bars:,} ({doji_bars/len(df)*100:.1f}%)")
|
||||
|
||||
# Consecutive moves
|
||||
direction = np.where(df['close'] > df['open'], 1, -1)
|
||||
direction_changes = np.diff(direction) != 0
|
||||
avg_consecutive = len(direction) / (direction_changes.sum() + 1)
|
||||
|
||||
print(f" Avg consecutive: {avg_consecutive:.1f} bars")
|
||||
|
||||
# Volume
|
||||
total_volume = df['volume'].sum()
|
||||
avg_volume = df['volume'].mean()
|
||||
|
||||
print(f"\n💹 Volume:")
|
||||
print(f" Total: {total_volume:,.0f}")
|
||||
print(f" Average: {avg_volume:,.0f}")
|
||||
print(f" Max: {df['volume'].max():,.0f}")
|
||||
|
||||
# Price levels (quintiles)
|
||||
quintiles = clean_prices.quantile([0.0, 0.25, 0.5, 0.75, 1.0])
|
||||
|
||||
print(f"\n📊 Price Distribution (Quintiles):")
|
||||
print(f" Min (0%): ${quintiles.iloc[0]:.2f}")
|
||||
print(f" Q1 (25%): ${quintiles.iloc[1]:.2f}")
|
||||
print(f" Median: ${quintiles.iloc[2]:.2f}")
|
||||
print(f" Q3 (75%): ${quintiles.iloc[3]:.2f}")
|
||||
print(f" Max (100%): ${quintiles.iloc[4]:.2f}")
|
||||
|
||||
# Trading sessions (rough estimate based on volume patterns)
|
||||
# High volume = regular session, low volume = extended hours
|
||||
volume_threshold = avg_volume * 0.5
|
||||
regular_hours = (df['volume'] > volume_threshold).sum()
|
||||
extended_hours = (df['volume'] <= volume_threshold).sum()
|
||||
|
||||
print(f"\n⏰ Estimated Trading Hours:")
|
||||
print(f" Regular hours: {regular_hours:,} bars ({regular_hours/len(df)*100:.1f}%)")
|
||||
print(f" Extended hours: {extended_hours:,} bars ({extended_hours/len(df)*100:.1f}%)")
|
||||
|
||||
# Regime classification (detailed)
|
||||
print(f"\n🎯 Regime Classification:")
|
||||
|
||||
# Trend strength
|
||||
time_idx = np.arange(len(clean_prices))
|
||||
trend_corr = np.corrcoef(time_idx, clean_prices)[0, 1]
|
||||
|
||||
if abs(trend_corr) > 0.7:
|
||||
trend_str = "STRONG TREND" + (" UP" if trend_corr > 0 else " DOWN")
|
||||
elif abs(trend_corr) > 0.3:
|
||||
trend_str = "MODERATE TREND" + (" UP" if trend_corr > 0 else " DOWN")
|
||||
else:
|
||||
trend_str = "NO CLEAR TREND (Ranging)"
|
||||
|
||||
print(f" Trend: {trend_str} (corr: {trend_corr:.3f})")
|
||||
|
||||
# Volatility
|
||||
volatility = returns.std() * np.sqrt(1440) # Annualized
|
||||
if volatility > 0.20:
|
||||
vol_str = "HIGH VOLATILITY"
|
||||
elif volatility > 0.10:
|
||||
vol_str = "MODERATE VOLATILITY"
|
||||
else:
|
||||
vol_str = "LOW VOLATILITY"
|
||||
|
||||
print(f" Volatility: {vol_str} ({volatility:.4f} annualized)")
|
||||
|
||||
# Range
|
||||
if range_pct > 2.0:
|
||||
range_str = "WIDE RANGE"
|
||||
elif range_pct > 1.0:
|
||||
range_str = "MODERATE RANGE"
|
||||
else:
|
||||
range_str = "NARROW RANGE"
|
||||
|
||||
print(f" Range: {range_str} ({range_pct:.2f}%)")
|
||||
|
||||
return {
|
||||
'date': date,
|
||||
'label': label,
|
||||
'net_change_pct': net_change_pct,
|
||||
'range_pct': range_pct,
|
||||
'trend_correlation': trend_corr,
|
||||
'volatility': volatility,
|
||||
'up_bars_pct': up_bars/len(df)*100,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
"""Analyze all files."""
|
||||
print("=" * 80)
|
||||
print("ES Futures Detailed Price Action Analysis")
|
||||
print("=" * 80)
|
||||
|
||||
results = []
|
||||
|
||||
for file_config in FILES:
|
||||
path = file_config["path"]
|
||||
date = file_config["date"]
|
||||
label = file_config["label"]
|
||||
|
||||
if not Path(path).exists():
|
||||
print(f"\n❌ File not found: {path}")
|
||||
continue
|
||||
|
||||
try:
|
||||
store = db.DBNStore.from_file(path)
|
||||
df = store.to_df()
|
||||
|
||||
result = analyze_price_action(df, date, label)
|
||||
results.append(result)
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ Error analyzing {date}: {e}")
|
||||
|
||||
# Summary table
|
||||
if results:
|
||||
print("\n" + "=" * 80)
|
||||
print("SUMMARY TABLE")
|
||||
print("=" * 80)
|
||||
print()
|
||||
|
||||
df_results = pd.DataFrame(results)
|
||||
|
||||
print("Date | Label | Net Chg | Range | Trend | Volatility | Up Bars")
|
||||
print("-" * 80)
|
||||
for _, row in df_results.iterrows():
|
||||
print(f"{row['date']} | {row['label']:8s} | {row['net_change_pct']:+6.2f}% | {row['range_pct']:5.2f}% | {row['trend_correlation']:+7.3f} | {row['volatility']:10.4f} | {row['up_bars_pct']:6.1f}%")
|
||||
|
||||
print()
|
||||
print("💡 Regime Interpretation:")
|
||||
print(" • Trending: |trend_correlation| > 0.7")
|
||||
print(" • Ranging: range < 2% and |trend_correlation| < 0.3")
|
||||
print(" • Volatile: volatility > 0.15")
|
||||
print()
|
||||
print("✅ Analysis complete!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
193
scripts/python/analysis/generate_backtest_summary.py
Normal file
193
scripts/python/analysis/generate_backtest_summary.py
Normal file
@@ -0,0 +1,193 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Generate Quick Summary Statistics for Backtest Results
|
||||
Creates a concise reference card for production deployment
|
||||
"""
|
||||
|
||||
import json
|
||||
import statistics
|
||||
|
||||
# Load results
|
||||
with open('results/comprehensive_backtest_results_20251014_143309.json', 'r') as f:
|
||||
results = json.load(f)
|
||||
|
||||
def generate_production_reference():
|
||||
"""Generate production reference card"""
|
||||
|
||||
# Filter to active, high-quality models
|
||||
active_models = [r for r in results if r['total_trades'] > 50]
|
||||
|
||||
# Top 5 by multiple criteria
|
||||
top_sharpe = sorted(active_models, key=lambda x: x['sharpe_ratio'], reverse=True)[:5]
|
||||
top_pnl = sorted(active_models, key=lambda x: x['total_pnl'], reverse=True)[:5]
|
||||
top_calmar = sorted([r for r in active_models if r['calmar_ratio'] > 0],
|
||||
key=lambda x: x['calmar_ratio'], reverse=True)[:5]
|
||||
|
||||
print("\n" + "="*80)
|
||||
print("PRODUCTION REFERENCE CARD - TOP 5 MODELS BY METRIC")
|
||||
print("="*80)
|
||||
|
||||
print("\n🏆 TOP 5 BY SHARPE RATIO (RISK-ADJUSTED)")
|
||||
print("-" * 80)
|
||||
for i, m in enumerate(top_sharpe, 1):
|
||||
print(f"{i}. {m['model_name']:25} | Sharpe: {m['sharpe_ratio']:6.2f} | "
|
||||
f"WR: {m['win_rate']:5.1f}% | PnL: ${m['total_pnl']:8.2f} | "
|
||||
f"Trades: {m['total_trades']:3d}")
|
||||
|
||||
print("\n💰 TOP 5 BY TOTAL PNL (ABSOLUTE RETURNS)")
|
||||
print("-" * 80)
|
||||
for i, m in enumerate(top_pnl, 1):
|
||||
print(f"{i}. {m['model_name']:25} | PnL: ${m['total_pnl']:8.2f} | "
|
||||
f"Sharpe: {m['sharpe_ratio']:6.2f} | WR: {m['win_rate']:5.1f}% | "
|
||||
f"Trades: {m['total_trades']:3d}")
|
||||
|
||||
print("\n🛡️ TOP 5 BY CALMAR RATIO (RETURN/DRAWDOWN)")
|
||||
print("-" * 80)
|
||||
for i, m in enumerate(top_calmar, 1):
|
||||
print(f"{i}. {m['model_name']:25} | Calmar: {m['calmar_ratio']:8.2f} | "
|
||||
f"DD: {m['max_drawdown']*100:6.4f}% | PnL: ${m['total_pnl']:8.2f}")
|
||||
|
||||
# Consistent performers (all-around strong)
|
||||
consistent = [r for r in active_models if
|
||||
r['win_rate'] > 50 and
|
||||
r['profit_factor'] and r['profit_factor'] > 2 and
|
||||
r['calmar_ratio'] > 5 and
|
||||
r['sharpe_ratio'] > 3]
|
||||
|
||||
print("\n⭐ TIER 1 CONSISTENT PERFORMERS (WR>50%, PF>2, Calmar>5, Sharpe>3)")
|
||||
print("-" * 80)
|
||||
print(f"Total: {len(consistent)} models")
|
||||
|
||||
# Sort by composite score
|
||||
for m in consistent:
|
||||
m['composite_score'] = (m['sharpe_ratio'] + m['win_rate']/10 +
|
||||
m['calmar_ratio']/100) / 3
|
||||
|
||||
consistent_sorted = sorted(consistent, key=lambda x: x['composite_score'], reverse=True)[:10]
|
||||
|
||||
for i, m in enumerate(consistent_sorted, 1):
|
||||
print(f"{i:2d}. {m['model_name']:25} | "
|
||||
f"Sharpe: {m['sharpe_ratio']:5.2f} | WR: {m['win_rate']:5.1f}% | "
|
||||
f"PnL: ${m['total_pnl']:7.2f} | Calmar: {m['calmar_ratio']:7.1f}")
|
||||
|
||||
# Production ensemble recommendation
|
||||
print("\n" + "="*80)
|
||||
print("RECOMMENDED PRODUCTION ENSEMBLE (8 MODELS)")
|
||||
print("="*80)
|
||||
|
||||
# 5 Tier 1 + 3 Tier 2
|
||||
tier1 = consistent_sorted[:5]
|
||||
tier2_candidates = [m for m in top_pnl if m not in tier1][:3]
|
||||
|
||||
print("\n📊 Tier 1: Consistent Performers (70% allocation)")
|
||||
for i, m in enumerate(tier1, 1):
|
||||
allocation = 14.0 # 70% / 5 models
|
||||
print(f"{i}. {m['model_name']:25} | {allocation:4.1f}% capital | "
|
||||
f"Sharpe: {m['sharpe_ratio']:5.2f} | WR: {m['win_rate']:5.1f}%")
|
||||
|
||||
print("\n🚀 Tier 2: High Return (30% allocation)")
|
||||
for i, m in enumerate(tier2_candidates, 1):
|
||||
allocation = 10.0 # 30% / 3 models
|
||||
print(f"{i}. {m['model_name']:25} | {allocation:4.1f}% capital | "
|
||||
f"PnL: ${m['total_pnl']:7.2f} | Sharpe: {m['sharpe_ratio']:5.2f}")
|
||||
|
||||
# Expected ensemble performance
|
||||
print("\n" + "="*80)
|
||||
print("EXPECTED ENSEMBLE PERFORMANCE")
|
||||
print("="*80)
|
||||
|
||||
tier1_sharpe = statistics.mean([m['sharpe_ratio'] for m in tier1])
|
||||
tier1_wr = statistics.mean([m['win_rate'] for m in tier1])
|
||||
tier1_pnl = statistics.mean([m['total_pnl'] for m in tier1])
|
||||
|
||||
tier2_sharpe = statistics.mean([m['sharpe_ratio'] for m in tier2_candidates])
|
||||
tier2_wr = statistics.mean([m['win_rate'] for m in tier2_candidates])
|
||||
tier2_pnl = statistics.mean([m['total_pnl'] for m in tier2_candidates])
|
||||
|
||||
ensemble_sharpe = 0.7 * tier1_sharpe + 0.3 * tier2_sharpe
|
||||
ensemble_wr = 0.7 * tier1_wr + 0.3 * tier2_wr
|
||||
ensemble_pnl = 0.7 * tier1_pnl + 0.3 * tier2_pnl
|
||||
|
||||
print(f"\nTier 1 Average: Sharpe {tier1_sharpe:.2f}, WR {tier1_wr:.1f}%, PnL ${tier1_pnl:.2f}")
|
||||
print(f"Tier 2 Average: Sharpe {tier2_sharpe:.2f}, WR {tier2_wr:.1f}%, PnL ${tier2_pnl:.2f}")
|
||||
print(f"\nWeighted Ensemble: Sharpe {ensemble_sharpe:.2f}, WR {ensemble_wr:.1f}%, PnL ${ensemble_pnl:.2f}")
|
||||
|
||||
# Monthly return projection
|
||||
monthly_return_pct = (ensemble_pnl / 90) * 30 # Scale to 30 days
|
||||
annual_return_pct = monthly_return_pct * 12
|
||||
|
||||
print(f"\nProjected Returns (90-day backtest scaled):")
|
||||
print(f" Monthly: {monthly_return_pct:.1f}% (on $10K = ${monthly_return_pct * 100:.2f}/month)")
|
||||
print(f" Annual: {annual_return_pct:.1f}% (not compounded)")
|
||||
|
||||
# Risk metrics
|
||||
ensemble_dd = statistics.mean([m['max_drawdown'] for m in tier1 + tier2_candidates])
|
||||
print(f"\nRisk Metrics:")
|
||||
print(f" Expected Max Drawdown: {ensemble_dd*100:.3f}%")
|
||||
print(f" Expected Calmar Ratio: {(monthly_return_pct/30)/(ensemble_dd*100):.1f}")
|
||||
|
||||
print("\n" + "="*80)
|
||||
print("PRODUCTION RISK LIMITS")
|
||||
print("="*80)
|
||||
print("""
|
||||
1. Per-Model Max Drawdown: 1.0% (kill switch)
|
||||
2. Per-Model Min Win Rate: 55% (rolling 100 trades)
|
||||
3. Per-Model Min Sharpe: 2.0 (rolling 50 trades)
|
||||
4. Ensemble Max Drawdown: 2.0% (flatten all)
|
||||
5. Daily Loss Limit: -3% (halt trading for 24h)
|
||||
6. Trade Frequency: 10-30 trades/day per model
|
||||
7. Position Sizing: 2% risk per trade
|
||||
8. Max Correlation: 0.7 between models
|
||||
""")
|
||||
|
||||
print("="*80)
|
||||
print("DEPLOYMENT CHECKLIST")
|
||||
print("="*80)
|
||||
print("""
|
||||
☐ Week 1: Out-of-sample validation (Jan-Mar 2025 data)
|
||||
☐ Week 2: Production risk framework implementation
|
||||
☐ Week 3: Ensemble system build + unit tests
|
||||
☐ Week 4: Paper trading (target: Sharpe >2.0, WR >55%)
|
||||
☐ Week 5: Limited live ($10K, Tier 1 only)
|
||||
☐ Week 6: Daily monitoring (require >3% weekly return)
|
||||
☐ Week 7: Add Tier 2 ($5K additional)
|
||||
☐ Week 8: Scale to $50K if 10%+ return, <3% DD
|
||||
☐ Week 9: Full production ($100K, 8-model ensemble)
|
||||
☐ Week 10: Automated monitoring dashboard
|
||||
☐ Week 11: Monthly retraining cycle begin
|
||||
☐ Week 12: Operations playbook documentation
|
||||
""")
|
||||
|
||||
def generate_model_matrix():
|
||||
"""Generate model selection matrix"""
|
||||
|
||||
print("\n" + "="*80)
|
||||
print("MODEL SELECTION MATRIX")
|
||||
print("="*80)
|
||||
|
||||
models = [r for r in results if r['total_trades'] > 50]
|
||||
|
||||
# Categorize by characteristics
|
||||
categories = {
|
||||
'High Sharpe (>7)': [m for m in models if m['sharpe_ratio'] > 7],
|
||||
'High PnL (>$80)': [m for m in models if m['total_pnl'] > 80],
|
||||
'Low Drawdown (<0.01%)': [m for m in models if m['max_drawdown'] < 0.0001],
|
||||
'High Win Rate (>58%)': [m for m in models if m['win_rate'] > 58],
|
||||
'Low Frequency (<20/day)': [m for m in models if m['trade_frequency'] < 20],
|
||||
'Balanced (50-60% WR, 100-500 trades)': [m for m in models if
|
||||
50 < m['win_rate'] < 60 and 100 < m['total_trades'] < 500]
|
||||
}
|
||||
|
||||
for category, category_models in categories.items():
|
||||
print(f"\n{category}: {len(category_models)} models")
|
||||
for m in sorted(category_models, key=lambda x: x['sharpe_ratio'], reverse=True)[:3]:
|
||||
print(f" • {m['model_name']:25} | Sharpe: {m['sharpe_ratio']:5.2f} | "
|
||||
f"WR: {m['win_rate']:5.1f}% | PnL: ${m['total_pnl']:7.2f}")
|
||||
|
||||
if __name__ == '__main__':
|
||||
generate_production_reference()
|
||||
generate_model_matrix()
|
||||
|
||||
print("\n" + "="*80)
|
||||
print("SUMMARY GENERATION COMPLETE")
|
||||
print("="*80 + "\n")
|
||||
35
scripts/python/analysis/parse_test_results.py
Normal file
35
scripts/python/analysis/parse_test_results.py
Normal file
@@ -0,0 +1,35 @@
|
||||
import re
|
||||
|
||||
with open('test_output.log', 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Find all test result summaries
|
||||
pattern = r'test result: (ok|FAILED)\. (\d+) passed; (\d+) failed; (\d+) ignored'
|
||||
matches = re.findall(pattern, content)
|
||||
|
||||
total_passed = 0
|
||||
total_failed = 0
|
||||
total_ignored = 0
|
||||
failed_suites = 0
|
||||
passed_suites = 0
|
||||
|
||||
for match in matches:
|
||||
status, passed, failed, ignored = match
|
||||
total_passed += int(passed)
|
||||
total_failed += int(failed)
|
||||
total_ignored += int(ignored)
|
||||
if status == 'FAILED':
|
||||
failed_suites += 1
|
||||
else:
|
||||
passed_suites += 1
|
||||
|
||||
print(f"Total Test Suites: {passed_suites + failed_suites}")
|
||||
print(f" Passed Suites: {passed_suites}")
|
||||
print(f" Failed Suites: {failed_suites}")
|
||||
print()
|
||||
print(f"Total Tests Executed: {total_passed + total_failed}")
|
||||
print(f" Passed: {total_passed}")
|
||||
print(f" Failed: {total_failed}")
|
||||
print(f" Ignored: {total_ignored}")
|
||||
print()
|
||||
print(f"Success Rate: {total_passed / (total_passed + total_failed) * 100:.2f}%")
|
||||
128
scripts/python/analysis/parse_wave_141_results.py
Executable file
128
scripts/python/analysis/parse_wave_141_results.py
Executable file
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Parse Wave 141 test results and generate comprehensive report."""
|
||||
|
||||
import re
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
|
||||
def parse_test_results(filename):
|
||||
"""Parse test results from cargo test output."""
|
||||
|
||||
with open(filename, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Find all test result blocks
|
||||
# Pattern: "test result: ok. X passed; Y failed; Z ignored; W measured; A filtered out"
|
||||
result_pattern = re.compile(
|
||||
r'test result: (\w+)\. (\d+) passed; (\d+) failed; (\d+) ignored; (\d+) measured; (\d+) filtered out'
|
||||
)
|
||||
|
||||
results = []
|
||||
total_passed = 0
|
||||
total_failed = 0
|
||||
total_ignored = 0
|
||||
|
||||
for match in result_pattern.finditer(content):
|
||||
status = match.group(1)
|
||||
passed = int(match.group(2))
|
||||
failed = int(match.group(3))
|
||||
ignored = int(match.group(4))
|
||||
measured = int(match.group(5))
|
||||
filtered = int(match.group(6))
|
||||
|
||||
total_passed += passed
|
||||
total_failed += failed
|
||||
total_ignored += ignored
|
||||
|
||||
results.append({
|
||||
'status': status,
|
||||
'passed': passed,
|
||||
'failed': failed,
|
||||
'ignored': ignored,
|
||||
'measured': measured,
|
||||
'filtered': filtered
|
||||
})
|
||||
|
||||
# Find compilation errors
|
||||
error_pattern = re.compile(r'^error\[E\d+\]:', re.MULTILINE)
|
||||
compilation_errors = len(error_pattern.findall(content))
|
||||
|
||||
# Check for compilation failure
|
||||
compile_failed_pattern = re.compile(r'error: could not compile')
|
||||
compile_failures = compile_failed_pattern.findall(content)
|
||||
|
||||
return {
|
||||
'results': results,
|
||||
'total_passed': total_passed,
|
||||
'total_failed': total_failed,
|
||||
'total_ignored': total_ignored,
|
||||
'compilation_errors': compilation_errors,
|
||||
'compile_failures': len(compile_failures),
|
||||
'num_test_suites': len(results)
|
||||
}
|
||||
|
||||
def generate_report(data):
|
||||
"""Generate comprehensive test report."""
|
||||
|
||||
total_tests = data['total_passed'] + data['total_failed']
|
||||
pass_rate = (data['total_passed'] / total_tests * 100) if total_tests > 0 else 0
|
||||
|
||||
print("=" * 80)
|
||||
print("WAVE 141 FULL WORKSPACE TEST RESULTS")
|
||||
print("=" * 80)
|
||||
print()
|
||||
|
||||
print("SUMMARY:")
|
||||
print(f" Test Suites Run: {data['num_test_suites']}")
|
||||
print(f" Total Tests: {total_tests}")
|
||||
print(f" Tests Passed: {data['total_passed']}")
|
||||
print(f" Tests Failed: {data['total_failed']}")
|
||||
print(f" Tests Ignored: {data['total_ignored']}")
|
||||
print(f" Pass Rate: {pass_rate:.1f}%")
|
||||
print()
|
||||
|
||||
print("COMPILATION STATUS:")
|
||||
print(f" Compilation Errors: {data['compilation_errors']}")
|
||||
print(f" Failed Compiles: {data['compile_failures']}")
|
||||
print()
|
||||
|
||||
if data['compile_failures'] > 0:
|
||||
print("⚠️ WARNING: Some tests failed to compile!")
|
||||
print(" The saturation_point_tests crate has 6 compilation errors")
|
||||
print()
|
||||
|
||||
print("WAVE 140 BASELINE COMPARISON:")
|
||||
print(f" Wave 140: 430/456 passing (94.2%)")
|
||||
print(f" Wave 141: {data['total_passed']}/{total_tests} passing ({pass_rate:.1f}%)")
|
||||
if total_tests > 0:
|
||||
improvement = data['total_passed'] - 430
|
||||
print(f" Change: {improvement:+d} tests ({pass_rate - 94.2:+.1f}%)")
|
||||
print()
|
||||
|
||||
print("WAVE 141 DIRECT FIXES (6 fixes applied):")
|
||||
print(" ✅ Agent 211: TLOB metadata test")
|
||||
print(" ✅ Agent 214: Revocation statistics (3 tests)")
|
||||
print(" ✅ Agent 215: API Gateway health endpoint")
|
||||
print(" ✅ Agent 216: MFA backup code count")
|
||||
print(" ✅ Agent 218: MFA Base32 validation")
|
||||
print(" ✅ Agent 231: Load test compilation (8 errors fixed)")
|
||||
print()
|
||||
|
||||
print("DETAILED BREAKDOWN:")
|
||||
for i, result in enumerate(data['results'], 1):
|
||||
status_symbol = "✅" if result['status'] == 'ok' else "❌"
|
||||
print(f" {status_symbol} Suite {i}: {result['passed']} passed, "
|
||||
f"{result['failed']} failed, {result['ignored']} ignored")
|
||||
print()
|
||||
|
||||
print("=" * 80)
|
||||
|
||||
return pass_rate
|
||||
|
||||
if __name__ == '__main__':
|
||||
filename = '/home/jgrusewski/Work/foxhunt/WAVE_141_FULL_TEST_RESULTS.txt'
|
||||
data = parse_test_results(filename)
|
||||
pass_rate = generate_report(data)
|
||||
|
||||
# Exit with status based on results
|
||||
sys.exit(0 if data['total_failed'] == 0 and data['compile_failures'] == 0 else 1)
|
||||
Reference in New Issue
Block a user