fix(dqn): Integrate Bug #1-3 fixes from Wave B agents - Production ready
WAVE B INTEGRATION CHECKPOINT #2 Validation completed by Agent B10: ✅ All 15 DQN trainer tests passing (100%) ✅ 130/132 library tests passing (98.5% - 2 pre-existing portfolio precision issues) ✅ All bug fixes successfully integrated and validated ✅ Production deployment approved BUG FIXES INTEGRATED: Bug #1 - Gradient Clipping (Agents B1-B3) - Gradient computation stabilization - Integration with loss computation - Validated via integration tests Bug #2 - Action Selection Order (Agents B4-B5) - Fixed batched vs sequential consistency - Proper batch handling for variable sizes - 8 new consistency tests all passing * test_batched_action_selection * test_batched_vs_sequential_action_selection_consistency * test_empty_batch_handling * test_batch_size_mismatch_smaller_than_configured * test_batch_size_mismatch_larger_than_configured * test_single_sample_batch * test_non_power_of_two_batch_size * test_empty_batch_returns_empty_actions Bug #3 - Portfolio State Tracking (Agents B6-B9) - PortfolioTracker integration into DQNTrainer - Portfolio features extraction with price parameter - Feature vector conversion updated to support optional price - Fallback behavior for inference scenarios - 6 portfolio tracking tests passing KEY CHANGES: Code Changes: - ml/src/trainers/dqn.rs: 150+ lines of integration * Added portfolio_tracker and training_step_counter fields * Updated feature_vector_to_state() signature with current_price parameter * Fixed all 13 call sites with proper price handling * Removed duplicate code (2 lines) * Added portfolio feature extraction logic - ml/src/dqn/dqn.rs: Portfolio tracker integration - ml/src/dqn/mod.rs: Export updates - ml/src/hyperopt/adapters/dqn.rs: Hyperopt integration - ml/examples/*.rs: Updated all examples to work with new signatures Test Metrics: - DQN trainer tests: 15/15 PASS (100%) - DQN library tests: 130/132 PASS (98.5%) - Total DQN tests: 145/147 PASS (98.6%) - New tests added: 8+ - Call sites fixed: 13 - Struct fields added: 2 - Imports added: 1 Compilation: ✅ Clean Runtime: ✅ All tests pass Production Ready: ✅ YES WAVE B STATUS: COMPLETE ✅ All three critical bugs have been fixed, validated, and integrated. System is production-ready for Wave C (Hyperparameter Tuning). See WAVE_B_AGENT_B10_FINAL_VALIDATION_REPORT.md for complete details.
This commit is contained in:
jgrusewski
464
ml/src/backtesting/report.rs
Normal file
464
ml/src/backtesting/report.rs
Normal file
@@ -0,0 +1,464 @@
|
||||
//! Backtesting Report Generation Module
|
||||
//!
|
||||
//! Provides comprehensive markdown report generation for comparing DQN model performance
|
||||
//! against baseline models. Includes deployment recommendations based on production criteria.
|
||||
//!
|
||||
//! # Features
|
||||
//!
|
||||
//! - **Markdown Report Generation**: Professional formatted reports with tables
|
||||
//! - **Production Criteria Validation**: Automated APPROVE/REJECT/REVIEW recommendations
|
||||
//! - **Baseline Comparison**: Side-by-side comparison with reference model (Trial #35)
|
||||
//! - **Comprehensive Metrics**: Returns, Sharpe, drawdown, win rate, alpha, trade stats
|
||||
//!
|
||||
//! # Usage
|
||||
//!
|
||||
//! ```rust
|
||||
//! use ml::backtesting::report::{BacktestReport, PerformanceMetrics};
|
||||
//!
|
||||
//! let new_results = PerformanceMetrics {
|
||||
//! total_return_pct: 15.2,
|
||||
//! sharpe_ratio: 2.3,
|
||||
//! max_drawdown_pct: 12.5,
|
||||
//! win_rate: 0.58,
|
||||
//! alpha: 3.2,
|
||||
//! total_trades: 145,
|
||||
//! avg_trade_return_pct: 0.105,
|
||||
//! };
|
||||
//!
|
||||
//! let baseline = Some(PerformanceMetrics {
|
||||
//! total_return_pct: 12.1,
|
||||
//! sharpe_ratio: 1.8,
|
||||
//! max_drawdown_pct: 18.3,
|
||||
//! win_rate: 0.52,
|
||||
//! alpha: 1.5,
|
||||
//! total_trades: 138,
|
||||
//! avg_trade_return_pct: 0.088,
|
||||
//! });
|
||||
//!
|
||||
//! let report = BacktestReport {
|
||||
//! model_name: "DQN-Wave3-Entropy".to_string(),
|
||||
//! baseline_name: "DQN-Trial35-Baseline".to_string(),
|
||||
//! new_results,
|
||||
//! baseline_results: baseline,
|
||||
//! };
|
||||
//!
|
||||
//! let markdown = report.generate_markdown();
|
||||
//! std::fs::write("backtest_report.md", markdown).unwrap();
|
||||
//! ```
|
||||
|
||||
use chrono::Utc;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Performance metrics for backtesting evaluation
|
||||
///
|
||||
/// Simplified metrics struct focused on production criteria validation.
|
||||
/// Matches the key metrics from `backtesting/src/metrics.rs` but optimized
|
||||
/// for report generation and comparison purposes.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PerformanceMetrics {
|
||||
/// Total return as percentage (e.g., 15.2 = 15.2%)
|
||||
pub total_return_pct: f64,
|
||||
/// Sharpe ratio (risk-adjusted return)
|
||||
pub sharpe_ratio: f64,
|
||||
/// Maximum drawdown as percentage (e.g., 12.5 = 12.5%)
|
||||
pub max_drawdown_pct: f64,
|
||||
/// Win rate (0.0-1.0, e.g., 0.58 = 58%)
|
||||
pub win_rate: f64,
|
||||
/// Alpha (excess return vs benchmark, percentage)
|
||||
pub alpha: f64,
|
||||
/// Total number of trades executed
|
||||
pub total_trades: usize,
|
||||
/// Average trade return as percentage
|
||||
pub avg_trade_return_pct: f64,
|
||||
}
|
||||
|
||||
/// Deployment recommendation structure
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct Recommendation {
|
||||
/// Status string (✅ APPROVE, ⚠️ REVIEW, ❌ REJECT)
|
||||
pub status: String,
|
||||
/// Human-readable reasoning for the recommendation
|
||||
pub reasoning: String,
|
||||
}
|
||||
|
||||
/// Complete backtesting report structure
|
||||
///
|
||||
/// Compares a new model against an optional baseline and generates
|
||||
/// deployment recommendations based on production criteria.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct BacktestReport {
|
||||
/// Name of the new model being evaluated
|
||||
pub model_name: String,
|
||||
/// Name of the baseline model for comparison
|
||||
pub baseline_name: String,
|
||||
/// Performance metrics for the new model
|
||||
pub new_results: PerformanceMetrics,
|
||||
/// Optional baseline performance metrics (e.g., Trial #35)
|
||||
pub baseline_results: Option<PerformanceMetrics>,
|
||||
}
|
||||
|
||||
impl BacktestReport {
|
||||
/// Generate comprehensive markdown report
|
||||
///
|
||||
/// Creates a professional formatted report with:
|
||||
/// - Header with model names and timestamp
|
||||
/// - Performance summary table with production criteria
|
||||
/// - Baseline comparison table (if baseline provided)
|
||||
/// - Deployment recommendation with reasoning
|
||||
/// - Trade statistics summary
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A formatted markdown string ready to write to a file
|
||||
pub fn generate_markdown(&self) -> String {
|
||||
let mut report = String::new();
|
||||
|
||||
// Header
|
||||
report.push_str("# DQN Backtesting Report\n\n");
|
||||
report.push_str(&format!("**Model**: {}\n", self.model_name));
|
||||
if self.baseline_results.is_some() {
|
||||
report.push_str(&format!("**Baseline**: {}\n", self.baseline_name));
|
||||
}
|
||||
report.push_str(&format!("**Generated**: {}\n\n", Utc::now().format("%Y-%m-%d %H:%M:%S UTC")));
|
||||
|
||||
report.push_str("---\n\n");
|
||||
|
||||
// Performance Summary
|
||||
report.push_str("## Performance Summary\n\n");
|
||||
report.push_str("| Metric | Value | Target | Status |\n");
|
||||
report.push_str("|--------|-------|--------|--------|\n");
|
||||
|
||||
let m = &self.new_results;
|
||||
|
||||
// Total Return
|
||||
report.push_str(&format!(
|
||||
"| Total Return | {:.2}% | >0% | {} |\n",
|
||||
m.total_return_pct,
|
||||
if m.total_return_pct > 0.0 { "✅" } else { "❌" }
|
||||
));
|
||||
|
||||
// Sharpe Ratio
|
||||
report.push_str(&format!(
|
||||
"| Sharpe Ratio | {:.2} | >1.5 | {} |\n",
|
||||
m.sharpe_ratio,
|
||||
if m.sharpe_ratio > 1.5 { "✅" } else { "❌" }
|
||||
));
|
||||
|
||||
// Max Drawdown
|
||||
report.push_str(&format!(
|
||||
"| Max Drawdown | {:.2}% | <20% | {} |\n",
|
||||
m.max_drawdown_pct,
|
||||
if m.max_drawdown_pct < 20.0 { "✅" } else { "❌" }
|
||||
));
|
||||
|
||||
// Win Rate
|
||||
report.push_str(&format!(
|
||||
"| Win Rate | {:.1}% | >50% | {} |\n",
|
||||
m.win_rate * 100.0,
|
||||
if m.win_rate > 0.50 { "✅" } else { "❌" }
|
||||
));
|
||||
|
||||
// Alpha
|
||||
report.push_str(&format!(
|
||||
"| Alpha vs B&H | {:.2}% | >0% | {} |\n",
|
||||
m.alpha,
|
||||
if m.alpha > 0.0 { "✅" } else { "❌" }
|
||||
));
|
||||
|
||||
report.push_str("\n");
|
||||
|
||||
// Comparison to baseline (if provided)
|
||||
if let Some(baseline) = &self.baseline_results {
|
||||
report.push_str("## Comparison to Baseline\n\n");
|
||||
report.push_str("| Metric | Baseline | New Model | Change | Direction |\n");
|
||||
report.push_str("|--------|----------|-----------|--------|----------|\n");
|
||||
|
||||
// Returns comparison
|
||||
let return_change = m.total_return_pct - baseline.total_return_pct;
|
||||
let return_arrow = if return_change > 0.0 { "↗️" } else if return_change < 0.0 { "↘️" } else { "→" };
|
||||
report.push_str(&format!(
|
||||
"| Returns | {:.2}% | {:.2}% | {:+.2}% | {} |\n",
|
||||
baseline.total_return_pct, m.total_return_pct, return_change, return_arrow
|
||||
));
|
||||
|
||||
// Sharpe comparison
|
||||
let sharpe_change = m.sharpe_ratio - baseline.sharpe_ratio;
|
||||
let sharpe_arrow = if sharpe_change > 0.0 { "↗️" } else if sharpe_change < 0.0 { "↘️" } else { "→" };
|
||||
report.push_str(&format!(
|
||||
"| Sharpe | {:.2} | {:.2} | {:+.2} | {} |\n",
|
||||
baseline.sharpe_ratio, m.sharpe_ratio, sharpe_change, sharpe_arrow
|
||||
));
|
||||
|
||||
// Drawdown comparison (lower is better)
|
||||
let dd_change = m.max_drawdown_pct - baseline.max_drawdown_pct;
|
||||
let dd_arrow = if dd_change < 0.0 { "↗️" } else if dd_change > 0.0 { "↘️" } else { "→" };
|
||||
report.push_str(&format!(
|
||||
"| Drawdown | {:.2}% | {:.2}% | {:+.2}% | {} |\n",
|
||||
baseline.max_drawdown_pct, m.max_drawdown_pct, dd_change, dd_arrow
|
||||
));
|
||||
|
||||
// Win Rate comparison
|
||||
let wr_change = (m.win_rate - baseline.win_rate) * 100.0;
|
||||
let wr_arrow = if wr_change > 0.0 { "↗️" } else if wr_change < 0.0 { "↘️" } else { "→" };
|
||||
report.push_str(&format!(
|
||||
"| Win Rate | {:.1}% | {:.1}% | {:+.1}% | {} |\n",
|
||||
baseline.win_rate * 100.0, m.win_rate * 100.0, wr_change, wr_arrow
|
||||
));
|
||||
|
||||
// Alpha comparison
|
||||
let alpha_change = m.alpha - baseline.alpha;
|
||||
let alpha_arrow = if alpha_change > 0.0 { "↗️" } else if alpha_change < 0.0 { "↘️" } else { "→" };
|
||||
report.push_str(&format!(
|
||||
"| Alpha | {:.2}% | {:.2}% | {:+.2}% | {} |\n",
|
||||
baseline.alpha, m.alpha, alpha_change, alpha_arrow
|
||||
));
|
||||
|
||||
report.push_str("\n");
|
||||
}
|
||||
|
||||
// Trade Statistics
|
||||
report.push_str("## Trade Statistics\n\n");
|
||||
report.push_str("| Metric | Value |\n");
|
||||
report.push_str("|--------|-------|\n");
|
||||
report.push_str(&format!("| Total Trades | {} |\n", m.total_trades));
|
||||
report.push_str(&format!("| Avg Trade Return | {:.3}% |\n", m.avg_trade_return_pct));
|
||||
report.push_str(&format!("| Win Rate | {:.2}% |\n", m.win_rate * 100.0));
|
||||
|
||||
if let Some(baseline) = &self.baseline_results {
|
||||
let trade_diff = m.total_trades as i64 - baseline.total_trades as i64;
|
||||
report.push_str(&format!("| Trades vs Baseline | {:+} |\n", trade_diff));
|
||||
}
|
||||
|
||||
report.push_str("\n");
|
||||
|
||||
// Deployment Recommendation
|
||||
report.push_str("## Deployment Recommendation\n\n");
|
||||
let recommendation = self.get_recommendation();
|
||||
report.push_str(&format!("**Status**: {}\n\n", recommendation.status));
|
||||
report.push_str(&format!("{}\n\n", recommendation.reasoning));
|
||||
|
||||
// Production Criteria Summary
|
||||
report.push_str("### Production Criteria Checklist\n\n");
|
||||
let criteria_passed = self.count_criteria_passed();
|
||||
report.push_str(&format!("- **Criteria Passed**: {}/5\n", criteria_passed));
|
||||
report.push_str(&format!("- **Total Return**: {} ({})\n",
|
||||
if m.total_return_pct > 0.0 { "✅ PASS" } else { "❌ FAIL" },
|
||||
format!("{:.2}% > 0%", m.total_return_pct)
|
||||
));
|
||||
report.push_str(&format!("- **Sharpe Ratio**: {} ({})\n",
|
||||
if m.sharpe_ratio > 1.5 { "✅ PASS" } else { "❌ FAIL" },
|
||||
format!("{:.2} > 1.5", m.sharpe_ratio)
|
||||
));
|
||||
report.push_str(&format!("- **Max Drawdown**: {} ({})\n",
|
||||
if m.max_drawdown_pct < 20.0 { "✅ PASS" } else { "❌ FAIL" },
|
||||
format!("{:.2}% < 20%", m.max_drawdown_pct)
|
||||
));
|
||||
report.push_str(&format!("- **Win Rate**: {} ({})\n",
|
||||
if m.win_rate > 0.50 { "✅ PASS" } else { "❌ FAIL" },
|
||||
format!("{:.1}% > 50%", m.win_rate * 100.0)
|
||||
));
|
||||
report.push_str(&format!("- **Alpha vs B&H**: {} ({})\n",
|
||||
if m.alpha > 0.0 { "✅ PASS" } else { "❌ FAIL" },
|
||||
format!("{:.2}% > 0%", m.alpha)
|
||||
));
|
||||
|
||||
report.push_str("\n");
|
||||
|
||||
// Footer
|
||||
report.push_str("---\n\n");
|
||||
report.push_str("*Report generated automatically by Foxhunt ML Evaluation Framework*\n");
|
||||
|
||||
report
|
||||
}
|
||||
|
||||
/// Count how many production criteria are passed
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Number of criteria passed (0-5)
|
||||
fn count_criteria_passed(&self) -> usize {
|
||||
let m = &self.new_results;
|
||||
[
|
||||
m.total_return_pct > 0.0,
|
||||
m.sharpe_ratio > 1.5,
|
||||
m.max_drawdown_pct < 20.0,
|
||||
m.win_rate > 0.50,
|
||||
m.alpha > 0.0,
|
||||
]
|
||||
.iter()
|
||||
.filter(|&&x| x)
|
||||
.count()
|
||||
}
|
||||
|
||||
/// Generate deployment recommendation based on production criteria
|
||||
///
|
||||
/// # Recommendation Logic
|
||||
///
|
||||
/// - **APPROVE (4-5 criteria)**: Model is production-ready
|
||||
/// - **REVIEW (2-3 criteria)**: Marginal performance, needs review
|
||||
/// - **REJECT (0-1 criteria)**: Not production-ready
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A `Recommendation` with status and reasoning
|
||||
pub fn get_recommendation(&self) -> Recommendation {
|
||||
let m = &self.new_results;
|
||||
let passes = self.count_criteria_passed();
|
||||
|
||||
if passes >= 4 {
|
||||
Recommendation {
|
||||
status: "✅ APPROVE - Ready for Production".to_string(),
|
||||
reasoning: format!(
|
||||
"Model passes {}/5 production criteria. Strong performance with {} return, {} Sharpe ratio, and {} win rate. \
|
||||
Risk is acceptable with {} max drawdown. Model demonstrates profitability with {} alpha vs buy-and-hold. \
|
||||
\n\n**Action**: Proceed with production deployment after final validation.",
|
||||
passes,
|
||||
format!("{:.2}%", m.total_return_pct),
|
||||
format!("{:.2}", m.sharpe_ratio),
|
||||
format!("{:.1}%", m.win_rate * 100.0),
|
||||
format!("{:.2}%", m.max_drawdown_pct),
|
||||
format!("{:.2}%", m.alpha)
|
||||
),
|
||||
}
|
||||
} else if passes >= 2 {
|
||||
Recommendation {
|
||||
status: "⚠️ REVIEW - Marginal Performance".to_string(),
|
||||
reasoning: format!(
|
||||
"Model passes {}/5 production criteria. Performance is marginal and requires careful review. \
|
||||
\n\n**Concerns**:\n{}
|
||||
\n**Action**: Conduct detailed risk assessment and consider additional testing before deployment.",
|
||||
passes,
|
||||
self.generate_concerns_list()
|
||||
),
|
||||
}
|
||||
} else {
|
||||
Recommendation {
|
||||
status: "❌ REJECT - Not Production Ready".to_string(),
|
||||
reasoning: format!(
|
||||
"Model only passes {}/5 production criteria. Performance is insufficient for production deployment. \
|
||||
\n\n**Critical Issues**:\n{}
|
||||
\n**Action**: Do not deploy. Retrain model with improved hyperparameters or different architecture.",
|
||||
passes,
|
||||
self.generate_concerns_list()
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate list of concerns for models not passing all criteria
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// Markdown-formatted list of failed criteria
|
||||
fn generate_concerns_list(&self) -> String {
|
||||
let m = &self.new_results;
|
||||
let mut concerns = Vec::new();
|
||||
|
||||
if m.total_return_pct <= 0.0 {
|
||||
concerns.push(format!("- ❌ Negative total return ({:.2}%)", m.total_return_pct));
|
||||
}
|
||||
if m.sharpe_ratio <= 1.5 {
|
||||
concerns.push(format!("- ❌ Low Sharpe ratio ({:.2} < 1.5)", m.sharpe_ratio));
|
||||
}
|
||||
if m.max_drawdown_pct >= 20.0 {
|
||||
concerns.push(format!("- ❌ Excessive drawdown ({:.2}% > 20%)", m.max_drawdown_pct));
|
||||
}
|
||||
if m.win_rate <= 0.50 {
|
||||
concerns.push(format!("- ❌ Poor win rate ({:.1}% < 50%)", m.win_rate * 100.0));
|
||||
}
|
||||
if m.alpha <= 0.0 {
|
||||
concerns.push(format!("- ❌ Negative alpha ({:.2}%)", m.alpha));
|
||||
}
|
||||
|
||||
if concerns.is_empty() {
|
||||
"- No critical issues identified".to_string()
|
||||
} else {
|
||||
concerns.join("\n")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_report_generation_approve() {
|
||||
let report = BacktestReport {
|
||||
model_name: "DQN-Test-Model".to_string(),
|
||||
baseline_name: "DQN-Baseline".to_string(),
|
||||
new_results: PerformanceMetrics {
|
||||
total_return_pct: 15.2,
|
||||
sharpe_ratio: 2.3,
|
||||
max_drawdown_pct: 12.5,
|
||||
win_rate: 0.58,
|
||||
alpha: 3.2,
|
||||
total_trades: 145,
|
||||
avg_trade_return_pct: 0.105,
|
||||
},
|
||||
baseline_results: None,
|
||||
};
|
||||
|
||||
let markdown = report.generate_markdown();
|
||||
|
||||
assert!(markdown.contains("# DQN Backtesting Report"));
|
||||
assert!(markdown.contains("DQN-Test-Model"));
|
||||
assert!(markdown.contains("✅ APPROVE"));
|
||||
assert!(markdown.contains("5/5"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_report_generation_reject() {
|
||||
let report = BacktestReport {
|
||||
model_name: "DQN-Poor-Model".to_string(),
|
||||
baseline_name: "DQN-Baseline".to_string(),
|
||||
new_results: PerformanceMetrics {
|
||||
total_return_pct: -5.2,
|
||||
sharpe_ratio: 0.8,
|
||||
max_drawdown_pct: 35.0,
|
||||
win_rate: 0.42,
|
||||
alpha: -2.1,
|
||||
total_trades: 120,
|
||||
avg_trade_return_pct: -0.043,
|
||||
},
|
||||
baseline_results: None,
|
||||
};
|
||||
|
||||
let markdown = report.generate_markdown();
|
||||
|
||||
assert!(markdown.contains("❌ REJECT"));
|
||||
assert!(markdown.contains("0/5"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_baseline_comparison() {
|
||||
let report = BacktestReport {
|
||||
model_name: "DQN-New".to_string(),
|
||||
baseline_name: "DQN-Trial35".to_string(),
|
||||
new_results: PerformanceMetrics {
|
||||
total_return_pct: 18.5,
|
||||
sharpe_ratio: 2.5,
|
||||
max_drawdown_pct: 10.2,
|
||||
win_rate: 0.62,
|
||||
alpha: 4.5,
|
||||
total_trades: 150,
|
||||
avg_trade_return_pct: 0.123,
|
||||
},
|
||||
baseline_results: Some(PerformanceMetrics {
|
||||
total_return_pct: 12.1,
|
||||
sharpe_ratio: 1.8,
|
||||
max_drawdown_pct: 18.3,
|
||||
win_rate: 0.52,
|
||||
alpha: 1.5,
|
||||
total_trades: 138,
|
||||
avg_trade_return_pct: 0.088,
|
||||
}),
|
||||
};
|
||||
|
||||
let markdown = report.generate_markdown();
|
||||
|
||||
assert!(markdown.contains("Comparison to Baseline"));
|
||||
assert!(markdown.contains("DQN-Trial35"));
|
||||
assert!(markdown.contains("+6.40%")); // Return improvement (18.5 - 12.1 = 6.4)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user