//! Component 5 Usage Example //! //! Demonstrates how to use the metrics calculator in the DQN evaluation pipeline. //! //! # Usage //! //! ```bash //! # This is a usage example, not a runnable binary //! # Include the module in evaluate_dqn.rs and use as shown below //! ``` use anyhow::Result; use serde::{Deserialize, Serialize}; // Import Component 5 (in production, this would be a module) // mod component5; // use component5::*; /// Example: Basic Usage /// /// Shows how to calculate metrics from a small set of inference results. fn example_basic_usage() -> Result<()> { // Simulated inference results from DQN model let results = vec![ DQNInferenceResult { action: 0, // BUY q_values: [1.25, -0.50, 0.10], // BUY has highest Q-value latency_us: 310, }, DQNInferenceResult { action: 0, // BUY q_values: [1.30, -0.40, 0.20], // BUY still best latency_us: 320, }, DQNInferenceResult { action: 2, // HOLD q_values: [0.80, -0.60, 0.90], // HOLD now best latency_us: 305, }, DQNInferenceResult { action: 1, // SELL q_values: [-0.20, 1.50, 0.30], // SELL best (regime shift) latency_us: 315, }, ]; // Calculate comprehensive metrics let metrics = calculate_metrics(&results)?; // Print summary println!("=== DQN Evaluation Metrics ==="); println!("Total bars evaluated: {}", metrics.total_bars); println!(); println!("Action Distribution:"); println!( " BUY: {} ({:.1}%)", metrics.action_distribution.buy_count, metrics.action_distribution.buy_pct ); println!( " SELL: {} ({:.1}%)", metrics.action_distribution.sell_count, metrics.action_distribution.sell_pct ); println!( " HOLD: {} ({:.1}%)", metrics.action_distribution.hold_count, metrics.action_distribution.hold_pct ); println!(); println!("Average Q-Values:"); println!(" BUY: {:.4}", metrics.avg_q_values.buy_avg); println!(" SELL: {:.4}", metrics.avg_q_values.sell_avg); println!(" HOLD: {:.4}", metrics.avg_q_values.hold_avg); println!(); println!("Latency Statistics:"); println!(" Mean: {:.2} μs", metrics.latency_stats.mean_us); println!(" Median: {} μs", metrics.latency_stats.median_us); println!(" P95: {} μs", metrics.latency_stats.p95_us); println!(" P99: {} μs", metrics.latency_stats.p99_us); println!( " Range: {} - {} μs", metrics.latency_stats.min_us, metrics.latency_stats.max_us ); println!(); println!("Policy Consistency:"); println!( " Switches: {} / {} bars", metrics.policy_consistency.total_switches, metrics.total_bars - 1 ); println!( " Rate: {:.1}%", metrics.policy_consistency.switch_rate * 100.0 ); println!(" Status: {}", metrics.policy_consistency.interpretation); Ok(()) } /// Example: Production Validation /// /// Shows how to validate metrics against production thresholds. fn example_production_validation(metrics: &EvaluationMetrics) -> Result<()> { println!("=== Production Readiness Check ==="); // Check 1: Latency P99 < 5,000μs (real-time constraint) let latency_ok = metrics.latency_stats.p99_us < 5_000; println!( "✓ Latency P99 < 5,000μs: {} (actual: {} μs) {}", latency_ok, metrics.latency_stats.p99_us, if latency_ok { "PASS ✅" } else { "FAIL ❌" } ); // Check 2: Policy consistency is moderate (10-30%) let consistency_ok = metrics.policy_consistency.switch_rate >= 0.10 && metrics.policy_consistency.switch_rate <= 0.30; println!( "✓ Policy switch rate 10-30%: {} (actual: {:.1}%) {}", consistency_ok, metrics.policy_consistency.switch_rate * 100.0, if consistency_ok { "PASS ✅" } else { "FAIL ❌" } ); // Check 3: No extreme action bias (each action >5%) let buy_ok = metrics.action_distribution.buy_pct >= 5.0; let sell_ok = metrics.action_distribution.sell_pct >= 5.0; let hold_ok = metrics.action_distribution.hold_pct >= 5.0; let balance_ok = buy_ok && sell_ok && hold_ok; println!( "✓ Balanced actions (each >5%): {} (BUY={:.1}%, SELL={:.1}%, HOLD={:.1}%) {}", balance_ok, metrics.action_distribution.buy_pct, metrics.action_distribution.sell_pct, metrics.action_distribution.hold_pct, if balance_ok { "PASS ✅" } else { "WARN ⚠️" } ); // Check 4: Q-values are finite (no NaN/Inf) let q_ok = metrics.avg_q_values.buy_avg.is_finite() && metrics.avg_q_values.sell_avg.is_finite() && metrics.avg_q_values.hold_avg.is_finite(); println!( "✓ Q-values finite: {} {}", q_ok, if q_ok { "PASS ✅" } else { "FAIL ❌" } ); println!(); let all_ok = latency_ok && consistency_ok && q_ok; if all_ok { println!("🎉 Model is PRODUCTION READY!"); } else { println!("⚠️ Model requires further tuning before production deployment"); } Ok(()) } /// Example: JSON Export /// /// Shows how to serialize metrics to JSON for CI/CD pipelines. fn example_json_export(metrics: &EvaluationMetrics) -> Result<()> { // Serialize to JSON let json = serde_json::to_string_pretty(&metrics)?; println!("=== JSON Export ==="); println!("{}", json); // In production, write to file: // std::fs::write("evaluation_metrics.json", json)?; Ok(()) } /// Example: Integration in Evaluation Loop /// /// Shows how Component 5 integrates with the full DQN evaluation pipeline. async fn example_full_pipeline() -> Result<()> { // Component 1: Load model // let model = load_dqn_model("/tmp/dqn_final_model.safetensors", device)?; // Component 2: Load data // let bars = load_ohlcv_from_parquet("test_data/ES_FUT_unseen.parquet")?; // Component 3: Compute features // let features = compute_features(&bars, warmup_bars)?; // Component 4: Run inference loop let mut results: Vec = Vec::new(); // Simulated inference loop (in production, this would iterate over features) for _bar_idx in 0..100 { // Start timer let start = std::time::Instant::now(); // Run DQN inference // let q_values = model.forward(&features[bar_idx])?; // Simulated Q-values let q_values = [0.8, -0.2, 0.1]; // Select action (argmax) let action = q_values .iter() .enumerate() .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap()) .map(|(idx, _)| idx) .unwrap(); // Record latency let latency_us = start.elapsed().as_micros() as u64; // Store result results.push(DQNInferenceResult { action, q_values, latency_us, }); } // Component 5: Calculate metrics let metrics = calculate_metrics(&results)?; // Component 6: Validate against production thresholds example_production_validation(&metrics)?; // Component 7: Export to JSON (optional) if let Ok(json) = serde_json::to_string_pretty(&metrics) { std::fs::write("evaluation_metrics.json", json)?; println!("✓ Metrics exported to evaluation_metrics.json"); } Ok(()) } // ============================================================================ // Supporting Structures (copied from Component 5 for this example) // ============================================================================ #[derive(Debug, Clone, Serialize, Deserialize)] struct DQNInferenceResult { pub action: usize, pub q_values: [f64; 3], pub latency_us: u64, } #[derive(Debug, Clone, Serialize, Deserialize)] struct EvaluationMetrics { pub total_bars: usize, pub action_distribution: ActionDistribution, pub avg_q_values: AvgQValues, pub latency_stats: LatencyStats, pub policy_consistency: PolicyConsistency, } #[derive(Debug, Clone, Serialize, Deserialize)] struct ActionDistribution { pub buy_count: usize, pub sell_count: usize, pub hold_count: usize, pub buy_pct: f64, pub sell_pct: f64, pub hold_pct: f64, } #[derive(Debug, Clone, Serialize, Deserialize)] struct AvgQValues { pub buy_avg: f64, pub sell_avg: f64, pub hold_avg: f64, } #[derive(Debug, Clone, Serialize, Deserialize)] struct LatencyStats { pub mean_us: f64, pub median_us: u64, pub p50_us: u64, pub p95_us: u64, pub p99_us: u64, pub min_us: u64, pub max_us: u64, } #[derive(Debug, Clone, Serialize, Deserialize)] struct PolicyConsistency { pub total_switches: usize, pub switch_rate: f64, pub interpretation: String, } // Placeholder for calculate_metrics (real implementation in Component 5) fn calculate_metrics(_results: &[DQNInferenceResult]) -> Result { // In production, this would call the real implementation unimplemented!("Use the real calculate_metrics from Component 5") } fn main() { println!("This is a usage example file, not a runnable binary."); println!("See the example functions above for how to use Component 5."); }