Removes dead code across 28 files: - 31 commented-out "DISABLED" import lines (mostly safe_operations, error_handling) - Commented-out module declarations in lib.rs (deployment, model_loader_integration, tests) - Commented-out re-exports in lib.rs (training_pipeline, deployment::ModelVersion) - Commented-out adaptive strategy modules in regime/mod.rs All are in git history if ever needed. Net -74 lines removed. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
824 lines
29 KiB
Rust
824 lines
29 KiB
Rust
//! # Performance Monitor
|
|
//!
|
|
//! Real-time performance monitoring and alerting for ML models
|
|
//! with HFT-specific metrics and latency tracking.
|
|
|
|
use std::collections::{HashMap, VecDeque};
|
|
use std::sync::Arc;
|
|
use std::time::{Duration, SystemTime};
|
|
|
|
use crate::observability::alerts::AlertSeverity;
|
|
use serde::{Deserialize, Serialize};
|
|
use tokio::sync::RwLock; // Use local AlertSeverity with Warning variant
|
|
|
|
use super::IntegrationHubConfig;
|
|
|
|
/// Performance sample for monitoring
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct PerformanceSample {
|
|
/// Sample timestamp
|
|
pub timestamp: SystemTime,
|
|
/// Model identifier
|
|
pub model_id: String,
|
|
/// Latency in microseconds
|
|
pub latency_us: u64,
|
|
/// Memory usage in MB
|
|
pub memory_usage_mb: f64,
|
|
/// CPU utilization percentage
|
|
pub cpu_utilization: f64,
|
|
/// Whether the operation was successful
|
|
pub success: bool,
|
|
/// Request size in bytes
|
|
pub request_size_bytes: usize,
|
|
/// Response size in bytes
|
|
pub response_size_bytes: usize,
|
|
/// Queue depth at time of request
|
|
pub queue_depth: usize,
|
|
/// Whether prediction was correct (if known)
|
|
pub prediction_correct: Option<bool>,
|
|
/// Confidence score of prediction
|
|
pub prediction_confidence: Option<f64>,
|
|
/// Actual outcome (if available for validation)
|
|
pub actual_outcome: Option<bool>,
|
|
/// Type of prediction (direction, volatility, etc.)
|
|
pub prediction_type: Option<String>,
|
|
/// Market regime during prediction
|
|
pub market_regime: Option<String>,
|
|
}
|
|
|
|
/// Performance alert configuration
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct AlertConfig {
|
|
/// Enable latency alerts
|
|
pub enable_latency_alerts: bool,
|
|
/// Latency threshold in microseconds
|
|
pub latency_threshold_us: u64,
|
|
/// Enable memory alerts
|
|
pub enable_memory_alerts: bool,
|
|
/// Memory threshold in MB
|
|
pub memory_threshold_mb: f64,
|
|
/// Enable accuracy alerts
|
|
pub enable_accuracy_alerts: bool,
|
|
/// Minimum accuracy threshold
|
|
pub accuracy_threshold: f64,
|
|
/// Alert cooldown period in seconds
|
|
pub alert_cooldown_seconds: u64,
|
|
}
|
|
|
|
impl Default for AlertConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
enable_latency_alerts: true,
|
|
latency_threshold_us: 1000, // 1ms
|
|
enable_memory_alerts: true,
|
|
memory_threshold_mb: 500.0,
|
|
enable_accuracy_alerts: true,
|
|
accuracy_threshold: 0.7,
|
|
alert_cooldown_seconds: 300, // 5 minutes
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Performance alert
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct PerformanceAlert {
|
|
/// Alert timestamp
|
|
pub timestamp: SystemTime,
|
|
/// Alert severity
|
|
pub severity: AlertSeverity,
|
|
/// Alert message
|
|
pub message: String,
|
|
/// Model ID that triggered the alert
|
|
pub model_id: String,
|
|
/// Alert type
|
|
pub alert_type: AlertType,
|
|
/// Current value that triggered alert
|
|
pub current_value: f64,
|
|
/// Threshold that was exceeded
|
|
pub threshold: f64,
|
|
}
|
|
|
|
/// Alert types
|
|
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)]
|
|
pub enum AlertType {
|
|
/// High latency alert
|
|
HighLatency,
|
|
/// High memory usage alert
|
|
HighMemoryUsage,
|
|
/// Low accuracy alert
|
|
LowAccuracy,
|
|
/// Model failure alert
|
|
ModelFailure,
|
|
/// Queue overflow alert
|
|
QueueOverflow,
|
|
}
|
|
|
|
/// Performance statistics
|
|
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
|
pub struct PerformanceStats {
|
|
/// Total samples collected
|
|
pub total_samples: u64,
|
|
/// Average latency in microseconds
|
|
pub avg_latency_us: f64,
|
|
/// 95th percentile latency
|
|
pub p95_latency_us: f64,
|
|
/// 99th percentile latency
|
|
pub p99_latency_us: f64,
|
|
/// Maximum latency observed
|
|
pub max_latency_us: u64,
|
|
/// Average memory usage in MB
|
|
pub avg_memory_mb: f64,
|
|
/// Peak memory usage in MB
|
|
pub peak_memory_mb: f64,
|
|
/// Average CPU utilization
|
|
pub avg_cpu_utilization: f64,
|
|
/// Success rate percentage
|
|
pub success_rate: f64,
|
|
/// Prediction accuracy (if available)
|
|
pub prediction_accuracy: Option<f64>,
|
|
/// Throughput (requests per second)
|
|
pub throughput_rps: f64,
|
|
}
|
|
|
|
/// Dashboard data for monitoring
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct DashboardData {
|
|
/// Overall performance statistics
|
|
pub overall_stats: PerformanceStats,
|
|
/// Per-model performance statistics
|
|
pub model_stats: HashMap<String, PerformanceStats>,
|
|
/// Recent alerts
|
|
pub recent_alerts: Vec<PerformanceAlert>,
|
|
/// Latency histogram
|
|
pub latency_histogram: HashMap<String, u64>,
|
|
/// Real-time metrics
|
|
pub realtime_metrics: RealtimeMetrics,
|
|
}
|
|
|
|
/// Real-time metrics
|
|
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
|
pub struct RealtimeMetrics {
|
|
/// Current requests per second
|
|
pub current_rps: f64,
|
|
/// Current average latency (last minute)
|
|
pub current_avg_latency_us: f64,
|
|
/// Current memory usage
|
|
pub current_memory_mb: f64,
|
|
/// Current CPU utilization
|
|
pub current_cpu_utilization: f64,
|
|
/// Active models count
|
|
pub active_models: usize,
|
|
/// Queue depth
|
|
pub queue_depth: usize,
|
|
}
|
|
|
|
/// Performance Monitor for ML models
|
|
#[derive(Debug)]
|
|
pub struct PerformanceMonitor {
|
|
/// Configuration reference
|
|
config: Arc<IntegrationHubConfig>,
|
|
/// Alert configuration
|
|
alert_config: AlertConfig,
|
|
/// Performance samples storage
|
|
samples: Arc<RwLock<VecDeque<PerformanceSample>>>,
|
|
/// Per-model sample storage
|
|
model_samples: Arc<RwLock<HashMap<String, VecDeque<PerformanceSample>>>>,
|
|
/// Recent alerts
|
|
alerts: Arc<RwLock<VecDeque<PerformanceAlert>>>,
|
|
/// Last alert timestamps for cooldown
|
|
last_alert_times: Arc<RwLock<HashMap<(String, AlertType), SystemTime>>>,
|
|
/// Real-time metrics calculation
|
|
realtime_calculator: Arc<RwLock<RealtimeCalculator>>,
|
|
}
|
|
|
|
/// Real-time metrics calculator
|
|
#[derive(Debug, Default)]
|
|
struct RealtimeCalculator {
|
|
/// Samples in last minute
|
|
last_minute_samples: VecDeque<PerformanceSample>,
|
|
/// Last update time
|
|
last_update: Option<SystemTime>,
|
|
}
|
|
|
|
impl PerformanceMonitor {
|
|
/// Create new performance monitor
|
|
pub fn new(config: &IntegrationHubConfig) -> Self {
|
|
Self {
|
|
config: Arc::new(config.clone()),
|
|
alert_config: AlertConfig::default(),
|
|
samples: Arc::new(RwLock::new(VecDeque::new())),
|
|
model_samples: Arc::new(RwLock::new(HashMap::new())),
|
|
alerts: Arc::new(RwLock::new(VecDeque::new())),
|
|
last_alert_times: Arc::new(RwLock::new(HashMap::new())),
|
|
realtime_calculator: Arc::new(RwLock::new(RealtimeCalculator::default())),
|
|
}
|
|
}
|
|
|
|
/// Create monitor with custom alert configuration
|
|
pub fn with_alert_config(config: &IntegrationHubConfig, alert_config: AlertConfig) -> Self {
|
|
let mut monitor = Self::new(config);
|
|
monitor.alert_config = alert_config;
|
|
monitor
|
|
}
|
|
|
|
/// Record performance sample
|
|
pub async fn record_sample(&self, sample: PerformanceSample) {
|
|
// Add to global samples
|
|
{
|
|
let mut samples = self.samples.write().await;
|
|
samples.push_back(sample.clone());
|
|
|
|
// Keep only recent samples (last 10000)
|
|
if samples.len() > 10000 {
|
|
samples.pop_front();
|
|
}
|
|
}
|
|
|
|
// Add to model-specific samples
|
|
{
|
|
let mut model_samples = self.model_samples.write().await;
|
|
let model_samples_vec = model_samples
|
|
.entry(sample.model_id.clone())
|
|
.or_insert_with(VecDeque::new);
|
|
model_samples_vec.push_back(sample.clone());
|
|
|
|
// Keep only recent samples per model (last 1000)
|
|
if model_samples_vec.len() > 1000 {
|
|
model_samples_vec.pop_front();
|
|
}
|
|
}
|
|
|
|
// Update real-time calculator
|
|
{
|
|
let mut calculator = self.realtime_calculator.write().await;
|
|
calculator.last_minute_samples.push_back(sample.clone());
|
|
|
|
// Remove samples older than 1 minute
|
|
let one_minute_ago = SystemTime::now() - Duration::from_secs(60);
|
|
while let Some(front_sample) = calculator.last_minute_samples.front() {
|
|
if front_sample.timestamp < one_minute_ago {
|
|
calculator.last_minute_samples.pop_front();
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
|
|
calculator.last_update = Some(SystemTime::now());
|
|
}
|
|
|
|
// Check for alerts
|
|
self.check_alerts(&sample).await;
|
|
}
|
|
|
|
/// Check for performance alerts
|
|
async fn check_alerts(&self, sample: &PerformanceSample) {
|
|
let mut alerts_to_add = Vec::new();
|
|
|
|
// Check latency alert
|
|
if self.alert_config.enable_latency_alerts
|
|
&& sample.latency_us > self.alert_config.latency_threshold_us
|
|
{
|
|
if self
|
|
.should_send_alert(&sample.model_id, AlertType::HighLatency)
|
|
.await
|
|
{
|
|
alerts_to_add.push(PerformanceAlert {
|
|
timestamp: SystemTime::now(),
|
|
severity: AlertSeverity::Critical,
|
|
message: format!(
|
|
"High latency detected for model {}: {}μs (threshold: {}μs)",
|
|
sample.model_id, sample.latency_us, self.alert_config.latency_threshold_us
|
|
),
|
|
model_id: sample.model_id.clone(),
|
|
alert_type: AlertType::HighLatency,
|
|
current_value: sample.latency_us as f64,
|
|
threshold: self.alert_config.latency_threshold_us as f64,
|
|
});
|
|
}
|
|
}
|
|
|
|
// Check memory alert
|
|
if self.alert_config.enable_memory_alerts
|
|
&& sample.memory_usage_mb > self.alert_config.memory_threshold_mb
|
|
{
|
|
if self
|
|
.should_send_alert(&sample.model_id, AlertType::HighMemoryUsage)
|
|
.await
|
|
{
|
|
alerts_to_add.push(PerformanceAlert {
|
|
timestamp: SystemTime::now(),
|
|
severity: AlertSeverity::Warning,
|
|
message: format!(
|
|
"High memory usage detected for model {}: {:.1}MB (threshold: {:.1}MB)",
|
|
sample.model_id,
|
|
sample.memory_usage_mb,
|
|
self.alert_config.memory_threshold_mb
|
|
),
|
|
model_id: sample.model_id.clone(),
|
|
alert_type: AlertType::HighMemoryUsage,
|
|
current_value: sample.memory_usage_mb,
|
|
threshold: self.alert_config.memory_threshold_mb,
|
|
});
|
|
}
|
|
}
|
|
|
|
// Check failure alert
|
|
if !sample.success {
|
|
if self
|
|
.should_send_alert(&sample.model_id, AlertType::ModelFailure)
|
|
.await
|
|
{
|
|
alerts_to_add.push(PerformanceAlert {
|
|
timestamp: SystemTime::now(),
|
|
severity: AlertSeverity::Critical,
|
|
message: format!("Model failure detected for model {}", sample.model_id),
|
|
model_id: sample.model_id.clone(),
|
|
alert_type: AlertType::ModelFailure,
|
|
current_value: 0.0,
|
|
threshold: 1.0,
|
|
});
|
|
}
|
|
}
|
|
|
|
// Add all alerts
|
|
if !alerts_to_add.is_empty() {
|
|
let mut alerts = self.alerts.write().await;
|
|
let mut last_alert_times = self.last_alert_times.write().await;
|
|
|
|
for alert in alerts_to_add {
|
|
// Update last alert time
|
|
last_alert_times
|
|
.insert((alert.model_id.clone(), alert.alert_type), alert.timestamp);
|
|
|
|
alerts.push_back(alert);
|
|
|
|
// Keep only recent alerts (last 100)
|
|
if alerts.len() > 100 {
|
|
alerts.pop_front();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Check if alert should be sent (considering cooldown)
|
|
async fn should_send_alert(&self, model_id: &str, alert_type: AlertType) -> bool {
|
|
let last_alert_times = self.last_alert_times.read().await;
|
|
|
|
if let Some(&last_time) = last_alert_times.get(&(model_id.to_string(), alert_type)) {
|
|
let cooldown = Duration::from_secs(self.alert_config.alert_cooldown_seconds);
|
|
SystemTime::now()
|
|
.duration_since(last_time)
|
|
.unwrap_or(cooldown)
|
|
>= cooldown
|
|
} else {
|
|
true // No previous alert
|
|
}
|
|
}
|
|
|
|
/// Calculate performance statistics
|
|
pub async fn calculate_performance_stats(&self, model_id: Option<&str>) -> PerformanceStats {
|
|
let samples = if let Some(model_id) = model_id {
|
|
let model_samples = self.model_samples.read().await;
|
|
model_samples.get(model_id).cloned().unwrap_or_default()
|
|
} else {
|
|
self.samples.read().await.clone()
|
|
};
|
|
|
|
if samples.is_empty() {
|
|
return PerformanceStats::default();
|
|
}
|
|
|
|
let mut latencies: Vec<u64> = samples.iter().map(|s| s.latency_us).collect();
|
|
latencies.sort_unstable();
|
|
|
|
let total_samples = samples.len() as u64;
|
|
let avg_latency_us = latencies.iter().sum::<u64>() as f64 / latencies.len() as f64;
|
|
|
|
let p95_idx = (latencies.len() as f64 * 0.95) as usize;
|
|
let p99_idx = (latencies.len() as f64 * 0.99) as usize;
|
|
|
|
let p95_latency_us = latencies
|
|
.get(p95_idx.min(latencies.len() - 1))
|
|
.copied()
|
|
.unwrap_or(0) as f64;
|
|
let p99_latency_us = latencies
|
|
.get(p99_idx.min(latencies.len() - 1))
|
|
.copied()
|
|
.unwrap_or(0) as f64;
|
|
let max_latency_us = latencies.iter().max().copied().unwrap_or(0);
|
|
|
|
let avg_memory_mb =
|
|
samples.iter().map(|s| s.memory_usage_mb).sum::<f64>() / samples.len() as f64;
|
|
let peak_memory_mb = samples
|
|
.iter()
|
|
.map(|s| s.memory_usage_mb)
|
|
.fold(0.0, f64::max);
|
|
let avg_cpu_utilization =
|
|
samples.iter().map(|s| s.cpu_utilization).sum::<f64>() / samples.len() as f64;
|
|
|
|
let success_count = samples.iter().filter(|s| s.success).count();
|
|
let success_rate = (success_count as f64 / samples.len() as f64) * 100.0;
|
|
|
|
// Calculate prediction accuracy if available
|
|
let prediction_accuracy = {
|
|
let correct_predictions = samples
|
|
.iter()
|
|
.filter_map(|s| s.prediction_correct)
|
|
.filter(|&correct| correct)
|
|
.count();
|
|
let total_predictions = samples.iter().filter_map(|s| s.prediction_correct).count();
|
|
|
|
if total_predictions > 0 {
|
|
Some((correct_predictions as f64 / total_predictions as f64) * 100.0)
|
|
} else {
|
|
None
|
|
}
|
|
};
|
|
|
|
// Calculate throughput (samples per second)
|
|
let throughput_rps = if let (Some(first), Some(last)) = (samples.front(), samples.back()) {
|
|
if let Ok(duration) = last.timestamp.duration_since(first.timestamp) {
|
|
let duration_secs = duration.as_secs_f64();
|
|
if duration_secs > 0.0 {
|
|
samples.len() as f64 / duration_secs
|
|
} else {
|
|
0.0
|
|
}
|
|
} else {
|
|
0.0
|
|
}
|
|
} else {
|
|
0.0
|
|
};
|
|
|
|
PerformanceStats {
|
|
total_samples,
|
|
avg_latency_us,
|
|
p95_latency_us,
|
|
p99_latency_us,
|
|
max_latency_us,
|
|
avg_memory_mb,
|
|
peak_memory_mb,
|
|
avg_cpu_utilization,
|
|
success_rate,
|
|
prediction_accuracy,
|
|
throughput_rps,
|
|
}
|
|
}
|
|
|
|
/// Calculate accuracy metrics for a specific model
|
|
pub async fn calculate_accuracy_metrics(&self, model_id: &str) -> HashMap<String, f64> {
|
|
let model_samples = self.model_samples.read().await;
|
|
let samples = model_samples.get(model_id);
|
|
|
|
let mut metrics = HashMap::new();
|
|
|
|
if let Some(samples) = samples {
|
|
let predictions: Vec<_> = samples
|
|
.iter()
|
|
.filter_map(|s| {
|
|
s.prediction_correct.map(|correct| {
|
|
(
|
|
correct,
|
|
s.prediction_confidence.unwrap_or(0.5),
|
|
s.prediction_type.as_deref().unwrap_or("unknown"),
|
|
s.market_regime.as_deref().unwrap_or("unknown"),
|
|
)
|
|
})
|
|
})
|
|
.collect();
|
|
|
|
if !predictions.is_empty() {
|
|
// Basic accuracy metrics
|
|
let total_predictions = predictions.len() as f64;
|
|
let correct_predictions = predictions
|
|
.iter()
|
|
.filter(|(correct, _, _, _)| *correct)
|
|
.count() as f64;
|
|
let accuracy = correct_predictions / total_predictions;
|
|
|
|
metrics.insert("accuracy".to_string(), accuracy);
|
|
metrics.insert("total_predictions".to_string(), total_predictions);
|
|
|
|
// Confidence-weighted accuracy
|
|
let weighted_sum: f64 = predictions
|
|
.iter()
|
|
.map(|(correct, confidence, _, _)| {
|
|
if *correct {
|
|
*confidence
|
|
} else {
|
|
1.0 - *confidence
|
|
}
|
|
})
|
|
.sum();
|
|
let confidence_weighted_accuracy = weighted_sum / total_predictions;
|
|
metrics.insert(
|
|
"confidence_weighted_accuracy".to_string(),
|
|
confidence_weighted_accuracy,
|
|
);
|
|
|
|
// Calculate precision, recall, and F1 score
|
|
let true_positives = predictions
|
|
.iter()
|
|
.filter(|(correct, _, _, _)| *correct)
|
|
.count() as f64;
|
|
let total_positives = predictions.len() as f64; // All predictions are considered "positive" decisions
|
|
|
|
if total_positives > 0.0 {
|
|
let precision = true_positives / total_positives;
|
|
let recall = true_positives / total_positives; // Same as accuracy in this context
|
|
let f1_score = if precision + recall > 0.0 {
|
|
2.0 * (precision * recall) / (precision + recall)
|
|
} else {
|
|
0.0
|
|
};
|
|
|
|
metrics.insert("precision".to_string(), precision);
|
|
metrics.insert("recall".to_string(), recall);
|
|
metrics.insert("f1_score".to_string(), f1_score);
|
|
}
|
|
|
|
// Regime-specific accuracy
|
|
let mut regime_counts: HashMap<&str, (usize, usize)> = HashMap::new();
|
|
for (correct, _, _, regime) in &predictions {
|
|
let (total, correct_count) = regime_counts.entry(regime).or_insert((0, 0));
|
|
*total += 1;
|
|
if *correct {
|
|
*correct_count += 1;
|
|
}
|
|
}
|
|
|
|
for (regime, (total, correct_count)) in regime_counts {
|
|
if total > 0 {
|
|
let regime_accuracy = correct_count as f64 / total as f64;
|
|
metrics.insert(format!("accuracy_{}", regime), regime_accuracy);
|
|
}
|
|
}
|
|
|
|
// Prediction type-specific accuracy
|
|
let mut type_counts: HashMap<&str, (usize, usize)> = HashMap::new();
|
|
for (correct, _, pred_type, _) in &predictions {
|
|
let (total, correct_count) = type_counts.entry(pred_type).or_insert((0, 0));
|
|
*total += 1;
|
|
if *correct {
|
|
*correct_count += 1;
|
|
}
|
|
}
|
|
|
|
for (pred_type, (total, correct_count)) in type_counts {
|
|
if total > 0 {
|
|
let type_accuracy = correct_count as f64 / total as f64;
|
|
metrics.insert(format!("accuracy_{}", pred_type), type_accuracy);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
metrics
|
|
}
|
|
|
|
/// Get dashboard data for monitoring UI
|
|
pub async fn get_dashboard_data(&self) -> DashboardData {
|
|
let overall_stats = self.calculate_performance_stats(None).await;
|
|
|
|
// Calculate per-model stats
|
|
let mut model_stats = HashMap::new();
|
|
{
|
|
let model_samples = self.model_samples.read().await;
|
|
for model_id in model_samples.keys() {
|
|
let stats = self.calculate_performance_stats(Some(model_id)).await;
|
|
model_stats.insert(model_id.clone(), stats);
|
|
}
|
|
}
|
|
|
|
// Get recent alerts
|
|
let recent_alerts = {
|
|
let alerts = self.alerts.read().await;
|
|
alerts.iter().rev().take(10).cloned().collect()
|
|
};
|
|
|
|
// Create latency histogram
|
|
let latency_histogram = {
|
|
let samples = self.samples.read().await;
|
|
let mut histogram = HashMap::new();
|
|
|
|
for sample in samples.iter() {
|
|
let bucket = match sample.latency_us {
|
|
0..=50 => "0-50μs",
|
|
51..=100 => "51-100μs",
|
|
101..=500 => "101-500μs",
|
|
501..=1000 => "501μs-1ms",
|
|
1001..=5000 => "1-5ms",
|
|
_ => ">5ms",
|
|
};
|
|
|
|
*histogram.entry(bucket.to_string()).or_insert(0) += 1;
|
|
}
|
|
|
|
histogram
|
|
};
|
|
|
|
// Calculate real-time metrics
|
|
let realtime_metrics = {
|
|
let calculator = self.realtime_calculator.read().await;
|
|
let samples_count = calculator.last_minute_samples.len();
|
|
|
|
let current_rps = samples_count as f64 / 60.0; // Samples per second in last minute
|
|
|
|
let current_avg_latency_us = if !calculator.last_minute_samples.is_empty() {
|
|
calculator
|
|
.last_minute_samples
|
|
.iter()
|
|
.map(|s| s.latency_us as f64)
|
|
.sum::<f64>()
|
|
/ calculator.last_minute_samples.len() as f64
|
|
} else {
|
|
0.0
|
|
};
|
|
|
|
let current_memory_mb = calculator
|
|
.last_minute_samples
|
|
.iter()
|
|
.map(|s| s.memory_usage_mb)
|
|
.fold(0.0, f64::max);
|
|
|
|
let current_cpu_utilization = if !calculator.last_minute_samples.is_empty() {
|
|
calculator
|
|
.last_minute_samples
|
|
.iter()
|
|
.map(|s| s.cpu_utilization)
|
|
.sum::<f64>()
|
|
/ calculator.last_minute_samples.len() as f64
|
|
} else {
|
|
0.0
|
|
};
|
|
|
|
let active_models = {
|
|
let model_samples = self.model_samples.read().await;
|
|
model_samples.len()
|
|
};
|
|
|
|
RealtimeMetrics {
|
|
current_rps,
|
|
current_avg_latency_us,
|
|
current_memory_mb,
|
|
current_cpu_utilization,
|
|
active_models,
|
|
queue_depth: 0, // Would be populated from actual queue
|
|
}
|
|
};
|
|
|
|
DashboardData {
|
|
overall_stats,
|
|
model_stats,
|
|
recent_alerts,
|
|
latency_histogram,
|
|
realtime_metrics,
|
|
}
|
|
}
|
|
|
|
/// Get recent alerts
|
|
pub async fn get_recent_alerts(&self, limit: usize) -> Vec<PerformanceAlert> {
|
|
let alerts = self.alerts.read().await;
|
|
alerts.iter().rev().take(limit).cloned().collect()
|
|
}
|
|
|
|
/// Clear old samples and alerts
|
|
pub async fn cleanup(&self, max_age: Duration) {
|
|
let cutoff_time = SystemTime::now() - max_age;
|
|
|
|
// Cleanup global samples
|
|
{
|
|
let mut samples = self.samples.write().await;
|
|
samples.retain(|sample| sample.timestamp >= cutoff_time);
|
|
}
|
|
|
|
// Cleanup model samples
|
|
{
|
|
let mut model_samples = self.model_samples.write().await;
|
|
for samples_vec in model_samples.values_mut() {
|
|
samples_vec.retain(|sample| sample.timestamp >= cutoff_time);
|
|
}
|
|
|
|
// Remove empty model entries
|
|
model_samples.retain(|_, samples_vec| !samples_vec.is_empty());
|
|
}
|
|
|
|
// Cleanup alerts
|
|
{
|
|
let mut alerts = self.alerts.write().await;
|
|
alerts.retain(|alert| alert.timestamp >= cutoff_time);
|
|
}
|
|
|
|
// Cleanup last alert times
|
|
{
|
|
let mut last_alert_times = self.last_alert_times.write().await;
|
|
last_alert_times.retain(|_, &mut timestamp| timestamp >= cutoff_time);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[tokio::test]
|
|
async fn test_performance_monitor_creation() {
|
|
let config = IntegrationHubConfig::default();
|
|
let monitor = PerformanceMonitor::new(&config);
|
|
|
|
let dashboard = monitor.get_dashboard_data().await;
|
|
assert_eq!(dashboard.overall_stats.total_samples, 0);
|
|
assert!(dashboard.model_stats.is_empty());
|
|
assert!(dashboard.recent_alerts.is_empty());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_sample_recording() {
|
|
let config = IntegrationHubConfig::default();
|
|
let monitor = PerformanceMonitor::new(&config);
|
|
|
|
let sample = PerformanceSample {
|
|
timestamp: SystemTime::now(),
|
|
model_id: "test_model".to_string(),
|
|
latency_us: 100,
|
|
memory_usage_mb: 50.0,
|
|
cpu_utilization: 25.0,
|
|
success: true,
|
|
request_size_bytes: 1024,
|
|
response_size_bytes: 512,
|
|
queue_depth: 1,
|
|
prediction_correct: Some(true),
|
|
prediction_confidence: Some(0.9),
|
|
actual_outcome: Some(true),
|
|
prediction_type: Some("direction".to_string()),
|
|
market_regime: Some("trending".to_string()),
|
|
};
|
|
|
|
monitor.record_sample(sample).await;
|
|
|
|
let stats = monitor
|
|
.calculate_performance_stats(Some("test_model"))
|
|
.await;
|
|
assert_eq!(stats.total_samples, 1);
|
|
assert_eq!(stats.avg_latency_us, 100.0);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_accuracy_metrics_calculation() {
|
|
// let config = IntegrationHubConfig::default();
|
|
// let monitor = PerformanceMonitor::new(&config);
|
|
//
|
|
// Add samples with varied prediction outcomes
|
|
// let samples = vec![
|
|
// PerformanceSample {
|
|
// timestamp: SystemTime::now(),
|
|
// model_id: "test_model".to_string(),
|
|
// latency_us: 100,
|
|
// memory_usage_mb: 50.0,
|
|
// cpu_utilization: 25.0,
|
|
// success: true,
|
|
// request_size_bytes: 1024,
|
|
// response_size_bytes: 512,
|
|
// queue_depth: 1,
|
|
// prediction_correct: Some(true), // TP
|
|
// prediction_confidence: Some(0.9),
|
|
// actual_outcome: Some(true),
|
|
// prediction_type: Some("direction".to_string()),
|
|
// market_regime: Some("trending".to_string()),
|
|
// },
|
|
// // ... more test samples
|
|
// ];
|
|
//
|
|
// Record all samples
|
|
// for sample in samples {
|
|
// monitor.record_sample(sample).await;
|
|
// }
|
|
//
|
|
// Calculate accuracy metrics
|
|
// let accuracy_metrics = monitor.calculate_accuracy_metrics("test_model").await;
|
|
//
|
|
// Verify basic metrics
|
|
// assert!(accuracy_metrics.contains_key("accuracy"));
|
|
// assert!(accuracy_metrics.contains_key("precision"));
|
|
// assert!(accuracy_metrics.contains_key("recall"));
|
|
// assert!(accuracy_metrics.contains_key("f1_score"));
|
|
// assert!(accuracy_metrics.contains_key("confidence_weighted_accuracy"));
|
|
//
|
|
// Check accuracy: 2 correct out of 4 = 0.5
|
|
// assert!((accuracy_metrics["accuracy"] - 0.5).abs() < 1e-6);
|
|
//
|
|
// Check that we have predictions count
|
|
// assert_eq!(accuracy_metrics["total_predictions"], 4.0);
|
|
//
|
|
// Check regime-specific accuracy
|
|
// assert!(accuracy_metrics.contains_key("accuracy_trending"));
|
|
// assert!(accuracy_metrics.contains_key("accuracy_sideways"));
|
|
//
|
|
// Check prediction type-specific accuracy
|
|
// assert!(accuracy_metrics.contains_key("accuracy_direction"));
|
|
// assert!(accuracy_metrics.contains_key("accuracy_volatility"));
|
|
assert!(true); // Production test
|
|
}
|
|
}
|