Files
foxhunt/ml/src/integration/performance_monitor.rs
jgrusewski bdf5b690b7 cleanup(ml): remove 31 disabled imports and commented-out module blocks
Removes dead code across 28 files:
- 31 commented-out "DISABLED" import lines (mostly safe_operations, error_handling)
- Commented-out module declarations in lib.rs (deployment, model_loader_integration, tests)
- Commented-out re-exports in lib.rs (training_pipeline, deployment::ModelVersion)
- Commented-out adaptive strategy modules in regime/mod.rs

All are in git history if ever needed. Net -74 lines removed.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-20 18:23:41 +01:00

824 lines
29 KiB
Rust

//! # Performance Monitor
//!
//! Real-time performance monitoring and alerting for ML models
//! with HFT-specific metrics and latency tracking.
use std::collections::{HashMap, VecDeque};
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use crate::observability::alerts::AlertSeverity;
use serde::{Deserialize, Serialize};
use tokio::sync::RwLock; // Use local AlertSeverity with Warning variant
use super::IntegrationHubConfig;
/// Performance sample for monitoring
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PerformanceSample {
/// Sample timestamp
pub timestamp: SystemTime,
/// Model identifier
pub model_id: String,
/// Latency in microseconds
pub latency_us: u64,
/// Memory usage in MB
pub memory_usage_mb: f64,
/// CPU utilization percentage
pub cpu_utilization: f64,
/// Whether the operation was successful
pub success: bool,
/// Request size in bytes
pub request_size_bytes: usize,
/// Response size in bytes
pub response_size_bytes: usize,
/// Queue depth at time of request
pub queue_depth: usize,
/// Whether prediction was correct (if known)
pub prediction_correct: Option<bool>,
/// Confidence score of prediction
pub prediction_confidence: Option<f64>,
/// Actual outcome (if available for validation)
pub actual_outcome: Option<bool>,
/// Type of prediction (direction, volatility, etc.)
pub prediction_type: Option<String>,
/// Market regime during prediction
pub market_regime: Option<String>,
}
/// Performance alert configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AlertConfig {
/// Enable latency alerts
pub enable_latency_alerts: bool,
/// Latency threshold in microseconds
pub latency_threshold_us: u64,
/// Enable memory alerts
pub enable_memory_alerts: bool,
/// Memory threshold in MB
pub memory_threshold_mb: f64,
/// Enable accuracy alerts
pub enable_accuracy_alerts: bool,
/// Minimum accuracy threshold
pub accuracy_threshold: f64,
/// Alert cooldown period in seconds
pub alert_cooldown_seconds: u64,
}
impl Default for AlertConfig {
fn default() -> Self {
Self {
enable_latency_alerts: true,
latency_threshold_us: 1000, // 1ms
enable_memory_alerts: true,
memory_threshold_mb: 500.0,
enable_accuracy_alerts: true,
accuracy_threshold: 0.7,
alert_cooldown_seconds: 300, // 5 minutes
}
}
}
/// Performance alert
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PerformanceAlert {
/// Alert timestamp
pub timestamp: SystemTime,
/// Alert severity
pub severity: AlertSeverity,
/// Alert message
pub message: String,
/// Model ID that triggered the alert
pub model_id: String,
/// Alert type
pub alert_type: AlertType,
/// Current value that triggered alert
pub current_value: f64,
/// Threshold that was exceeded
pub threshold: f64,
}
/// Alert types
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)]
pub enum AlertType {
/// High latency alert
HighLatency,
/// High memory usage alert
HighMemoryUsage,
/// Low accuracy alert
LowAccuracy,
/// Model failure alert
ModelFailure,
/// Queue overflow alert
QueueOverflow,
}
/// Performance statistics
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct PerformanceStats {
/// Total samples collected
pub total_samples: u64,
/// Average latency in microseconds
pub avg_latency_us: f64,
/// 95th percentile latency
pub p95_latency_us: f64,
/// 99th percentile latency
pub p99_latency_us: f64,
/// Maximum latency observed
pub max_latency_us: u64,
/// Average memory usage in MB
pub avg_memory_mb: f64,
/// Peak memory usage in MB
pub peak_memory_mb: f64,
/// Average CPU utilization
pub avg_cpu_utilization: f64,
/// Success rate percentage
pub success_rate: f64,
/// Prediction accuracy (if available)
pub prediction_accuracy: Option<f64>,
/// Throughput (requests per second)
pub throughput_rps: f64,
}
/// Dashboard data for monitoring
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DashboardData {
/// Overall performance statistics
pub overall_stats: PerformanceStats,
/// Per-model performance statistics
pub model_stats: HashMap<String, PerformanceStats>,
/// Recent alerts
pub recent_alerts: Vec<PerformanceAlert>,
/// Latency histogram
pub latency_histogram: HashMap<String, u64>,
/// Real-time metrics
pub realtime_metrics: RealtimeMetrics,
}
/// Real-time metrics
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct RealtimeMetrics {
/// Current requests per second
pub current_rps: f64,
/// Current average latency (last minute)
pub current_avg_latency_us: f64,
/// Current memory usage
pub current_memory_mb: f64,
/// Current CPU utilization
pub current_cpu_utilization: f64,
/// Active models count
pub active_models: usize,
/// Queue depth
pub queue_depth: usize,
}
/// Performance Monitor for ML models
#[derive(Debug)]
pub struct PerformanceMonitor {
/// Configuration reference
config: Arc<IntegrationHubConfig>,
/// Alert configuration
alert_config: AlertConfig,
/// Performance samples storage
samples: Arc<RwLock<VecDeque<PerformanceSample>>>,
/// Per-model sample storage
model_samples: Arc<RwLock<HashMap<String, VecDeque<PerformanceSample>>>>,
/// Recent alerts
alerts: Arc<RwLock<VecDeque<PerformanceAlert>>>,
/// Last alert timestamps for cooldown
last_alert_times: Arc<RwLock<HashMap<(String, AlertType), SystemTime>>>,
/// Real-time metrics calculation
realtime_calculator: Arc<RwLock<RealtimeCalculator>>,
}
/// Real-time metrics calculator
#[derive(Debug, Default)]
struct RealtimeCalculator {
/// Samples in last minute
last_minute_samples: VecDeque<PerformanceSample>,
/// Last update time
last_update: Option<SystemTime>,
}
impl PerformanceMonitor {
/// Create new performance monitor
pub fn new(config: &IntegrationHubConfig) -> Self {
Self {
config: Arc::new(config.clone()),
alert_config: AlertConfig::default(),
samples: Arc::new(RwLock::new(VecDeque::new())),
model_samples: Arc::new(RwLock::new(HashMap::new())),
alerts: Arc::new(RwLock::new(VecDeque::new())),
last_alert_times: Arc::new(RwLock::new(HashMap::new())),
realtime_calculator: Arc::new(RwLock::new(RealtimeCalculator::default())),
}
}
/// Create monitor with custom alert configuration
pub fn with_alert_config(config: &IntegrationHubConfig, alert_config: AlertConfig) -> Self {
let mut monitor = Self::new(config);
monitor.alert_config = alert_config;
monitor
}
/// Record performance sample
pub async fn record_sample(&self, sample: PerformanceSample) {
// Add to global samples
{
let mut samples = self.samples.write().await;
samples.push_back(sample.clone());
// Keep only recent samples (last 10000)
if samples.len() > 10000 {
samples.pop_front();
}
}
// Add to model-specific samples
{
let mut model_samples = self.model_samples.write().await;
let model_samples_vec = model_samples
.entry(sample.model_id.clone())
.or_insert_with(VecDeque::new);
model_samples_vec.push_back(sample.clone());
// Keep only recent samples per model (last 1000)
if model_samples_vec.len() > 1000 {
model_samples_vec.pop_front();
}
}
// Update real-time calculator
{
let mut calculator = self.realtime_calculator.write().await;
calculator.last_minute_samples.push_back(sample.clone());
// Remove samples older than 1 minute
let one_minute_ago = SystemTime::now() - Duration::from_secs(60);
while let Some(front_sample) = calculator.last_minute_samples.front() {
if front_sample.timestamp < one_minute_ago {
calculator.last_minute_samples.pop_front();
} else {
break;
}
}
calculator.last_update = Some(SystemTime::now());
}
// Check for alerts
self.check_alerts(&sample).await;
}
/// Check for performance alerts
async fn check_alerts(&self, sample: &PerformanceSample) {
let mut alerts_to_add = Vec::new();
// Check latency alert
if self.alert_config.enable_latency_alerts
&& sample.latency_us > self.alert_config.latency_threshold_us
{
if self
.should_send_alert(&sample.model_id, AlertType::HighLatency)
.await
{
alerts_to_add.push(PerformanceAlert {
timestamp: SystemTime::now(),
severity: AlertSeverity::Critical,
message: format!(
"High latency detected for model {}: {}μs (threshold: {}μs)",
sample.model_id, sample.latency_us, self.alert_config.latency_threshold_us
),
model_id: sample.model_id.clone(),
alert_type: AlertType::HighLatency,
current_value: sample.latency_us as f64,
threshold: self.alert_config.latency_threshold_us as f64,
});
}
}
// Check memory alert
if self.alert_config.enable_memory_alerts
&& sample.memory_usage_mb > self.alert_config.memory_threshold_mb
{
if self
.should_send_alert(&sample.model_id, AlertType::HighMemoryUsage)
.await
{
alerts_to_add.push(PerformanceAlert {
timestamp: SystemTime::now(),
severity: AlertSeverity::Warning,
message: format!(
"High memory usage detected for model {}: {:.1}MB (threshold: {:.1}MB)",
sample.model_id,
sample.memory_usage_mb,
self.alert_config.memory_threshold_mb
),
model_id: sample.model_id.clone(),
alert_type: AlertType::HighMemoryUsage,
current_value: sample.memory_usage_mb,
threshold: self.alert_config.memory_threshold_mb,
});
}
}
// Check failure alert
if !sample.success {
if self
.should_send_alert(&sample.model_id, AlertType::ModelFailure)
.await
{
alerts_to_add.push(PerformanceAlert {
timestamp: SystemTime::now(),
severity: AlertSeverity::Critical,
message: format!("Model failure detected for model {}", sample.model_id),
model_id: sample.model_id.clone(),
alert_type: AlertType::ModelFailure,
current_value: 0.0,
threshold: 1.0,
});
}
}
// Add all alerts
if !alerts_to_add.is_empty() {
let mut alerts = self.alerts.write().await;
let mut last_alert_times = self.last_alert_times.write().await;
for alert in alerts_to_add {
// Update last alert time
last_alert_times
.insert((alert.model_id.clone(), alert.alert_type), alert.timestamp);
alerts.push_back(alert);
// Keep only recent alerts (last 100)
if alerts.len() > 100 {
alerts.pop_front();
}
}
}
}
/// Check if alert should be sent (considering cooldown)
async fn should_send_alert(&self, model_id: &str, alert_type: AlertType) -> bool {
let last_alert_times = self.last_alert_times.read().await;
if let Some(&last_time) = last_alert_times.get(&(model_id.to_string(), alert_type)) {
let cooldown = Duration::from_secs(self.alert_config.alert_cooldown_seconds);
SystemTime::now()
.duration_since(last_time)
.unwrap_or(cooldown)
>= cooldown
} else {
true // No previous alert
}
}
/// Calculate performance statistics
pub async fn calculate_performance_stats(&self, model_id: Option<&str>) -> PerformanceStats {
let samples = if let Some(model_id) = model_id {
let model_samples = self.model_samples.read().await;
model_samples.get(model_id).cloned().unwrap_or_default()
} else {
self.samples.read().await.clone()
};
if samples.is_empty() {
return PerformanceStats::default();
}
let mut latencies: Vec<u64> = samples.iter().map(|s| s.latency_us).collect();
latencies.sort_unstable();
let total_samples = samples.len() as u64;
let avg_latency_us = latencies.iter().sum::<u64>() as f64 / latencies.len() as f64;
let p95_idx = (latencies.len() as f64 * 0.95) as usize;
let p99_idx = (latencies.len() as f64 * 0.99) as usize;
let p95_latency_us = latencies
.get(p95_idx.min(latencies.len() - 1))
.copied()
.unwrap_or(0) as f64;
let p99_latency_us = latencies
.get(p99_idx.min(latencies.len() - 1))
.copied()
.unwrap_or(0) as f64;
let max_latency_us = latencies.iter().max().copied().unwrap_or(0);
let avg_memory_mb =
samples.iter().map(|s| s.memory_usage_mb).sum::<f64>() / samples.len() as f64;
let peak_memory_mb = samples
.iter()
.map(|s| s.memory_usage_mb)
.fold(0.0, f64::max);
let avg_cpu_utilization =
samples.iter().map(|s| s.cpu_utilization).sum::<f64>() / samples.len() as f64;
let success_count = samples.iter().filter(|s| s.success).count();
let success_rate = (success_count as f64 / samples.len() as f64) * 100.0;
// Calculate prediction accuracy if available
let prediction_accuracy = {
let correct_predictions = samples
.iter()
.filter_map(|s| s.prediction_correct)
.filter(|&correct| correct)
.count();
let total_predictions = samples.iter().filter_map(|s| s.prediction_correct).count();
if total_predictions > 0 {
Some((correct_predictions as f64 / total_predictions as f64) * 100.0)
} else {
None
}
};
// Calculate throughput (samples per second)
let throughput_rps = if let (Some(first), Some(last)) = (samples.front(), samples.back()) {
if let Ok(duration) = last.timestamp.duration_since(first.timestamp) {
let duration_secs = duration.as_secs_f64();
if duration_secs > 0.0 {
samples.len() as f64 / duration_secs
} else {
0.0
}
} else {
0.0
}
} else {
0.0
};
PerformanceStats {
total_samples,
avg_latency_us,
p95_latency_us,
p99_latency_us,
max_latency_us,
avg_memory_mb,
peak_memory_mb,
avg_cpu_utilization,
success_rate,
prediction_accuracy,
throughput_rps,
}
}
/// Calculate accuracy metrics for a specific model
pub async fn calculate_accuracy_metrics(&self, model_id: &str) -> HashMap<String, f64> {
let model_samples = self.model_samples.read().await;
let samples = model_samples.get(model_id);
let mut metrics = HashMap::new();
if let Some(samples) = samples {
let predictions: Vec<_> = samples
.iter()
.filter_map(|s| {
s.prediction_correct.map(|correct| {
(
correct,
s.prediction_confidence.unwrap_or(0.5),
s.prediction_type.as_deref().unwrap_or("unknown"),
s.market_regime.as_deref().unwrap_or("unknown"),
)
})
})
.collect();
if !predictions.is_empty() {
// Basic accuracy metrics
let total_predictions = predictions.len() as f64;
let correct_predictions = predictions
.iter()
.filter(|(correct, _, _, _)| *correct)
.count() as f64;
let accuracy = correct_predictions / total_predictions;
metrics.insert("accuracy".to_string(), accuracy);
metrics.insert("total_predictions".to_string(), total_predictions);
// Confidence-weighted accuracy
let weighted_sum: f64 = predictions
.iter()
.map(|(correct, confidence, _, _)| {
if *correct {
*confidence
} else {
1.0 - *confidence
}
})
.sum();
let confidence_weighted_accuracy = weighted_sum / total_predictions;
metrics.insert(
"confidence_weighted_accuracy".to_string(),
confidence_weighted_accuracy,
);
// Calculate precision, recall, and F1 score
let true_positives = predictions
.iter()
.filter(|(correct, _, _, _)| *correct)
.count() as f64;
let total_positives = predictions.len() as f64; // All predictions are considered "positive" decisions
if total_positives > 0.0 {
let precision = true_positives / total_positives;
let recall = true_positives / total_positives; // Same as accuracy in this context
let f1_score = if precision + recall > 0.0 {
2.0 * (precision * recall) / (precision + recall)
} else {
0.0
};
metrics.insert("precision".to_string(), precision);
metrics.insert("recall".to_string(), recall);
metrics.insert("f1_score".to_string(), f1_score);
}
// Regime-specific accuracy
let mut regime_counts: HashMap<&str, (usize, usize)> = HashMap::new();
for (correct, _, _, regime) in &predictions {
let (total, correct_count) = regime_counts.entry(regime).or_insert((0, 0));
*total += 1;
if *correct {
*correct_count += 1;
}
}
for (regime, (total, correct_count)) in regime_counts {
if total > 0 {
let regime_accuracy = correct_count as f64 / total as f64;
metrics.insert(format!("accuracy_{}", regime), regime_accuracy);
}
}
// Prediction type-specific accuracy
let mut type_counts: HashMap<&str, (usize, usize)> = HashMap::new();
for (correct, _, pred_type, _) in &predictions {
let (total, correct_count) = type_counts.entry(pred_type).or_insert((0, 0));
*total += 1;
if *correct {
*correct_count += 1;
}
}
for (pred_type, (total, correct_count)) in type_counts {
if total > 0 {
let type_accuracy = correct_count as f64 / total as f64;
metrics.insert(format!("accuracy_{}", pred_type), type_accuracy);
}
}
}
}
metrics
}
/// Get dashboard data for monitoring UI
pub async fn get_dashboard_data(&self) -> DashboardData {
let overall_stats = self.calculate_performance_stats(None).await;
// Calculate per-model stats
let mut model_stats = HashMap::new();
{
let model_samples = self.model_samples.read().await;
for model_id in model_samples.keys() {
let stats = self.calculate_performance_stats(Some(model_id)).await;
model_stats.insert(model_id.clone(), stats);
}
}
// Get recent alerts
let recent_alerts = {
let alerts = self.alerts.read().await;
alerts.iter().rev().take(10).cloned().collect()
};
// Create latency histogram
let latency_histogram = {
let samples = self.samples.read().await;
let mut histogram = HashMap::new();
for sample in samples.iter() {
let bucket = match sample.latency_us {
0..=50 => "0-50μs",
51..=100 => "51-100μs",
101..=500 => "101-500μs",
501..=1000 => "501μs-1ms",
1001..=5000 => "1-5ms",
_ => ">5ms",
};
*histogram.entry(bucket.to_string()).or_insert(0) += 1;
}
histogram
};
// Calculate real-time metrics
let realtime_metrics = {
let calculator = self.realtime_calculator.read().await;
let samples_count = calculator.last_minute_samples.len();
let current_rps = samples_count as f64 / 60.0; // Samples per second in last minute
let current_avg_latency_us = if !calculator.last_minute_samples.is_empty() {
calculator
.last_minute_samples
.iter()
.map(|s| s.latency_us as f64)
.sum::<f64>()
/ calculator.last_minute_samples.len() as f64
} else {
0.0
};
let current_memory_mb = calculator
.last_minute_samples
.iter()
.map(|s| s.memory_usage_mb)
.fold(0.0, f64::max);
let current_cpu_utilization = if !calculator.last_minute_samples.is_empty() {
calculator
.last_minute_samples
.iter()
.map(|s| s.cpu_utilization)
.sum::<f64>()
/ calculator.last_minute_samples.len() as f64
} else {
0.0
};
let active_models = {
let model_samples = self.model_samples.read().await;
model_samples.len()
};
RealtimeMetrics {
current_rps,
current_avg_latency_us,
current_memory_mb,
current_cpu_utilization,
active_models,
queue_depth: 0, // Would be populated from actual queue
}
};
DashboardData {
overall_stats,
model_stats,
recent_alerts,
latency_histogram,
realtime_metrics,
}
}
/// Get recent alerts
pub async fn get_recent_alerts(&self, limit: usize) -> Vec<PerformanceAlert> {
let alerts = self.alerts.read().await;
alerts.iter().rev().take(limit).cloned().collect()
}
/// Clear old samples and alerts
pub async fn cleanup(&self, max_age: Duration) {
let cutoff_time = SystemTime::now() - max_age;
// Cleanup global samples
{
let mut samples = self.samples.write().await;
samples.retain(|sample| sample.timestamp >= cutoff_time);
}
// Cleanup model samples
{
let mut model_samples = self.model_samples.write().await;
for samples_vec in model_samples.values_mut() {
samples_vec.retain(|sample| sample.timestamp >= cutoff_time);
}
// Remove empty model entries
model_samples.retain(|_, samples_vec| !samples_vec.is_empty());
}
// Cleanup alerts
{
let mut alerts = self.alerts.write().await;
alerts.retain(|alert| alert.timestamp >= cutoff_time);
}
// Cleanup last alert times
{
let mut last_alert_times = self.last_alert_times.write().await;
last_alert_times.retain(|_, &mut timestamp| timestamp >= cutoff_time);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_performance_monitor_creation() {
let config = IntegrationHubConfig::default();
let monitor = PerformanceMonitor::new(&config);
let dashboard = monitor.get_dashboard_data().await;
assert_eq!(dashboard.overall_stats.total_samples, 0);
assert!(dashboard.model_stats.is_empty());
assert!(dashboard.recent_alerts.is_empty());
}
#[tokio::test]
async fn test_sample_recording() {
let config = IntegrationHubConfig::default();
let monitor = PerformanceMonitor::new(&config);
let sample = PerformanceSample {
timestamp: SystemTime::now(),
model_id: "test_model".to_string(),
latency_us: 100,
memory_usage_mb: 50.0,
cpu_utilization: 25.0,
success: true,
request_size_bytes: 1024,
response_size_bytes: 512,
queue_depth: 1,
prediction_correct: Some(true),
prediction_confidence: Some(0.9),
actual_outcome: Some(true),
prediction_type: Some("direction".to_string()),
market_regime: Some("trending".to_string()),
};
monitor.record_sample(sample).await;
let stats = monitor
.calculate_performance_stats(Some("test_model"))
.await;
assert_eq!(stats.total_samples, 1);
assert_eq!(stats.avg_latency_us, 100.0);
}
#[tokio::test]
async fn test_accuracy_metrics_calculation() {
// let config = IntegrationHubConfig::default();
// let monitor = PerformanceMonitor::new(&config);
//
// Add samples with varied prediction outcomes
// let samples = vec![
// PerformanceSample {
// timestamp: SystemTime::now(),
// model_id: "test_model".to_string(),
// latency_us: 100,
// memory_usage_mb: 50.0,
// cpu_utilization: 25.0,
// success: true,
// request_size_bytes: 1024,
// response_size_bytes: 512,
// queue_depth: 1,
// prediction_correct: Some(true), // TP
// prediction_confidence: Some(0.9),
// actual_outcome: Some(true),
// prediction_type: Some("direction".to_string()),
// market_regime: Some("trending".to_string()),
// },
// // ... more test samples
// ];
//
// Record all samples
// for sample in samples {
// monitor.record_sample(sample).await;
// }
//
// Calculate accuracy metrics
// let accuracy_metrics = monitor.calculate_accuracy_metrics("test_model").await;
//
// Verify basic metrics
// assert!(accuracy_metrics.contains_key("accuracy"));
// assert!(accuracy_metrics.contains_key("precision"));
// assert!(accuracy_metrics.contains_key("recall"));
// assert!(accuracy_metrics.contains_key("f1_score"));
// assert!(accuracy_metrics.contains_key("confidence_weighted_accuracy"));
//
// Check accuracy: 2 correct out of 4 = 0.5
// assert!((accuracy_metrics["accuracy"] - 0.5).abs() < 1e-6);
//
// Check that we have predictions count
// assert_eq!(accuracy_metrics["total_predictions"], 4.0);
//
// Check regime-specific accuracy
// assert!(accuracy_metrics.contains_key("accuracy_trending"));
// assert!(accuracy_metrics.contains_key("accuracy_sideways"));
//
// Check prediction type-specific accuracy
// assert!(accuracy_metrics.contains_key("accuracy_direction"));
// assert!(accuracy_metrics.contains_key("accuracy_volatility"));
assert!(true); // Production test
}
}