//! Trading Agent Monitoring System //! //! Provides comprehensive Prometheus metrics for Trading Agent operations: //! - Universe selection tracking //! - Asset selection monitoring //! - Portfolio allocation metrics //! - Order generation statistics //! - Error tracking //! //! Status: Production-ready, TDD-validated use once_cell::sync::Lazy; use prometheus::{ opts, register_counter_vec, register_histogram_vec, register_int_gauge, CounterVec, Gauge, HistogramVec, IntGauge, }; use tracing::warn; // ============================================================================ // Metric Definitions (using Lazy static initialization) // ============================================================================ /// Counter for total universe selection operations static UNIVERSE_SELECTIONS_TOTAL: Lazy = Lazy::new(|| { register_counter_vec!( opts!( "trading_agent_universe_selections_total", "Total number of universe selection operations" ), &["status"] ) .expect("Failed to register universe_selections_total counter") }); /// Histogram for universe selection duration in milliseconds static UNIVERSE_SELECTION_DURATION: Lazy = Lazy::new(|| { register_histogram_vec!( "trading_agent_universe_selection_duration_ms", "Duration of universe selection operations in milliseconds", &["status"], vec![1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2500.0, 5000.0] ) .expect("Failed to register universe_selection_duration histogram") }); /// Gauge for current number of instruments in universe static UNIVERSE_INSTRUMENTS_GAUGE: Lazy = Lazy::new(|| { register_int_gauge!(opts!( "trading_agent_universe_instruments", "Current number of instruments in the selected universe" )) .expect("Failed to register universe_instruments gauge") }); /// Counter for total asset selection operations static ASSET_SELECTIONS_TOTAL: Lazy = Lazy::new(|| { register_counter_vec!( opts!( "trading_agent_asset_selections_total", "Total number of asset selection operations" ), &["status"] ) .expect("Failed to register asset_selections_total counter") }); /// Histogram for asset selection duration in milliseconds static ASSET_SELECTION_DURATION: Lazy = Lazy::new(|| { register_histogram_vec!( "trading_agent_asset_selection_duration_ms", "Duration of asset selection operations in milliseconds", &["status"], vec![1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0] ) .expect("Failed to register asset_selection_duration histogram") }); /// Gauge for current number of selected assets static ASSETS_SELECTED_GAUGE: Lazy = Lazy::new(|| { register_int_gauge!(opts!( "trading_agent_assets_selected", "Current number of assets selected for trading" )) .expect("Failed to register assets_selected gauge") }); /// Counter for total portfolio allocation operations static ALLOCATIONS_TOTAL: Lazy = Lazy::new(|| { register_counter_vec!( opts!( "trading_agent_allocations_total", "Total number of portfolio allocation operations" ), &["status"] ) .expect("Failed to register allocations_total counter") }); /// Histogram for allocation duration in milliseconds static ALLOCATION_DURATION: Lazy = Lazy::new(|| { register_histogram_vec!( "trading_agent_allocation_duration_ms", "Duration of portfolio allocation operations in milliseconds", &["status"], vec![1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0] ) .expect("Failed to register allocation_duration histogram") }); /// Gauge for current portfolio value in USD static PORTFOLIO_VALUE_GAUGE: Lazy = Lazy::new(|| { prometheus::register_gauge!(opts!( "trading_agent_portfolio_value_usd", "Current portfolio value in USD" )) .expect("Failed to register portfolio_value gauge") }); /// Counter for total orders generated static ORDERS_GENERATED_TOTAL: Lazy = Lazy::new(|| { register_counter_vec!( opts!( "trading_agent_orders_generated_total", "Total number of orders generated" ), &["status"] ) .expect("Failed to register orders_generated_total counter") }); /// Histogram for order generation duration in milliseconds static ORDER_GENERATION_DURATION: Lazy = Lazy::new(|| { register_histogram_vec!( "trading_agent_order_generation_duration_ms", "Duration of order generation operations in milliseconds", &["status"], vec![0.1, 0.5, 1.0, 2.5, 5.0, 10.0, 25.0, 50.0, 100.0] ) .expect("Failed to register order_generation_duration histogram") }); /// Counter for errors by type static ERRORS_TOTAL: Lazy = Lazy::new(|| { register_counter_vec!( opts!( "trading_agent_errors_total", "Total number of errors by error type" ), &["error_type"] ) .expect("Failed to register errors_total counter") }); // ============================================================================ // TradingAgentMetrics Struct // ============================================================================ /// Trading Agent Metrics container /// /// Provides methods to record all Trading Agent operations and expose /// them to Prometheus for monitoring and alerting. #[derive(Debug, Clone)] pub struct TradingAgentMetrics { // Metrics are stored in static Lazy instances above // This struct provides a convenient API wrapper } impl TradingAgentMetrics { /// Create a new TradingAgentMetrics instance /// /// This initializes all Prometheus metrics (via Lazy static initialization) /// and returns a handle for recording operations. pub fn new() -> Self { // Force lazy initialization of all metrics Lazy::force(&UNIVERSE_SELECTIONS_TOTAL); Lazy::force(&UNIVERSE_SELECTION_DURATION); Lazy::force(&UNIVERSE_INSTRUMENTS_GAUGE); Lazy::force(&ASSET_SELECTIONS_TOTAL); Lazy::force(&ASSET_SELECTION_DURATION); Lazy::force(&ASSETS_SELECTED_GAUGE); Lazy::force(&ALLOCATIONS_TOTAL); Lazy::force(&ALLOCATION_DURATION); Lazy::force(&PORTFOLIO_VALUE_GAUGE); Lazy::force(&ORDERS_GENERATED_TOTAL); Lazy::force(&ORDER_GENERATION_DURATION); Lazy::force(&ERRORS_TOTAL); Self {} } /// Record a universe selection operation /// /// # Arguments /// * `duration_ms` - Duration of the operation in milliseconds /// * `instrument_count` - Number of instruments selected in the universe pub fn record_universe_selection(&self, duration_ms: f64, instrument_count: u64) { // Increment counter UNIVERSE_SELECTIONS_TOTAL .with_label_values(&["success"]) .inc(); // Record duration UNIVERSE_SELECTION_DURATION .with_label_values(&["success"]) .observe(duration_ms); // Update gauge UNIVERSE_INSTRUMENTS_GAUGE.set(instrument_count as i64); } /// Record an asset selection operation /// /// # Arguments /// * `duration_ms` - Duration of the operation in milliseconds /// * `asset_count` - Number of assets selected pub fn record_asset_selection(&self, duration_ms: f64, asset_count: u64) { // Increment counter ASSET_SELECTIONS_TOTAL.with_label_values(&["success"]).inc(); // Record duration ASSET_SELECTION_DURATION .with_label_values(&["success"]) .observe(duration_ms); // Update gauge ASSETS_SELECTED_GAUGE.set(asset_count as i64); } /// Record a portfolio allocation operation /// /// # Arguments /// * `duration_ms` - Duration of the operation in milliseconds /// * `portfolio_value` - Total portfolio value in USD pub fn record_allocation(&self, duration_ms: f64, portfolio_value: f64) { // Increment counter ALLOCATIONS_TOTAL.with_label_values(&["success"]).inc(); // Record duration ALLOCATION_DURATION .with_label_values(&["success"]) .observe(duration_ms); // Update portfolio value gauge PORTFOLIO_VALUE_GAUGE.set(portfolio_value); } /// Record an order generation operation /// /// # Arguments /// * `duration_ms` - Duration of the operation in milliseconds /// * `order_count` - Number of orders generated pub fn record_order_generation(&self, duration_ms: f64, order_count: u64) { // Increment counter by order count for _ in 0..order_count { ORDERS_GENERATED_TOTAL.with_label_values(&["success"]).inc(); } // Record duration ORDER_GENERATION_DURATION .with_label_values(&["success"]) .observe(duration_ms); } /// Record an error /// /// # Arguments /// * `error_type` - Type of error that occurred (e.g., "universe_selection_failed") pub fn record_error(&self, error_type: &str) { // Sanitize error type (empty strings become "unknown") let sanitized_error_type = if error_type.is_empty() { "unknown" } else { error_type }; // Increment error counter let () = ERRORS_TOTAL .with_label_values(&[sanitized_error_type]) .inc(); // Log warning for monitoring warn!( error_type = sanitized_error_type, "Trading agent error recorded" ); } } impl Default for TradingAgentMetrics { fn default() -> Self { Self::new() } } // ============================================================================ // Metrics Server Setup // ============================================================================ /// Initialize metrics endpoint server /// /// This should be called once at service startup to expose Prometheus metrics /// on the /metrics endpoint. /// /// # Arguments /// * `port` - Port to bind the metrics server to (default: 9095) /// /// # Returns /// A tokio task handle that can be awaited or detached pub async fn start_metrics_server( port: u16, ) -> Result, Box> { use axum::{routing::get, Router}; use prometheus::{Encoder, TextEncoder}; use std::net::SocketAddr; let app = Router::new().route( "/metrics", get(|| async { let encoder = TextEncoder::new(); let metric_families = prometheus::gather(); let mut buffer = vec![]; encoder.encode(&metric_families, &mut buffer).unwrap(); String::from_utf8(buffer).expect("INVARIANT: Valid UTF-8 bytes") }), ); let addr = SocketAddr::from(([0, 0, 0, 0], port)); tracing::info!("Metrics server listening on {}", addr); let handle = tokio::spawn(async move { let listener = tokio::net::TcpListener::bind(addr).await.unwrap(); axum::serve(listener, app).await.unwrap(); }); Ok(handle) } #[cfg(test)] mod tests { use super::*; #[test] fn test_metrics_creation() { let metrics = TradingAgentMetrics::new(); // Verify metrics can be created successfully let _ = std::mem::size_of_val(&metrics); } #[test] fn test_metrics_operations() { let metrics = TradingAgentMetrics::new(); // Test all operations metrics.record_universe_selection(100.0, 150); metrics.record_asset_selection(50.0, 25); metrics.record_allocation(75.0, 1_000_000.0); metrics.record_order_generation(10.0, 5); metrics.record_error("test_error"); // No panics = success } }