foxhunt/services/trading_agent_service/src/monitoring.rs

//! Trading Agent Monitoring System
//!
//! Provides comprehensive Prometheus metrics for Trading Agent operations:
//! - Universe selection tracking
//! - Asset selection monitoring
//! - Portfolio allocation metrics
//! - Order generation statistics
//! - Error tracking
//!
//! Status: Production-ready, TDD-validated

use once_cell::sync::Lazy;
use prometheus::{
    opts, register_counter_vec, register_histogram_vec, register_int_gauge, CounterVec, Gauge,
    HistogramVec, IntGauge,
};
use tracing::warn;

// ============================================================================
// Metric Definitions (using Lazy static initialization)
// ============================================================================

/// Counter for total universe selection operations
static UNIVERSE_SELECTIONS_TOTAL: Lazy<CounterVec> = Lazy::new(|| {
    register_counter_vec!(
        opts!(
            "trading_agent_universe_selections_total",
            "Total number of universe selection operations"
        ),
        &["status"]
    )
    .expect("Failed to register universe_selections_total counter")
});

/// Histogram for universe selection duration in milliseconds
static UNIVERSE_SELECTION_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
    register_histogram_vec!(
        "trading_agent_universe_selection_duration_ms",
        "Duration of universe selection operations in milliseconds",
        &["status"],
        vec![1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2500.0, 5000.0]
    )
    .expect("Failed to register universe_selection_duration histogram")
});

/// Gauge for current number of instruments in universe
static UNIVERSE_INSTRUMENTS_GAUGE: Lazy<IntGauge> = Lazy::new(|| {
    register_int_gauge!(opts!(
        "trading_agent_universe_instruments",
        "Current number of instruments in the selected universe"
    ))
    .expect("Failed to register universe_instruments gauge")
});

/// Counter for total asset selection operations
static ASSET_SELECTIONS_TOTAL: Lazy<CounterVec> = Lazy::new(|| {
    register_counter_vec!(
        opts!(
            "trading_agent_asset_selections_total",
            "Total number of asset selection operations"
        ),
        &["status"]
    )
    .expect("Failed to register asset_selections_total counter")
});

/// Histogram for asset selection duration in milliseconds
static ASSET_SELECTION_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
    register_histogram_vec!(
        "trading_agent_asset_selection_duration_ms",
        "Duration of asset selection operations in milliseconds",
        &["status"],
        vec![1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0]
    )
    .expect("Failed to register asset_selection_duration histogram")
});

/// Gauge for current number of selected assets
static ASSETS_SELECTED_GAUGE: Lazy<IntGauge> = Lazy::new(|| {
    register_int_gauge!(opts!(
        "trading_agent_assets_selected",
        "Current number of assets selected for trading"
    ))
    .expect("Failed to register assets_selected gauge")
});

/// Counter for total portfolio allocation operations
static ALLOCATIONS_TOTAL: Lazy<CounterVec> = Lazy::new(|| {
    register_counter_vec!(
        opts!(
            "trading_agent_allocations_total",
            "Total number of portfolio allocation operations"
        ),
        &["status"]
    )
    .expect("Failed to register allocations_total counter")
});

/// Histogram for allocation duration in milliseconds
static ALLOCATION_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
    register_histogram_vec!(
        "trading_agent_allocation_duration_ms",
        "Duration of portfolio allocation operations in milliseconds",
        &["status"],
        vec![1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0]
    )
    .expect("Failed to register allocation_duration histogram")
});

/// Gauge for current portfolio value in USD
static PORTFOLIO_VALUE_GAUGE: Lazy<Gauge> = Lazy::new(|| {
    prometheus::register_gauge!(opts!(
        "trading_agent_portfolio_value_usd",
        "Current portfolio value in USD"
    ))
    .expect("Failed to register portfolio_value gauge")
});

/// Counter for total orders generated
static ORDERS_GENERATED_TOTAL: Lazy<CounterVec> = Lazy::new(|| {
    register_counter_vec!(
        opts!(
            "trading_agent_orders_generated_total",
            "Total number of orders generated"
        ),
        &["status"]
    )
    .expect("Failed to register orders_generated_total counter")
});

/// Histogram for order generation duration in milliseconds
static ORDER_GENERATION_DURATION: Lazy<HistogramVec> = Lazy::new(|| {
    register_histogram_vec!(
        "trading_agent_order_generation_duration_ms",
        "Duration of order generation operations in milliseconds",
        &["status"],
        vec![0.1, 0.5, 1.0, 2.5, 5.0, 10.0, 25.0, 50.0, 100.0]
    )
    .expect("Failed to register order_generation_duration histogram")
});

/// Counter for errors by type
static ERRORS_TOTAL: Lazy<CounterVec> = Lazy::new(|| {
    register_counter_vec!(
        opts!(
            "trading_agent_errors_total",
            "Total number of errors by error type"
        ),
        &["error_type"]
    )
    .expect("Failed to register errors_total counter")
});

// ============================================================================
// TradingAgentMetrics Struct
// ============================================================================

/// Trading Agent Metrics container
///
/// Provides methods to record all Trading Agent operations and expose
/// them to Prometheus for monitoring and alerting.
#[derive(Debug, Clone)]
pub struct TradingAgentMetrics {
    // Metrics are stored in static Lazy instances above
    // This struct provides a convenient API wrapper
}

impl TradingAgentMetrics {
    /// Create a new TradingAgentMetrics instance
    ///
    /// This initializes all Prometheus metrics (via Lazy static initialization)
    /// and returns a handle for recording operations.
    pub fn new() -> Self {
        // Force lazy initialization of all metrics
        Lazy::force(&UNIVERSE_SELECTIONS_TOTAL);
        Lazy::force(&UNIVERSE_SELECTION_DURATION);
        Lazy::force(&UNIVERSE_INSTRUMENTS_GAUGE);
        Lazy::force(&ASSET_SELECTIONS_TOTAL);
        Lazy::force(&ASSET_SELECTION_DURATION);
        Lazy::force(&ASSETS_SELECTED_GAUGE);
        Lazy::force(&ALLOCATIONS_TOTAL);
        Lazy::force(&ALLOCATION_DURATION);
        Lazy::force(&PORTFOLIO_VALUE_GAUGE);
        Lazy::force(&ORDERS_GENERATED_TOTAL);
        Lazy::force(&ORDER_GENERATION_DURATION);
        Lazy::force(&ERRORS_TOTAL);

        Self {}
    }

    /// Record a universe selection operation
    ///
    /// # Arguments
    /// * `duration_ms` - Duration of the operation in milliseconds
    /// * `instrument_count` - Number of instruments selected in the universe
    pub fn record_universe_selection(&self, duration_ms: f64, instrument_count: u64) {
        // Increment counter
        UNIVERSE_SELECTIONS_TOTAL
            .with_label_values(&["success"])
            .inc();

        // Record duration
        UNIVERSE_SELECTION_DURATION
            .with_label_values(&["success"])
            .observe(duration_ms);

        // Update gauge
        UNIVERSE_INSTRUMENTS_GAUGE.set(instrument_count as i64);
    }

    /// Record an asset selection operation
    ///
    /// # Arguments
    /// * `duration_ms` - Duration of the operation in milliseconds
    /// * `asset_count` - Number of assets selected
    pub fn record_asset_selection(&self, duration_ms: f64, asset_count: u64) {
        // Increment counter
        ASSET_SELECTIONS_TOTAL.with_label_values(&["success"]).inc();

        // Record duration
        ASSET_SELECTION_DURATION
            .with_label_values(&["success"])
            .observe(duration_ms);

        // Update gauge
        ASSETS_SELECTED_GAUGE.set(asset_count as i64);
    }

    /// Record a portfolio allocation operation
    ///
    /// # Arguments
    /// * `duration_ms` - Duration of the operation in milliseconds
    /// * `portfolio_value` - Total portfolio value in USD
    pub fn record_allocation(&self, duration_ms: f64, portfolio_value: f64) {
        // Increment counter
        ALLOCATIONS_TOTAL.with_label_values(&["success"]).inc();

        // Record duration
        ALLOCATION_DURATION
            .with_label_values(&["success"])
            .observe(duration_ms);

        // Update portfolio value gauge
        PORTFOLIO_VALUE_GAUGE.set(portfolio_value);
    }

    /// Record an order generation operation
    ///
    /// # Arguments
    /// * `duration_ms` - Duration of the operation in milliseconds
    /// * `order_count` - Number of orders generated
    pub fn record_order_generation(&self, duration_ms: f64, order_count: u64) {
        // Increment counter by order count
        for _ in 0..order_count {
            ORDERS_GENERATED_TOTAL.with_label_values(&["success"]).inc();
        }

        // Record duration
        ORDER_GENERATION_DURATION
            .with_label_values(&["success"])
            .observe(duration_ms);
    }

    /// Record an error
    ///
    /// # Arguments
    /// * `error_type` - Type of error that occurred (e.g., "universe_selection_failed")
    pub fn record_error(&self, error_type: &str) {
        // Sanitize error type (empty strings become "unknown")
        let sanitized_error_type = if error_type.is_empty() {
            "unknown"
        } else {
            error_type
        };

        // Increment error counter
        let () = ERRORS_TOTAL
            .with_label_values(&[sanitized_error_type])
            .inc();

        // Log warning for monitoring
        warn!(
            error_type = sanitized_error_type,
            "Trading agent error recorded"
        );
    }
}

impl Default for TradingAgentMetrics {
    fn default() -> Self {
        Self::new()
    }
}

// ============================================================================
// Metrics Server Setup
// ============================================================================

/// Initialize metrics endpoint server
///
/// This should be called once at service startup to expose Prometheus metrics
/// on the /metrics endpoint.
///
/// # Arguments
/// * `port` - Port to bind the metrics server to (default: 9095)
///
/// # Returns
/// A tokio task handle that can be awaited or detached
pub async fn start_metrics_server(
    port: u16,
) -> Result<tokio::task::JoinHandle<()>, Box<dyn std::error::Error>> {
    use axum::{routing::get, Router};
    use prometheus::{Encoder, TextEncoder};
    use std::net::SocketAddr;

    let app = Router::new().route(
        "/metrics",
        get(|| async {
            let encoder = TextEncoder::new();
            let metric_families = prometheus::gather();
            let mut buffer = vec![];
            encoder.encode(&metric_families, &mut buffer).unwrap();
            String::from_utf8(buffer).expect("INVARIANT: Valid UTF-8 bytes")
        }),
    );

    let addr = SocketAddr::from(([0, 0, 0, 0], port));
    tracing::info!("Metrics server listening on {}", addr);

    let handle = tokio::spawn(async move {
        let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
        axum::serve(listener, app).await.unwrap();
    });

    Ok(handle)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_metrics_creation() {
        let metrics = TradingAgentMetrics::new();
        // Verify metrics can be created successfully
        let _ = std::mem::size_of_val(&metrics);
    }

    #[test]
    fn test_metrics_operations() {
        let metrics = TradingAgentMetrics::new();

        // Test all operations
        metrics.record_universe_selection(100.0, 150);
        metrics.record_asset_selection(50.0, 25);
        metrics.record_allocation(75.0, 1_000_000.0);
        metrics.record_order_generation(10.0, 5);
        metrics.record_error("test_error");

        // No panics = success
    }
}