foxhunt/services/api/src/metrics/proxy_metrics.rs

//! Backend Proxy Metrics
//!
//! Tracks performance and health for backend gRPC services:
//! - Trading Service
//! - Backtesting Service
//! - ML Training Service

use prometheus::{CounterVec, Histogram, HistogramOpts, HistogramVec, IntGaugeVec, Opts, Registry};

/// Backend proxy and routing metrics
pub struct ProxyMetrics {
    // === Backend Request Metrics ===
    /// Total requests to each backend service
    pub backend_requests_total: CounterVec,
    /// Successful backend requests
    pub backend_requests_success: CounterVec,
    /// Failed backend requests
    pub backend_requests_failure: CounterVec,

    // === Backend Latencies (milliseconds) ===
    /// Request latency to backend services
    pub backend_request_duration_ms: HistogramVec,
    /// gRPC connection establishment time
    pub backend_connection_duration_ms: HistogramVec,

    // === Circuit Breaker State ===
    /// Circuit breaker state per service (0=closed, 1=half-open, 2=open)
    pub circuit_breaker_state: IntGaugeVec,
    /// Circuit breaker trips (transitions to open)
    pub circuit_breaker_trips: CounterVec,
    /// Circuit breaker recoveries (transitions to closed)
    pub circuit_breaker_recoveries: CounterVec,

    // === Connection Pool ===
    /// Active connections per backend
    pub connection_pool_active: IntGaugeVec,
    /// Idle connections per backend
    pub connection_pool_idle: IntGaugeVec,
    /// Connection pool max size
    pub connection_pool_max: IntGaugeVec,
    /// Connection pool wait time (when exhausted)
    pub connection_pool_wait_duration_ms: HistogramVec,

    // === Health Checks ===
    /// Health check successes per service
    pub health_check_success: CounterVec,
    /// Health check failures per service
    pub health_check_failure: CounterVec,
    /// Health check latency
    pub health_check_duration_ms: HistogramVec,
    /// Current health status (0=unhealthy, 1=healthy)
    pub health_status: IntGaugeVec,

    // === gRPC Error Breakdown ===
    /// gRPC errors by status code and service
    pub grpc_errors: CounterVec,
    /// gRPC timeouts per service
    pub grpc_timeouts: CounterVec,

    // === Request Routing ===
    /// Requests routed to each method
    pub requests_by_method: CounterVec,
    /// Routing latency (method resolution)
    pub routing_duration_us: Histogram,
}

impl ProxyMetrics {
    /// Create new proxy metrics and register with Prometheus
    pub fn new(registry: &Registry) -> Result<Self, prometheus::Error> {
        // === Backend Request Metrics ===
        let backend_requests_total = CounterVec::new(
            Opts::new(
                "api_backend_requests_total",
                "Total requests to backend services",
            ),
            &["service"],
        )?;
        registry.register(Box::new(backend_requests_total.clone()))?;

        let backend_requests_success = CounterVec::new(
            Opts::new(
                "api_backend_requests_success",
                "Successful backend requests",
            ),
            &["service"],
        )?;
        registry.register(Box::new(backend_requests_success.clone()))?;

        let backend_requests_failure = CounterVec::new(
            Opts::new(
                "api_backend_requests_failure",
                "Failed backend requests",
            ),
            &["service", "error_type"],
        )?;
        registry.register(Box::new(backend_requests_failure.clone()))?;

        // === Backend Latencies (milliseconds) ===
        let backend_request_duration_ms = HistogramVec::new(
            HistogramOpts::new(
                "api_backend_request_duration_milliseconds",
                "Backend request latency in milliseconds",
            )
            .buckets(vec![
                1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0,
            ]),
            &["service", "method"],
        )?;
        registry.register(Box::new(backend_request_duration_ms.clone()))?;

        let backend_connection_duration_ms = HistogramVec::new(
            HistogramOpts::new(
                "api_backend_connection_duration_milliseconds",
                "gRPC connection establishment time in milliseconds",
            )
            .buckets(vec![1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0]),
            &["service"],
        )?;
        registry.register(Box::new(backend_connection_duration_ms.clone()))?;

        // === Circuit Breaker State ===
        let circuit_breaker_state = IntGaugeVec::new(
            Opts::new(
                "api_circuit_breaker_state",
                "Circuit breaker state (0=closed, 1=half-open, 2=open)",
            ),
            &["service"],
        )?;
        registry.register(Box::new(circuit_breaker_state.clone()))?;

        let circuit_breaker_trips = CounterVec::new(
            Opts::new(
                "api_circuit_breaker_trips",
                "Circuit breaker trips to open state",
            ),
            &["service"],
        )?;
        registry.register(Box::new(circuit_breaker_trips.clone()))?;

        let circuit_breaker_recoveries = CounterVec::new(
            Opts::new(
                "api_circuit_breaker_recoveries",
                "Circuit breaker recoveries to closed state",
            ),
            &["service"],
        )?;
        registry.register(Box::new(circuit_breaker_recoveries.clone()))?;

        // === Connection Pool ===
        let connection_pool_active = IntGaugeVec::new(
            Opts::new(
                "api_connection_pool_active",
                "Active connections per backend",
            ),
            &["service"],
        )?;
        registry.register(Box::new(connection_pool_active.clone()))?;

        let connection_pool_idle = IntGaugeVec::new(
            Opts::new(
                "api_connection_pool_idle",
                "Idle connections per backend",
            ),
            &["service"],
        )?;
        registry.register(Box::new(connection_pool_idle.clone()))?;

        let connection_pool_max = IntGaugeVec::new(
            Opts::new(
                "api_connection_pool_max",
                "Connection pool max size per backend",
            ),
            &["service"],
        )?;
        registry.register(Box::new(connection_pool_max.clone()))?;

        let connection_pool_wait_duration_ms = HistogramVec::new(
            HistogramOpts::new(
                "api_connection_pool_wait_duration_milliseconds",
                "Connection pool wait time when exhausted",
            )
            .buckets(vec![1.0, 5.0, 10.0, 25.0, 50.0, 100.0]),
            &["service"],
        )?;
        registry.register(Box::new(connection_pool_wait_duration_ms.clone()))?;

        // === Health Checks ===
        let health_check_success = CounterVec::new(
            Opts::new(
                "api_health_check_success",
                "Health check successes per service",
            ),
            &["service"],
        )?;
        registry.register(Box::new(health_check_success.clone()))?;

        let health_check_failure = CounterVec::new(
            Opts::new(
                "api_health_check_failure",
                "Health check failures per service",
            ),
            &["service"],
        )?;
        registry.register(Box::new(health_check_failure.clone()))?;

        let health_check_duration_ms = HistogramVec::new(
            HistogramOpts::new(
                "api_health_check_duration_milliseconds",
                "Health check latency in milliseconds",
            )
            .buckets(vec![10.0, 50.0, 100.0, 250.0, 500.0, 1000.0]),
            &["service"],
        )?;
        registry.register(Box::new(health_check_duration_ms.clone()))?;

        let health_status = IntGaugeVec::new(
            Opts::new(
                "api_health_status",
                "Current health status (0=unhealthy, 1=healthy)",
            ),
            &["service"],
        )?;
        registry.register(Box::new(health_status.clone()))?;

        // === gRPC Error Breakdown ===
        let grpc_errors = CounterVec::new(
            Opts::new("api_grpc_errors", "gRPC errors by status code"),
            &["service", "status_code"],
        )?;
        registry.register(Box::new(grpc_errors.clone()))?;

        let grpc_timeouts = CounterVec::new(
            Opts::new("api_grpc_timeouts", "gRPC timeouts per service"),
            &["service"],
        )?;
        registry.register(Box::new(grpc_timeouts.clone()))?;

        // === Request Routing ===
        let requests_by_method = CounterVec::new(
            Opts::new("api_requests_by_method", "Requests by gRPC method"),
            &["service", "method"],
        )?;
        registry.register(Box::new(requests_by_method.clone()))?;

        let routing_duration_us = Histogram::with_opts(
            HistogramOpts::new(
                "api_routing_duration_microseconds",
                "Routing decision latency in microseconds",
            )
            .buckets(vec![0.1, 0.5, 1.0, 2.0, 5.0, 10.0]),
        )?;
        registry.register(Box::new(routing_duration_us.clone()))?;

        Ok(Self {
            backend_requests_total,
            backend_requests_success,
            backend_requests_failure,
            backend_request_duration_ms,
            backend_connection_duration_ms,
            circuit_breaker_state,
            circuit_breaker_trips,
            circuit_breaker_recoveries,
            connection_pool_active,
            connection_pool_idle,
            connection_pool_max,
            connection_pool_wait_duration_ms,
            health_check_success,
            health_check_failure,
            health_check_duration_ms,
            health_status,
            grpc_errors,
            grpc_timeouts,
            requests_by_method,
            routing_duration_us,
        })
    }

    /// Record successful backend request
    pub fn record_backend_success(&self, service: &str, method: &str, duration_ms: f64) {
        self.backend_requests_total
            .with_label_values(&[service])
            .inc();
        self.backend_requests_success
            .with_label_values(&[service])
            .inc();
        self.backend_request_duration_ms
            .with_label_values(&[service, method])
            .observe(duration_ms);
        self.requests_by_method
            .with_label_values(&[service, method])
            .inc();
    }

    /// Record failed backend request
    pub fn record_backend_failure(&self, service: &str, error_type: &str) {
        self.backend_requests_total
            .with_label_values(&[service])
            .inc();
        self.backend_requests_failure
            .with_label_values(&[service, error_type])
            .inc();
    }

    /// Update circuit breaker state (0=closed, 1=half-open, 2=open)
    pub fn update_circuit_breaker_state(&self, service: &str, state: i64) {
        self.circuit_breaker_state
            .with_label_values(&[service])
            .set(state);
    }

    /// Record circuit breaker trip
    pub fn record_circuit_breaker_trip(&self, service: &str) {
        self.circuit_breaker_trips
            .with_label_values(&[service])
            .inc();
        self.update_circuit_breaker_state(service, 2); // 2 = open
    }

    /// Record circuit breaker recovery
    pub fn record_circuit_breaker_recovery(&self, service: &str) {
        self.circuit_breaker_recoveries
            .with_label_values(&[service])
            .inc();
        self.update_circuit_breaker_state(service, 0); // 0 = closed
    }

    /// Update connection pool stats
    pub fn update_connection_pool(&self, service: &str, active: i64, idle: i64, max: i64) {
        self.connection_pool_active
            .with_label_values(&[service])
            .set(active);
        self.connection_pool_idle
            .with_label_values(&[service])
            .set(idle);
        self.connection_pool_max
            .with_label_values(&[service])
            .set(max);
    }

    /// Update health status (0=unhealthy, 1=healthy)
    pub fn update_health_status(&self, service: &str, healthy: bool) {
        self.health_status
            .with_label_values(&[service])
            .set(if healthy { 1 } else { 0 });
    }
}