# Prometheus Alert Rules for API Gateway # # Critical alerts for authentication, proxy, and configuration groups: - name: api_gateway_auth interval: 10s rules: # Auth SLA Violation: >10μs latency - alert: AuthLatencySLAViolation expr: histogram_quantile(0.99, rate(api_gateway_auth_total_duration_microseconds_bucket[1m])) > 10 for: 1m labels: severity: critical component: auth annotations: summary: "API Gateway auth latency exceeded 10μs SLA" description: "p99 auth latency is {{ $value }}μs (target: <10μs)" # High auth failure rate - alert: HighAuthFailureRate expr: 100 * rate(api_gateway_auth_requests_failure[5m]) / rate(api_gateway_auth_requests_total[5m]) > 10 for: 2m labels: severity: warning component: auth annotations: summary: "High authentication failure rate" description: "Auth failure rate is {{ $value }}% (threshold: 10%)" # Redis connection failure - alert: RedisConnectionFailure expr: rate(api_gateway_auth_errors_redis_failure[1m]) > 0 for: 1m labels: severity: critical component: auth annotations: summary: "JWT revocation Redis connection failed" description: "Redis errors detected: {{ $value }}/s" # JWT revocation cache size explosion - alert: RevocationCacheSizeExplosion expr: api_gateway_revoked_tokens_cached > 100000 for: 5m labels: severity: warning component: auth annotations: summary: "JWT revocation cache size excessive" description: "Revoked tokens cached: {{ $value }} (threshold: 100k)" # Cache hit rate too low - alert: LowCacheHitRate expr: | 100 * rate(api_gateway_rbac_cache_hits[5m]) / (rate(api_gateway_rbac_cache_hits[5m]) + rate(api_gateway_rbac_cache_misses[5m])) < 90 for: 5m labels: severity: warning component: auth annotations: summary: "RBAC cache hit rate below 90%" description: "Cache hit rate is {{ $value }}% (target: >90%)" - name: api_gateway_proxy interval: 10s rules: # Circuit breaker open - alert: CircuitBreakerOpen expr: api_gateway_circuit_breaker_state > 1.5 for: 1m labels: severity: critical component: proxy annotations: summary: "Circuit breaker open for {{ $labels.service }}" description: "Backend service {{ $labels.service }} circuit breaker is open" # Backend service unhealthy - alert: BackendServiceUnhealthy expr: api_gateway_health_status == 0 for: 2m labels: severity: critical component: proxy annotations: summary: "Backend service {{ $labels.service }} unhealthy" description: "Health checks failing for {{ $labels.service }}" # High backend latency - alert: HighBackendLatency expr: histogram_quantile(0.99, rate(api_gateway_backend_request_duration_milliseconds_bucket[1m])) > 100 for: 3m labels: severity: warning component: proxy annotations: summary: "High latency to {{ $labels.service }}" description: "p99 latency to {{ $labels.service }} is {{ $value }}ms (threshold: 100ms)" # Connection pool exhaustion - alert: ConnectionPoolExhaustion expr: | 100 * api_gateway_connection_pool_active / api_gateway_connection_pool_max > 90 for: 5m labels: severity: warning component: proxy annotations: summary: "Connection pool nearly exhausted for {{ $labels.service }}" description: "Pool utilization: {{ $value }}% (threshold: 90%)" - name: api_gateway_config interval: 10s rules: # NOTIFY listener disconnected - alert: NotifyListenerDisconnected expr: api_gateway_notify_listener_connected == 0 for: 1m labels: severity: critical component: config annotations: summary: "PostgreSQL NOTIFY listener disconnected" description: "Hot-reload capability lost - configuration changes will not propagate" # High config reload latency - alert: HighConfigReloadLatency expr: histogram_quantile(0.95, rate(api_gateway_config_reload_duration_milliseconds_bucket[1m])) > 100 for: 5m labels: severity: warning component: config annotations: summary: "Slow configuration reload" description: "p95 config reload latency is {{ $value }}ms (threshold: 100ms)" # Config validation failures - alert: ConfigValidationFailures expr: rate(api_gateway_config_validation_failure[5m]) > 0 for: 2m labels: severity: warning component: config annotations: summary: "Configuration validation failures detected" description: "Invalid config updates: {{ $value }}/s" - name: api_gateway_rate_limiting interval: 10s rules: # Excessive rate limiting - alert: ExcessiveRateLimiting expr: rate(api_gateway_auth_errors_rate_limited[1m]) > 10 for: 5m labels: severity: warning component: rate_limiting annotations: summary: "High rate limit rejection rate" description: "Rate limit rejections: {{ $value }}/s (may indicate DDoS or misconfiguration)"