foxhunt/monitoring/prometheus/alerts/system_alerts.yml

# Prometheus Alert Rules for System-Level Monitoring
#
# Infrastructure, database, cache, and network alerts

groups:
  - name: system_resources
    interval: 15s
    rules:
      # CPU utilization critical across all services
      - alert: SystemCPUUtilizationCritical
        expr: |
          100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) > 90
        for: 2m
        labels:
          severity: critical
          component: system
        annotations:
          summary: "System CPU utilization critically high"
          description: "CPU usage {{ $value }}% on {{ $labels.instance }} (threshold: 90%)"
          impact: "All services performance severely degraded"
          action: "1. Identify CPU-intensive processes 2. Scale resources 3. Kill non-essential services"

      # Memory utilization critical
      - alert: SystemMemoryUtilizationCritical
        expr: |
          100 * (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) /
          node_memory_MemTotal_bytes > 95
        for: 2m
        labels:
          severity: critical
          component: system
        annotations:
          summary: "System memory utilization critically high"
          description: "Memory usage {{ $value }}% on {{ $labels.instance }} (threshold: 95%)"
          impact: "Risk of OOM kills and system instability"
          action: "1. Identify memory leaks 2. Restart services 3. Scale memory"

      # Memory utilization warning
      - alert: SystemMemoryUtilizationHigh
        expr: |
          100 * (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) /
          node_memory_MemTotal_bytes > 85
        for: 5m
        labels:
          severity: warning
          component: system
        annotations:
          summary: "System memory utilization high"
          description: "Memory usage {{ $value }}% on {{ $labels.instance }} (threshold: 85%)"

      # Disk space critical
      - alert: DiskSpaceCritical
        expr: |
          100 * node_filesystem_avail_bytes{fstype!="tmpfs",mountpoint!~".*/proc|.*/sys|.*/run.*"} /
          node_filesystem_size_bytes{fstype!="tmpfs",mountpoint!~".*/proc|.*/sys|.*/run.*"} < 5
        for: 5m
        labels:
          severity: critical
          component: system
        annotations:
          summary: "Disk space critically low"
          description: "Only {{ $value }}% space remaining on {{ $labels.mountpoint }} (threshold: 5%)"
          impact: "Risk of service failures due to disk full"
          action: "1. Clean logs 2. Archive data 3. Expand disk"

      # Disk space low
      - alert: DiskSpaceLow
        expr: |
          100 * node_filesystem_avail_bytes{fstype!="tmpfs",mountpoint!~".*/proc|.*/sys|.*/run.*"} /
          node_filesystem_size_bytes{fstype!="tmpfs",mountpoint!~".*/proc|.*/sys|.*/run.*"} < 15
        for: 10m
        labels:
          severity: warning
          component: system
        annotations:
          summary: "Disk space low"
          description: "{{ $value }}% space remaining on {{ $labels.mountpoint }} (threshold: 15%)"

      # System load high
      - alert: SystemLoadHigh
        expr: node_load5 / count(node_cpu_seconds_total{mode="idle"}) without(cpu,mode) > 2
        for: 5m
        labels:
          severity: warning
          component: system
        annotations:
          summary: "System load high"
          description: "5-minute load average {{ $value }} on {{ $labels.instance }}"
          impact: "System under heavy load - performance degradation likely"

      # Disk I/O utilization high
      - alert: DiskIOUtilizationHigh
        expr: |
          100 * rate(node_disk_io_time_seconds_total[2m]) > 80
        for: 5m
        labels:
          severity: warning
          component: system
        annotations:
          summary: "Disk I/O utilization high"
          description: "Disk {{ $labels.device }} I/O {{ $value }}% (threshold: 80%)"
          impact: "Disk operations may be slow"

  - name: network_connectivity
    interval: 10s
    rules:
      # Network packet loss high
      - alert: NetworkPacketLossHigh
        expr: |
          100 * rate(node_network_transmit_drop_total[1m]) /
          rate(node_network_transmit_packets_total[1m]) > 1
        for: 2m
        labels:
          severity: critical
          component: network
        annotations:
          summary: "Network packet loss high"
          description: "Packet loss {{ $value }}% on {{ $labels.device }} (threshold: 1%)"
          impact: "Network reliability degraded"

      # Network errors detected
      - alert: NetworkErrorsDetected
        expr: rate(node_network_transmit_errs_total[5m]) > 10
        for: 2m
        labels:
          severity: warning
          component: network
        annotations:
          summary: "Network transmission errors detected"
          description: "{{ $value }} errors/sec on {{ $labels.device }}"

      # Network interface down
      - alert: NetworkInterfaceDown
        expr: node_network_up{device!~"lo|veth.*"} == 0
        for: 1m
        labels:
          severity: critical
          component: network
        annotations:
          summary: "Network interface down"
          description: "Interface {{ $labels.device }} is down on {{ $labels.instance }}"
          impact: "Potential connectivity loss"

  - name: database_postgresql
    interval: 10s
    rules:
      # PostgreSQL down
      - alert: PostgreSQLDown
        expr: up{job="postgresql"} == 0
        for: 30s
        labels:
          severity: critical
          component: database
        annotations:
          summary: "PostgreSQL database is DOWN"
          description: "Database unreachable for >30 seconds"
          impact: "All services unable to access persistent data"
          action: "1. Check database service 2. Restart if necessary 3. Verify network"
          runbook_url: "https://docs.foxhunt.io/runbooks/postgres-down"

      # PostgreSQL connection pool exhaustion
      - alert: PostgreSQLConnectionPoolExhaustion
        expr: |
          100 * pg_stat_database_numbackends / pg_settings_max_connections > 80
        for: 2m
        labels:
          severity: critical
          component: database
        annotations:
          summary: "PostgreSQL connection pool nearly exhausted"
          description: "{{ $value }}% of max connections used (threshold: 80%)"
          impact: "New database connections may fail"
          action: "1. Identify connection leaks 2. Restart services 3. Increase max_connections"

      # PostgreSQL replication lag high
      - alert: PostgreSQLReplicationLagHigh
        expr: pg_replication_lag_seconds > 30
        for: 2m
        labels:
          severity: warning
          component: database
        annotations:
          summary: "PostgreSQL replication lag high"
          description: "Replication lag {{ $value }}s (threshold: 30s)"
          impact: "Replica data may be stale"

      # PostgreSQL slow queries
      - alert: PostgreSQLSlowQueries
        expr: rate(pg_stat_activity_max_tx_duration[5m]) > 10
        for: 5m
        labels:
          severity: warning
          component: database
        annotations:
          summary: "PostgreSQL slow queries detected"
          description: "Long-running queries detected: {{ $value }}s"
          impact: "Database performance degraded"

      # PostgreSQL deadlocks
      - alert: PostgreSQLDeadlocks
        expr: rate(pg_stat_database_deadlocks[5m]) > 0
        for: 2m
        labels:
          severity: warning
          component: database
        annotations:
          summary: "PostgreSQL deadlocks detected"
          description: "{{ $value }} deadlocks/sec"
          impact: "Transaction conflicts occurring"

      # PostgreSQL cache hit rate low
      - alert: PostgreSQLCacheHitRateLow
        expr: |
          100 * pg_stat_database_blks_hit /
          (pg_stat_database_blks_hit + pg_stat_database_blks_read) < 90
        for: 10m
        labels:
          severity: warning
          component: database
        annotations:
          summary: "PostgreSQL cache hit rate low"
          description: "Cache hit rate {{ $value }}% (threshold: 90%)"
          impact: "Increased disk I/O and slower queries"

  - name: cache_redis
    interval: 10s
    rules:
      # Redis down
      - alert: RedisDown
        expr: up{job="redis"} == 0
        for: 30s
        labels:
          severity: critical
          component: cache
        annotations:
          summary: "Redis cache is DOWN"
          description: "Redis unreachable for >30 seconds"
          impact: "JWT revocation and caching unavailable"
          action: "1. Check Redis service 2. Restart if necessary 3. Verify network"
          runbook_url: "https://docs.foxhunt.io/runbooks/redis-down"

      # Redis memory usage critical
      - alert: RedisMemoryUsageCritical
        expr: |
          100 * redis_memory_used_bytes / redis_memory_max_bytes > 95
        for: 2m
        labels:
          severity: critical
          component: cache
        annotations:
          summary: "Redis memory usage critically high"
          description: "Memory usage {{ $value }}% (threshold: 95%)"
          impact: "Risk of evictions or OOM"
          action: "1. Clear expired keys 2. Increase memory 3. Review retention policies"

      # Redis memory usage high
      - alert: RedisMemoryUsageHigh
        expr: |
          100 * redis_memory_used_bytes / redis_memory_max_bytes > 85
        for: 5m
        labels:
          severity: warning
          component: cache
        annotations:
          summary: "Redis memory usage high"
          description: "Memory usage {{ $value }}% (threshold: 85%)"

      # Redis evicted keys high
      - alert: RedisEvictedKeysHigh
        expr: rate(redis_evicted_keys_total[5m]) > 100
        for: 5m
        labels:
          severity: warning
          component: cache
        annotations:
          summary: "Redis evicting keys due to memory pressure"
          description: "Evicting {{ $value }} keys/sec (threshold: 100)"
          impact: "Cache effectiveness reduced"

      # Redis connection count high
      - alert: RedisConnectionCountHigh
        expr: redis_connected_clients > 1000
        for: 5m
        labels:
          severity: warning
          component: cache
        annotations:
          summary: "Redis connection count high"
          description: "{{ $value }} connected clients (threshold: 1000)"

      # Redis hit rate low
      - alert: RedisHitRateLow
        expr: |
          100 * rate(redis_keyspace_hits_total[5m]) /
          (rate(redis_keyspace_hits_total[5m]) + rate(redis_keyspace_misses_total[5m])) < 80
        for: 10m
        labels:
          severity: warning
          component: cache
        annotations:
          summary: "Redis cache hit rate low"
          description: "Hit rate {{ $value }}% (threshold: 80%)"
          impact: "Increased backend load"

  - name: service_health
    interval: 5s
    rules:
      # Any service down
      - alert: ServiceDown
        expr: up{job=~".*_service"} == 0
        for: 30s
        labels:
          severity: critical
          component: service
        annotations:
          summary: "Service {{ $labels.job }} is DOWN"
          description: "Service unreachable for >30 seconds"
          impact: "Service unavailable - operations affected"
          runbook_url: "https://docs.foxhunt.io/runbooks/service-down"

      # Service restart detected
      - alert: ServiceRestarted
        expr: changes(process_start_time_seconds{job=~".*_service"}[5m]) > 0
        for: 0s
        labels:
          severity: info
          component: service
        annotations:
          summary: "Service {{ $labels.job }} restarted"
          description: "Service restarted - monitoring for stability"

  - name: monitoring_infrastructure
    interval: 30s
    rules:
      # Prometheus target down
      - alert: PrometheusTargetDown
        expr: up == 0
        for: 2m
        labels:
          severity: warning
          component: monitoring
        annotations:
          summary: "Prometheus target {{ $labels.job }} is down"
          description: "Cannot scrape metrics from {{ $labels.instance }}"
          impact: "Monitoring visibility reduced"

      # Prometheus scrape duration high
      - alert: PrometheusScrapeDurationHigh
        expr: scrape_duration_seconds > 10
        for: 5m
        labels:
          severity: warning
          component: monitoring
        annotations:
          summary: "Prometheus scrape duration high"
          description: "Scraping {{ $labels.job }} takes {{ $value }}s (threshold: 10s)"

      # Prometheus storage usage high
      - alert: PrometheusStorageUsageHigh
        expr: |
          100 * prometheus_tsdb_storage_blocks_bytes /
          prometheus_tsdb_retention_limit_bytes > 85
        for: 10m
        labels:
          severity: warning
          component: monitoring
        annotations:
          summary: "Prometheus storage usage high"
          description: "Storage {{ $value }}% full (threshold: 85%)"
          action: "1. Review retention policies 2. Archive old data 3. Expand storage"