foxhunt/config/prometheus/rules/foxhunt-alerts.yml

# Foxhunt HFT Trading System - Alert Rules
groups:
  - name: foxhunt-trading-alerts
    rules:
      # Trading System Health
      - alert: TradingServiceDown
        expr: up{job=~"foxhunt-.*"} == 0
        for: 10s
        labels:
          severity: critical
        annotations:
          summary: "Foxhunt service {{ $labels.job }} is down"
          description: "Service {{ $labels.job }} has been down for more than 10 seconds."

      # Latency Alerts
      - alert: HighTradingLatency
        expr: foxhunt_order_processing_duration_seconds{quantile="0.95"} > 0.001
        for: 30s
        labels:
          severity: critical
        annotations:
          summary: "High trading latency detected"
          description: "95th percentile order processing latency is {{ $value }}s, above 1ms threshold."

      - alert: HighMarketDataLatency
        expr: foxhunt_market_data_latency_seconds{quantile="0.95"} > 0.0005
        for: 15s
        labels:
          severity: warning
        annotations:
          summary: "High market data latency"
          description: "95th percentile market data latency is {{ $value }}s, above 500μs threshold."

      # Risk Management Alerts
      - alert: PositionLimitBreached
        expr: foxhunt_position_size_total > foxhunt_position_limit_total
        for: 0s
        labels:
          severity: critical
        annotations:
          summary: "Position limit breached"
          description: "Total position size ({{ $value }}) exceeds configured limit."

      - alert: HighDrawdown
        expr: foxhunt_portfolio_drawdown_percent > 5
        for: 0s
        labels:
          severity: critical
        annotations:
          summary: "High portfolio drawdown"
          description: "Portfolio drawdown is {{ $value }}%, exceeding 5% threshold."

      - alert: RiskServiceUnresponsive
        expr: increase(foxhunt_risk_check_failures_total[1m]) > 5
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Risk management service failures"
          description: "Risk check failures have increased by {{ $value }} in the last minute."

      # Performance Alerts
      - alert: HighCPUUsage
        expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) > 80
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "High CPU usage"
          description: "CPU usage is {{ $value }}% on {{ $labels.instance }}."

      - alert: HighMemoryUsage
        expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "High memory usage"
          description: "Memory usage is {{ $value }}% on {{ $labels.instance }}."

      - alert: DiskSpaceLow
        expr: (node_filesystem_avail_bytes / node_filesystem_size_bytes) * 100 < 10
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Low disk space"
          description: "Disk space is {{ $value }}% available on {{ $labels.instance }}."

  - name: foxhunt-database-alerts
    rules:
      # Database Health
      - alert: PostgreSQLDown
        expr: up{job="postgres"} == 0
        for: 30s
        labels:
          severity: critical
        annotations:
          summary: "PostgreSQL is down"
          description: "PostgreSQL database has been down for more than 30 seconds."

      - alert: RedisDown
        expr: up{job="redis"} == 0
        for: 30s
        labels:
          severity: critical
        annotations:
          summary: "Redis is down"
          description: "Redis cache has been down for more than 30 seconds."

      - alert: InfluxDBDown
        expr: up{job="influxdb"} == 0
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "InfluxDB is down"
          description: "InfluxDB time-series database has been down for more than 1 minute."

      # Database Performance
      - alert: HighDatabaseConnections
        expr: postgres_stat_database_numbackends > 150
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "High PostgreSQL connections"
          description: "PostgreSQL has {{ $value }} active connections, approaching limit."

      - alert: SlowDatabaseQueries
        expr: postgres_stat_statements_mean_time_ms > 100
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "Slow database queries detected"
          description: "Average query time is {{ $value }}ms, above 100ms threshold."

  - name: foxhunt-trading-metrics
    rules:
      # Trading Volume and Activity
      - alert: LowTradingVolume
        expr: rate(foxhunt_trades_total[5m]) < 0.1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Low trading volume"
          description: "Trading volume has been below 0.1 trades/second for 5 minutes."

      - alert: OrderRejectionsHigh
        expr: rate(foxhunt_orders_rejected_total[1m]) > 0.05
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "High order rejection rate"
          description: "Order rejection rate is {{ $value }} orders/second, above normal threshold."

      # Market Data Quality
      - alert: MarketDataStale
        expr: time() - foxhunt_last_market_data_timestamp > 5
        for: 0s
        labels:
          severity: critical
        annotations:
          summary: "Stale market data"
          description: "Market data is {{ $value }} seconds old, exceeding 5 second threshold."

      - alert: MarketDataGaps
        expr: increase(foxhunt_market_data_gaps_total[1m]) > 3
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "Market data gaps detected"
          description: "{{ $value }} market data gaps detected in the last minute."