# AlertManager Configuration for Foxhunt # # Routes alerts to appropriate notification channels global: resolve_timeout: 5m slack_api_url: 'https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK' # Alert routing tree route: receiver: 'default' group_by: ['alertname', 'cluster', 'service'] group_wait: 10s group_interval: 5m repeat_interval: 4h # Route alerts based on severity routes: # Critical alerts - immediate notification - match: severity: critical receiver: 'critical-alerts' group_wait: 0s repeat_interval: 1h # Warning alerts - less urgent - match: severity: warning receiver: 'warning-alerts' group_wait: 30s repeat_interval: 4h # Auth-specific alerts - match: component: auth receiver: 'auth-alerts' group_by: ['alertname'] # Backend proxy alerts - match: component: proxy receiver: 'backend-alerts' # Configuration alerts - match: component: config receiver: 'config-alerts' # Alert receivers (notification channels) receivers: # Default receiver (logs only) - name: 'default' webhook_configs: - url: 'http://localhost:9090/api/v1/alerts' # Critical alerts - multiple channels - name: 'critical-alerts' slack_configs: - channel: '#foxhunt-critical' title: '🚨 CRITICAL: {{ .GroupLabels.alertname }}' text: '{{ range .Alerts }}{{ .Annotations.summary }}\n{{ .Annotations.description }}{{ end }}' send_resolved: true # PagerDuty for on-call rotation pagerduty_configs: - service_key: 'YOUR_PAGERDUTY_SERVICE_KEY' description: '{{ .GroupLabels.alertname }}' # Warning alerts - Slack only - name: 'warning-alerts' slack_configs: - channel: '#foxhunt-warnings' title: '⚠️ Warning: {{ .GroupLabels.alertname }}' text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}' send_resolved: true # Auth-specific alerts - name: 'auth-alerts' slack_configs: - channel: '#foxhunt-auth' title: '🔐 Auth Alert: {{ .GroupLabels.alertname }}' text: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}' # Backend alerts - name: 'backend-alerts' slack_configs: - channel: '#foxhunt-backend' title: '🔌 Backend Alert: {{ .GroupLabels.alertname }}' text: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}' # Config alerts - name: 'config-alerts' slack_configs: - channel: '#foxhunt-config' title: '⚙️ Config Alert: {{ .GroupLabels.alertname }}' text: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}' # Inhibition rules (suppress redundant alerts) inhibit_rules: # If circuit breaker is open, suppress high latency alerts - source_match: alertname: 'CircuitBreakerOpen' target_match: alertname: 'HighBackendLatency' equal: ['service'] # If backend is unhealthy, suppress other backend alerts - source_match: alertname: 'BackendServiceUnhealthy' target_match_re: alertname: 'HighBackendLatency|CircuitBreakerOpen' equal: ['service'] # If NOTIFY listener is down, suppress config alerts - source_match: alertname: 'NotifyListenerDisconnected' target_match_re: alertname: 'HighConfigReloadLatency|ConfigValidationFailures'