foxhunt/monitoring/alertmanager/alertmanager.yml

# AlertManager Configuration for Foxhunt
#
# Routes alerts to appropriate notification channels

global:
  resolve_timeout: 5m
  slack_api_url: 'https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK'
  pagerduty_url: 'https://events.pagerduty.com/v2/enqueue'

# Alert routing tree
route:
  receiver: 'default'
  group_by: ['alertname', 'cluster', 'service']
  group_wait: 10s
  group_interval: 5m
  repeat_interval: 4h

  # Route alerts based on severity
  routes:
    # Critical ensemble alerts - IMMEDIATE PagerDuty + Slack
    - match:
        severity: critical
        component: ensemble
      receiver: 'ensemble-critical'
      group_wait: 0s
      repeat_interval: 30m
      continue: false

    # Warning ensemble alerts - Slack only
    - match:
        severity: warning
        component: ensemble
      receiver: 'ensemble-warnings'
      group_wait: 15s
      repeat_interval: 2h
      continue: false

    # Info ensemble alerts (A/B test results) - Slack only
    - match:
        severity: info
        component: ensemble
      receiver: 'ensemble-info'
      group_wait: 5m
      repeat_interval: 24h
      continue: false

    # Critical alerts (non-ensemble) - PagerDuty + Slack
    - match:
        severity: critical
      receiver: 'critical-alerts'
      group_wait: 0s
      repeat_interval: 1h

    # Warning alerts - less urgent
    - match:
        severity: warning
      receiver: 'warning-alerts'
      group_wait: 30s
      repeat_interval: 4h

    # Auth-specific alerts
    - match:
        component: auth
      receiver: 'auth-alerts'
      group_by: ['alertname']

    # Backend proxy alerts
    - match:
        component: proxy
      receiver: 'backend-alerts'

    # Configuration alerts
    - match:
        component: config
      receiver: 'config-alerts'

# Alert receivers (notification channels)
receivers:
  # Default receiver (logs only)
  - name: 'default'
    webhook_configs:
      - url: 'http://localhost:9090/api/v1/alerts'

  # ========== ENSEMBLE ALERTS ==========
  # Ensemble critical alerts - PagerDuty + Slack
  - name: 'ensemble-critical'
    slack_configs:
      - channel: '#foxhunt-ensemble-critical'
        title: '🚨 ENSEMBLE CRITICAL: {{ .GroupLabels.alertname }}'
        text: |
          *Alert:* {{ .GroupLabels.alertname }}
          *Type:* {{ .CommonLabels.alert_type }}
          *Symbol:* {{ .CommonLabels.symbol }}
          {{ range .Alerts }}
          *Summary:* {{ .Annotations.summary }}
          *Description:* {{ .Annotations.description }}
          *Impact:* {{ .Annotations.impact }}
          *Action Required:*
          {{ .Annotations.action }}
          *Runbook:* {{ .Annotations.runbook_url }}
          {{ end }}
        send_resolved: true
        color: '{{ if eq .Status "firing" }}danger{{ else }}good{{ end }}'

    pagerduty_configs:
      - routing_key: 'YOUR_PAGERDUTY_ENSEMBLE_INTEGRATION_KEY'
        severity: 'critical'
        description: '{{ .GroupLabels.alertname }}: {{ .CommonAnnotations.summary }}'
        details:
          alert_type: '{{ .CommonLabels.alert_type }}'
          symbol: '{{ .CommonLabels.symbol }}'
          impact: '{{ .CommonAnnotations.impact }}'
          action: '{{ .CommonAnnotations.action }}'
          runbook_url: '{{ .CommonAnnotations.runbook_url }}'
        client: 'Foxhunt Ensemble Monitoring'
        client_url: 'http://localhost:3000/d/ensemble-ml-production'

  # Ensemble warning alerts - Slack only
  - name: 'ensemble-warnings'
    slack_configs:
      - channel: '#foxhunt-ensemble-warnings'
        title: '⚠️ ENSEMBLE WARNING: {{ .GroupLabels.alertname }}'
        text: |
          *Alert:* {{ .GroupLabels.alertname }}
          *Type:* {{ .CommonLabels.alert_type }}
          *Symbol:* {{ .CommonLabels.symbol }}
          {{ range .Alerts }}
          *Summary:* {{ .Annotations.summary }}
          *Description:* {{ .Annotations.description }}
          {{ if .Annotations.action }}*Action:* {{ .Annotations.action }}{{ end }}
          {{ end }}
        send_resolved: true
        color: 'warning'

  # Ensemble info alerts (A/B tests, model updates)
  - name: 'ensemble-info'
    slack_configs:
      - channel: '#foxhunt-ensemble-info'
        title: 'ℹ️ ENSEMBLE INFO: {{ .GroupLabels.alertname }}'
        text: |
          *Alert:* {{ .GroupLabels.alertname }}
          {{ range .Alerts }}
          *Summary:* {{ .Annotations.summary }}
          *Description:* {{ .Annotations.description }}
          {{ end }}
        send_resolved: true
        color: 'good'

  # ========== GENERAL ALERTS ==========
  # Critical alerts (non-ensemble) - PagerDuty + Slack
  - name: 'critical-alerts'
    slack_configs:
      - channel: '#foxhunt-critical'
        title: '🚨 CRITICAL: {{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.summary }}\n{{ .Annotations.description }}{{ end }}'
        send_resolved: true

    # PagerDuty for on-call rotation
    pagerduty_configs:
      - routing_key: 'YOUR_PAGERDUTY_SERVICE_KEY'
        severity: 'critical'
        description: '{{ .GroupLabels.alertname }}'

  # Warning alerts - Slack only
  - name: 'warning-alerts'
    slack_configs:
      - channel: '#foxhunt-warnings'
        title: '⚠️ Warning: {{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'
        send_resolved: true

  # Auth-specific alerts
  - name: 'auth-alerts'
    slack_configs:
      - channel: '#foxhunt-auth'
        title: '🔐 Auth Alert: {{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}'

  # Backend alerts
  - name: 'backend-alerts'
    slack_configs:
      - channel: '#foxhunt-backend'
        title: '🔌 Backend Alert: {{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}'

  # Config alerts
  - name: 'config-alerts'
    slack_configs:
      - channel: '#foxhunt-config'
        title: '⚙️ Config Alert: {{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}'

# Inhibition rules (suppress redundant alerts)
inhibit_rules:
  # ========== ENSEMBLE INHIBITION RULES ==========
  # If cascade failure detected, suppress individual model failures
  - source_match:
      alertname: 'EnsembleCascadeFailureDetected'
    target_match:
      alertname: 'EnsembleModelFailureDetected'
    equal: ['cluster']

  # If Sharpe ratio drop critical, suppress warning
  - source_match:
      alertname: 'EnsembleSharpeRatioDropCritical'
    target_match:
      alertname: 'EnsembleSharpeRatioDropWarning'
    equal: ['symbol']

  # If high memory, suppress latency alerts (latency caused by swapping)
  - source_match:
      alertname: 'EnsembleServiceMemoryHigh'
    target_match:
      alertname: 'EnsembleAggregationLatencyP99High'
    equal: ['instance']

  # If checkpoint rollback rate high, suppress individual swap failures
  - source_match:
      alertname: 'EnsembleCheckpointRollbackRateHigh'
    target_match:
      alertname: 'EnsembleCheckpointSwapFailed'
    equal: ['cluster']

  # If high disagreement critical, suppress warning
  - source_match:
      alertname: 'EnsembleHighDisagreementCritical'
    target_match:
      alertname: 'EnsembleHighDisagreementWarning'
    equal: ['symbol']

  # If low confidence + high disagreement, suppress both individual alerts
  - source_match:
      alertname: 'EnsembleLowConfidenceHighDisagreement'
    target_match_re:
      alertname: 'EnsembleHighDisagreementCritical|EnsembleHighDisagreementWarning'
    equal: ['symbol']

  # ========== GENERAL INHIBITION RULES ==========
  # If circuit breaker is open, suppress high latency alerts
  - source_match:
      alertname: 'CircuitBreakerOpen'
    target_match:
      alertname: 'HighBackendLatency'
    equal: ['service']

  # If backend is unhealthy, suppress other backend alerts
  - source_match:
      alertname: 'BackendServiceUnhealthy'
    target_match_re:
      alertname: 'HighBackendLatency|CircuitBreakerOpen'
    equal: ['service']

  # If NOTIFY listener is down, suppress config alerts
  - source_match:
      alertname: 'NotifyListenerDisconnected'
    target_match_re:
      alertname: 'HighConfigReloadLatency|ConfigValidationFailures'