foxhunt/config/prometheus/alertmanager-production.yml

# AlertManager Production Configuration for Foxhunt HFT Trading System
# Agent H5: Production Alerting Configuration
# Created: 2025-10-18

global:
  resolve_timeout: 5m
  # Slack webhook (replace with actual webhook URL in production)
  slack_api_url: 'https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK'
  # SMTP for email alerts
  smtp_smarthost: 'localhost:587'
  smtp_from: 'alerts@foxhunt.local'
  smtp_require_tls: true

# Templates for rich alert formatting
templates:
  - '/etc/alertmanager/templates/*.tmpl'

# ============================================================================
# ROUTING CONFIGURATION
# ============================================================================
route:
  # Default settings
  receiver: 'default-webhook'
  group_by: ['alertname', 'severity', 'component', 'service']
  group_wait: 10s        # Wait 10s to batch alerts
  group_interval: 5m     # Wait 5m before sending more from same group
  repeat_interval: 4h    # Resend after 4h if still firing

  # Hierarchical routing based on severity and component
  routes:
    # ========================================================================
    # CRITICAL ALERTS - IMMEDIATE RESPONSE
    # ========================================================================

    # Critical latency alerts - P99 > 100ms
    - match:
        severity: critical
        component: latency
      receiver: 'critical-latency'
      group_wait: 0s
      group_interval: 1m
      repeat_interval: 15m
      continue: false

    # Critical service down alerts
    - match:
        severity: critical
        component: availability
      receiver: 'critical-service-down'
      group_wait: 0s
      group_interval: 30s
      repeat_interval: 5m
      continue: false

    # Critical memory growth alerts
    - match:
        severity: critical
        component: memory
      receiver: 'critical-memory'
      group_wait: 0s
      group_interval: 2m
      repeat_interval: 10m
      continue: false

    # Critical trading/risk alerts
    - match:
        severity: critical
        component: risk
      receiver: 'critical-risk'
      group_wait: 0s
      group_interval: 30s
      repeat_interval: 5m
      continue: false

    - match:
        severity: critical
        component: trading
      receiver: 'critical-trading'
      group_wait: 0s
      group_interval: 1m
      repeat_interval: 10m
      continue: false

    # Critical database alerts
    - match:
        severity: critical
        component: database
      receiver: 'critical-database'
      group_wait: 0s
      group_interval: 1m
      repeat_interval: 10m
      continue: false

    # All other critical alerts
    - match:
        severity: critical
      receiver: 'critical-generic'
      group_wait: 5s
      group_interval: 2m
      repeat_interval: 30m
      continue: false

    # ========================================================================
    # WARNING ALERTS - REVIEW WITHIN HOURS
    # ========================================================================

    # Warning error rate alerts
    - match:
        severity: warning
        component: errors
      receiver: 'warning-errors'
      group_wait: 30s
      group_interval: 5m
      repeat_interval: 2h
      continue: false

    # Warning resource alerts (CPU, disk)
    - match:
        severity: warning
        component: cpu
      receiver: 'warning-resources'
      group_wait: 1m
      group_interval: 5m
      repeat_interval: 4h
      continue: false

    - match:
        severity: warning
        component: disk
      receiver: 'warning-resources'
      group_wait: 1m
      group_interval: 5m
      repeat_interval: 4h
      continue: false

    # Warning ML alerts
    - match:
        severity: warning
        component: ml
      receiver: 'warning-ml'
      group_wait: 1m
      group_interval: 10m
      repeat_interval: 4h
      continue: false

    # All other warnings
    - match:
        severity: warning
      receiver: 'warning-generic'
      group_wait: 1m
      group_interval: 10m
      repeat_interval: 6h

# ============================================================================
# ALERT RECEIVERS (Notification Channels)
# ============================================================================
receivers:
  # Default webhook receiver
  - name: 'default-webhook'
    webhook_configs:
      - url: 'http://localhost:5001/webhook'
        send_resolved: true

  # ========================================================================
  # CRITICAL ALERT RECEIVERS
  # ========================================================================

  # Critical Latency Alerts (P99 > 100ms)
  - name: 'critical-latency'
    slack_configs:
      - channel: '#foxhunt-critical-latency'
        username: 'Foxhunt Alerting'
        icon_emoji: ':rotating_light:'
        title: '🚨 CRITICAL LATENCY: {{ .GroupLabels.alertname }}'
        title_link: 'http://localhost:3000/d/foxhunt-performance'
        text: |
          *Service:* {{ .CommonLabels.service }}
          *Component:* {{ .CommonLabels.component }}
          {{ range .Alerts }}
          *Summary:* {{ .Annotations.summary }}
          *Description:* {{ .Annotations.description }}
          *Runbook:* {{ .Annotations.runbook_url }}
          {{ end }}
        send_resolved: true
        color: '{{ if eq .Status "firing" }}danger{{ else }}good{{ end }}'

    webhook_configs:
      - url: 'http://localhost:5001/critical-latency'
        send_resolved: true

  # Critical Service Down
  - name: 'critical-service-down'
    slack_configs:
      - channel: '#foxhunt-critical-outages'
        username: 'Foxhunt Alerting'
        icon_emoji: ':fire:'
        title: '🔥 SERVICE DOWN: {{ .GroupLabels.alertname }}'
        title_link: 'http://localhost:3000/d/foxhunt-services'
        text: |
          *PRODUCTION OUTAGE*
          {{ range .Alerts }}
          *Service:* {{ .Labels.job }}
          *Instance:* {{ .Labels.instance }}
          *Summary:* {{ .Annotations.summary }}
          *Description:* {{ .Annotations.description }}
          *Runbook:* {{ .Annotations.runbook_url }}
          {{ end }}
        send_resolved: true
        color: '{{ if eq .Status "firing" }}danger{{ else }}good{{ end }}'

    email_configs:
      - to: 'oncall@foxhunt.local'
        subject: '🔥 CRITICAL: {{ .GroupLabels.alertname }} - SERVICE DOWN'
        html: |
          <h2 style="color: red;">PRODUCTION OUTAGE</h2>
          {{ range .Alerts }}
          <p><strong>Service:</strong> {{ .Labels.job }}</p>
          <p><strong>Instance:</strong> {{ .Labels.instance }}</p>
          <p><strong>Summary:</strong> {{ .Annotations.summary }}</p>
          <p><strong>Description:</strong> {{ .Annotations.description }}</p>
          <p><strong>Runbook:</strong> <a href="{{ .Annotations.runbook_url }}">{{ .Annotations.runbook_url }}</a></p>
          {{ end }}
        headers:
          Priority: 'urgent'

    webhook_configs:
      - url: 'http://localhost:5001/critical-service-down'
        send_resolved: true

  # Critical Memory Growth/Leaks
  - name: 'critical-memory'
    slack_configs:
      - channel: '#foxhunt-critical-memory'
        username: 'Foxhunt Alerting'
        icon_emoji: ':chart_with_upwards_trend:'
        title: '🚨 CRITICAL MEMORY: {{ .GroupLabels.alertname }}'
        title_link: 'http://localhost:3000/d/foxhunt-resources'
        text: |
          *Service:* {{ .CommonLabels.job }}
          {{ range .Alerts }}
          *Summary:* {{ .Annotations.summary }}
          *Description:* {{ .Annotations.description }}
          *Runbook:* {{ .Annotations.runbook_url }}
          {{ end }}
        send_resolved: true
        color: '{{ if eq .Status "firing" }}danger{{ else }}good{{ end }}'

    webhook_configs:
      - url: 'http://localhost:5001/critical-memory'
        send_resolved: true

  # Critical Risk Management
  - name: 'critical-risk'
    slack_configs:
      - channel: '#foxhunt-critical-risk'
        username: 'Foxhunt Alerting'
        icon_emoji: ':warning:'
        title: '🚨 CRITICAL RISK: {{ .GroupLabels.alertname }}'
        title_link: 'http://localhost:3000/d/foxhunt-risk'
        text: |
          *IMMEDIATE ACTION REQUIRED*
          {{ range .Alerts }}
          *Summary:* {{ .Annotations.summary }}
          *Description:* {{ .Annotations.description }}
          *Runbook:* {{ .Annotations.runbook_url }}
          {{ end }}
        send_resolved: true
        color: '{{ if eq .Status "firing" }}danger{{ else }}good{{ end }}'

    email_configs:
      - to: 'risk-team@foxhunt.local'
        subject: '🚨 CRITICAL RISK: {{ .GroupLabels.alertname }}'
        html: |
          <h2 style="color: red;">CRITICAL RISK ALERT</h2>
          {{ range .Alerts }}
          <p><strong>Summary:</strong> {{ .Annotations.summary }}</p>
          <p><strong>Description:</strong> {{ .Annotations.description }}</p>
          <p><strong>Runbook:</strong> <a href="{{ .Annotations.runbook_url }}">{{ .Annotations.runbook_url }}</a></p>
          {{ end }}

    webhook_configs:
      - url: 'http://localhost:5001/critical-risk'
        send_resolved: true

  # Critical Trading
  - name: 'critical-trading'
    slack_configs:
      - channel: '#foxhunt-critical-trading'
        username: 'Foxhunt Alerting'
        icon_emoji: ':moneybag:'
        title: '🚨 CRITICAL TRADING: {{ .GroupLabels.alertname }}'
        title_link: 'http://localhost:3000/d/foxhunt-trading'
        text: |
          {{ range .Alerts }}
          *Summary:* {{ .Annotations.summary }}
          *Description:* {{ .Annotations.description }}
          *Runbook:* {{ .Annotations.runbook_url }}
          {{ end }}
        send_resolved: true
        color: '{{ if eq .Status "firing" }}danger{{ else }}good{{ end }}'

    webhook_configs:
      - url: 'http://localhost:5001/critical-trading'
        send_resolved: true

  # Critical Database
  - name: 'critical-database'
    slack_configs:
      - channel: '#foxhunt-critical-database'
        username: 'Foxhunt Alerting'
        icon_emoji: ':floppy_disk:'
        title: '🚨 CRITICAL DATABASE: {{ .GroupLabels.alertname }}'
        title_link: 'http://localhost:3000/d/foxhunt-database'
        text: |
          {{ range .Alerts }}
          *Summary:* {{ .Annotations.summary }}
          *Description:* {{ .Annotations.description }}
          *Runbook:* {{ .Annotations.runbook_url }}
          {{ end }}
        send_resolved: true
        color: '{{ if eq .Status "firing" }}danger{{ else }}good{{ end }}'

    webhook_configs:
      - url: 'http://localhost:5001/critical-database'
        send_resolved: true

  # Generic Critical Alerts
  - name: 'critical-generic'
    slack_configs:
      - channel: '#foxhunt-critical'
        username: 'Foxhunt Alerting'
        icon_emoji: ':rotating_light:'
        title: '🚨 CRITICAL: {{ .GroupLabels.alertname }}'
        text: |
          *Component:* {{ .CommonLabels.component }}
          {{ range .Alerts }}
          *Summary:* {{ .Annotations.summary }}
          *Description:* {{ .Annotations.description }}
          {{ if .Annotations.runbook_url }}*Runbook:* {{ .Annotations.runbook_url }}{{ end }}
          {{ end }}
        send_resolved: true
        color: '{{ if eq .Status "firing" }}danger{{ else }}good{{ end }}'

    webhook_configs:
      - url: 'http://localhost:5001/critical'
        send_resolved: true

  # ========================================================================
  # WARNING ALERT RECEIVERS
  # ========================================================================

  # Warning Error Rates
  - name: 'warning-errors'
    slack_configs:
      - channel: '#foxhunt-warnings-errors'
        username: 'Foxhunt Alerting'
        icon_emoji: ':warning:'
        title: '⚠️ Warning: {{ .GroupLabels.alertname }}'
        text: |
          *Service:* {{ .CommonLabels.service }}
          {{ range .Alerts }}
          *Summary:* {{ .Annotations.summary }}
          *Description:* {{ .Annotations.description }}
          {{ if .Annotations.runbook_url }}*Runbook:* {{ .Annotations.runbook_url }}{{ end }}
          {{ end }}
        send_resolved: true
        color: 'warning'

  # Warning Resources (CPU, Disk)
  - name: 'warning-resources'
    slack_configs:
      - channel: '#foxhunt-warnings-resources'
        username: 'Foxhunt Alerting'
        icon_emoji: ':bar_chart:'
        title: '⚠️ Resource Warning: {{ .GroupLabels.alertname }}'
        text: |
          *Component:* {{ .CommonLabels.component }}
          {{ range .Alerts }}
          *Summary:* {{ .Annotations.summary }}
          *Description:* {{ .Annotations.description }}
          {{ if .Annotations.runbook_url }}*Runbook:* {{ .Annotations.runbook_url }}{{ end }}
          {{ end }}
        send_resolved: true
        color: 'warning'

  # Warning ML
  - name: 'warning-ml'
    slack_configs:
      - channel: '#foxhunt-warnings-ml'
        username: 'Foxhunt Alerting'
        icon_emoji: ':robot_face:'
        title: '⚠️ ML Warning: {{ .GroupLabels.alertname }}'
        text: |
          {{ range .Alerts }}
          *Summary:* {{ .Annotations.summary }}
          *Description:* {{ .Annotations.description }}
          {{ if .Annotations.runbook_url }}*Runbook:* {{ .Annotations.runbook_url }}{{ end }}
          {{ end }}
        send_resolved: true
        color: 'warning'

  # Generic Warnings
  - name: 'warning-generic'
    slack_configs:
      - channel: '#foxhunt-warnings'
        username: 'Foxhunt Alerting'
        icon_emoji: ':warning:'
        title: '⚠️ Warning: {{ .GroupLabels.alertname }}'
        text: |
          {{ range .Alerts }}
          *Summary:* {{ .Annotations.summary }}
          *Description:* {{ .Annotations.description }}
          {{ end }}
        send_resolved: true
        color: 'warning'

# ============================================================================
# INHIBITION RULES (Suppress Redundant Alerts)
# ============================================================================
inhibit_rules:
  # If service is down, suppress all other alerts from that service
  - source_match:
      alertname: 'CriticalServiceDown'
    target_match_re:
      alertname: '.*'
    equal: ['job']

  # If system health is degraded, suppress individual service alerts
  - source_match:
      alertname: 'DegradedSystemHealth'
    target_match_re:
      alertname: '(APIGatewayDown|TradingServiceDown|BacktestingServiceDown|MLTrainingServiceDown)'
    equal: ['cluster']

  # If critical memory, suppress warning memory alerts
  - source_match:
      alertname: 'CriticalMemoryUsageAbsolute'
    target_match:
      component: 'memory'
    equal: ['job']

  # If critical memory growth, suppress absolute memory alerts
  - source_match:
      alertname: 'CriticalMemoryGrowth'
    target_match:
      alertname: 'CriticalMemoryUsageAbsolute'
    equal: ['job']

  # If system memory pressure, suppress process memory alerts
  - source_match:
      alertname: 'CriticalSystemMemoryPressure'
    target_match_re:
      alertname: '(CriticalMemoryGrowth|CriticalMemoryUsageAbsolute)'
    equal: ['instance']

  # If database is down, suppress slow query and connection alerts
  - source_match:
      alertname: 'CriticalPostgreSQLDown'
    target_match_re:
      alertname: '(SlowDatabaseQueries|PostgreSQLConnectionPoolExhaustion)'
    equal: ['cluster']

  # If market data is stale, suppress risk check failures (may be related)
  - source_match:
      alertname: 'CriticalMarketDataStale'
    target_match:
      alertname: 'RiskCheckFailures'
    equal: ['cluster']

  # If position limit breached, suppress order rejection alerts
  - source_match:
      alertname: 'CriticalPositionLimitBreach'
    target_match:
      alertname: 'HighOrderRejectionRate'
    equal: ['cluster']

  # If alert storm, suppress individual monitoring alerts
  - source_match:
      alertname: 'AlertStorm'
    target_match:
      component: 'monitoring'
    equal: ['cluster']

  # If disk space critical, suppress warning
  - source_match:
      alertname: 'DiskSpaceCritical'
    target_match:
      alertname: 'DiskSpaceLow'
    equal: ['instance', 'mountpoint']