foxhunt/config/monitoring/alertmanager-hft-production.yml

# FOXHUNT HFT ALERTMANAGER CONFIGURATION
# Production-grade alerting with sub-second response times
# Optimized for high-frequency trading critical path monitoring

global:
  # SMTP configuration for email alerts
  smtp_smarthost: 'smtp.company.com:587'
  smtp_from: 'foxhunt-alerts@company.com'
  smtp_auth_username: 'foxhunt-alerts@company.com'
  smtp_auth_password: '${SMTP_PASSWORD}'
  smtp_require_tls: true
  
  # Slack webhook for instant notifications
  slack_api_url: '${SLACK_WEBHOOK_URL}'
  
  # PagerDuty integration for on-call escalation
  pagerduty_url: 'https://events.pagerduty.com/v2/enqueue'
  
  # Webhook timeout - keep short for responsiveness
  http_config:
    timeout: 500ms

# Routing configuration with priority-based escalation
route:
  group_by: ['alertname', 'cluster', 'service', 'severity']
  group_wait: 0s          # No delay for HFT alerts
  group_interval: 1s      # Minimal grouping interval
  repeat_interval: 30s    # Re-send critical alerts frequently
  receiver: 'default'
  
  routes:
    # ULTRA-CRITICAL: Trading system latency alerts (immediate)
    - match:
        severity: critical
        component: trading
        alertname: OrderSubmissionLatencyHigh
      receiver: 'ultra-critical-trading'
      group_wait: 0s
      group_interval: 0s
      repeat_interval: 15s
      continue: true

    # ULTRA-CRITICAL: Trading service down (immediate)
    - match:
        severity: critical
        component: trading
        alertname: TradingServiceDown
      receiver: 'ultra-critical-trading'
      group_wait: 0s
      group_interval: 0s
      repeat_interval: 10s
      continue: true

    # CRITICAL: Risk management alerts (sub-second)
    - match:
        severity: critical
        component: risk
      receiver: 'critical-risk'
      group_wait: 0s
      group_interval: 100ms
      repeat_interval: 30s
      continue: true

    # CRITICAL: System performance alerts (1 second)
    - match:
        severity: critical
        component: system
      receiver: 'critical-system'
      group_wait: 100ms
      group_interval: 1s
      repeat_interval: 1m

    # CRITICAL: Network/connectivity alerts (sub-second)
    - match:
        severity: critical
        component: network
      receiver: 'critical-network'
      group_wait: 0s
      group_interval: 200ms
      repeat_interval: 30s

    # HIGH: Performance degradation (2 seconds)
    - match:
        severity: high
      receiver: 'high-priority'
      group_wait: 1s
      group_interval: 5s
      repeat_interval: 5m

    # WARNING: Standard alerts (10 seconds)
    - match:
        severity: warning
      receiver: 'warning-alerts'
      group_wait: 10s
      group_interval: 30s
      repeat_interval: 30m

    # INFO: Informational alerts (30 seconds)
    - match:
        severity: info
      receiver: 'info-alerts'
      group_wait: 30s
      group_interval: 2m
      repeat_interval: 2h

# Alert receivers with escalation chains
receivers:
  # Default fallback
  - name: 'default'
    slack_configs:
      - channel: '#general-alerts'
        title: 'Foxhunt Alert'
        text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'
        send_resolved: true

  # ULTRA-CRITICAL: Trading system alerts with immediate multi-channel escalation
  - name: 'ultra-critical-trading'
    # Immediate Slack notification to trading team
    slack_configs:
      - channel: '#trading-critical'
        username: 'FoxhuntUrgent'
        title: '🚨 ULTRA-CRITICAL: {{ .GroupLabels.alertname }}'
        title_link: '{{ .ExternalURL }}'
        text: |
          *IMMEDIATE ACTION REQUIRED*
          
          Alert: {{ range .Alerts }}{{ .Annotations.summary }}{{ end }}
          Impact: {{ range .Alerts }}{{ .Annotations.impact }}{{ end }}
          Action: {{ range .Alerts }}{{ .Annotations.action }}{{ end }}
          
          Latency Target: <50μs | Current: {{ range .Alerts }}{{ .Annotations.current_value }}{{ end }}
          
          Dashboard: http://grafana:3000/d/trading-performance
        color: 'danger'
        send_resolved: true
        actions:
          - type: button
            text: 'View Dashboard'
            url: 'http://grafana:3000/d/trading-performance'
          - type: button
            text: 'Emergency Stop'
            url: 'http://trading-service:8080/emergency-stop'
            style: 'danger'
    
    # Immediate email to trading team and executives
    email_configs:
      - to: 'trading-team@company.com,cto@company.com,risk-manager@company.com'
        subject: '🚨 ULTRA-CRITICAL: {{ .GroupLabels.alertname }} - Immediate Action Required'
        body: |
          ULTRA-CRITICAL TRADING ALERT - IMMEDIATE ACTION REQUIRED
          
          Time: {{ range .Alerts }}{{ .StartsAt.Format "2006-01-02 15:04:05 MST" }}{{ end }}
          Alert: {{ range .Alerts }}{{ .Annotations.summary }}{{ end }}
          
          PERFORMANCE IMPACT:
          {{ range .Alerts }}{{ .Annotations.impact }}{{ end }}
          
          REQUIRED ACTION:
          {{ range .Alerts }}{{ .Annotations.action }}{{ end }}
          
          CURRENT STATUS:
          - Latency: {{ range .Alerts }}{{ .Annotations.current_value }}{{ end }}
          - Target: <50μs (EXCEEDED)
          - Service: {{ .GroupLabels.service }}
          
          IMMEDIATE RESPONSE REQUIRED:
          1. Check trading service health: http://trading-service:8080/health
          2. Review dashboard: http://grafana:3000/d/trading-performance
          3. Consider emergency trading halt if necessary
          
          This alert will repeat every 15 seconds until resolved.
        headers:
          Priority: 'urgent'
          Importance: 'high'
          X-Priority: '1'
    
    # Immediate PagerDuty escalation
    pagerduty_configs:
      - service_key: '${PAGERDUTY_TRADING_SERVICE_KEY}'
        description: 'ULTRA-CRITICAL: {{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'
        severity: 'critical'
        details:
          alert: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'
          impact: '{{ range .Alerts }}{{ .Annotations.impact }}{{ end }}'
          action: '{{ range .Alerts }}{{ .Annotations.action }}{{ end }}'
          current_value: '{{ range .Alerts }}{{ .Annotations.current_value }}{{ end }}'
          service: '{{ .GroupLabels.service }}'
          dashboard: 'http://grafana:3000/d/trading-performance'
        client: 'FoxhuntHFT'
        client_url: '{{ .ExternalURL }}'
    
    # SMS/Voice alerts for maximum urgency (webhook to SMS service)
    webhook_configs:
      - url: '${SMS_WEBHOOK_URL}'
        send_resolved: false
        http_config:
          timeout: 200ms
        body: |
          {
            "message": "FOXHUNT CRITICAL: {{ range .Alerts }}{{ .Annotations.summary }}{{ end }}. Latency exceeded 50μs. Immediate action required.",
            "numbers": ["${TRADING_MANAGER_PHONE}", "${CTO_PHONE}"],
            "priority": "urgent"
          }

  # CRITICAL: Risk management alerts
  - name: 'critical-risk'
    slack_configs:
      - channel: '#risk-critical'
        title: '🚨 CRITICAL RISK: {{ .GroupLabels.alertname }}'
        text: |
          Alert: {{ range .Alerts }}{{ .Annotations.summary }}{{ end }}
          Impact: {{ range .Alerts }}{{ .Annotations.impact }}{{ end }}
          
          Risk Dashboard: http://grafana:3000/d/risk-management
        color: 'danger'
        send_resolved: true
    
    email_configs:
      - to: 'risk-team@company.com,cro@company.com'
        subject: '🚨 CRITICAL RISK: {{ .GroupLabels.alertname }}'
        body: |
          CRITICAL RISK ALERT
          
          Alert: {{ range .Alerts }}{{ .Annotations.summary }}{{ end }}
          Description: {{ range .Alerts }}{{ .Annotations.description }}{{ end }}
          Impact: {{ range .Alerts }}{{ .Annotations.impact }}{{ end }}
          
          Risk Dashboard: http://grafana:3000/d/risk-management
    
    pagerduty_configs:
      - service_key: '${PAGERDUTY_RISK_SERVICE_KEY}'
        description: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'
        severity: 'critical'

  # CRITICAL: System performance alerts
  - name: 'critical-system'
    slack_configs:
      - channel: '#ops-critical'
        title: '⚠️ CRITICAL SYSTEM: {{ .GroupLabels.alertname }}'
        text: |
          System Issue: {{ range .Alerts }}{{ .Annotations.summary }}{{ end }}
          
          System Dashboard: http://grafana:3000/d/system-health
        color: 'warning'
    
    email_configs:
      - to: 'ops-team@company.com'
        subject: '⚠️ CRITICAL SYSTEM: {{ .GroupLabels.alertname }}'

  # CRITICAL: Network/connectivity alerts  
  - name: 'critical-network'
    slack_configs:
      - channel: '#network-critical'
        title: '🌐 NETWORK CRITICAL: {{ .GroupLabels.alertname }}'
        text: |
          Network Issue: {{ range .Alerts }}{{ .Annotations.summary }}{{ end }}
          Broker Connectivity: {{ range .Alerts }}{{ .Annotations.broker_status }}{{ end }}
        color: 'danger'

  # HIGH: Performance degradation alerts
  - name: 'high-priority'
    slack_configs:
      - channel: '#performance-alerts'
        title: 'HIGH: {{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'
        color: 'warning'

  # WARNING: Standard operational alerts
  - name: 'warning-alerts'
    slack_configs:
      - channel: '#monitoring'
        title: 'WARNING: {{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'
        color: '#ff9500'

  # INFO: Informational alerts
  - name: 'info-alerts'
    slack_configs:
      - channel: '#info-alerts'
        title: 'INFO: {{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'
        color: 'good'

# Inhibition rules to prevent alert storms
inhibit_rules:
  # Inhibit all other alerts if trading service is completely down
  - source_match:
      alertname: TradingServiceDown
    target_match_re:
      component: trading
      alertname: '.*'
    equal: ['instance', 'service']

  # Inhibit individual latency alerts if overall system latency is high
  - source_match:
      alertname: SystemLatencyHigh
    target_match_re:
      alertname: '.*Latency.*'
    equal: ['instance']

  # Inhibit memory alerts if disk is full (likely cause)
  - source_match:
      alertname: DiskSpaceCritical
    target_match:
      alertname: HighMemoryUsage
    equal: ['instance']

  # Inhibit service-specific alerts if the whole node is down
  - source_match:
      alertname: NodeDown
    target_match_re:
      alertname: '(ServiceDown|HighLatency|.*Error)'
    equal: ['instance']

  # Inhibit broker-specific alerts if all brokers are down
  - source_match:
      alertname: AllBrokersDown
    target_match_re:
      component: broker
      alertname: '.*'
    equal: ['cluster']

# Mute rules for maintenance windows
mute_time_intervals:
  - name: maintenance-window
    time_intervals:
      - times:
          - start_time: '02:00'
            end_time: '04:00'
        weekdays: ['sunday']
        months: ['1:12']

# Notification rate limiting to prevent spam
notification_rate_limit:
  # Global rate limit: max 100 notifications per minute
  global: 100
  # Per-receiver limits
  receivers:
    ultra-critical-trading: 20  # Allow more frequent critical alerts
    critical-risk: 10
    critical-system: 5
    default: 3

# Template definitions for consistent formatting
templates:
  - '/etc/alertmanager/templates/*.tmpl'