foxhunt/config/monitoring/alertmanager-hft.yml

# FOXHUNT HFT ALERTMANAGER CONFIGURATION
# Real-time alerting for production trading system
# Generated for critical deployment

global:
  # SMTP configuration for email alerts
  smtp_smarthost: 'smtp.gmail.com:587'
  smtp_from: 'alerts@foxhunt.io'
  smtp_auth_username: 'alerts@foxhunt.io'
  smtp_auth_password: 'YOUR_SMTP_PASSWORD'
  smtp_require_tls: true

  # Slack webhook for instant notifications
  slack_api_url: 'https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK'

# Template files for custom alert formatting
templates:
  - '/etc/alertmanager/templates/*.tmpl'

# Route configuration - CRITICAL for HFT operations
route:
  # Default receiver for unmatched alerts
  receiver: 'hft-default'
  
  # Group alerts by service and severity
  group_by: ['alertname', 'service', 'severity']
  group_wait: 1s        # Wait 1 second before sending grouped alerts
  group_interval: 5s    # Wait 5 seconds between grouped alerts
  repeat_interval: 1m   # Repeat critical alerts every minute
  
  # Route critical alerts immediately
  routes:
  - match:
      severity: critical
    receiver: 'hft-critical-immediate'
    group_wait: 0s      # Send critical alerts immediately
    group_interval: 0s  # No grouping delay for critical
    repeat_interval: 30s # Repeat every 30 seconds
    
  - match:
      team: risk-management
    receiver: 'hft-risk-team'
    group_wait: 0s
    
  - match:
      team: trading
    receiver: 'hft-trading-team'
    group_wait: 1s
    
  - match:
      team: infrastructure
    receiver: 'hft-infra-team'
    group_wait: 5s

# Inhibit rules - prevent alert spam
inhibit_rules:
  # Inhibit warning alerts when critical alert is firing
  - source_match:
      severity: critical
    target_match:
      severity: warning
    equal: ['service', 'alertname']
    
  # Inhibit individual service alerts when trading halt is active
  - source_match:
      alertname: HFT_TradingHalt_Active
    target_match_re:
      alertname: HFT_.*
    equal: ['service']

# Receiver configurations
receivers:
  # Default receiver
  - name: 'hft-default'
    slack_configs:
    - channel: '#hft-alerts'
      title: 'HFT Alert - {{ .GroupLabels.alertname }}'
      text: |
        {{ range .Alerts }}
        *{{ .Annotations.summary }}*
        {{ .Annotations.description }}
        {{ end }}
      send_resolved: true
      
  # CRITICAL ALERTS - Multiple channels for maximum reliability
  - name: 'hft-critical-immediate'
    # Slack notification
    slack_configs:
    - channel: '#hft-critical'
      title: '🚨 CRITICAL HFT ALERT - {{ .GroupLabels.alertname }}'
      text: |
        ⚠️ **IMMEDIATE ACTION REQUIRED** ⚠️
        {{ range .Alerts }}
        *Alert:* {{ .Labels.alertname }}
        *Service:* {{ .Labels.service }}
        *Summary:* {{ .Annotations.summary }}
        *Details:* {{ .Annotations.description }}
        *Runbook:* {{ .Annotations.runbook }}
        *Time:* {{ .StartsAt.Format "2006-01-02 15:04:05 UTC" }}
        {{ end }}
      send_resolved: true
      color: 'danger'
      
    # Email notification
    email_configs:
    - to: 'trading-alerts@foxhunt.io'
      subject: '🚨 CRITICAL HFT ALERT - {{ .GroupLabels.alertname }}'
      html: |
        <h2 style="color: red;">CRITICAL HFT SYSTEM ALERT</h2>
        <p><strong>IMMEDIATE ACTION REQUIRED</strong></p>
        {{ range .Alerts }}
        <table border="1" cellpadding="5">
          <tr><td><strong>Alert</strong></td><td>{{ .Labels.alertname }}</td></tr>
          <tr><td><strong>Service</strong></td><td>{{ .Labels.service }}</td></tr>
          <tr><td><strong>Severity</strong></td><td>{{ .Labels.severity }}</td></tr>
          <tr><td><strong>Summary</strong></td><td>{{ .Annotations.summary }}</td></tr>
          <tr><td><strong>Description</strong></td><td>{{ .Annotations.description }}</td></tr>
          <tr><td><strong>Time</strong></td><td>{{ .StartsAt.Format "2006-01-02 15:04:05 UTC" }}</td></tr>
          {{ if .Annotations.runbook }}
          <tr><td><strong>Runbook</strong></td><td><a href="{{ .Annotations.runbook }}">{{ .Annotations.runbook }}</a></td></tr>
          {{ end }}
        </table>
        <br>
        {{ end }}
        
    # PagerDuty integration
    pagerduty_configs:
    - service_key: 'YOUR_PAGERDUTY_SERVICE_KEY'
      description: '{{ .GroupLabels.alertname }} - {{ .CommonAnnotations.summary }}'
      details:
        firing: '{{ .Alerts.Firing | len }}'
        resolved: '{{ .Alerts.Resolved | len }}'
        
  # Risk management team alerts
  - name: 'hft-risk-team'
    slack_configs:
    - channel: '#risk-management'
      title: '⚠️ Risk Management Alert - {{ .GroupLabels.alertname }}'
      text: |
        {{ range .Alerts }}
        *{{ .Annotations.summary }}*
        {{ .Annotations.description }}
        {{ end }}
    email_configs:
    - to: 'risk@foxhunt.io'
      subject: 'HFT Risk Alert - {{ .GroupLabels.alertname }}'
      
  # Trading team alerts  
  - name: 'hft-trading-team'
    slack_configs:
    - channel: '#trading-alerts'
      title: '📊 Trading Alert - {{ .GroupLabels.alertname }}'
      text: |
        {{ range .Alerts }}
        *{{ .Annotations.summary }}*
        {{ .Annotations.description }}
        {{ end }}
    email_configs:
    - to: 'trading@foxhunt.io'
      subject: 'HFT Trading Alert - {{ .GroupLabels.alertname }}'
      
  # Infrastructure team alerts
  - name: 'hft-infra-team'
    slack_configs:
    - channel: '#infrastructure'
      title: '🔧 Infrastructure Alert - {{ .GroupLabels.alertname }}'
      text: |
        {{ range .Alerts }}
        *{{ .Annotations.summary }}*
        {{ .Annotations.description }}
        {{ end }}
    email_configs:
    - to: 'infra@foxhunt.io'
      subject: 'HFT Infrastructure Alert - {{ .GroupLabels.alertname }}'