foxhunt/config/monitoring/hft-critical-alerts.yml

# FOXHUNT HFT CRITICAL ALERT RULES
# Production-grade alerting rules optimized for high-frequency trading
# Thresholds calibrated for sub-50μs latency requirements

groups:
  # ULTRA-CRITICAL: Trading Performance Alerts (0-5 second tolerance)
  - name: trading.ultra_critical
    interval: 1s  # Evaluate every second for ultra-fast response
    rules:
      - alert: OrderSubmissionLatencyHigh
        expr: |
          (
            histogram_quantile(0.99, rate(trading_order_processing_seconds_bucket[10s])) > 0.000050
            or
            histogram_quantile(0.95, rate(trading_order_processing_seconds_bucket[5s])) > 0.000040
          )
        for: 2s
        labels:
          severity: critical
          component: trading
          team: trading
          priority: ultra_high
        annotations:
          summary: "Order submission latency critically high"
          description: "P99 order latency is {{ $value | humanizeDuration }}, exceeding 50μs critical threshold"
          impact: "Trading strategy performance severely degraded - potential profit loss"
          action: "1. Check CPU affinity 2. Verify network latency 3. Review system resources 4. Consider emergency halt"
          current_value: "{{ $value | humanizeDuration }}"
          runbook_url: "https://docs.company.com/runbooks/trading-latency-high"

      - alert: TradingServiceDown
        expr: up{job="foxhunt-trading"} == 0
        for: 1s
        labels:
          severity: critical
          component: trading
          team: trading
          priority: ultra_high
        annotations:
          summary: "Trading service is DOWN"
          description: "Trading service has been unreachable for more than 1 second"
          impact: "All trading operations halted - immediate revenue impact"
          action: "1. Check service health 2. Restart if necessary 3. Verify broker connections 4. Escalate to on-call"
          runbook_url: "https://docs.company.com/runbooks/trading-service-down"

      - alert: OrderFillRateCollapse
        expr: |
          (
            rate(trading_orders_filled_total[1m]) / rate(trading_orders_submitted_total[1m])
          ) < 0.50
        for: 10s
        labels:
          severity: critical
          component: trading
          team: trading
          priority: ultra_high
        annotations:
          summary: "Order fill rate critically low"
          description: "Fill rate dropped to {{ $value | humanizePercentage }}, indicating severe execution issues"
          impact: "Strategy execution quality severely degraded"
          action: "1. Check broker connectivity 2. Review market conditions 3. Verify order parameters"

      - alert: TradingLatencySpike
        expr: |
          (
            rate(trading_total_latency_seconds_sum[30s]) / rate(trading_total_latency_seconds_count[30s])
          ) > 0.000100
        for: 5s
        labels:
          severity: critical
          component: trading
          team: trading
        annotations:
          summary: "Average trading latency spike detected"
          description: "Average latency is {{ $value | humanizeDuration }}, exceeding 100μs"
          impact: "Significant performance degradation"

  # CRITICAL: Risk Management Alerts (0-10 second tolerance)
  - name: risk.critical
    interval: 2s
    rules:
      - alert: RiskLimitBreach
        expr: trading_current_risk_exposure > trading_max_risk_limit
        for: 0s  # Immediate alert for risk breaches
        labels:
          severity: critical
          component: risk
          team: risk
        annotations:
          summary: "Risk limits breached"
          description: "Current risk exposure {{ $value }} exceeds maximum limit"
          impact: "Potential significant financial loss exposure"
          action: "1. Activate emergency risk controls 2. Begin position reduction 3. Alert CRO"

      - alert: VaRUtilizationCritical
        expr: (trading_daily_var_used / trading_daily_var_limit) > 0.95
        for: 5s
        labels:
          severity: critical
          component: risk
          team: risk
        annotations:
          summary: "VaR utilization exceeding 95%"
          description: "Daily VaR utilization is {{ $value | humanizePercentage }}"
          impact: "Approaching daily risk limits - potential trading halt"
          action: "1. Review position sizes 2. Consider risk reduction 3. Prepare for possible halt"

      - alert: DrawdownExcessive
        expr: trading_current_drawdown_pct > trading_max_drawdown_limit
        for: 10s
        labels:
          severity: critical
          component: risk
          team: risk
        annotations:
          summary: "Maximum drawdown exceeded"
          description: "Current drawdown {{ $value }}% exceeds maximum allowed"
          impact: "Strategy performance significantly degraded"

      - alert: PositionConcentrationHigh
        expr: max(trading_position_concentration_pct) > 40
        for: 30s
        labels:
          severity: critical
          component: risk
          team: risk
        annotations:
          summary: "Position concentration risk high"
          description: "Single position represents {{ $value }}% of portfolio"

  # CRITICAL: System Performance Alerts (1-30 second tolerance)
  - name: system.critical
    interval: 5s
    rules:
      - alert: CPUUtilizationCritical
        expr: |
          (
            100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[30s])) * 100)
          ) > 95
        for: 30s
        labels:
          severity: critical
          component: system
          team: operations
        annotations:
          summary: "CPU utilization critically high"
          description: "CPU usage is {{ $value }}% on {{ $labels.instance }}"
          impact: "System performance severely degraded - latency impact imminent"
          action: "1. Check process utilization 2. Scale resources 3. Kill non-essential processes"

      - alert: MemoryUtilizationCritical
        expr: |
          (
            (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes
          ) > 0.95
        for: 30s
        labels:
          severity: critical
          component: system
          team: operations
        annotations:
          summary: "Memory utilization critically high"
          description: "Memory usage is {{ $value | humanizePercentage }} on {{ $labels.instance }}"
          impact: "Risk of OOM kills and system instability"

      - alert: DiskSpaceCritical
        expr: |
          (
            node_filesystem_avail_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes{fstype!="tmpfs"}
          ) < 0.05
        for: 1m
        labels:
          severity: critical
          component: system
          team: operations
        annotations:
          summary: "Disk space critically low"
          description: "Only {{ $value | humanizePercentage }} disk space remaining on {{ $labels.instance }}"

      - alert: SystemLoadHigh
        expr: node_load15 > (node_cpu_seconds_total * 2)
        for: 2m
        labels:
          severity: critical
          component: system
          team: operations
        annotations:
          summary: "System load critically high"
          description: "15-minute load average is {{ $value }} on {{ $labels.instance }}"

  # CRITICAL: Network and Connectivity Alerts (1-15 second tolerance)
  - name: network.critical
    interval: 2s
    rules:
      - alert: BrokerConnectionDown
        expr: broker_connection_status{broker=~".*"} == 0
        for: 2s
        labels:
          severity: critical
          component: network
          team: trading
        annotations:
          summary: "Broker connection down"
          description: "Connection to {{ $labels.broker }} is down"
          impact: "Trading capacity reduced - potential execution issues"
          action: "1. Check network connectivity 2. Restart broker connection 3. Switch to backup"
          broker_status: "DOWN"

      - alert: NetworkLatencyHigh
        expr: |
          histogram_quantile(0.95, rate(network_latency_seconds_bucket[30s])) > 0.002
        for: 10s
        labels:
          severity: critical
          component: network
          team: operations
        annotations:
          summary: "Network latency critically high"
          description: "P95 network latency is {{ $value | humanizeDuration }}"
          impact: "Trading latency significantly impacted"

      - alert: MarketDataFeedDown
        expr: rate(market_data_messages_total[1m]) == 0
        for: 5s
        labels:
          severity: critical
          component: network
          team: trading
        annotations:
          summary: "Market data feed interruption"
          description: "No market data received for 5+ seconds"
          impact: "Trading decisions based on stale data - potential losses"

      - alert: PacketLossHigh
        expr: rate(node_network_transmit_drop_total[1m]) > 10
        for: 15s
        labels:
          severity: critical
          component: network
          team: operations
        annotations:
          summary: "High packet loss detected"
          description: "Packet loss rate: {{ $value }} packets/second"

  # HIGH: Performance Degradation Alerts (30 second - 5 minute tolerance)
  - name: performance.high
    interval: 10s
    rules:
      - alert: DatabaseQuerySlow
        expr: |
          histogram_quantile(0.95, rate(database_query_duration_seconds_bucket[2m])) > 0.050
        for: 1m
        labels:
          severity: high
          component: database
          team: operations
        annotations:
          summary: "Database queries running slow"
          description: "P95 query time is {{ $value | humanizeDuration }}"

      - alert: MetricsBufferFull
        expr: metrics_buffer_utilization_percent > 90
        for: 30s
        labels:
          severity: high
          component: monitoring
          team: operations
        annotations:
          summary: "Metrics buffer utilization high"
          description: "Buffer utilization at {{ $value }}%"
          impact: "Risk of losing metrics data"

      - alert: GarbageCollectionHigh
        expr: rate(go_gc_duration_seconds_sum[5m]) / rate(go_gc_duration_seconds_count[5m]) > 0.01
        for: 2m
        labels:
          severity: high
          component: system
          team: operations
        annotations:
          summary: "High garbage collection overhead"
          description: "GC overhead is {{ $value | humanizeDuration }} per collection"

  # WARNING: Operational Alerts (5-30 minute tolerance)
  - name: operational.warning
    interval: 30s
    rules:
      - alert: HighOrderRejectionRate
        expr: |
          (
            rate(trading_orders_rejected_total[5m]) / rate(trading_orders_submitted_total[5m])
          ) > 0.05
        for: 2m
        labels:
          severity: warning
          component: trading
          team: trading
        annotations:
          summary: "High order rejection rate"
          description: "{{ $value | humanizePercentage }} of orders being rejected"

      - alert: SlowStartupTime
        expr: trading_service_startup_duration_seconds > 30
        for: 0s
        labels:
          severity: warning
          component: trading
          team: operations
        annotations:
          summary: "Trading service startup time excessive"
          description: "Service took {{ $value }} seconds to start"

      - alert: ConfigurationDrift
        expr: trading_config_version != on() trading_expected_config_version
        for: 5m
        labels:
          severity: warning
          component: config
          team: operations
        annotations:
          summary: "Configuration version mismatch detected"

  # INFO: System Health and Maintenance
  - name: system.info
    interval: 1m
    rules:
      - alert: ServiceRestart
        expr: changes(process_start_time_seconds[10m]) > 0
        for: 0s
        labels:
          severity: info
          component: system
          team: operations
        annotations:
          summary: "Service {{ $labels.job }} restarted"
          description: "Service restarted at {{ $value | humanizeTimestamp }}"

      - alert: HighTradeVolume
        expr: rate(trading_orders_submitted_total[1h]) > 10000
        for: 5m
        labels:
          severity: info
          component: trading
          team: trading
        annotations:
          summary: "High trading volume detected"
          description: "Processing {{ $value }} orders per second"