foxhunt/deployment/monitoring/alerts/hft-alerts.yml

groups:
  - name: foxhunt-hft-critical
    interval: 1s
    rules:
      # Ultra-critical latency alert
      - alert: CriticalLatencyBreach
        expr: foxhunt_order_latency_microseconds > 30
        for: 1s
        labels:
          severity: critical
          component: core
          impact: trading
        annotations:
          summary: "CRITICAL: Order latency exceeded 30μs threshold"
          description: "Current latency: {{ $value }}μs. Immediate action required."
          runbook_url: "https://docs.foxhunt.io/runbooks/latency-breach"
          
      - alert: TradingSystemDown
        expr: up{job="foxhunt-core"} == 0
        for: 5s
        labels:
          severity: critical
          component: core
          impact: trading
        annotations:
          summary: "CRITICAL: Core trading system is down"
          description: "The core trading service is not responding"
          runbook_url: "https://docs.foxhunt.io/runbooks/system-down"
          
      - alert: TLISystemDown
        expr: up{job="foxhunt-tli"} == 0
        for: 10s
        labels:
          severity: critical
          component: tli
          impact: trading
        annotations:
          summary: "CRITICAL: Trading Layer Interface is down"
          description: "TLI service is not responding - trading operations halted"
          
      - alert: PerformanceDegradation
        expr: rate(foxhunt_orders_processed_total[1m]) < 1000
        for: 30s
        labels:
          severity: warning
          component: core
          impact: performance
        annotations:
          summary: "Order processing rate below threshold"
          description: "Current rate: {{ $value }} orders/min, threshold: 1000"
          
      - alert: HighErrorRate
        expr: rate(foxhunt_orders_failed_total[1m]) / rate(foxhunt_orders_total[1m]) > 0.01
        for: 15s
        labels:
          severity: critical
          component: core
          impact: trading
        annotations:
          summary: "High order failure rate detected"
          description: "Error rate: {{ $value | humanizePercentage }}"

  - name: foxhunt-risk-management
    interval: 5s
    rules:
      - alert: RiskLimitBreach
        expr: foxhunt_position_risk_ratio > 0.8
        for: 10s
        labels:
          severity: critical
          component: risk
          impact: compliance
        annotations:
          summary: "Risk limit approaching breach"
          description: "Current risk ratio: {{ $value }}"
          
      - alert: VaRExceeded
        expr: foxhunt_portfolio_var_ratio > 1.0
        for: 5s
        labels:
          severity: critical
          component: risk
          impact: compliance
        annotations:
          summary: "Value at Risk exceeded"
          description: "VaR ratio: {{ $value }}"
          
      - alert: RiskServiceDown
        expr: up{job="foxhunt-risk"} == 0
        for: 30s
        labels:
          severity: high
          component: risk
          impact: compliance
        annotations:
          summary: "Risk management service is down"
          description: "Risk calculations are not available"

  - name: foxhunt-ml-services
    interval: 10s
    rules:
      - alert: MLModelDegraded
        expr: foxhunt_ml_model_accuracy < 0.85
        for: 60s
        labels:
          severity: warning
          component: ml
          impact: strategy
        annotations:
          summary: "ML model accuracy degraded"
          description: "Model accuracy: {{ $value }}"
          
      - alert: GPUUtilizationLow
        expr: foxhunt_gpu_utilization < 0.3
        for: 300s
        labels:
          severity: info
          component: ml
          impact: efficiency
        annotations:
          summary: "Low GPU utilization detected"
          description: "GPU utilization: {{ $value | humanizePercentage }}"
          
      - alert: MLServiceDown
        expr: up{job="foxhunt-ml"} == 0
        for: 60s
        labels:
          severity: high
          component: ml
          impact: strategy
        annotations:
          summary: "ML service is down"
          description: "Machine learning predictions are not available"

  - name: foxhunt-data-connectivity
    interval: 5s
    rules:
      - alert: DataFeedDisconnected
        expr: foxhunt_data_feed_connected == 0
        for: 15s
        labels:
          severity: critical
          component: data
          impact: trading
        annotations:
          summary: "Market data feed disconnected"
          description: "Data feed status: disconnected"
          
      - alert: DataLatencyHigh
        expr: foxhunt_data_latency_milliseconds > 100
        for: 30s
        labels:
          severity: warning
          component: data
          impact: performance
        annotations:
          summary: "High data latency detected"
          description: "Data latency: {{ $value }}ms"
          
      - alert: DataServiceDown
        expr: up{job="foxhunt-data"} == 0
        for: 30s
        labels:
          severity: high
          component: data
          impact: trading
        annotations:
          summary: "Data service is down"
          description: "Market data ingestion is not available"

  - name: foxhunt-system-resources
    interval: 15s
    rules:
      - alert: HighMemoryUsage
        expr: process_resident_memory_bytes{job=~"foxhunt-.*"} > 4e9
        for: 60s
        labels:
          severity: warning
          component: system
          impact: performance
        annotations:
          summary: "High memory usage detected"
          description: "Memory usage: {{ $value | humanizeBytes }} on {{ $labels.job }}"
          
      - alert: CPUAffinityLost
        expr: foxhunt_cpu_affinity_configured == 0
        for: 30s
        labels:
          severity: high
          component: system
          impact: performance
        annotations:
          summary: "CPU affinity configuration lost"
          description: "Service is not bound to designated CPU cores"
          
      - alert: DiskSpaceLow
        expr: node_filesystem_avail_bytes{mountpoint="/opt/foxhunt"} / node_filesystem_size_bytes{mountpoint="/opt/foxhunt"} < 0.1
        for: 60s
        labels:
          severity: critical
          component: system
          impact: operations
        annotations:
          summary: "Low disk space on Foxhunt directory"
          description: "Available space: {{ $value | humanizePercentage }}"