foxhunt/config/monitoring/hft-alerts.yml

# Foxhunt HFT Trading System - Prometheus Alerting Rules
# Critical alerts for high-frequency trading production environment

groups:
  # =======================
  # CRITICAL HFT ALERTS
  # =======================
  - name: hft_critical
    rules:
      # Trading Engine Down
      - alert: TradingEngineDown
        expr: up{job="trading-engine"} == 0
        for: 5s
        labels:
          severity: critical
          component: trading_engine
          impact: trading_halt
        annotations:
          summary: "Trading Engine is DOWN"
          description: "Trading Engine has been down for {{ $value }} seconds. All trading operations are halted."
          runbook_url: "https://docs.foxhunt.com/runbooks/trading-engine-down"
          action: "Immediate intervention required"

      # Market Data Feed Failure
      - alert: MarketDataFeedDown
        expr: up{job="market-data"} == 0
        for: 10s
        labels:
          severity: critical
          component: market_data
          impact: blind_trading
        annotations:
          summary: "Market Data Feed is DOWN"
          description: "Market data feed has been down for {{ $value }} seconds. Trading without market data is extremely dangerous."
          runbook_url: "https://docs.foxhunt.com/runbooks/market-data-down"
          action: "Stop all trading immediately"

      # Risk Management System Failure
      - alert: RiskManagementDown
        expr: up{job="risk-management"} == 0
        for: 5s
        labels:
          severity: critical
          component: risk_management
          impact: uncontrolled_risk
        annotations:
          summary: "Risk Management System is DOWN"
          description: "Risk management system has been down for {{ $value }} seconds. Trading without risk controls is prohibited."
          runbook_url: "https://docs.foxhunt.com/runbooks/risk-management-down"
          action: "Emergency trading halt"

      # High Latency Alert
      - alert: TradingLatencyHigh
        expr: histogram_quantile(0.95, foxhunt_trading_latency_seconds_bucket{operation="order_placement"}) > 0.001
        for: 30s
        labels:
          severity: critical
          component: trading_engine
          impact: competitive_disadvantage
        annotations:
          summary: "Trading latency is critically high"
          description: "95th percentile order placement latency is {{ $value }}s (>1ms). HFT advantage is compromised."
          runbook_url: "https://docs.foxhunt.com/runbooks/high-latency"
          action: "Investigate performance bottlenecks"

      # Position Limit Breach
      - alert: PositionLimitBreach
        expr: foxhunt_position_size_usd / foxhunt_position_limit_usd > 0.95
        for: 0s
        labels:
          severity: critical
          component: risk_management
          impact: regulatory_breach
        annotations:
          summary: "Position limit nearly breached"
          description: "Current position is {{ $value | humanizePercentage }} of limit. Immediate action required."
          runbook_url: "https://docs.foxhunt.com/runbooks/position-limits"
          action: "Reduce positions immediately"

      # Loss Limit Breach
      - alert: LossLimitBreach
        expr: foxhunt_daily_pnl_usd < foxhunt_loss_limit_usd
        for: 0s
        labels:
          severity: critical
          component: risk_management
          impact: financial_loss
        annotations:
          summary: "Daily loss limit breached"
          description: "Daily P&L is ${{ $value }}, breaching loss limit. Trading must be halted."
          runbook_url: "https://docs.foxhunt.com/runbooks/loss-limits"
          action: "Emergency trading halt"

  # =======================
  # HIGH PRIORITY ALERTS
  # =======================
  - name: hft_high
    rules:
      # Database Connection Issues
      - alert: DatabaseConnectionHigh
        expr: foxhunt_database_connections_active / foxhunt_database_connections_max > 0.85
        for: 1m
        labels:
          severity: high
          component: database
          impact: performance_degradation
        annotations:
          summary: "Database connection pool nearly exhausted"
          description: "Database connections are at {{ $value | humanizePercentage }} of maximum."
          runbook_url: "https://docs.foxhunt.com/runbooks/database-connections"

      # Memory Usage High
      - alert: MemoryUsageHigh
        expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.90
        for: 2m
        labels:
          severity: high
          component: system
          impact: performance_degradation
        annotations:
          summary: "Memory usage is critically high"
          description: "Memory usage is {{ $value | humanizePercentage }} on {{ $labels.instance }}."
          runbook_url: "https://docs.foxhunt.com/runbooks/memory-usage"

      # CPU Usage High
      - alert: CPUUsageHigh
        expr: 100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 90
        for: 5m
        labels:
          severity: high
          component: system
          impact: performance_degradation
        annotations:
          summary: "CPU usage is critically high"
          description: "CPU usage is {{ $value | printf \"%.1f\" }}% on {{ $labels.instance }}."
          runbook_url: "https://docs.foxhunt.com/runbooks/cpu-usage"

      # Disk Usage High
      - alert: DiskUsageHigh
        expr: (1 - (node_filesystem_avail_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes{fstype!="tmpfs"})) > 0.85
        for: 5m
        labels:
          severity: high
          component: system
          impact: storage_failure
        annotations:
          summary: "Disk usage is critically high"
          description: "Disk usage is {{ $value | humanizePercentage }} on {{ $labels.instance }} {{ $labels.mountpoint }}."
          runbook_url: "https://docs.foxhunt.com/runbooks/disk-usage"

      # Order Rejection Rate High
      - alert: OrderRejectionRateHigh
        expr: rate(foxhunt_orders_rejected_total[5m]) / rate(foxhunt_orders_total[5m]) > 0.05
        for: 2m
        labels:
          severity: high
          component: trading_engine
          impact: trading_inefficiency
        annotations:
          summary: "Order rejection rate is high"
          description: "{{ $value | humanizePercentage }} of orders are being rejected."
          runbook_url: "https://docs.foxhunt.com/runbooks/order-rejections"

      # Market Data Lag
      - alert: MarketDataLag
        expr: foxhunt_market_data_lag_seconds > 0.1
        for: 1m
        labels:
          severity: high
          component: market_data
          impact: stale_data
        annotations:
          summary: "Market data lag is high"
          description: "Market data is lagging by {{ $value }}s. Trading decisions may be based on stale data."
          runbook_url: "https://docs.foxhunt.com/runbooks/market-data-lag"

  # =======================
  # MEDIUM PRIORITY ALERTS  
  # =======================
  - name: hft_medium
    rules:
      # Service Restart
      - alert: ServiceRestarted
        expr: increase(process_start_time_seconds[10m]) > 0
        for: 0s
        labels:
          severity: medium
          component: service
          impact: disruption
        annotations:
          summary: "Service {{ $labels.job }} restarted"
          description: "Service {{ $labels.job }} on {{ $labels.instance }} has restarted."
          runbook_url: "https://docs.foxhunt.com/runbooks/service-restarts"

      # Network Latency High
      - alert: NetworkLatencyHigh
        expr: foxhunt_network_latency_seconds > 0.01
        for: 5m
        labels:
          severity: medium
          component: network
          impact: performance_impact
        annotations:
          summary: "Network latency is elevated"
          description: "Network latency is {{ $value }}s, which may impact HFT performance."
          runbook_url: "https://docs.foxhunt.com/runbooks/network-latency"

      # Backup Failure
      - alert: BackupFailed
        expr: time() - foxhunt_last_backup_timestamp > 86400
        for: 1h
        labels:
          severity: medium
          component: backup
          impact: data_risk
        annotations:
          summary: "Database backup has failed"
          description: "Last successful backup was {{ $value | humanizeDuration }} ago."
          runbook_url: "https://docs.foxhunt.com/runbooks/backup-failure"

      # SSL Certificate Expiry
      - alert: SSLCertificateExpiringSoon
        expr: (ssl_certificate_expiry_timestamp - time()) / 86400 < 30
        for: 1h
        labels:
          severity: medium
          component: security
          impact: service_disruption
        annotations:
          summary: "SSL certificate expires soon"
          description: "SSL certificate for {{ $labels.instance }} expires in {{ $value }} days."
          runbook_url: "https://docs.foxhunt.com/runbooks/ssl-renewal"

  # =======================
  # BUSINESS LOGIC ALERTS
  # =======================
  - name: hft_business
    rules:
      # Trading Volume Anomaly
      - alert: TradingVolumeAnomalyHigh
        expr: rate(foxhunt_trades_total[5m]) > (avg_over_time(rate(foxhunt_trades_total[5m])[1h:5m]) * 3)
        for: 2m
        labels:
          severity: medium
          component: trading_engine
          impact: business_anomaly
        annotations:
          summary: "Trading volume is unusually high"
          description: "Current trading volume is {{ $value }} trades/sec, which is 3x the hourly average."
          runbook_url: "https://docs.foxhunt.com/runbooks/volume-anomaly"

      - alert: TradingVolumeAnomalyLow
        expr: rate(foxhunt_trades_total[5m]) < (avg_over_time(rate(foxhunt_trades_total[5m])[1h:5m]) * 0.1)
        for: 10m
        labels:
          severity: medium
          component: trading_engine
          impact: business_anomaly
        annotations:
          summary: "Trading volume is unusually low"
          description: "Current trading volume is {{ $value }} trades/sec, which is 10% of the hourly average."
          runbook_url: "https://docs.foxhunt.com/runbooks/volume-anomaly"

      # P&L Volatility
      - alert: PnLVolatilityHigh
        expr: stddev_over_time(foxhunt_pnl_usd[1h]) > 10000
        for: 30m
        labels:
          severity: medium
          component: risk_management
          impact: financial_risk
        annotations:
          summary: "P&L volatility is high"
          description: "P&L standard deviation over 1 hour is ${{ $value }}, indicating high volatility."
          runbook_url: "https://docs.foxhunt.com/runbooks/pnl-volatility"

      # Drawdown Alert
      - alert: DrawdownHigh
        expr: foxhunt_drawdown_percent > 0.05
        for: 15m
        labels:
          severity: high
          component: risk_management
          impact: financial_risk
        annotations:
          summary: "Portfolio drawdown is high"
          description: "Current drawdown is {{ $value | humanizePercentage }}, exceeding comfort zone."
          runbook_url: "https://docs.foxhunt.com/runbooks/drawdown"

  # =======================
  # INFRASTRUCTURE ALERTS
  # =======================
  - name: hft_infrastructure
    rules:
      # Docker Container Down
      - alert: DockerContainerDown
        expr: up{job=~".*-exporter"} == 0
        for: 1m
        labels:
          severity: high
          component: infrastructure
          impact: service_disruption
        annotations:
          summary: "Docker container is down"
          description: "Container {{ $labels.job }} on {{ $labels.instance }} is down."
          runbook_url: "https://docs.foxhunt.com/runbooks/container-down"

      # Redis Connection Issues
      - alert: RedisConnectionFailed
        expr: redis_connected_clients{job="redis"} == 0
        for: 30s
        labels:
          severity: critical
          component: redis
          impact: cache_failure
        annotations:
          summary: "Redis connection failed"
          description: "No clients connected to Redis. Cache functionality compromised."
          runbook_url: "https://docs.foxhunt.com/runbooks/redis-failure"

      # PostgreSQL Connection Issues
      - alert: PostgreSQLDown
        expr: pg_up{job="postgres"} == 0
        for: 30s
        labels:
          severity: critical
          component: postgresql
          impact: data_unavailable
        annotations:
          summary: "PostgreSQL is down"
          description: "PostgreSQL database is not responding. Data persistence compromised."
          runbook_url: "https://docs.foxhunt.com/runbooks/postgresql-down"

      # InfluxDB Issues
      - alert: InfluxDBDown
        expr: influxdb_up{job="influxdb"} == 0
        for: 1m
        labels:
          severity: high
          component: influxdb
          impact: metrics_loss
        annotations:
          summary: "InfluxDB is down"
          description: "InfluxDB is not responding. Time-series data collection compromised."
          runbook_url: "https://docs.foxhunt.com/runbooks/influxdb-down"

  # =======================
  # SECURITY ALERTS
  # =======================
  - name: hft_security
    rules:
      # Authentication Failures
      - alert: AuthenticationFailuresHigh
        expr: rate(foxhunt_auth_failures_total[5m]) > 5
        for: 2m
        labels:
          severity: high
          component: security
          impact: security_breach
        annotations:
          summary: "High rate of authentication failures"
          description: "{{ $value }} authentication failures per second detected."
          runbook_url: "https://docs.foxhunt.com/runbooks/auth-failures"

      # Unauthorized Access Attempts
      - alert: UnauthorizedAccessAttempts
        expr: rate(foxhunt_unauthorized_requests_total[5m]) > 1
        for: 1m
        labels:
          severity: critical
          component: security
          impact: security_breach
        annotations:
          summary: "Unauthorized access attempts detected"
          description: "{{ $value }} unauthorized requests per second from {{ $labels.source_ip }}."
          runbook_url: "https://docs.foxhunt.com/runbooks/unauthorized-access"

      # API Rate Limiting
      - alert: APIRateLimitExceeded
        expr: rate(foxhunt_api_rate_limit_exceeded_total[5m]) > 0.1
        for: 5m
        labels:
          severity: medium
          component: api
          impact: service_degradation
        annotations:
          summary: "API rate limits are being exceeded"
          description: "API rate limits exceeded {{ $value }} times per second."
          runbook_url: "https://docs.foxhunt.com/runbooks/rate-limiting"

  # =======================
  # MONITORING ALERTS
  # =======================
  - name: hft_monitoring
    rules:
      # Prometheus Target Down
      - alert: PrometheusTargetDown
        expr: up == 0
        for: 2m
        labels:
          severity: medium
          component: monitoring
          impact: observability_loss
        annotations:
          summary: "Prometheus target is down"
          description: "{{ $labels.job }} target {{ $labels.instance }} has been down for more than 2 minutes."
          runbook_url: "https://docs.foxhunt.com/runbooks/prometheus-target-down"

      # Prometheus Configuration Reload Failed
      - alert: PrometheusConfigReloadFailed
        expr: prometheus_config_last_reload_successful == 0
        for: 5m
        labels:
          severity: high
          component: monitoring
          impact: configuration_error
        annotations:
          summary: "Prometheus configuration reload failed"
          description: "Prometheus configuration reload has failed. Monitoring may not reflect latest config."
          runbook_url: "https://docs.foxhunt.com/runbooks/prometheus-config-reload"

      # Alert Manager Down
      - alert: AlertManagerDown
        expr: up{job="alertmanager"} == 0
        for: 5m
        labels:
          severity: high
          component: monitoring
          impact: alert_delivery_failure
        annotations:
          summary: "AlertManager is down"
          description: "AlertManager has been down for more than 5 minutes. Alerts will not be delivered."
          runbook_url: "https://docs.foxhunt.com/runbooks/alertmanager-down"

      # High Alert Rate
      - alert: HighAlertRate
        expr: rate(prometheus_notifications_total[5m]) > 10
        for: 10m
        labels:
          severity: medium
          component: monitoring
          impact: alert_fatigue
        annotations:
          summary: "High rate of alerts being generated"
          description: "{{ $value }} alerts per second are being generated. Check for alert storms."
          runbook_url: "https://docs.foxhunt.com/runbooks/alert-storms"