foxhunt/infra/k8s/monitoring/prometheus-rules-broker.yaml

# Broker Gateway alert rules — discovered automatically by Prometheus Operator
# Apply: kubectl apply -f prometheus-rules-broker.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: foxhunt-broker-alerts
  namespace: foxhunt
  labels:
    app.kubernetes.io/part-of: foxhunt
    role: alert-rules
spec:
  groups:
    - name: broker_gateway_fix_session
      interval: 30s
      rules:
        # FIX session disconnected >60s
        - alert: BrokerGatewayFIXSessionDisconnected
          expr: broker_gateway_fix_session_status == 0
          for: 60s
          labels:
            severity: critical
            service: broker_gateway_service
            component: fix_session
            impact: trading_halted
          annotations:
            summary: "FIX session {{ $labels.session_id }} DISCONNECTED"
            description: "FIX session down >60s. All order routing halted."
            action: "Check FIX engine logs, verify network to AMP Futures gateway"

        # Order latency P95 > 100ms
        - alert: BrokerGatewayHighOrderLatency
          expr: |
            histogram_quantile(0.95,
              sum by(le, order_type) (rate(broker_gateway_order_latency_seconds_bucket[5m]))
            ) * 1000 > 100
          for: 5m
          labels:
            severity: warning
            service: broker_gateway_service
            component: order_routing
            impact: performance_degraded
          annotations:
            summary: "Order latency P95 > 100ms for {{ $labels.order_type }}"

        # Error rate > 5%
        - alert: BrokerGatewayHighErrorRate
          expr: |
            (
              sum by(severity) (rate(broker_gateway_error_total[5m]))
              /
              (sum(rate(broker_gateway_orders_submitted_total[5m])) + 0.001)
            ) > 0.05
          for: 5m
          labels:
            severity: warning
            service: broker_gateway_service
            component: error_handling
            impact: reliability_degraded
          annotations:
            summary: "Error rate {{ $value | humanizePercentage }} ({{ $labels.severity }})"

        # No orders in 10m during market hours
        - alert: BrokerGatewayNoOrderActivity
          expr: |
            (time() - broker_gateway_last_order_time) > 600
            and
            (hour() >= 9 and hour() < 16)
          for: 1m
          labels:
            severity: warning
            service: broker_gateway_service
            component: order_flow
            impact: trading_inactive
          annotations:
            summary: "No order activity for 10+ minutes during market hours"

        # Position mismatch > $10K
        - alert: BrokerGatewayPositionMismatch
          expr: |
            abs(
              broker_gateway_position_value_usd
              -
              on(symbol, account_id) group_left()
              trading_service_position_value_usd
            ) > 10000
          for: 2m
          labels:
            severity: critical
            service: broker_gateway_service
            component: position_management
            impact: data_integrity
          annotations:
            summary: "Position mismatch ${{ $value | humanize }} on {{ $labels.symbol }}"
            action: "HALT trading on affected symbol, reconcile with broker"

        # FIX sequence number gap
        - alert: BrokerGatewayFIXSequenceGap
          expr: rate(broker_gateway_sequence_number_gap_total[5m]) > 0
          for: 1m
          labels:
            severity: warning
            service: broker_gateway_service
            component: fix_session
            impact: message_loss
          annotations:
            summary: "FIX sequence gap on {{ $labels.session_id }} ({{ $value | humanize }} gaps/sec)"

    - name: broker_gateway_performance
      interval: 60s
      rules:
        # High FIX heartbeat RTT
        - alert: BrokerGatewayHighFIXHeartbeatRTT
          expr: broker_gateway_fix_heartbeat_rtt_ms > 50
          for: 5m
          labels:
            severity: warning
            service: broker_gateway_service
            component: fix_session
            impact: network_latency
          annotations:
            summary: "FIX heartbeat RTT {{ $value }}ms on {{ $labels.session_id }}"

        # High order rejection rate per symbol
        - alert: BrokerGatewayHighOrderRejectionRate
          expr: |
            (
              sum by(symbol) (rate(broker_gateway_orders_rejected_total[5m]))
              /
              (sum by(symbol) (rate(broker_gateway_orders_submitted_total[5m])) + 0.001)
            ) > 0.10
          for: 5m
          labels:
            severity: warning
            service: broker_gateway_service
            component: order_validation
            impact: order_flow_disrupted
          annotations:
            summary: "Order rejection rate {{ $value | humanizePercentage }} for {{ $labels.symbol }}"

        # Slow database queries
        - alert: BrokerGatewaySlowDatabaseQueries
          expr: |
            histogram_quantile(0.95,
              sum by(le) (rate(database_query_duration_seconds_bucket{service="broker_gateway_service"}[5m]))
            ) > 0.100
          for: 5m
          labels:
            severity: warning
            service: broker_gateway_service
            component: database
            impact: performance_degraded
          annotations:
            summary: "Database query P95 > 100ms"

        # High margin usage
        - alert: BrokerGatewayHighMarginUsage
          expr: |
            (
              broker_gateway_margin_used_usd
              / (broker_gateway_cash_balance_usd + 0.001)
            ) > 0.80
          for: 5m
          labels:
            severity: warning
            service: broker_gateway_service
            component: risk_management
            impact: margin_pressure
          annotations:
            summary: "Margin usage {{ $value | humanizePercentage }} for {{ $labels.account_id }}"