foxhunt/monitoring/prometheus/alerts/ml_trading_alerts.yml

# Prometheus Alert Rules for ML Trading Operations
#
# Alerts for ML model predictions, order execution, and ensemble behavior.
# Coordinates with ml_training_alerts.yml (training) and ensemble_ml_alerts.yml (aggregation).

groups:
  - name: ml_trading_accuracy
    interval: 60s
    rules:
      # Model accuracy below production threshold
      - alert: MLModelLowAccuracy
        expr: ml_prediction_accuracy{model_id!="test"} < 55
        for: 1h
        labels:
          severity: warning
          component: ml_trading
          alert_type: performance
        annotations:
          summary: "ML model {{ $labels.model_id }} accuracy below 55%"
          description: "Model {{ $labels.model_id }} prediction accuracy is {{ $value }}% (threshold: 55%)"
          impact: "Model performance degraded - trading signals less reliable"
          action: |
            1. Check model drift metrics
            2. Review recent data quality
            3. Compare with other models
            4. Consider retraining or disabling model
          runbook_url: "https://docs.foxhunt.io/runbooks/ml-low-accuracy"

      # Win rate below profitability threshold
      - alert: MLModelLowWinRate
        expr: ml_model_win_rate{model_id!="test"} < 0.55
        for: 2h
        labels:
          severity: warning
          component: ml_trading
          alert_type: performance
        annotations:
          summary: "Model {{ $labels.model_id }} win rate below 55%"
          description: "Win rate is {{ $value | humanizePercentage }} (threshold: 55%)"
          impact: "Unprofitable trading - capital at risk"
          action: |
            1. Review trade history for patterns
            2. Check market conditions (regime shift?)
            3. Analyze model confidence scores
            4. Consider reducing position sizes or disabling
          dashboard_url: "https://grafana.foxhunt.io/d/ml-trading/model-performance"

      # Sharpe ratio below target
      - alert: MLModelLowSharpeRatio
        expr: ml_model_sharpe_ratio{model_id!="test"} < 1.5
        for: 6h
        labels:
          severity: info
          component: ml_trading
          alert_type: performance
        annotations:
          summary: "Model {{ $labels.model_id }} Sharpe ratio below 1.5"
          description: "Sharpe ratio is {{ $value }} (target: >1.5)"
          impact: "Risk-adjusted returns suboptimal"
          action: |
            1. Compare with benchmark models
            2. Analyze volatility patterns
            3. Review position sizing strategy
            4. Monitor over longer timeframe

  - name: ml_trading_predictions
    interval: 10s
    rules:
      # Model stopped making predictions (stale)
      - alert: MLModelStalePredictions
        expr: (time() - ml_model_last_prediction_time{model_id!="test"}) > 3600
        for: 5m
        labels:
          severity: warning
          component: ml_trading
          alert_type: availability
        annotations:
          summary: "Model {{ $labels.model_id }} has not made predictions in >1h"
          description: "Last prediction was {{ $value | humanizeDuration }} ago (threshold: 1h)"
          impact: "Model not generating signals - missing trading opportunities"
          action: |
            1. Check model health status
            2. Verify data pipeline connectivity
            3. Review inference service logs
            4. Restart model if necessary
          runbook_url: "https://docs.foxhunt.io/runbooks/stale-predictions"

      # Low prediction confidence (median)
      - alert: MLPredictionConfidenceLow
        expr: |
          histogram_quantile(0.50, sum by (model_id, le) (rate(ml_predictions_confidence_bucket[5m]))) < 0.70
        for: 10m
        labels:
          severity: info
          component: ml_trading
          alert_type: quality
        annotations:
          summary: "Model {{ $labels.model_id }} median confidence < 0.70"
          description: "Median prediction confidence is {{ $value }} (threshold: 0.70)"
          impact: "High model uncertainty - signals less reliable"
          action: |
            1. Check if market regime changed
            2. Review feature distribution
            3. Compare with historical confidence
            4. Consider reducing position sizes

      # Prediction rate anomaly (too low)
      - alert: MLPredictionRateLow
        expr: |
          sum by (model_id) (rate(ml_predictions_total[5m])) < 0.01
        for: 15m
        labels:
          severity: warning
          component: ml_trading
          alert_type: availability
        annotations:
          summary: "Model {{ $labels.model_id }} prediction rate abnormally low"
          description: "Prediction rate is {{ $value }}/sec (threshold: 0.01/sec)"
          impact: "Model not actively generating signals"
          action: |
            1. Check inference service health
            2. Verify data feed connectivity
            3. Review model load errors
            4. Check CPU/GPU utilization

  - name: ml_trading_orders
    interval: 10s
    rules:
      # High order rejection rate
      - alert: MLOrderRejectionRateHigh
        expr: |
          100 * (
            sum by (model_id, symbol) (rate(ml_orders_rejected_total[5m]))
            /
            sum by (model_id, symbol) (rate(ml_orders_submitted_total[5m]))
          ) > 10
        for: 5m
        labels:
          severity: warning
          component: ml_trading
          alert_type: execution
        annotations:
          summary: "High order rejection rate for {{ $labels.model_id }} on {{ $labels.symbol }}"
          description: "Rejection rate is {{ $value }}% (threshold: 10%)"
          impact: "Many orders being rejected - missing trades"
          action: |
            1. Check rejection reasons (see ml_orders_rejected_total labels)
            2. Review risk limits for {{ $labels.symbol }}
            3. Verify margin availability
            4. Check market hours and trading halts
            5. Analyze price validity checks
          dashboard_url: "https://grafana.foxhunt.io/d/ml-trading/order-execution"

      # Low order fill rate
      - alert: MLOrderFillRateLow
        expr: |
          100 * (
            sum by (model_id, symbol) (rate(ml_orders_filled_total[5m]))
            /
            sum by (model_id, symbol) (rate(ml_orders_submitted_total[5m]))
          ) < 80
        for: 10m
        labels:
          severity: info
          component: ml_trading
          alert_type: execution
        annotations:
          summary: "Low fill rate for {{ $labels.model_id }} on {{ $labels.symbol }}"
          description: "Fill rate is {{ $value }}% (threshold: 80%)"
          impact: "Poor order execution - slippage may be high"
          action: |
            1. Review order types (market vs limit)
            2. Check liquidity on {{ $labels.symbol }}
            3. Analyze order size relative to market depth
            4. Consider adjusting execution strategy

      # Risk limit rejections spike
      - alert: MLOrderRiskLimitRejections
        expr: |
          rate(ml_orders_rejected_total{reason="risk_limit"}[5m]) > 0.1
        for: 2m
        labels:
          severity: critical
          component: ml_trading
          alert_type: risk
        annotations:
          summary: "Risk limit rejections for {{ $labels.model_id }} on {{ $labels.symbol }}"
          description: "{{ $value }}/sec orders rejected due to risk limits"
          impact: "Model hitting risk constraints - capital protection active"
          action: |
            1. Review current exposure for {{ $labels.symbol }}
            2. Check if position limits are appropriate
            3. Verify VaR calculations
            4. Consider temporary model disablement if excessive
          priority: high

  - name: ml_trading_ensemble
    interval: 30s
    rules:
      # High ensemble disagreement rate
      - alert: MLEnsembleHighDisagreement
        expr: ml_ensemble_agreement_rate{symbol!=""} < 0.5
        for: 10m
        labels:
          severity: info
          component: ml_trading
          alert_type: ensemble
        annotations:
          summary: "High ensemble disagreement on {{ $labels.symbol }}"
          description: "Model agreement rate is {{ $value | humanizePercentage }} (threshold: 50%)"
          impact: "Models disagree significantly - market uncertainty or data issues"
          action: |
            1. Check if market regime changed
            2. Review data quality metrics
            3. Compare individual model predictions
            4. Consider reducing position sizes
            5. Monitor for regime shift
          dashboard_url: "https://grafana.foxhunt.io/d/ensemble/disagreement-analysis"

      # Frequent high disagreement events
      - alert: MLEnsembleDisagreementEventsFrequent
        expr: |
          rate(ml_ensemble_disagreement_events{threshold="0.7"}[5m]) > 0.3
        for: 10m
        labels:
          severity: warning
          component: ml_trading
          alert_type: ensemble
        annotations:
          summary: "Frequent high disagreement events on {{ $labels.symbol }}"
          description: "{{ $value }}/sec disagreement events (threshold: 0.3/sec)"
          impact: "Models frequently conflicting - possible regime shift"
          action: |
            1. Investigate market conditions
            2. Check for data anomalies
            3. Review model drift scores
            4. Consider ensemble rebalancing
            5. Alert trading desk

      # Ensemble voting stopped
      - alert: MLEnsembleVotingStopped
        expr: |
          rate(ml_ensemble_votes_total[5m]) == 0
        for: 15m
        labels:
          severity: critical
          component: ml_trading
          alert_type: availability
        annotations:
          summary: "Ensemble voting stopped for {{ $labels.symbol }}"
          description: "No ensemble votes in last 15 minutes"
          impact: "Ensemble system not generating signals - trading halted"
          action: |
            1. Check ensemble coordinator service
            2. Verify individual model health
            3. Review aggregation service logs
            4. Restart ensemble if necessary
          priority: high
          runbook_url: "https://docs.foxhunt.io/runbooks/ensemble-down"

  - name: ml_trading_latency
    interval: 10s
    rules:
      # Inference latency P99 high
      - alert: MLInferenceLatencyHigh
        expr: |
          histogram_quantile(0.99, sum by (model_id, le) (rate(ml_model_inference_latency_bucket[5m]))) > 1000
        for: 5m
        labels:
          severity: warning
          component: ml_trading
          alert_type: performance
        annotations:
          summary: "Model {{ $labels.model_id }} P99 inference latency > 1ms"
          description: "P99 latency is {{ $value }}μs (threshold: 1000μs)"
          impact: "Slow inference - trading signal delays"
          action: |
            1. Check GPU utilization
            2. Review model load (batch size)
            3. Profile inference bottlenecks
            4. Consider model optimization
          dashboard_url: "https://grafana.foxhunt.io/d/ml-trading/inference-latency"

      # Inference latency P99 critical
      - alert: MLInferenceLatencyCritical
        expr: |
          histogram_quantile(0.99, sum by (model_id, le) (rate(ml_model_inference_latency_bucket[5m]))) > 5000
        for: 2m
        labels:
          severity: critical
          component: ml_trading
          alert_type: performance
        annotations:
          summary: "Model {{ $labels.model_id }} P99 inference latency > 5ms"
          description: "P99 latency is {{ $value }}μs (threshold: 5000μs)"
          impact: "Critical inference delays - real-time trading compromised"
          action: |
            1. IMMEDIATE: Check system resources
            2. Reduce model load if possible
            3. Consider failover to faster model
            4. Alert on-call engineer
          priority: high

  - name: ml_trading_risk
    interval: 30s
    rules:
      # Large drawdown detected
      - alert: MLModelLargeDrawdown
        expr: ml_model_max_drawdown{model_id!="test"} < -5000
        for: 1h
        labels:
          severity: critical
          component: ml_trading
          alert_type: risk
        annotations:
          summary: "Model {{ $labels.model_id }} experiencing large drawdown"
          description: "Maximum drawdown is ${{ $value }} (threshold: -$5000)"
          impact: "Significant capital loss - risk controls may need adjustment"
          action: |
            1. IMMEDIATE: Review current positions
            2. Consider reducing model allocation
            3. Analyze losing trades for patterns
            4. Check if stop-losses are working
            5. Alert risk management team
          priority: high
          dashboard_url: "https://grafana.foxhunt.io/d/ml-trading/risk-metrics"

      # Negative cumulative PnL
      - alert: MLModelNegativePnL
        expr: ml_model_cumulative_pnl{model_id!="test"} < 0
        for: 24h
        labels:
          severity: warning
          component: ml_trading
          alert_type: risk
        annotations:
          summary: "Model {{ $labels.model_id }} cumulative PnL negative"
          description: "Cumulative PnL is ${{ $value }}"
          impact: "Model unprofitable - consider disabling"
          action: |
            1. Review 24h trading history
            2. Compare with other models
            3. Check market conditions
            4. Consider model retraining
            5. Evaluate continued deployment

  - name: ml_trading_healthcheck
    interval: 30s
    rules:
      # No predictions from any model
      - alert: MLTradingSystemDown
        expr: |
          sum(rate(ml_predictions_total[5m])) == 0
        for: 10m
        labels:
          severity: critical
          component: ml_trading
          alert_type: availability
        annotations:
          summary: "ML trading system not generating predictions"
          description: "No predictions from any model in last 10 minutes"
          impact: "Complete trading system outage - no signals generated"
          action: |
            1. IMMEDIATE: Check trading service health
            2. Verify all model services running
            3. Check data pipeline connectivity
            4. Review system logs
            5. Alert on-call engineer
            6. Consider manual trading fallback
          priority: emergency
          runbook_url: "https://docs.foxhunt.io/runbooks/ml-system-down"

      # No orders submitted despite predictions
      - alert: MLOrderSubmissionFailure
        expr: |
          (sum(rate(ml_predictions_total{action!="hold"}[5m])) > 0.01) and
          (sum(rate(ml_orders_submitted_total[5m])) == 0)
        for: 5m
        labels:
          severity: critical
          component: ml_trading
          alert_type: execution
        annotations:
          summary: "Models generating signals but no orders submitted"
          description: "{{ $value }} buy/sell predictions but 0 orders submitted"
          impact: "Order execution broken - missing all trades"
          action: |
            1. IMMEDIATE: Check order submission service
            2. Verify exchange connectivity
            3. Review risk system status
            4. Check order validation logic
            5. Alert on-call engineer
          priority: emergency