foxhunt/monitoring/prometheus/alerts/ensemble_ml_alerts.yml

# Prometheus Alert Rules for Ensemble ML Production Monitoring
#
# Critical alerts for 6-model ensemble (DQN, PPO, MAMBA-2, TFT, TLOB)
# Monitors: Sharpe ratio, disagreement, latency, model failures, cascades
#
# See: ENSEMBLE_METRICS_QUICK_REFERENCE.md
# Runbook: ENSEMBLE_RUNBOOK.md

groups:
  # =============================================================================
  # GROUP 1: PERFORMANCE DEGRADATION ALERTS
  # =============================================================================
  - name: ensemble_performance_degradation
    interval: 15s
    rules:
      # CRITICAL: Sharpe ratio drops >50% (15min window)
      - alert: EnsembleSharpeRatioDropCritical
        expr: |
          (
            (
              sum(rate(ensemble_model_pnl_contribution_dollars_sum[15m])) /
              sqrt(sum(rate(ensemble_model_pnl_contribution_dollars_count[15m])))
            ) /
            (
              sum(rate(ensemble_model_pnl_contribution_dollars_sum[15m] offset 1h)) /
              sqrt(sum(rate(ensemble_model_pnl_contribution_dollars_count[15m] offset 1h)))
            )
          ) < 0.5
        for: 15m
        labels:
          severity: critical
          component: ensemble
          alert_type: performance
        annotations:
          summary: "Ensemble Sharpe ratio dropped >50% in 15 minutes"
          description: "Current Sharpe ratio is {{ $value | humanizePercentage }} of baseline (threshold: 50%)"
          impact: "Strategy profitability severely degraded - potential regime shift or model drift"
          action: |
            1. Check Grafana dashboard: http://localhost:3000/d/ensemble-ml-production
            2. Review model disagreement rates
            3. Check for market regime changes
            4. Consider reducing position sizes or switching to defensive mode
            5. Review model weights - any single model dominating?
          runbook_url: "https://docs.foxhunt.io/runbooks/ensemble-sharpe-drop"

      # WARNING: Sharpe ratio drops 25-50% (15min window)
      - alert: EnsembleSharpeRatioDropWarning
        expr: |
          (
            (
              sum(rate(ensemble_model_pnl_contribution_dollars_sum[15m])) /
              sqrt(sum(rate(ensemble_model_pnl_contribution_dollars_count[15m])))
            ) /
            (
              sum(rate(ensemble_model_pnl_contribution_dollars_sum[15m] offset 1h)) /
              sqrt(sum(rate(ensemble_model_pnl_contribution_dollars_count[15m] offset 1h)))
            )
          ) < 0.75
        for: 15m
        labels:
          severity: warning
          component: ensemble
          alert_type: performance
        annotations:
          summary: "Ensemble Sharpe ratio degraded by 25-50%"
          description: "Sharpe ratio {{ $value | humanizePercentage }} of baseline (threshold: 75%)"
          impact: "Strategy performance degrading - early warning"
          action: "Monitor closely - review model weights and disagreement patterns"

      # CRITICAL: Win rate collapse (<40%)
      - alert: EnsembleWinRateCollapse
        expr: |
          100 * (
            sum(rate(ensemble_predictions_total{action="buy"}[15m])) +
            sum(rate(ensemble_predictions_total{action="sell"}[15m]))
          ) /
          sum(rate(ensemble_predictions_total[15m])) < 40
        for: 15m
        labels:
          severity: critical
          component: ensemble
          alert_type: performance
        annotations:
          summary: "Ensemble win rate collapsed below 40%"
          description: "Win rate: {{ $value }}% (threshold: 40%)"
          impact: "Strategy losing money - immediate intervention required"
          action: |
            1. STOP trading immediately if drawdown >10%
            2. Review prediction confidence levels
            3. Check model weights - any model contributing negative P&L?
            4. Switch to single best-performing model
          runbook_url: "https://docs.foxhunt.io/runbooks/ensemble-win-rate-collapse"

      # CRITICAL: Negative P&L trending over 30min
      - alert: EnsembleNegativePnLTrend
        expr: |
          sum(rate(ensemble_model_pnl_contribution_dollars_sum[30m])) < -1000
        for: 30m
        labels:
          severity: critical
          component: ensemble
          alert_type: performance
        annotations:
          summary: "Ensemble generating consistent negative P&L"
          description: "30-min P&L trend: ${{ $value | humanize }} (threshold: -$1000)"
          impact: "Active capital loss - trading system hemorrhaging money"
          action: |
            1. IMMEDIATE: Halt automated trading
            2. Review all active positions
            3. Identify losing model(s) via P&L attribution
            4. Switch to manual trading or single-model mode
            5. Escalate to ML team for model investigation
          runbook_url: "https://docs.foxhunt.io/runbooks/ensemble-negative-pnl"

  # =============================================================================
  # GROUP 2: MODEL DISAGREEMENT & UNCERTAINTY
  # =============================================================================
  - name: ensemble_disagreement_alerts
    interval: 5s
    rules:
      # CRITICAL: High disagreement >70% sustained for 5min
      - alert: EnsembleHighDisagreementCritical
        expr: ensemble_disagreement_rate > 0.7
        for: 5m
        labels:
          severity: critical
          component: ensemble
          alert_type: disagreement
        annotations:
          summary: "Critical model disagreement >70% for 5 minutes"
          description: "Disagreement rate {{ $value | humanizePercentage }} on {{ $labels.symbol }} (threshold: 70%)"
          impact: "Models fundamentally disagree - prediction reliability compromised"
          action: |
            1. REDUCE position sizes by 50% immediately
            2. Check for market regime shift (Grafana regime detection panel)
            3. Review individual model predictions for {{ $labels.symbol }}
            4. Consider pausing trading for {{ $labels.symbol }} if disagreement persists >15min
            5. Check recent news/events for {{ $labels.symbol }}
          runbook_url: "https://docs.foxhunt.io/runbooks/ensemble-high-disagreement"

      # WARNING: Moderate disagreement 50-70% sustained for 5min
      - alert: EnsembleHighDisagreementWarning
        expr: ensemble_disagreement_rate > 0.5 and ensemble_disagreement_rate <= 0.7
        for: 5m
        labels:
          severity: warning
          component: ensemble
          alert_type: disagreement
        annotations:
          summary: "Elevated model disagreement 50-70%"
          description: "Disagreement {{ $value | humanizePercentage }} on {{ $labels.symbol }} (threshold: 50%)"
          impact: "Models showing divergence - increased prediction uncertainty"
          action: |
            1. Monitor closely - review model weights
            2. Check ensemble confidence scores
            3. Consider tightening stop-losses for {{ $labels.symbol }}
          runbook_url: "https://docs.foxhunt.io/runbooks/ensemble-disagreement"

      # CRITICAL: Disagreement spike rate (rapid increase)
      - alert: EnsembleDisagreementSpikeRate
        expr: |
          rate(ensemble_high_disagreement_total{threshold="0.7"}[5m]) > 10
        for: 5m
        labels:
          severity: critical
          component: ensemble
          alert_type: disagreement
        annotations:
          summary: "Rapid increase in high-disagreement predictions"
          description: "{{ $value }} high-disagreement events/sec (threshold: 10)"
          impact: "Market conditions rapidly changing - model consensus breaking down"
          action: |
            1. PAUSE new position entries immediately
            2. Close positions with low confidence (<0.6)
            3. Review market volatility indicators
            4. Check for flash crash or news events
          runbook_url: "https://docs.foxhunt.io/runbooks/ensemble-disagreement-spike"

      # CRITICAL: Low confidence + high disagreement (worst case)
      - alert: EnsembleLowConfidenceHighDisagreement
        expr: |
          (ensemble_confidence_score < 0.6) and (ensemble_disagreement_rate > 0.7)
        for: 2m
        labels:
          severity: critical
          component: ensemble
          alert_type: uncertainty
        annotations:
          summary: "Low confidence AND high disagreement detected"
          description: "Confidence {{ $labels.confidence }} / Disagreement {{ $labels.disagreement }} on {{ $labels.symbol }}"
          impact: "Models uncertain AND disagreeing - prediction reliability at minimum"
          action: |
            1. HALT all trading for {{ $labels.symbol }} immediately
            2. Switch to observation mode (log predictions without execution)
            3. Manually review market conditions
            4. Do NOT resume automated trading until confidence >0.7 AND disagreement <0.5
          runbook_url: "https://docs.foxhunt.io/runbooks/ensemble-uncertainty"

  # =============================================================================
  # GROUP 3: LATENCY & PERFORMANCE
  # =============================================================================
  - name: ensemble_latency_alerts
    interval: 15s
    rules:
      # CRITICAL: Aggregation latency P99 >50μs for 1min
      - alert: EnsembleAggregationLatencyP99High
        expr: |
          histogram_quantile(0.99, rate(ensemble_aggregation_latency_microseconds_bucket[1m])) > 50
        for: 1m
        labels:
          severity: critical
          component: ensemble
          alert_type: latency
        annotations:
          summary: "Ensemble aggregation P99 latency >50μs SLA violation"
          description: "P99 latency {{ $value }}μs for {{ $labels.aggregation_method }} (threshold: 50μs)"
          impact: "HFT execution delays - potential alpha decay and slippage"
          action: |
            1. Check CPU utilization on trading service
            2. Review concurrent prediction load
            3. Check for contention in model inference
            4. Consider scaling to additional instances
            5. Review Grafana latency breakdown by model
          runbook_url: "https://docs.foxhunt.io/runbooks/ensemble-latency-high"

      # WARNING: Aggregation latency P99 25-50μs for 1min
      - alert: EnsembleAggregationLatencyP99Warning
        expr: |
          histogram_quantile(0.99, rate(ensemble_aggregation_latency_microseconds_bucket[1m])) > 25
        for: 1m
        labels:
          severity: warning
          component: ensemble
          alert_type: latency
        annotations:
          summary: "Ensemble aggregation P99 latency elevated"
          description: "P99 latency {{ $value }}μs (target: <25μs)"
          impact: "Performance degrading - monitor for further increases"

      # WARNING: Aggregation latency P95 >25μs for 2min
      - alert: EnsembleAggregationLatencyP95High
        expr: |
          histogram_quantile(0.95, rate(ensemble_aggregation_latency_microseconds_bucket[2m])) > 25
        for: 2m
        labels:
          severity: warning
          component: ensemble
          alert_type: latency
        annotations:
          summary: "Ensemble aggregation P95 latency elevated"
          description: "P95 latency {{ $value }}μs (target: <10μs)"

      # CRITICAL: Prediction throughput collapse
      - alert: EnsemblePredictionThroughputCollapse
        expr: rate(ensemble_predictions_total[1m]) < 10
        for: 2m
        labels:
          severity: critical
          component: ensemble
          alert_type: performance
        annotations:
          summary: "Ensemble prediction throughput collapsed"
          description: "{{ $value }} predictions/sec (expected: >100)"
          impact: "Ensemble not generating predictions - trading halted"
          action: |
            1. Check trading service health status
            2. Review model loading status (any models failed?)
            3. Check inference queue depth
            4. Review logs for errors
          runbook_url: "https://docs.foxhunt.io/runbooks/ensemble-throughput-collapse"

  # =============================================================================
  # GROUP 4: MODEL FAILURES & CASCADE DETECTION
  # =============================================================================
  - name: ensemble_model_failures
    interval: 5s
    rules:
      # CRITICAL: Any model failed to load or predict (immediate)
      - alert: EnsembleModelFailureDetected
        expr: |
          (
            checkpoint_swaps_total{status="failed"} > 0
          ) or (
            sum(rate(ensemble_predictions_total[1m])) by (model_id) == 0
          )
        for: 0s
        labels:
          severity: critical
          component: ensemble
          alert_type: model_failure
        annotations:
          summary: "Model failure detected in ensemble"
          description: "Model {{ $labels.model_id }} failed (checkpoint swap or prediction)"
          impact: "Ensemble operating with reduced capacity - prediction quality degraded"
          action: |
            1. Identify failed model: {{ $labels.model_id }}
            2. Check trading service logs for error details
            3. Verify checkpoint file integrity in MinIO
            4. Attempt manual checkpoint reload
            5. If reload fails, remove model from ensemble temporarily
            6. Escalate to ML team if issue persists
          runbook_url: "https://docs.foxhunt.io/runbooks/ensemble-model-failure"

      # CRITICAL: Multiple models failed (cascade failure)
      - alert: EnsembleCascadeFailureDetected
        expr: |
          count(
            sum(rate(ensemble_predictions_total[1m])) by (model_id) == 0
          ) >= 2
        for: 30s
        labels:
          severity: critical
          component: ensemble
          alert_type: cascade_failure
        annotations:
          summary: "CASCADE FAILURE: Multiple models failed simultaneously"
          description: "{{ $value }} models not producing predictions"
          impact: "CRITICAL: Ensemble integrity compromised - predictions unreliable"
          action: |
            1. IMMEDIATE: Halt all automated trading
            2. Switch to manual trading mode
            3. Review trading service logs for common failure cause
            4. Check infrastructure: GPU, memory, disk, network
            5. Verify MinIO checkpoint storage accessibility
            6. Page on-call ML engineer immediately
            7. Do NOT resume automated trading until >=4 models operational
          runbook_url: "https://docs.foxhunt.io/runbooks/ensemble-cascade-failure"

      # CRITICAL: High checkpoint rollback rate (>10% over 10min)
      - alert: EnsembleCheckpointRollbackRateHigh
        expr: |
          100 * (
            sum(rate(checkpoint_swaps_total{status="rollback"}[10m])) /
            sum(rate(checkpoint_swaps_total[10m]))
          ) > 10
        for: 10m
        labels:
          severity: critical
          component: ensemble
          alert_type: checkpoint_quality
        annotations:
          summary: "High checkpoint rollback rate detected"
          description: "{{ $value }}% of checkpoint swaps rolled back (threshold: 10%)"
          impact: "New checkpoints failing canary validation - model quality issues"
          action: |
            1. Review recent checkpoint training quality metrics
            2. Check canary validation failure reasons in logs
            3. Investigate training data quality issues
            4. Review hyperparameter tuning results
            5. Consider pausing automated checkpoint updates
            6. Escalate to ML training team
          runbook_url: "https://docs.foxhunt.io/runbooks/ensemble-checkpoint-rollback"

      # WARNING: Single model checkpoint swap failed
      - alert: EnsembleCheckpointSwapFailed
        expr: |
          rate(checkpoint_swaps_total{status="failed"}[5m]) > 0
        for: 5m
        labels:
          severity: warning
          component: ensemble
          alert_type: checkpoint
        annotations:
          summary: "Checkpoint swap failed for model {{ $labels.model_id }}"
          description: "Failed to load new checkpoint - using previous version"
          impact: "Model {{ $labels.model_id }} not updated - operating on stale checkpoint"
          action: |
            1. Check MinIO checkpoint availability
            2. Verify checkpoint file integrity
            3. Review disk space and memory availability
            4. Attempt manual checkpoint load
          runbook_url: "https://docs.foxhunt.io/runbooks/ensemble-checkpoint-swap-failed"

  # =============================================================================
  # GROUP 5: MODEL WEIGHT & CONTRIBUTION ANOMALIES
  # =============================================================================
  - name: ensemble_weight_anomalies
    interval: 30s
    rules:
      # CRITICAL: Single model dominating (>70% weight)
      - alert: EnsembleSingleModelDominance
        expr: |
          max(ensemble_model_weight) by (symbol) > 0.7
        for: 10m
        labels:
          severity: critical
          component: ensemble
          alert_type: weight_imbalance
        annotations:
          summary: "Single model dominating ensemble decisions"
          description: "Model {{ $labels.model_id }} has {{ $value | humanizePercentage }} weight on {{ $labels.symbol }} (threshold: 70%)"
          impact: "Ensemble diversity lost - effectively operating as single model"
          action: |
            1. Review why other models have low weights
            2. Check P&L attribution - is dominant model actually best performer?
            3. Verify other models are producing valid predictions
            4. Consider manual weight rebalancing if issue persists
            5. May indicate other models underperforming or failing silently
          runbook_url: "https://docs.foxhunt.io/runbooks/ensemble-model-dominance"

      # WARNING: Model weights not summing to 1.0 (coordination issue)
      - alert: EnsembleWeightSumAnomalous
        expr: |
          abs(sum(ensemble_model_weight) by (symbol) - 1.0) > 0.1
        for: 5m
        labels:
          severity: warning
          component: ensemble
          alert_type: weight_coordination
        annotations:
          summary: "Ensemble model weights not summing to 1.0"
          description: "Weight sum {{ $value }} on {{ $labels.symbol }} (expected: 1.0 ± 0.1)"
          impact: "Weight normalization issue - ensemble coordination broken"
          action: |
            1. Check ensemble coordinator logic
            2. Review weight update algorithm
            3. Restart trading service if issue persists
          runbook_url: "https://docs.foxhunt.io/runbooks/ensemble-weight-sum-anomaly"

      # CRITICAL: Model contributing consistently negative P&L
      - alert: EnsembleModelNegativePnLContribution
        expr: |
          sum(rate(ensemble_model_pnl_contribution_dollars_sum[1h])) by (model_id, symbol) < -500
        for: 1h
        labels:
          severity: critical
          component: ensemble
          alert_type: model_performance
        annotations:
          summary: "Model {{ $labels.model_id }} contributing negative P&L"
          description: "1-hour P&L: ${{ $value | humanize }} on {{ $labels.symbol }} (threshold: -$500)"
          impact: "Model actively losing money - dragging down ensemble performance"
          action: |
            1. Reduce weight for {{ $labels.model_id }} to 0 temporarily
            2. Review model training quality
            3. Check for model drift or data distribution shift
            4. Consider removing model from ensemble until retrained
            5. Analyze prediction patterns for {{ $labels.model_id }}
          runbook_url: "https://docs.foxhunt.io/runbooks/ensemble-model-negative-pnl"

      # WARNING: Model stopped contributing (weight=0 for >15min)
      - alert: EnsembleModelZeroWeight
        expr: |
          ensemble_model_weight == 0
        for: 15m
        labels:
          severity: warning
          component: ensemble
          alert_type: model_weight
        annotations:
          summary: "Model {{ $labels.model_id }} has zero weight"
          description: "Model not contributing to ensemble decisions on {{ $labels.symbol }}"
          impact: "Reduced ensemble diversity - operating with fewer models"
          action: |
            1. Check if model is producing predictions
            2. Review recent P&L attribution for this model
            3. Verify model not marked as failed
            4. Check weight update algorithm logic

  # =============================================================================
  # GROUP 6: A/B TESTING & EXPERIMENTAL MODELS
  # =============================================================================
  - name: ensemble_ab_testing_alerts
    interval: 30s
    rules:
      # WARNING: A/B test assignment imbalance (>55/45 split)
      - alert: EnsembleABTestImbalance
        expr: |
          abs(
            (
              sum(rate(ab_test_assignments_total{group="treatment"}[10m])) /
              sum(rate(ab_test_assignments_total[10m]))
            ) - 0.5
          ) > 0.05
        for: 10m
        labels:
          severity: warning
          component: ensemble
          alert_type: ab_test
        annotations:
          summary: "A/B test assignment imbalance detected"
          description: "Treatment group {{ $value | humanizePercentage }} of assignments (expected: 50% ± 5%)"
          impact: "A/B test results may be biased - statistical validity compromised"
          action: |
            1. Review user assignment hashing algorithm
            2. Check for bias in traffic sources
            3. Verify random seed is properly set
            4. Consider restarting A/B test with corrected assignment

      # INFO: A/B test showing significant lift
      - alert: EnsembleABTestSignificantLift
        expr: |
          ab_test_metric_difference{metric="sharpe_ratio"} > 0.2
        for: 1h
        labels:
          severity: info
          component: ensemble
          alert_type: ab_test
        annotations:
          summary: "A/B test showing significant Sharpe ratio lift"
          description: "Treatment group {{ $value | humanizePercentage }} better than control"
          impact: "Positive signal - treatment variant may be ready for full rollout"
          action: |
            1. Review statistical significance (p-value <0.05)
            2. Check sample size (N >1000 per group)
            3. Verify lift is consistent across all symbols
            4. Consider graduating treatment to production

      # CRITICAL: A/B test showing significant negative impact
      - alert: EnsembleABTestNegativeImpact
        expr: |
          ab_test_metric_difference{metric="sharpe_ratio"} < -0.15
        for: 30m
        labels:
          severity: critical
          component: ensemble
          alert_type: ab_test
        annotations:
          summary: "A/B test treatment performing significantly worse"
          description: "Treatment group {{ $value | humanizePercentage }} worse than control"
          impact: "Experimental model/configuration actively degrading performance"
          action: |
            1. STOP A/B test immediately
            2. Revert all treatment group users to control
            3. Review what changed in treatment configuration
            4. Do NOT roll out treatment variant
            5. Investigate root cause before retrying

  # =============================================================================
  # GROUP 7: SYSTEM HEALTH & INFRASTRUCTURE
  # =============================================================================
  - name: ensemble_system_health
    interval: 15s
    rules:
      # CRITICAL: Trading service memory usage high (>85%)
      - alert: EnsembleServiceMemoryHigh
        expr: |
          100 * process_resident_memory_bytes{job="trading_service"} /
          node_memory_MemTotal_bytes > 85
        for: 2m
        labels:
          severity: critical
          component: ensemble
          alert_type: resources
        annotations:
          summary: "Trading service memory usage critical"
          description: "Memory usage {{ $value }}% (threshold: 85%)"
          impact: "Risk of OOM kill - service instability"
          action: |
            1. Check for memory leaks in ensemble coordinator
            2. Review loaded model sizes (MAMBA-2: 150-500MB, TFT: 1.5-2.5GB)
            3. Consider restarting service during low-traffic window
            4. Scale to larger instance if issue persists
          runbook_url: "https://docs.foxhunt.io/runbooks/ensemble-memory-high"

      # CRITICAL: GPU inference failing (if GPU-accelerated)
      - alert: EnsembleGPUInferenceFailure
        expr: |
          rate(ml_inference_gpu_errors_total[5m]) > 1
        for: 5m
        labels:
          severity: critical
          component: ensemble
          alert_type: gpu
        annotations:
          summary: "GPU inference errors detected"
          description: "{{ $value }} GPU errors/sec"
          impact: "GPU-accelerated models (MAMBA-2, TFT) may fail - falling back to CPU"
          action: |
            1. Check GPU health: nvidia-smi
            2. Review CUDA error messages in logs
            3. Verify GPU memory not exhausted
            4. Restart service to reset GPU state
            5. Check for driver issues

      # WARNING: Ensemble coordinator errors
      - alert: EnsembleCoordinatorErrors
        expr: |
          rate(ensemble_coordinator_errors_total[5m]) > 1
        for: 5m
        labels:
          severity: warning
          component: ensemble
          alert_type: errors
        annotations:
          summary: "Ensemble coordinator errors detected"
          description: "{{ $value }} errors/sec in ensemble coordination logic"
          impact: "Ensemble predictions may be incomplete or degraded"
          action: |
            1. Review trading service error logs
            2. Check for pattern in error types
            3. Verify all models loaded correctly
            4. Consider restarting service if errors persist

# =============================================================================
# INHIBITION RULES (defined in alertmanager.yml)
# =============================================================================
# - If EnsembleCascadeFailureDetected fires, suppress EnsembleModelFailureDetected
# - If EnsembleSharpeRatioDropCritical fires, suppress EnsembleSharpeRatioDropWarning
# - If EnsembleServiceMemoryHigh fires, suppress EnsembleAggregationLatencyP99High
# - If EnsembleCheckpointRollbackRateHigh fires, suppress EnsembleCheckpointSwapFailed