foxhunt/monitoring/prometheus/alerts/backtesting_alerts.yml

# Prometheus Alert Rules for Backtesting Service
#
# Alerts for strategy testing, performance analytics, and data replay

groups:
  - name: backtesting_performance
    interval: 10s
    rules:
      # Backtest execution time excessive
      - alert: BacktestExecutionTimeSlow
        expr: histogram_quantile(0.95, rate(backtesting_execution_duration_seconds_bucket[10m])) > 300
        for: 5m
        labels:
          severity: warning
          component: backtesting
        annotations:
          summary: "Backtest execution time excessive"
          description: "p95 execution time {{ $value }}s (threshold: 300s)"
          impact: "Strategy testing taking too long"

      # Data replay latency high
      - alert: DataReplayLatencyHigh
        expr: histogram_quantile(0.99, rate(backtesting_data_replay_microseconds_bucket[5m])) > 1000
        for: 3m
        labels:
          severity: warning
          component: backtesting
        annotations:
          summary: "Data replay latency high"
          description: "p99 replay latency {{ $value }}μs (threshold: 1ms)"
          impact: "Backtest simulation slower than expected"

      # Backtest throughput low
      - alert: BacktestThroughputLow
        expr: rate(backtesting_events_processed_total[5m]) < 1000
        for: 5m
        labels:
          severity: warning
          component: backtesting
        annotations:
          summary: "Backtest event processing throughput low"
          description: "Processing {{ $value }} events/sec (threshold: 1000)"

      # Strategy simulation errors
      - alert: StrategySimulationErrors
        expr: rate(backtesting_simulation_errors_total[5m]) > 0.5
        for: 2m
        labels:
          severity: warning
          component: backtesting
        annotations:
          summary: "Strategy simulation errors detected"
          description: "{{ $value }} simulation errors/sec"
          impact: "Backtest results may be unreliable"

  - name: backtesting_availability
    interval: 10s
    rules:
      # Backtesting service down
      - alert: BacktestingServiceDown
        expr: up{job="backtesting_service"} == 0
        for: 30s
        labels:
          severity: high
          component: backtesting
        annotations:
          summary: "Backtesting service is DOWN"
          description: "Service unreachable for >30 seconds"
          impact: "Strategy testing unavailable"
          runbook_url: "https://docs.foxhunt.io/runbooks/backtesting-service-down"

      # Active backtest count high
      - alert: ActiveBacktestCountHigh
        expr: backtesting_active_backtests > 10
        for: 10m
        labels:
          severity: warning
          component: backtesting
        annotations:
          summary: "High number of concurrent backtests"
          description: "{{ $value }} active backtests (threshold: 10)"
          impact: "System resources may be constrained"

      # Backtest queue depth high
      - alert: BacktestQueueDepthHigh
        expr: backtesting_queue_depth > 50
        for: 5m
        labels:
          severity: warning
          component: backtesting
        annotations:
          summary: "Backtest queue depth high"
          description: "{{ $value }} backtests queued (threshold: 50)"
          impact: "Backtest execution delays likely"

  - name: backtesting_data_quality
    interval: 15s
    rules:
      # Parquet file read errors
      - alert: ParquetFileReadErrors
        expr: rate(backtesting_parquet_read_errors_total[5m]) > 0.1
        for: 2m
        labels:
          severity: warning
          component: backtesting
        annotations:
          summary: "Parquet file read errors detected"
          description: "{{ $value }} read errors/sec"
          impact: "Historical data replay failing"
          action: "1. Check file integrity 2. Verify storage access 3. Review file formats"

      # Missing historical data
      - alert: MissingHistoricalData
        expr: backtesting_missing_data_points > 100
        for: 5m
        labels:
          severity: warning
          component: backtesting
        annotations:
          summary: "Missing historical data points detected"
          description: "{{ $value }} missing data points in backtest"
          impact: "Backtest results may have gaps"

      # Data timestamp inconsistency
      - alert: DataTimestampInconsistency
        expr: rate(backtesting_timestamp_errors_total[5m]) > 0
        for: 2m
        labels:
          severity: warning
          component: backtesting
        annotations:
          summary: "Data timestamp inconsistencies detected"
          description: "{{ $value }} timestamp errors/sec"
          impact: "Backtest timeline accuracy compromised"

  - name: backtesting_analytics
    interval: 15s
    rules:
      # Strategy Sharpe ratio low
      - alert: StrategyPerformancePoor
        expr: backtesting_strategy_sharpe_ratio < 1.0
        for: 0s
        labels:
          severity: info
          component: backtesting
        annotations:
          summary: "Strategy Sharpe ratio below target"
          description: "Strategy {{ $labels.strategy }} Sharpe ratio {{ $value }} (target: >1.0)"
          impact: "Strategy may not be profitable enough"

      # Excessive drawdown in backtest
      - alert: BacktestDrawdownExcessive
        expr: backtesting_max_drawdown_pct > 25
        for: 0s
        labels:
          severity: warning
          component: backtesting
        annotations:
          summary: "Backtest shows excessive drawdown"
          description: "Strategy {{ $labels.strategy }} max drawdown {{ $value }}% (threshold: 25%)"
          impact: "Strategy risk profile unacceptable"

      # Win rate too low
      - alert: BacktestWinRateLow
        expr: backtesting_win_rate_pct < 45
        for: 0s
        labels:
          severity: info
          component: backtesting
        annotations:
          summary: "Backtest win rate low"
          description: "Strategy {{ $labels.strategy }} win rate {{ $value }}% (threshold: 45%)"

      # PnL volatility high
      - alert: BacktestPnLVolatilityHigh
        expr: backtesting_pnl_volatility > 0.10
        for: 0s
        labels:
          severity: info
          component: backtesting
        annotations:
          summary: "Backtest PnL volatility high"
          description: "Strategy {{ $labels.strategy }} PnL volatility {{ $value }} (threshold: 0.10)"
          impact: "Strategy may be too risky"

  - name: backtesting_model_loader
    interval: 15s
    rules:
      # Model loading failures in backtest
      - alert: BacktestModelLoadingFailures
        expr: rate(backtesting_model_load_failures_total[5m]) > 0.1
        for: 2m
        labels:
          severity: critical
          component: backtesting
        annotations:
          summary: "ML model loading failures in backtesting"
          description: "{{ $value }} model load failures/sec"
          impact: "Cannot test ML-based strategies"
          action: "1. Check model files 2. Verify S3 access 3. Check model compatibility"

      # Model cache misses
      - alert: BacktestModelCacheMissesHigh
        expr: |
          100 * rate(backtesting_model_cache_misses[5m]) /
          (rate(backtesting_model_cache_hits[5m]) + rate(backtesting_model_cache_misses[5m])) > 30
        for: 5m
        labels:
          severity: warning
          component: backtesting
        annotations:
          summary: "Model cache miss rate high in backtesting"
          description: "Cache miss rate {{ $value }}% (threshold: 30%)"
          impact: "Increased backtest execution time"

      # Model inference errors during backtest
      - alert: BacktestModelInferenceErrors
        expr: rate(backtesting_model_inference_errors_total[5m]) > 1
        for: 2m
        labels:
          severity: warning
          component: backtesting
        annotations:
          summary: "ML inference errors during backtest"
          description: "{{ $value }} inference errors/sec"
          impact: "Strategy simulation results unreliable"

  - name: backtesting_resources
    interval: 15s
    rules:
      # High memory usage
      - alert: BacktestingServiceMemoryHigh
        expr: |
          100 * process_resident_memory_bytes{job="backtesting_service"} /
          node_memory_MemTotal_bytes > 85
        for: 5m
        labels:
          severity: warning
          component: backtesting
        annotations:
          summary: "Backtesting service memory usage high"
          description: "Memory usage {{ $value }}% (threshold: 85%)"
          impact: "Risk of OOM during large backtests"

      # High CPU usage
      - alert: BacktestingServiceCPUHigh
        expr: |
          100 * rate(process_cpu_seconds_total{job="backtesting_service"}[1m]) > 90
        for: 3m
        labels:
          severity: warning
          component: backtesting
        annotations:
          summary: "Backtesting service CPU usage high"
          description: "CPU usage {{ $value }}% (threshold: 90%)"
          impact: "Backtest execution may slow down"

      # Storage space low for results
      - alert: BacktestResultStorageLow
        expr: |
          100 * backtesting_results_storage_used_bytes /
          backtesting_results_storage_limit_bytes > 85
        for: 10m
        labels:
          severity: warning
          component: backtesting
        annotations:
          summary: "Backtest result storage usage high"
          description: "Storage {{ $value }}% full (threshold: 85%)"
          impact: "Cannot save new backtest results"
          action: "1. Archive old results 2. Clean temporary files 3. Increase storage"

  - name: backtesting_database
    interval: 15s
    rules:
      # Database connection pool exhaustion
      - alert: BacktestingDBPoolExhaustion
        expr: |
          100 * backtesting_db_connections_active /
          backtesting_db_connections_max > 80
        for: 2m
        labels:
          severity: warning
          component: backtesting
        annotations:
          summary: "Backtesting database connection pool nearly exhausted"
          description: "Pool utilization {{ $value }}% (threshold: 80%)"

      # Slow database queries
      - alert: BacktestingDBQueriesSlow
        expr: histogram_quantile(0.95, rate(backtesting_db_query_duration_seconds_bucket[5m])) > 1
        for: 3m
        labels:
          severity: warning
          component: backtesting
        annotations:
          summary: "Backtesting database queries slow"
          description: "p95 query time {{ $value }}s (threshold: 1s)"
          impact: "Backtest result persistence delayed"

      # Database write errors
      - alert: BacktestingDBWriteErrors
        expr: rate(backtesting_db_write_errors_total[5m]) > 0.1
        for: 2m
        labels:
          severity: warning
          component: backtesting
        annotations:
          summary: "Database write errors in backtesting"
          description: "{{ $value }} write errors/sec"
          impact: "Backtest results may not be saved"