foxhunt/monitoring/prometheus/alerts/ml_training_alerts.yml

# Prometheus Alert Rules for ML Training Service
#
# Alerts for model training, GPU utilization, and ML inference performance

groups:
  - name: ml_training_performance
    interval: 10s
    rules:
      # NaN values detected during training (CRITICAL)
      - alert: TrainingNaNDetected
        expr: ml_training_nan_count > 0
        for: 30s
        labels:
          severity: critical
          component: ml
        annotations:
          summary: "NaN values detected during training"
          description: "Model {{ $labels.model_type }} job {{ $labels.job_id }} detected NaN in {{ $labels.tensor_type }}"
          impact: "Training unstable - model will produce invalid results"
          action: "1. Stop training immediately 2. Check learning rate 3. Review data normalization 4. Inspect gradient clipping"

      # Training slowdown detected
      - alert: TrainingSlowdown
        expr: ml_training_epochs_per_second < 0.1
        for: 5m
        labels:
          severity: warning
          component: ml
        annotations:
          summary: "Training speed degraded"
          description: "Model {{ $labels.model_type }} training at {{ $value }} epochs/sec (threshold: 0.1)"
          impact: "Training will take much longer than expected"
          action: "1. Check GPU utilization 2. Profile data loading 3. Review batch size"

      # ML inference latency high
      - alert: MLInferenceLatencyHigh
        expr: histogram_quantile(0.99, rate(ml_inference_duration_milliseconds_bucket[1m])) > 100
        for: 2m
        labels:
          severity: warning
          component: ml
        annotations:
          summary: "ML inference latency high"
          description: "p99 inference latency is {{ $value }}ms (threshold: 100ms)"
          impact: "Trading signal generation delayed"

      # Model prediction accuracy degraded
      - alert: MLModelAccuracyDegraded
        expr: ml_model_accuracy < 0.85
        for: 5m
        labels:
          severity: critical
          component: ml
        annotations:
          summary: "ML model accuracy degraded"
          description: "Model {{ $labels.model }} accuracy is {{ $value }} (threshold: 0.85)"
          impact: "Trading strategy performance severely degraded"
          action: "1. Check training data quality 2. Investigate model drift 3. Consider retraining"

      # Model prediction error rate high
      - alert: MLPredictionErrorRateHigh
        expr: |
          100 * rate(ml_prediction_errors_total[5m]) /
          rate(ml_predictions_total[5m]) > 5
        for: 2m
        labels:
          severity: warning
          component: ml
        annotations:
          summary: "ML prediction error rate high"
          description: "Error rate {{ $value }}% (threshold: 5%)"

      # Training iteration time high
      - alert: TrainingIterationTimeSlow
        expr: histogram_quantile(0.95, rate(ml_training_iteration_seconds_bucket[5m])) > 60
        for: 10m
        labels:
          severity: warning
          component: ml
        annotations:
          summary: "Training iterations running slow"
          description: "p95 iteration time is {{ $value }}s (threshold: 60s)"
          impact: "Model training taking longer than expected"

      # Model convergence stalled
      - alert: ModelConvergenceStalled
        expr: delta(ml_training_loss[10m]) > -0.001
        for: 10m
        labels:
          severity: warning
          component: ml
        annotations:
          summary: "Model training convergence stalled"
          description: "Training loss not improving: {{ $value }}"
          impact: "Model may not be learning properly"
          action: "1. Check hyperparameters 2. Verify training data 3. Consider stopping"

  - name: ml_training_availability
    interval: 10s
    rules:
      # ML training service down
      - alert: MLTrainingServiceDown
        expr: up{job="ml_training_service"} == 0
        for: 30s
        labels:
          severity: high
          component: ml
        annotations:
          summary: "ML training service is DOWN"
          description: "Service unreachable for >30 seconds"
          impact: "Model training and predictions unavailable"
          runbook_url: "https://docs.foxhunt.io/runbooks/ml-service-down"

      # Model loading failures
      - alert: ModelLoadingFailures
        expr: rate(ml_model_load_failures_total[5m]) > 0.1
        for: 2m
        labels:
          severity: critical
          component: ml
        annotations:
          summary: "ML model loading failures"
          description: "{{ $value }} model load failures/sec"
          impact: "Unable to serve predictions"
          action: "1. Check model files 2. Verify S3 connectivity 3. Check disk space"

      # Model cache misses high
      - alert: ModelCacheMissesHigh
        expr: |
          100 * rate(ml_model_cache_misses[5m]) /
          (rate(ml_model_cache_hits[5m]) + rate(ml_model_cache_misses[5m])) > 20
        for: 5m
        labels:
          severity: warning
          component: ml
        annotations:
          summary: "Model cache miss rate high"
          description: "Cache miss rate {{ $value }}% (threshold: 20%)"
          impact: "Increased model loading latency"

  - name: ml_gpu_resources
    interval: 15s
    rules:
      # GPU utilization low (waste of resources)
      - alert: GPUUtilizationLow
        expr: ml_gpu_utilization_percent < 30
        for: 10m
        labels:
          severity: info
          component: ml
        annotations:
          summary: "Low GPU utilization detected"
          description: "GPU {{ $labels.gpu_id }} utilization {{ $value }}% (threshold: 30%)"
          impact: "Underutilizing expensive GPU resources"

      # GPU utilization critical (near capacity)
      - alert: GPUUtilizationCritical
        expr: ml_gpu_utilization_percent > 95
        for: 5m
        labels:
          severity: warning
          component: ml
        annotations:
          summary: "GPU utilization critically high"
          description: "GPU {{ $labels.gpu_id }} utilization {{ $value }}% (threshold: 95%)"
          impact: "GPU at capacity - potential bottleneck"

      # GPU memory usage high
      - alert: GPUMemoryUsageHigh
        expr: |
          100 * ml_gpu_memory_used_bytes / ml_gpu_memory_total_bytes > 90
        for: 5m
        labels:
          severity: warning
          component: ml
        annotations:
          summary: "GPU memory usage high"
          description: "GPU {{ $labels.gpu_id }} memory {{ $value }}% (threshold: 90%)"
          impact: "Risk of OOM errors during training"

      # GPU memory exhausted (CRITICAL - immediate action required)
      - alert: GPUMemoryExhausted
        expr: |
          100 * ml_gpu_memory_used_bytes / ml_gpu_memory_total_bytes > 95
        for: 1m
        labels:
          severity: critical
          component: ml
        annotations:
          summary: "GPU memory critically exhausted"
          description: "GPU {{ $labels.gpu_id }} memory {{ $value }}% (threshold: 95%)"
          impact: "Imminent OOM - training will crash"
          action: "1. Reduce batch size 2. Enable gradient checkpointing 3. Clear GPU cache 4. Kill training job if necessary"

      # GPU temperature high
      - alert: GPUTemperatureHigh
        expr: ml_gpu_temperature_celsius > 85
        for: 2m
        labels:
          severity: critical
          component: ml
        annotations:
          summary: "GPU temperature critically high"
          description: "GPU {{ $labels.gpu_id }} temperature {{ $value }}°C (threshold: 85°C)"
          impact: "Risk of thermal throttling and hardware damage"
          action: "1. Check cooling 2. Reduce workload 3. Monitor temperature"

      # GPU errors detected
      - alert: GPUErrorsDetected
        expr: rate(ml_gpu_errors_total[5m]) > 0
        for: 1m
        labels:
          severity: critical
          component: ml
        annotations:
          summary: "GPU errors detected"
          description: "GPU {{ $labels.gpu_id }} reporting errors: {{ $value }}/sec"
          impact: "GPU hardware issues - training results unreliable"
          action: "1. Stop training 2. Check nvidia-smi 3. Contact hardware support"

  - name: ml_model_quality
    interval: 30s
    rules:
      # Model drift detected
      - alert: ModelDriftDetected
        expr: ml_model_drift_score > 0.15
        for: 5m
        labels:
          severity: warning
          component: ml
        annotations:
          summary: "ML model drift detected"
          description: "Model {{ $labels.model }} drift score {{ $value }} (threshold: 0.15)"
          impact: "Model predictions becoming less accurate"
          action: "1. Analyze recent data 2. Consider model retraining"

      # Feature distribution shift
      - alert: FeatureDistributionShift
        expr: ml_feature_distribution_distance > 0.20
        for: 10m
        labels:
          severity: warning
          component: ml
        annotations:
          summary: "Feature distribution shift detected"
          description: "Feature {{ $labels.feature }} distance {{ $value }} (threshold: 0.20)"
          impact: "Input data pattern has changed significantly"

      # Prediction confidence low
      - alert: PredictionConfidenceLow
        expr: histogram_quantile(0.50, rate(ml_prediction_confidence_bucket[5m])) < 0.70
        for: 10m
        labels:
          severity: warning
          component: ml
        annotations:
          summary: "ML prediction confidence low"
          description: "Median prediction confidence {{ $value }} (threshold: 0.70)"
          impact: "Model uncertainty high - signals less reliable"

  - name: ml_data_pipeline
    interval: 15s
    rules:
      # Training data stale
      - alert: TrainingDataStale
        expr: (time() - ml_training_data_last_updated_timestamp) > 86400
        for: 1h
        labels:
          severity: warning
          component: ml
        annotations:
          summary: "Training data not updated recently"
          description: "Training data is {{ $value | humanizeDuration }} old (threshold: 24h)"
          impact: "Models training on outdated data"

      # Feature engineering errors
      - alert: FeatureEngineeringErrors
        expr: rate(ml_feature_engineering_errors_total[5m]) > 1
        for: 2m
        labels:
          severity: warning
          component: ml
        annotations:
          summary: "Feature engineering errors detected"
          description: "{{ $value }} errors/sec in feature pipeline"
          impact: "Training data quality degraded"

      # Feature extraction latency high
      - alert: FeatureExtractionLatencyHigh
        expr: histogram_quantile(0.95, rate(ml_feature_extraction_seconds_bucket[5m])) > 5
        for: 5m
        labels:
          severity: warning
          component: ml
        annotations:
          summary: "Feature extraction latency high"
          description: "p95 extraction time {{ $value }}s (threshold: 5s)"

  - name: ml_storage
    interval: 15s
    rules:
      # S3 connection errors
      - alert: S3ConnectionErrors
        expr: rate(ml_s3_request_errors_total[5m]) > 1
        for: 2m
        labels:
          severity: warning
          component: ml
        annotations:
          summary: "S3 connection errors detected"
          description: "{{ $value }} S3 errors/sec"
          impact: "Model loading/saving operations failing"

      # Model checkpoint save failures
      - alert: CheckpointSaveFailures
        expr: rate(ml_checkpoint_save_failures_total[10m]) > 0
        for: 5m
        labels:
          severity: critical
          component: ml
        annotations:
          summary: "Model checkpoint save failures"
          description: "{{ $value }} checkpoint save failures"
          impact: "Training progress may be lost"
          action: "1. Check disk space 2. Verify S3 connectivity 3. Check permissions"

      # Model artifact storage usage high
      - alert: ModelStorageUsageHigh
        expr: |
          100 * ml_model_storage_used_bytes / ml_model_storage_limit_bytes > 85
        for: 10m
        labels:
          severity: warning
          component: ml
        annotations:
          summary: "Model artifact storage usage high"
          description: "Storage usage {{ $value }}% (threshold: 85%)"
          impact: "Risk of storage exhaustion"
          action: "1. Clean old model versions 2. Archive unused models"

  - name: ml_training_resources
    interval: 15s
    rules:
      # High memory usage during training
      - alert: MLServiceMemoryHigh
        expr: |
          100 * process_resident_memory_bytes{job="ml_training_service"} /
          node_memory_MemTotal_bytes > 85
        for: 5m
        labels:
          severity: warning
          component: ml
        annotations:
          summary: "ML service memory usage high"
          description: "Memory usage {{ $value }}% (threshold: 85%)"
          impact: "Risk of OOM during training"

      # High CPU usage (non-GPU workload)
      - alert: MLServiceCPUHigh
        expr: |
          100 * rate(process_cpu_seconds_total{job="ml_training_service"}[1m]) > 90
        for: 3m
        labels:
          severity: warning
          component: ml
        annotations:
          summary: "ML service CPU usage high"
          description: "CPU usage {{ $value }}% (threshold: 90%)"
          impact: "Possible CPU-bound operation or insufficient GPU utilization"

  - name: ml_automated_pipeline
    interval: 30s
    rules:
      # Automated training job stuck
      - alert: AutomatedTrainingJobStuck
        expr: |
          time() - ml_training_job_last_update_timestamp{status="running"} > 3600
        for: 10m
        labels:
          severity: critical
          component: ml
          alert_type: automation
        annotations:
          summary: "Automated training job stuck"
          description: "Job {{ $labels.job_id }} ({{ $labels.model_type }}) no progress for >1 hour"
          impact: "Automated ML pipeline blocked, downstream jobs queued"
          action: "1. Check job logs 2. Verify GPU availability 3. Consider killing stuck job 4. Restart automation queue"
          runbook_url: "https://docs.foxhunt.io/runbooks/stuck-job"

      # Cost budget exceeded
      - alert: MonthlyCostBudgetExceeded
        expr: ml_monthly_cost_projection_dollars > ml_monthly_budget_dollars
        for: 1h
        labels:
          severity: high
          component: ml
          alert_type: cost
        annotations:
          summary: "Monthly cost budget exceeded"
          description: "Projected cost ${{ $value }} exceeds budget ${{ $labels.budget }}"
          impact: "Cost overrun - budget constraints violated"
          action: "1. Review S3 retention policy 2. Pause non-critical training 3. Optimize GPU usage 4. Request budget increase"

      # S3 storage approaching 1TB limit
      - alert: S3StorageApproaching1TB
        expr: ml_model_storage_used_bytes > 9e11
        for: 1h
        labels:
          severity: warning
          component: ml
          alert_type: storage
        annotations:
          summary: "S3 storage approaching 1TB limit"
          description: "S3 storage {{ $value | humanize1024 }} (threshold: 900GB)"
          impact: "Storage costs increasing, potential quota limits"
          action: "1. Archive old model versions 2. Clean up unused checkpoints 3. Review retention policy"

      # Automated tuning job failure rate high
      - alert: AutomatedTuningFailureRateHigh
        expr: |
          100 * rate(ml_tuning_job_failures_total[1h]) /
          rate(ml_tuning_jobs_total[1h]) > 20
        for: 30m
        labels:
          severity: warning
          component: ml
          alert_type: automation
        annotations:
          summary: "Automated tuning failure rate high"
          description: "Tuning job failure rate {{ $value }}% (threshold: 20%)"
          impact: "Hyperparameter optimization unreliable"
          action: "1. Check search space configuration 2. Review failure logs 3. Verify GPU stability"

      # Data quality degradation
      - alert: TrainingDataQualityDegraded
        expr: ml_training_data_quality_score < 0.80
        for: 10m
        labels:
          severity: warning
          component: ml
          alert_type: data_quality
        annotations:
          summary: "Training data quality degraded"
          description: "Data quality score {{ $value }} (threshold: 0.80)"
          impact: "Model training on low-quality data"
          action: "1. Check data pipeline 2. Verify feature engineering 3. Review data sources"