foxhunt/config/prometheus/rules/wave_d_alerts.yml

# Wave D Regime Detection & Adaptive Strategies - Alert Rules
# Author: Agent M1 - Prometheus Alert Deployment
# Date: 2025-10-19
# System: Foxhunt HFT Trading System
# Version: Wave D (225 features)
#
# CRITICAL: These alerts monitor Wave D regime detection features and trigger
# rollback procedures when thresholds are exceeded.
#
# Rollback Levels:
#   Level 1: Feature-only rollback (Zero downtime, <1 minute)
#   Level 2: Database rollback (~5 minutes, planned downtime)
#   Level 3: Full rollback to Wave C (~15 minutes, full outage)
#
# Reference: ROLLBACK_PROCEDURES.md Section 6.3

groups:
  - name: wave_d_rollback_triggers
    interval: 30s
    rules:
      # ==================================================================
      # CRITICAL ALERTS - Immediate Action Required
      # ==================================================================

      # CRITICAL: Regime flip-flopping (>50 transitions/hour)
      # Trigger: Level 1 rollback
      # Impact: Excessive regime changes indicate unstable regime detection
      - alert: WaveDFlipFlopping
        expr: rate(regime_transitions_total[1h]) > 50
        for: 5m
        labels:
          severity: critical
          rollback_level: level_1
          component: wave_d_regime_detection
        annotations:
          summary: "Wave D flip-flopping detected ({{ $value }} transitions/hour)"
          description: |
            Regime detection is changing states >50 times/hour (actual: {{ $value | humanize }}).
            This indicates unstable regime classification that may degrade trading performance.

            **Immediate Action**: Execute Level 1 rollback (zero downtime, <1 minute)

            **Procedure**:
            1. Disable Wave D features via configuration
            2. Rebuild services with Wave C features (201 features)
            3. Rolling restart (zero downtime)
            4. Verify feature count = 201

            **Expected Impact**: None (Wave D data preserved for recovery)
          runbook: "https://github.com/foxhunt/runbooks/ROLLBACK_PROCEDURES.md#level-1-feature-only-rollback-zero-downtime"
          dashboard: "https://grafana.foxhunt.ai/d/wave-d-monitoring/regime-detection"

      # CRITICAL: False positives (>80% error rate)
      # Trigger: Level 1 rollback
      # Impact: Poor regime classification accuracy
      - alert: WaveDFalsePositives
        expr: (sum(regime_detection_errors_total) / sum(regime_detections_total)) > 0.80
        for: 10m
        labels:
          severity: critical
          rollback_level: level_1
          component: wave_d_regime_detection
        annotations:
          summary: "Wave D false positive rate >80% (actual: {{ $value | humanizePercentage }})"
          description: |
            Regime detection accuracy is below threshold ({{ $value | humanizePercentage }} error rate).
            Expected: <20% error rate. Actual: >80% error rate.

            This indicates that regime classifications are unreliable and should not be used for trading decisions.

            **Immediate Action**: Execute Level 1 rollback (zero downtime, <1 minute)

            **Root Cause Analysis**:
            - Check regime detection parameters (CUSUM thresholds, ADX levels)
            - Verify training data quality (DBN data integrity)
            - Review regime transition matrix (state probabilities)
            - Validate feature engineering (indices 201-224)
          runbook: "https://github.com/foxhunt/runbooks/ROLLBACK_PROCEDURES.md#level-1-feature-only-rollback-zero-downtime"
          dashboard: "https://grafana.foxhunt.ai/d/wave-d-monitoring/regime-accuracy"

      # CRITICAL: Data corruption (NaN/Inf values in features)
      # Trigger: Level 3 rollback (IMMEDIATE)
      # Impact: Corrupted data compromises system integrity
      - alert: WaveDDataCorruption
        expr: wave_d_features_nan_count > 0 OR wave_d_features_inf_count > 0
        for: 1m
        labels:
          severity: critical
          rollback_level: level_3
          component: wave_d_feature_extraction
        annotations:
          summary: "Wave D data corruption detected (NaN/Inf values in features)"
          description: |
            CRITICAL DATA INTEGRITY VIOLATION

            NaN count: {{ with query "wave_d_features_nan_count" }}{{ . | first | value }}{{ end }}
            Inf count: {{ with query "wave_d_features_inf_count" }}{{ . | first | value }}{{ end }}

            **IMMEDIATE LEVEL 3 ROLLBACK REQUIRED**

            Data integrity is compromised. NaN/Inf values will propagate through ML models
            and cause unpredictable trading behavior.

            **Emergency Procedure**:
            1. STOP all trading immediately
            2. Execute Level 3 rollback to Wave C baseline (~15 minutes)
            3. Restore database from last known good backup
            4. Root cause analysis (feature extraction bugs, data provider issues)
            5. Fix and validate before re-deployment

            **Data Loss**: All Wave D regime detection data will be PERMANENTLY DELETED.
          runbook: "https://github.com/foxhunt/runbooks/ROLLBACK_PROCEDURES.md#level-3-full-rollback-to-wave-c"
          dashboard: "https://grafana.foxhunt.ai/d/wave-d-monitoring/data-quality"

      # CRITICAL: System unavailable (>5 min downtime)
      # Trigger: Level 3 rollback
      # Impact: Trading system down, potential revenue loss
      - alert: FoxhuntSystemDown
        expr: up{job="foxhunt_services"} == 0
        for: 5m
        labels:
          severity: critical
          rollback_level: level_3
          component: system_health
        annotations:
          summary: "Foxhunt system unavailable for >5 minutes"
          description: |
            CRITICAL SYSTEM OUTAGE

            The Foxhunt trading system has been unavailable for >5 minutes.

            **Immediate Action**: Consider Level 3 rollback to Wave C baseline.

            **Investigation Steps**:
            1. Check service health endpoints (all 5 services)
            2. Review Docker logs for crash reports
            3. Verify database connectivity (PostgreSQL, Redis)
            4. Check resource utilization (CPU, memory, disk)
            5. If Wave D deployment is suspected, execute Level 3 rollback

            **Rollback Decision Matrix**:
            - If system was stable before Wave D deployment → Level 3 rollback
            - If infrastructure issue (database, network) → Fix infrastructure
            - If unknown cause → Level 3 rollback as safeguard
          runbook: "https://github.com/foxhunt/runbooks/ROLLBACK_PROCEDURES.md#level-3-full-rollback-to-wave-c"
          dashboard: "https://grafana.foxhunt.ai/d/system-health/service-uptime"

      # ==================================================================
      # WARNING ALERTS - Monitoring & Early Detection
      # ==================================================================

      # WARNING: Performance degradation (>2x latency)
      # Trigger: Level 1 rollback if persists for 15 minutes
      # Impact: Slower feature extraction may miss trading opportunities
      - alert: WaveDLatencyDegradation
        expr: histogram_quantile(0.99, rate(wave_d_feature_extraction_duration_seconds_bucket[5m])) > 0.002
        for: 15m
        labels:
          severity: warning
          rollback_level: level_1
          component: wave_d_feature_extraction
        annotations:
          summary: "Wave D feature extraction latency >2ms (>2x target)"
          description: |
            Wave D feature extraction P99 latency: {{ $value | humanizeDuration }}
            Target: <1ms (1000μs)
            Threshold: 2ms (2000μs) - 2x target

            Current performance is degraded beyond acceptable levels.

            **Action Required**:
            - Monitor for 15 minutes
            - If latency persists → Execute Level 1 rollback
            - If latency resolves → Continue monitoring

            **Performance Investigation**:
            1. Check CPU utilization (high load may slow feature extraction)
            2. Review regime detection algorithm efficiency (CUSUM, ADX calculations)
            3. Verify no memory leaks (RSS growth)
            4. Check database query performance (regime_states lookups)

            **Rollback Trigger**: If alert fires continuously for >15 minutes
          runbook: "https://github.com/foxhunt/runbooks/ROLLBACK_PROCEDURES.md#level-1-feature-only-rollback-zero-downtime"
          dashboard: "https://grafana.foxhunt.ai/d/wave-d-monitoring/performance-metrics"

      # WARNING: Memory leak detection (RSS growth >20%/hour)
      # Trigger: Level 1 rollback if confirmed memory leak
      # Impact: System instability, potential OOM crashes
      - alert: WaveDMemoryLeak
        expr: rate(process_resident_memory_bytes{job=~".*service"}[1h]) / process_resident_memory_bytes{job=~".*service"} > 0.20
        for: 1h
        labels:
          severity: warning
          rollback_level: level_1
          component: wave_d_memory_management
        annotations:
          summary: "Wave D memory leak detected (RSS growth >20%/hour on {{ $labels.job }})"
          description: |
            Service {{ $labels.job }} is experiencing memory growth >20%/hour.
            Current growth rate: {{ $value | humanizePercentage }}/hour

            This may indicate a memory leak in Wave D regime detection code.

            **Investigation Steps**:
            1. Review regime_states table size (unbounded growth?)
            2. Check adaptive_strategy_metrics retention (cleanup job running?)
            3. Verify no circular references in Rust code (Arc/Rc cycles)
            4. Monitor for 1 hour to confirm leak (not just warmup/cache)

            **Action Required**:
            - If memory continues growing → Execute Level 1 rollback
            - If memory stabilizes → Investigate and fix memory leak
            - If memory drops → False alarm (warmup/cache behavior)

            **Memory Leak Confirmation**:
            - Check `/proc/<pid>/status` for VmRSS growth
            - Use `valgrind` or `heaptrack` for leak analysis
            - Review Wave D code for potential leaks (agent allocations, caching)
          runbook: "https://github.com/foxhunt/runbooks/ROLLBACK_PROCEDURES.md#level-1-feature-only-rollback-zero-downtime"
          dashboard: "https://grafana.foxhunt.ai/d/wave-d-monitoring/memory-usage"

      # WARNING: Excessive regime coverage (>95%)
      # Trigger: Manual investigation
      # Impact: Regime detection may be overfitting or not discriminating
      - alert: WaveDRegimeCoverageHigh
        expr: sum(regime_states_count{regime=~"trending|ranging|volatile"}) / sum(regime_states_count) > 0.95
        for: 30m
        labels:
          severity: warning
          rollback_level: none
          component: wave_d_regime_detection
        annotations:
          summary: "Wave D regime coverage >95% for single regime type"
          description: |
            A single regime type is dominating classifications ({{ $value | humanizePercentage }} coverage).

            This may indicate:
            - Regime detection is not discriminating properly
            - Market conditions are genuinely uniform (rare)
            - Overfitting to training data (all bars classified as one regime)

            **Investigation Required**:
            1. Review regime transition matrix (state probabilities balanced?)
            2. Check CUSUM thresholds (too sensitive or too loose?)
            3. Verify ADX calculations (directional strength accurate?)
            4. Analyze market conditions (genuinely uniform or detection issue?)

            **No Rollback Required**: This is informational only.
            Consider retraining regime detection with more diverse data.
          runbook: "https://github.com/foxhunt/runbooks/WAVE_D_TROUBLESHOOTING.md#regime-coverage-issues"
          dashboard: "https://grafana.foxhunt.ai/d/wave-d-monitoring/regime-distribution"

      # WARNING: Low regime transition rate (<5/day)
      # Trigger: Manual investigation
      # Impact: Adaptive strategies may not activate if regimes don't change
      - alert: WaveDRegimeTransitionRateLow
        expr: rate(regime_transitions_total[24h]) < 5
        for: 2h
        labels:
          severity: warning
          rollback_level: none
          component: wave_d_regime_detection
        annotations:
          summary: "Wave D regime transition rate <5/day (actual: {{ $value | humanize }}/day)"
          description: |
            Regime transitions are occurring less frequently than expected.
            Target: 5-10 transitions/day. Actual: {{ $value | humanize }}/day.

            This may indicate:
            - Market is genuinely stable (low volatility period)
            - Regime detection thresholds too high (not sensitive enough)
            - Transition matrix biased towards current state (hysteresis too strong)

            **Impact**:
            - Adaptive strategies may not activate (position sizing, stop-loss adjustments)
            - Wave D benefits reduced if regimes don't change

            **Investigation**:
            1. Review transition matrix (probabilities too low?)
            2. Adjust CUSUM thresholds (increase sensitivity)
            3. Verify market volatility (genuinely low or detection issue?)

            **No Rollback Required**: This is informational only.
          runbook: "https://github.com/foxhunt/runbooks/WAVE_D_TROUBLESHOOTING.md#low-transition-rate"
          dashboard: "https://grafana.foxhunt.ai/d/wave-d-monitoring/transition-frequency"

      # WARNING: High regime detection error rate (>20%, <80%)
      # Trigger: Manual investigation
      # Impact: Moderate accuracy degradation (not critical yet)
      - alert: WaveDDetectionErrorsModerate
        expr: (sum(regime_detection_errors_total) / sum(regime_detections_total)) > 0.20 AND (sum(regime_detection_errors_total) / sum(regime_detections_total)) <= 0.80
        for: 30m
        labels:
          severity: warning
          rollback_level: none
          component: wave_d_regime_detection
        annotations:
          summary: "Wave D moderate error rate ({{ $value | humanizePercentage }})"
          description: |
            Regime detection error rate: {{ $value | humanizePercentage }}
            Target: <20%. Current: >20% but <80% (moderate degradation).

            **Action Required**:
            - Monitor for trend (error rate increasing or stable?)
            - If increasing towards 80% → Prepare for Level 1 rollback
            - If stable/decreasing → Investigate root cause

            **Investigation**:
            1. Review recent regime classifications (false positives/negatives?)
            2. Check feature quality (indices 201-224 values in expected ranges?)
            3. Verify training data representativeness (market regime distribution)
            4. Analyze transition matrix (probabilities match reality?)

            **Escalation**: If error rate exceeds 80%, WaveDFalsePositives alert will fire.
          runbook: "https://github.com/foxhunt/runbooks/WAVE_D_TROUBLESHOOTING.md#moderate-error-rate"
          dashboard: "https://grafana.foxhunt.ai/d/wave-d-monitoring/regime-accuracy"

# ==================================================================
# Wave D Alert Metrics Reference
# ==================================================================
#
# The following Prometheus metrics should be exposed by Wave D services:
#
# Regime Detection Metrics:
#   - regime_transitions_total (counter): Total regime transitions
#   - regime_detections_total (counter): Total regime detections
#   - regime_detection_errors_total (counter): Regime detection errors
#   - regime_states_count (gauge): Current regime state counts by type
#
# Feature Quality Metrics:
#   - wave_d_features_nan_count (gauge): NaN values in Wave D features
#   - wave_d_features_inf_count (gauge): Inf values in Wave D features
#   - wave_d_features_zero_count (gauge): Zero values in Wave D features
#
# Performance Metrics:
#   - wave_d_feature_extraction_duration_seconds (histogram): Feature extraction latency
#   - process_resident_memory_bytes (gauge): RSS memory usage
#
# Database Metrics:
#   - postgres_stat_user_tables_n_tup_ins{relname="regime_states"} (gauge): Regime states row count
#   - postgres_stat_user_tables_n_tup_ins{relname="regime_transitions"} (gauge): Regime transitions row count
#
# NOTE: If any metrics are missing, alerts will not fire. Ensure all Wave D
# services expose these metrics via Prometheus endpoints.
#
# Verification:
#   curl http://localhost:9091/metrics | grep wave_d
#   curl http://localhost:9092/metrics | grep regime
#
# ==================================================================
# Alert Testing
# ==================================================================
#
# To test alerts without production deployment:
#
# 1. Syntax validation:
#    promtool check rules /home/jgrusewski/Work/foxhunt/config/prometheus/rules/wave_d_alerts.yml
#
# 2. Unit tests (create wave_d_alerts.test.yml):
#    promtool test rules wave_d_alerts.test.yml
#
# 3. Manual trigger (Prometheus console):
#    # Set flip-flopping rate to 100
#    regime_transitions_total = 100
#    # Wait 5 minutes → WaveDFlipFlopping should fire
#
# 4. Alertmanager integration test:
#    curl -H "Content-Type: application/json" -d '[{"labels":{"alertname":"WaveDFlipFlopping","severity":"critical"}}]' http://localhost:9093/api/v1/alerts
#
# ==================================================================
# Rollback Procedures Quick Reference
# ==================================================================
#
# Level 1: Feature-only rollback (Zero downtime, <1 minute)
#   Triggered by: WaveDFlipFlopping, WaveDFalsePositives, WaveDLatencyDegradation
#   Command: sed -i 's/enable_wave_d_regime: true/enable_wave_d_regime: false/' ml/src/features/config.rs && cargo build --release
#   Impact: None (Wave D data preserved)
#
# Level 2: Database rollback (~5 minutes, planned downtime)
#   Triggered by: Database migration failures, data integrity issues (non-critical)
#   Command: sqlx migrate revert
#   Impact: Wave D data DELETED (regime_states, regime_transitions, adaptive_strategy_metrics)
#
# Level 3: Full rollback to Wave C (~15 minutes, full outage)
#   Triggered by: WaveDDataCorruption, FoxhuntSystemDown, catastrophic failures
#   Command: git checkout <wave-c-commit> && cargo build --release
#   Impact: Complete Wave D removal (code + data)
#
# ==================================================================