foxhunt/scripts/test_wave_d_alerts.sh

#!/bin/bash
# Wave D Alert Testing Script
# Agent: M1 - Prometheus Alert Deployment
# Date: 2025-10-19
#
# Purpose: Validate that Wave D Prometheus alerts are properly deployed and functional
#
# Usage: ./scripts/test_wave_d_alerts.sh

set -e

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
FOXHUNT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

echo "================================================================="
echo "Wave D Alert Deployment Validation"
echo "================================================================="
echo ""

# Test 1: Validate alert file exists
echo -n "Test 1: Checking alert file exists... "
if [ -f "$FOXHUNT_ROOT/config/prometheus/rules/wave_d_alerts.yml" ]; then
    echo -e "${GREEN}PASS${NC}"
else
    echo -e "${RED}FAIL${NC} - Alert file not found"
    exit 1
fi

# Test 2: Validate Docker container running
echo -n "Test 2: Checking Prometheus container... "
if docker ps | grep -q foxhunt-prometheus; then
    echo -e "${GREEN}PASS${NC}"
else
    echo -e "${RED}FAIL${NC} - Prometheus container not running"
    echo "Run: docker-compose up -d prometheus"
    exit 1
fi

# Test 3: Validate alert syntax
echo -n "Test 3: Validating alert syntax... "
SYNTAX_CHECK=$(docker exec foxhunt-prometheus promtool check rules /etc/prometheus/rules/wave_d_alerts.yml 2>&1)
if echo "$SYNTAX_CHECK" | grep -q "SUCCESS"; then
    RULE_COUNT=$(echo "$SYNTAX_CHECK" | grep -oP '\d+(?= rules found)')
    echo -e "${GREEN}PASS${NC} - $RULE_COUNT rules found"
else
    echo -e "${RED}FAIL${NC}"
    echo "$SYNTAX_CHECK"
    exit 1
fi

# Test 4: Validate alert rules loaded
echo -n "Test 4: Checking alerts loaded in Prometheus... "
LOADED_ALERTS=$(curl -s http://localhost:9090/api/v1/rules 2>/dev/null | jq '.data.groups[] | select(.name == "wave_d_rollback_triggers") | .rules | length' 2>/dev/null || echo "0")
if [ "$LOADED_ALERTS" -eq 9 ]; then
    echo -e "${GREEN}PASS${NC} - 9/9 alerts loaded"
else
    echo -e "${YELLOW}WARNING${NC} - Expected 9 alerts, found $LOADED_ALERTS"
    echo "Run: docker exec foxhunt-prometheus kill -HUP 1"
fi

# Test 5: Validate Prometheus is healthy
echo -n "Test 5: Checking Prometheus health... "
HEALTH=$(curl -s http://localhost:9090/-/healthy 2>/dev/null || echo "FAIL")
if echo "$HEALTH" | grep -q "Prometheus is Healthy"; then
    echo -e "${GREEN}PASS${NC}"
else
    echo -e "${RED}FAIL${NC} - Prometheus unhealthy"
    exit 1
fi

# Test 6: List all Wave D alerts
echo ""
echo "Test 6: Wave D Alert Inventory"
echo "================================"
curl -s http://localhost:9090/api/v1/rules 2>/dev/null | \
    jq -r '.data.groups[] | select(.name == "wave_d_rollback_triggers") | .rules[] |
    "\(.name) - \(.labels.severity) - Rollback: \(.labels.rollback_level)"' 2>/dev/null | \
    while read -r line; do
        if echo "$line" | grep -q "critical"; then
            echo -e "${RED}[CRITICAL]${NC} $line"
        else
            echo -e "${YELLOW}[WARNING]${NC}  $line"
        fi
    done

# Test 7: Validate critical alerts have runbooks
echo ""
echo -n "Test 7: Checking runbook URLs... "
CRITICAL_WITHOUT_RUNBOOK=$(curl -s http://localhost:9090/api/v1/rules 2>/dev/null | \
    jq '[.data.groups[] | select(.name == "wave_d_rollback_triggers") | .rules[] |
    select(.labels.severity == "critical") | select(.annotations.runbook == null or .annotations.runbook == "")] | length' 2>/dev/null || echo "0")

if [ "$CRITICAL_WITHOUT_RUNBOOK" -eq 0 ]; then
    echo -e "${GREEN}PASS${NC} - All critical alerts have runbooks"
else
    echo -e "${RED}FAIL${NC} - $CRITICAL_WITHOUT_RUNBOOK critical alerts missing runbooks"
    exit 1
fi

# Test 8: Validate rollback_level labels
echo -n "Test 8: Checking rollback_level labels... "
ALERTS_WITHOUT_ROLLBACK_LEVEL=$(curl -s http://localhost:9090/api/v1/rules 2>/dev/null | \
    jq '[.data.groups[] | select(.name == "wave_d_rollback_triggers") | .rules[] |
    select(.labels.rollback_level == null or .labels.rollback_level == "")] | length' 2>/dev/null || echo "9")

if [ "$ALERTS_WITHOUT_ROLLBACK_LEVEL" -eq 0 ]; then
    echo -e "${GREEN}PASS${NC} - All alerts have rollback_level labels"
else
    echo -e "${YELLOW}WARNING${NC} - $ALERTS_WITHOUT_ROLLBACK_LEVEL alerts missing rollback_level"
fi

# Test 9: Check current alert states
echo ""
echo "Test 9: Current Alert States"
echo "============================="
FIRING_ALERTS=$(curl -s http://localhost:9090/api/v1/alerts 2>/dev/null | \
    jq -r '.data.alerts[] | select(.labels.component | startswith("wave_d")) |
    "\(.labels.alertname): \(.state)"' 2>/dev/null || echo "No alerts")

if [ "$FIRING_ALERTS" = "No alerts" ]; then
    echo -e "${GREEN}✓ No Wave D alerts firing (system healthy)${NC}"
else
    echo "$FIRING_ALERTS" | while read -r line; do
        if echo "$line" | grep -q "firing"; then
            echo -e "${RED}⚠ $line${NC}"
        elif echo "$line" | grep -q "pending"; then
            echo -e "${YELLOW}⏳ $line${NC}"
        else
            echo -e "${GREEN}✓ $line${NC}"
        fi
    done
fi

# Test 10: Check metrics availability
echo ""
echo "Test 10: Wave D Metrics Availability"
echo "====================================="

METRICS_TO_CHECK=(
    "regime_transitions_total"
    "regime_detections_total"
    "regime_detection_errors_total"
    "wave_d_features_nan_count"
    "wave_d_features_inf_count"
    "wave_d_feature_extraction_duration_seconds"
)

MISSING_METRICS=0
for metric in "${METRICS_TO_CHECK[@]}"; do
    RESULT=$(curl -s "http://localhost:9090/api/v1/query?query=$metric" 2>/dev/null | jq -r '.data.result | length' 2>/dev/null || echo "0")
    if [ "$RESULT" -gt 0 ]; then
        echo -e "${GREEN}✓ $metric${NC} - Found $RESULT series"
    else
        echo -e "${YELLOW}⚠ $metric${NC} - NOT FOUND (alert will not fire)"
        MISSING_METRICS=$((MISSING_METRICS + 1))
    fi
done

if [ $MISSING_METRICS -gt 0 ]; then
    echo ""
    echo -e "${YELLOW}WARNING:${NC} $MISSING_METRICS metrics not found. Alerts will not fire until Wave D services expose these metrics."
    echo "See: WAVE_D_ALERTS_DEPLOYMENT_GUIDE.md - Metrics Instrumentation Checklist"
fi

# Summary
echo ""
echo "================================================================="
echo "Validation Summary"
echo "================================================================="
echo -e "Alert File:        ${GREEN}✓${NC} Exists"
echo -e "Prometheus:        ${GREEN}✓${NC} Running"
echo -e "Alert Syntax:      ${GREEN}✓${NC} Valid"
echo -e "Alerts Loaded:     ${GREEN}✓${NC} $LOADED_ALERTS/9"
echo -e "Runbook URLs:      ${GREEN}✓${NC} All critical alerts have runbooks"
echo -e "Rollback Labels:   ${GREEN}✓${NC} All alerts have rollback_level"

if [ $MISSING_METRICS -gt 0 ]; then
    echo -e "Metrics:           ${YELLOW}⚠${NC} $MISSING_METRICS/$((${#METRICS_TO_CHECK[@]})) missing"
    echo ""
    echo -e "${YELLOW}ACTION REQUIRED:${NC} Implement missing metrics before Wave D production deployment."
else
    echo -e "Metrics:           ${GREEN}✓${NC} All required metrics available"
    echo ""
    echo -e "${GREEN}SUCCESS:${NC} Wave D alerts are ready for production!"
fi

echo "================================================================="