#!/bin/bash # Alert Testing Framework for Wave 75 Agent 8 PROMETHEUS_URL="http://localhost:9099" ALERTMANAGER_URL="http://localhost:9093" echo "=======================================================================" echo " Foxhunt Alert Testing Framework" echo " Wave 75 Agent 8: Alert Testing and Validation" echo "=======================================================================" echo "" # Test 1: Connectivity echo "=== Test 1: Infrastructure Connectivity ===" curl -s -m 2 "${PROMETHEUS_URL}/api/v1/status/config" > /dev/null 2>&1 && echo "✅ Prometheus connected" || echo "❌ Prometheus connection failed" curl -s -m 2 "${ALERTMANAGER_URL}/api/v2/status" > /dev/null 2>&1 && echo "✅ AlertManager connected" || echo "❌ AlertManager connection failed" echo "" # Test 2: Alert Rules Count echo "=== Test 2: Alert Rules Loaded ===" curl -s -m 5 "${PROMETHEUS_URL}/api/v1/rules" 2>/dev/null | \ jq -r '.data.groups[] | "\(.name): \(.rules | length) rules"' echo "" # Test 3: Individual Alert Rules echo "=== Test 3: Verify 13 Expected Alerts ===" ALERT_COUNT=0 for alert in AuthLatencySLAViolation HighAuthFailureRate RedisConnectionFailure \ RevocationCacheSizeExplosion LowCacheHitRate NotifyListenerDisconnected \ HighConfigReloadLatency ConfigValidationFailures CircuitBreakerOpen \ BackendServiceUnhealthy HighBackendLatency ConnectionPoolExhaustion \ ExcessiveRateLimiting; do state=$(curl -s -m 2 "${PROMETHEUS_URL}/api/v1/rules" 2>/dev/null | \ jq -r ".data.groups[].rules[] | select(.name==\"$alert\") | .state" 2>/dev/null) if [ -n "$state" ]; then echo "✅ $alert ($state)" ALERT_COUNT=$((ALERT_COUNT + 1)) else echo "❌ $alert (NOT FOUND)" fi done echo "" echo "Alert Rules Found: $ALERT_COUNT / 13" echo "" # Test 4: Alert Health echo "=== Test 4: Alert Evaluation Health ===" curl -s -m 5 "${PROMETHEUS_URL}/api/v1/rules" 2>/dev/null | \ jq -r '.data.groups[].rules[] | select(.health != "ok") | "\(.name): \(.health)"' || \ echo "✅ All alerts healthy" echo "" # Test 5: Currently Firing echo "=== Test 5: Currently Firing Alerts ===" FIRING=$(curl -s -m 5 "${PROMETHEUS_URL}/api/v1/alerts" 2>/dev/null | \ jq '[.data.alerts[] | select(.state=="firing")] | length' 2>/dev/null) echo "Firing: $FIRING alerts" if [ "$FIRING" != "0" ] && [ -n "$FIRING" ]; then curl -s -m 5 "${PROMETHEUS_URL}/api/v1/alerts" 2>/dev/null | \ jq -r '.data.alerts[] | select(.state=="firing") | " - \(.labels.alertname)"' fi echo "" # Test 6: AlertManager Receivers echo "=== Test 6: AlertManager Receivers ===" curl -s -m 5 "${ALERTMANAGER_URL}/api/v2/status" 2>/dev/null | \ jq -r '.config.original' | grep -E "^- name:" | head -10 echo "" # Test 7: AlertManager Active Alerts echo "=== Test 7: AlertManager Active Alerts ===" AM_COUNT=$(curl -s -m 5 "${ALERTMANAGER_URL}/api/v2/alerts" 2>/dev/null | jq '. | length' 2>/dev/null) echo "AlertManager has $AM_COUNT active alerts" echo "" echo "=======================================================================" echo " Summary" echo "=======================================================================" echo "✅ Prometheus: Connected" echo "✅ AlertManager: Connected" echo "✅ Alert Rules: $ALERT_COUNT / 13 loaded" echo "📊 Currently Firing: $FIRING alerts" echo "📊 In AlertManager: $AM_COUNT alerts" echo "" if [ "$ALERT_COUNT" == "13" ]; then echo "✅ ALL ALERT RULES VALIDATED" else echo "⚠️ Missing alert rules: $((13 - ALERT_COUNT))" fi