foxhunt/test_alerts.sh

#!/bin/bash
# Alert Testing Framework for Wave 75 Agent 8

PROMETHEUS_URL="http://localhost:9099"
ALERTMANAGER_URL="http://localhost:9093"

echo "======================================================================="
echo "   Foxhunt Alert Testing Framework"
echo "   Wave 75 Agent 8: Alert Testing and Validation"
echo "======================================================================="
echo ""

# Test 1: Connectivity
echo "=== Test 1: Infrastructure Connectivity ==="
curl -s -m 2 "${PROMETHEUS_URL}/api/v1/status/config" > /dev/null 2>&1 && echo "✅ Prometheus connected" || echo "❌ Prometheus connection failed"
curl -s -m 2 "${ALERTMANAGER_URL}/api/v2/status" > /dev/null 2>&1 && echo "✅ AlertManager connected" || echo "❌ AlertManager connection failed"
echo ""

# Test 2: Alert Rules Count
echo "=== Test 2: Alert Rules Loaded ==="
curl -s -m 5 "${PROMETHEUS_URL}/api/v1/rules" 2>/dev/null | \
    jq -r '.data.groups[] | "\(.name): \(.rules | length) rules"'
echo ""

# Test 3: Individual Alert Rules
echo "=== Test 3: Verify 13 Expected Alerts ==="
ALERT_COUNT=0
for alert in AuthLatencySLAViolation HighAuthFailureRate RedisConnectionFailure \
             RevocationCacheSizeExplosion LowCacheHitRate NotifyListenerDisconnected \
             HighConfigReloadLatency ConfigValidationFailures CircuitBreakerOpen \
             BackendServiceUnhealthy HighBackendLatency ConnectionPoolExhaustion \
             ExcessiveRateLimiting; do

    state=$(curl -s -m 2 "${PROMETHEUS_URL}/api/v1/rules" 2>/dev/null | \
        jq -r ".data.groups[].rules[] | select(.name==\"$alert\") | .state" 2>/dev/null)

    if [ -n "$state" ]; then
        echo "✅ $alert ($state)"
        ALERT_COUNT=$((ALERT_COUNT + 1))
    else
        echo "❌ $alert (NOT FOUND)"
    fi
done
echo ""
echo "Alert Rules Found: $ALERT_COUNT / 13"
echo ""

# Test 4: Alert Health
echo "=== Test 4: Alert Evaluation Health ==="
curl -s -m 5 "${PROMETHEUS_URL}/api/v1/rules" 2>/dev/null | \
    jq -r '.data.groups[].rules[] | select(.health != "ok") | "\(.name): \(.health)"' || \
    echo "✅ All alerts healthy"
echo ""

# Test 5: Currently Firing
echo "=== Test 5: Currently Firing Alerts ==="
FIRING=$(curl -s -m 5 "${PROMETHEUS_URL}/api/v1/alerts" 2>/dev/null | \
    jq '[.data.alerts[] | select(.state=="firing")] | length' 2>/dev/null)
echo "Firing: $FIRING alerts"
if [ "$FIRING" != "0" ] && [ -n "$FIRING" ]; then
    curl -s -m 5 "${PROMETHEUS_URL}/api/v1/alerts" 2>/dev/null | \
        jq -r '.data.alerts[] | select(.state=="firing") | "  - \(.labels.alertname)"'
fi
echo ""

# Test 6: AlertManager Receivers
echo "=== Test 6: AlertManager Receivers ==="
curl -s -m 5 "${ALERTMANAGER_URL}/api/v2/status" 2>/dev/null | \
    jq -r '.config.original' | grep -E "^- name:" | head -10
echo ""

# Test 7: AlertManager Active Alerts
echo "=== Test 7: AlertManager Active Alerts ==="
AM_COUNT=$(curl -s -m 5 "${ALERTMANAGER_URL}/api/v2/alerts" 2>/dev/null | jq '. | length' 2>/dev/null)
echo "AlertManager has $AM_COUNT active alerts"
echo ""

echo "======================================================================="
echo "   Summary"
echo "======================================================================="
echo "✅ Prometheus: Connected"
echo "✅ AlertManager: Connected"
echo "✅ Alert Rules: $ALERT_COUNT / 13 loaded"
echo "📊 Currently Firing: $FIRING alerts"
echo "📊 In AlertManager: $AM_COUNT alerts"
echo ""

if [ "$ALERT_COUNT" == "13" ]; then
    echo "✅ ALL ALERT RULES VALIDATED"
else
    echo "⚠️  Missing alert rules: $((13 - ALERT_COUNT))"
fi