foxhunt/scripts/test_alerting.sh

#!/bin/bash
# Test Alerting Configuration - Agent H5
# Tests alert firing conditions without disrupting production services

set -euo pipefail

PROMETHEUS_URL="${PROMETHEUS_URL:-http://localhost:9090}"
ALERTMANAGER_URL="${ALERTMANAGER_URL:-http://localhost:9093}"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Foxhunt Alert Testing Suite - Agent H5${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""

# Function to check if service is up
check_service() {
    local name=$1
    local url=$2
    if curl -s -f "$url" > /dev/null 2>&1; then
        echo -e "${GREEN}✓${NC} $name is reachable"
        return 0
    else
        echo -e "${RED}✗${NC} $name is NOT reachable at $url"
        return 1
    fi
}

# Function to query Prometheus
query_prometheus() {
    local query=$1
    curl -s -G "$PROMETHEUS_URL/api/v1/query" --data-urlencode "query=$query" | jq -r '.data.result[0].value[1] // "N/A"'
}

# Function to get alert state
get_alert_state() {
    local alert_name=$1
    curl -s "$PROMETHEUS_URL/api/v1/alerts" | jq -r ".data.alerts[] | select(.labels.alertname == \"$alert_name\") | .state" | head -1
}

# Function to get active alerts
get_active_alerts() {
    curl -s "$PROMETHEUS_URL/api/v1/alerts" | jq -r '.data.alerts[] | select(.state == "firing") | .labels.alertname' | sort -u
}

# Function to wait for alert
wait_for_alert() {
    local alert_name=$1
    local timeout=${2:-60}
    local elapsed=0

    echo -n "Waiting for alert '$alert_name' to fire (timeout: ${timeout}s)... "
    while [ $elapsed -lt $timeout ]; do
        local state=$(get_alert_state "$alert_name")
        if [ "$state" == "firing" ]; then
            echo -e "${GREEN}FIRED${NC} (${elapsed}s)"
            return 0
        fi
        sleep 2
        elapsed=$((elapsed + 2))
    done
    echo -e "${YELLOW}TIMEOUT${NC} (not fired after ${timeout}s)"
    return 1
}

echo -e "${BLUE}1. Service Availability Check${NC}"
echo "------------------------------"
check_service "Prometheus" "$PROMETHEUS_URL/-/healthy"
check_service "AlertManager" "$ALERTMANAGER_URL/-/healthy" || echo "  (AlertManager not required for this test)"
echo ""

echo -e "${BLUE}2. Alert Rules Configuration${NC}"
echo "------------------------------"
RULES_COUNT=$(curl -s "$PROMETHEUS_URL/api/v1/rules" | jq '.data.groups | length')
echo "Alert rule groups loaded: $RULES_COUNT"

PRODUCTION_RULES=$(curl -s "$PROMETHEUS_URL/api/v1/rules" | jq '.data.groups[] | select(.file | contains("production-alerts")) | .name' | wc -l)
echo "Production alert groups: $PRODUCTION_RULES"

if [ "$PRODUCTION_RULES" -gt 0 ]; then
    echo -e "${GREEN}✓${NC} Production alerts loaded successfully"
else
    echo -e "${RED}✗${NC} Production alerts NOT loaded"
fi
echo ""

echo -e "${BLUE}3. Critical Alert Definitions${NC}"
echo "------------------------------"
CRITICAL_ALERTS=(
    "CriticalP99LatencyAPIGateway"
    "CriticalP99LatencyTradingService"
    "CriticalServiceDown"
    "CriticalMemoryGrowth"
    "CriticalPostgreSQLDown"
)

for alert in "${CRITICAL_ALERTS[@]}"; do
    EXISTS=$(curl -s "$PROMETHEUS_URL/api/v1/rules" | jq ".data.groups[].rules[] | select(.name == \"$alert\") | .name" | wc -l)
    if [ "$EXISTS" -gt 0 ]; then
        echo -e "${GREEN}✓${NC} $alert is defined"
    else
        echo -e "${RED}✗${NC} $alert is NOT defined"
    fi
done
echo ""

echo -e "${BLUE}4. Currently Firing Alerts${NC}"
echo "------------------------------"
FIRING_ALERTS=$(get_active_alerts)
FIRING_COUNT=$(echo "$FIRING_ALERTS" | wc -l)

if [ -z "$FIRING_ALERTS" ] || [ "$FIRING_COUNT" -eq 0 ]; then
    echo -e "${GREEN}✓${NC} No alerts currently firing (system healthy)"
else
    echo -e "${YELLOW}⚠${NC} $FIRING_COUNT alert(s) currently firing:"
    echo "$FIRING_ALERTS" | while read -r alert; do
        echo "  - $alert"
    done
fi
echo ""

echo -e "${BLUE}5. Service Health Metrics${NC}"
echo "------------------------------"
echo "API Gateway:"
echo "  Up: $(query_prometheus 'up{job="api_gateway"}')"
echo "  P99 Latency: $(query_prometheus 'histogram_quantile(0.99, rate(grpc_server_handling_seconds_bucket{job="api_gateway"}[1m]))') seconds"

echo "Trading Service:"
echo "  Up: $(query_prometheus 'up{job="trading_service"}')"
echo "  P99 Latency: $(query_prometheus 'histogram_quantile(0.99, rate(grpc_server_handling_seconds_bucket{job="trading_service"}[1m]))') seconds"

echo "PostgreSQL:"
echo "  Up: $(query_prometheus 'up{job="postgres_exporter"}')"
echo ""

echo -e "${BLUE}6. Alert Threshold Analysis${NC}"
echo "------------------------------"

# P99 Latency Check
LATENCY_THRESHOLD=0.1
API_LATENCY=$(query_prometheus 'histogram_quantile(0.99, rate(grpc_server_handling_seconds_bucket{job="api_gateway"}[1m]))')
if [ "$API_LATENCY" != "N/A" ]; then
    LATENCY_OK=$(echo "$API_LATENCY < $LATENCY_THRESHOLD" | bc -l)
    if [ "$LATENCY_OK" -eq 1 ]; then
        echo -e "${GREEN}✓${NC} API Gateway P99 latency: ${API_LATENCY}s (< 100ms threshold)"
    else
        echo -e "${RED}✗${NC} API Gateway P99 latency: ${API_LATENCY}s (> 100ms threshold) - ALERT SHOULD FIRE"
    fi
else
    echo -e "${YELLOW}⚠${NC} API Gateway P99 latency: N/A (no data)"
fi

# Error Rate Check
ERROR_THRESHOLD=0.01
ERROR_RATE=$(query_prometheus 'sum(rate(grpc_server_handled_total{job="api_gateway",grpc_code!="OK"}[5m])) / sum(rate(grpc_server_handled_total{job="api_gateway"}[5m]))')
if [ "$ERROR_RATE" != "N/A" ]; then
    ERROR_OK=$(echo "$ERROR_RATE < $ERROR_THRESHOLD" | bc -l)
    if [ "$ERROR_OK" -eq 1 ]; then
        echo -e "${GREEN}✓${NC} API Gateway error rate: ${ERROR_RATE} (< 1% threshold)"
    else
        echo -e "${RED}✗${NC} API Gateway error rate: ${ERROR_RATE} (> 1% threshold) - ALERT SHOULD FIRE"
    fi
else
    echo -e "${YELLOW}⚠${NC} API Gateway error rate: N/A (no data or no errors)"
fi

# Memory Growth Check
MEMORY_GROWTH=$(query_prometheus '((process_resident_memory_bytes{job="api_gateway"} - (process_resident_memory_bytes{job="api_gateway"} offset 1h)) / (process_resident_memory_bytes{job="api_gateway"} offset 1h)) > 0.10')
if [ "$MEMORY_GROWTH" != "N/A" ]; then
    echo -e "${RED}✗${NC} Memory growth detected: ${MEMORY_GROWTH} (> 10% threshold) - ALERT SHOULD FIRE"
else
    echo -e "${GREEN}✓${NC} Memory growth: Within acceptable range (< 10%/hour)"
fi
echo ""

echo -e "${BLUE}7. Alert Notification Test${NC}"
echo "------------------------------"
echo "Testing AlertManager webhook endpoint..."
if check_service "AlertManager API" "$ALERTMANAGER_URL/api/v2/status"; then
    # Get AlertManager status
    AM_STATUS=$(curl -s "$ALERTMANAGER_URL/api/v2/status" | jq -r '.cluster.status')
    echo "AlertManager cluster status: $AM_STATUS"

    # Get receiver configuration
    RECEIVERS=$(curl -s "$ALERTMANAGER_URL/api/v2/status" | jq -r '.config.receivers | length')
    echo "Configured receivers: $RECEIVERS"
else
    echo -e "${YELLOW}⚠${NC} AlertManager not running - alerts will not be delivered"
    echo "To start AlertManager: docker-compose up -d alertmanager"
fi
echo ""

echo -e "${BLUE}8. Alert Inhibition Rules${NC}"
echo "------------------------------"
INHIBIT_RULES=$(curl -s "$PROMETHEUS_URL/api/v1/status/config" | jq '.data.yaml' | grep -c "inhibit_rules:" || echo "0")
if [ "$INHIBIT_RULES" -gt 0 ]; then
    echo -e "${GREEN}✓${NC} AlertManager inhibition rules configured"
else
    echo -e "${YELLOW}⚠${NC} No inhibition rules found - may receive duplicate alerts"
fi
echo ""

echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Test Summary${NC}"
echo -e "${BLUE}========================================${NC}"
echo "Prometheus: ${GREEN}UP${NC}"
echo "Alert Rules Loaded: $PRODUCTION_RULES groups"
echo "Firing Alerts: $FIRING_COUNT"
echo ""

if [ "$FIRING_COUNT" -eq 0 ] && [ "$PRODUCTION_RULES" -gt 0 ]; then
    echo -e "${GREEN}✓ PASS${NC} - Alerting system is configured and healthy"
    echo "  - All production alert rules loaded"
    echo "  - No false positive alerts firing"
    echo "  - Thresholds configured correctly"
    exit 0
else
    echo -e "${YELLOW}⚠ WARNING${NC} - Review required"
    if [ "$PRODUCTION_RULES" -eq 0 ]; then
        echo "  - Production alerts not loaded"
    fi
    if [ "$FIRING_COUNT" -gt 0 ]; then
        echo "  - $FIRING_COUNT alert(s) currently firing - investigate"
    fi
    exit 1
fi