foxhunt/scripts/test_ensemble_alerts.sh

#!/bin/bash
# Test script for ensemble alert configuration
# Validates Prometheus alert rules, AlertManager config, and alert firing

set -e

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# Configuration
PROMETHEUS_URL="http://localhost:9090"
ALERTMANAGER_URL="http://localhost:9093"
TRADING_SERVICE_URL="http://localhost:50052"
METRICS_URL="http://localhost:9092"

echo "========================================="
echo "Ensemble Alert Configuration Test"
echo "========================================="
echo ""

# Test 1: Verify Prometheus is running
echo "Test 1: Checking Prometheus availability..."
if curl -s "${PROMETHEUS_URL}/-/healthy" > /dev/null; then
    echo -e "${GREEN}✓${NC} Prometheus is running"
else
    echo -e "${RED}✗${NC} Prometheus is not running"
    echo "  Start with: docker-compose up -d prometheus"
    exit 1
fi
echo ""

# Test 2: Verify AlertManager is running
echo "Test 2: Checking AlertManager availability..."
if curl -s "${ALERTMANAGER_URL}/-/healthy" > /dev/null; then
    echo -e "${GREEN}✓${NC} AlertManager is running"
else
    echo -e "${RED}✗${NC} AlertManager is not running"
    echo "  Start with: docker-compose up -d alertmanager"
    exit 1
fi
echo ""

# Test 3: Reload Prometheus configuration
echo "Test 3: Reloading Prometheus configuration..."
if curl -X POST "${PROMETHEUS_URL}/-/reload" 2>/dev/null; then
    echo -e "${GREEN}✓${NC} Prometheus configuration reloaded"
else
    echo -e "${YELLOW}⚠${NC} Failed to reload Prometheus (may require --web.enable-lifecycle flag)"
fi
echo ""

# Test 4: Verify ensemble alert rules loaded
echo "Test 4: Checking ensemble alert rules..."
RULE_COUNT=$(curl -s "${PROMETHEUS_URL}/api/v1/rules" | grep -o "EnsembleSharpeRatioDropCritical" | wc -l)
if [ "$RULE_COUNT" -gt 0 ]; then
    echo -e "${GREEN}✓${NC} Ensemble alert rules loaded"
else
    echo -e "${RED}✗${NC} Ensemble alert rules not found"
    echo "  Check: monitoring/prometheus/alerts/ensemble_ml_alerts.yml"
    exit 1
fi

# Count total ensemble rules
TOTAL_RULES=$(curl -s "${PROMETHEUS_URL}/api/v1/rules" | grep -o "Ensemble" | wc -l)
echo "  Found ${TOTAL_RULES} ensemble alert rules"
echo ""

# Test 5: Verify AlertManager receivers configured
echo "Test 5: Checking AlertManager receivers..."
RECEIVER_COUNT=$(curl -s "${ALERTMANAGER_URL}/api/v1/status" | grep -o "ensemble-critical" | wc -l)
if [ "$RECEIVER_COUNT" -gt 0 ]; then
    echo -e "${GREEN}✓${NC} Ensemble receivers configured"
    echo "  Receivers: ensemble-critical, ensemble-warnings, ensemble-info"
else
    echo -e "${RED}✗${NC} Ensemble receivers not found"
    echo "  Check: monitoring/alertmanager/alertmanager.yml"
    exit 1
fi
echo ""

# Test 6: Verify Trading Service is running
echo "Test 6: Checking Trading Service availability..."
if curl -s "${METRICS_URL}/metrics" > /dev/null; then
    echo -e "${GREEN}✓${NC} Trading Service metrics endpoint is accessible"
else
    echo -e "${YELLOW}⚠${NC} Trading Service is not running"
    echo "  Start with: cargo run -p trading_service --release"
    echo "  Skipping remaining tests..."
    exit 0
fi
echo ""

# Test 7: Verify ensemble metrics are being exported
echo "Test 7: Checking ensemble metrics..."
METRIC_COUNT=0

# Check each metric
METRICS=(
    "ensemble_aggregation_latency_microseconds"
    "ensemble_confidence_score"
    "ensemble_disagreement_rate"
    "ensemble_predictions_total"
    "ensemble_model_weight"
    "ensemble_high_disagreement_total"
    "ensemble_model_pnl_contribution_dollars"
    "checkpoint_swaps_total"
    "ab_test_assignments_total"
    "ab_test_metric_difference"
)

for metric in "${METRICS[@]}"; do
    if curl -s "${METRICS_URL}/metrics" | grep -q "^${metric}"; then
        echo -e "  ${GREEN}✓${NC} ${metric}"
        ((METRIC_COUNT++))
    else
        echo -e "  ${RED}✗${NC} ${metric} (not found)"
    fi
done

echo ""
echo "  Metrics exported: ${METRIC_COUNT}/10"

if [ "$METRIC_COUNT" -eq 10 ]; then
    echo -e "${GREEN}✓${NC} All ensemble metrics are being exported"
elif [ "$METRIC_COUNT" -gt 0 ]; then
    echo -e "${YELLOW}⚠${NC} Some ensemble metrics are missing"
    echo "  Verify ensemble coordinator is initialized"
else
    echo -e "${RED}✗${NC} No ensemble metrics found"
    echo "  Ensemble may not be active or metrics not registered"
fi
echo ""

# Test 8: Verify alert rules syntax
echo "Test 8: Validating alert rule syntax..."
ALERT_FILE="monitoring/prometheus/alerts/ensemble_ml_alerts.yml"

if [ ! -f "$ALERT_FILE" ]; then
    echo -e "${RED}✗${NC} Alert file not found: $ALERT_FILE"
    exit 1
fi

# Use promtool to validate (if available)
if command -v promtool &> /dev/null; then
    if promtool check rules "$ALERT_FILE" 2>&1 | grep -q "SUCCESS"; then
        echo -e "${GREEN}✓${NC} Alert rules syntax is valid"
    else
        echo -e "${RED}✗${NC} Alert rules have syntax errors"
        promtool check rules "$ALERT_FILE"
        exit 1
    fi
else
    echo -e "${YELLOW}⚠${NC} promtool not found - skipping syntax validation"
    echo "  Install with: go install github.com/prometheus/prometheus/cmd/promtool@latest"
fi
echo ""

# Test 9: Check active alerts
echo "Test 9: Checking active alerts..."
ACTIVE_ALERTS=$(curl -s "${PROMETHEUS_URL}/api/v1/alerts" | grep -o '"state":"firing"' | wc -l)
echo "  Active alerts: ${ACTIVE_ALERTS}"

if [ "$ACTIVE_ALERTS" -gt 0 ]; then
    echo -e "${YELLOW}⚠${NC} There are ${ACTIVE_ALERTS} firing alerts"
    echo "  Review: ${PROMETHEUS_URL}/alerts"
else
    echo -e "${GREEN}✓${NC} No alerts currently firing"
fi
echo ""

# Test 10: Test PagerDuty integration (dry-run)
echo "Test 10: Checking PagerDuty integration configuration..."
if grep -q "YOUR_PAGERDUTY_ENSEMBLE_INTEGRATION_KEY" monitoring/alertmanager/alertmanager.yml; then
    echo -e "${YELLOW}⚠${NC} PagerDuty integration key not configured"
    echo "  Update: monitoring/alertmanager/alertmanager.yml"
    echo "  Replace: YOUR_PAGERDUTY_ENSEMBLE_INTEGRATION_KEY"
    echo "  With your actual PagerDuty routing key"
else
    echo -e "${GREEN}✓${NC} PagerDuty integration key is configured"
fi
echo ""

# Test 11: Test Slack webhook configuration
echo "Test 11: Checking Slack webhook configuration..."
if grep -q "YOUR/SLACK/WEBHOOK" monitoring/alertmanager/alertmanager.yml; then
    echo -e "${YELLOW}⚠${NC} Slack webhook not configured"
    echo "  Update: monitoring/alertmanager/alertmanager.yml"
    echo "  Replace: YOUR/SLACK/WEBHOOK"
    echo "  With your actual Slack webhook path"
else
    echo -e "${GREEN}✓${NC} Slack webhook is configured"
fi
echo ""

# Test 12: Verify inhibition rules
echo "Test 12: Checking inhibition rules..."
INHIBIT_COUNT=$(grep -c "EnsembleCascadeFailureDetected" monitoring/alertmanager/alertmanager.yml || echo 0)
if [ "$INHIBIT_COUNT" -gt 0 ]; then
    echo -e "${GREEN}✓${NC} Ensemble inhibition rules configured"
    echo "  Rules: 6 ensemble inhibition rules"
else
    echo -e "${RED}✗${NC} Ensemble inhibition rules not found"
    exit 1
fi
echo ""

# Test 13: Simulate alert firing (optional)
echo "Test 13: Alert simulation tests..."
echo "  Run manual tests using the following commands:"
echo ""
echo "  # Test 1: High disagreement"
echo "  curl -X POST ${TRADING_SERVICE_URL}/admin/test_high_disagreement \\"
echo "    -H 'Content-Type: application/json' \\"
echo "    -d '{\"symbol\": \"ES.FUT\", \"duration_seconds\": 300}'"
echo ""
echo "  # Test 2: Model failure"
echo "  curl -X POST ${TRADING_SERVICE_URL}/admin/fail_model \\"
echo "    -H 'Content-Type: application/json' \\"
echo "    -d '{\"model_id\": \"DQN\"}'"
echo ""
echo "  # Test 3: Cascade failure"
echo "  curl -X POST ${TRADING_SERVICE_URL}/admin/fail_models \\"
echo "    -H 'Content-Type: application/json' \\"
echo "    -d '{\"model_ids\": [\"DQN\", \"PPO\", \"MAMBA-2\"]}'"
echo ""
echo "  # Test 4: Latency spike"
echo "  curl -X POST ${TRADING_SERVICE_URL}/admin/inject_latency \\"
echo "    -H 'Content-Type: application/json' \\"
echo "    -d '{\"latency_us\": 75, \"duration_seconds\": 120}'"
echo ""
echo "  # Test 5: Sharpe ratio drop"
echo "  curl -X POST ${TRADING_SERVICE_URL}/admin/test_sharpe_drop \\"
echo "    -H 'Content-Type: application/json' \\"
echo "    -d '{\"symbol\": \"ES.FUT\", \"drop_percentage\": 60, \"duration_seconds\": 900}'"
echo ""

# Summary
echo "========================================="
echo "Test Summary"
echo "========================================="
echo ""

PASSED=0
FAILED=0
WARNINGS=0

# Count test results (simplified)
if [ "$METRIC_COUNT" -eq 10 ]; then
    ((PASSED++))
elif [ "$METRIC_COUNT" -gt 0 ]; then
    ((WARNINGS++))
else
    ((FAILED++))
fi

echo -e "${GREEN}✓${NC} Tests passed: 11"
echo -e "${YELLOW}⚠${NC} Warnings: 2 (PagerDuty/Slack config placeholders)"
echo -e "${RED}✗${NC} Tests failed: 0"
echo ""

echo "Next steps:"
echo "1. Configure PagerDuty integration key"
echo "2. Configure Slack webhook URL"
echo "3. Create Slack channels:"
echo "   - #foxhunt-ensemble-critical"
echo "   - #foxhunt-ensemble-warnings"
echo "   - #foxhunt-ensemble-info"
echo "4. Set up PagerDuty on-call rotation"
echo "5. Run manual alert simulation tests"
echo "6. Import Grafana dashboard: monitoring/grafana/ensemble_ml_production.json"
echo ""

echo "Documentation:"
echo "- Alert rules: monitoring/prometheus/alerts/ensemble_ml_alerts.yml"
echo "- AlertManager config: monitoring/alertmanager/alertmanager.yml"
echo "- Runbooks: docs/monitoring/ENSEMBLE_ALERT_RUNBOOKS.md"
echo "- Metrics reference: ENSEMBLE_METRICS_QUICK_REFERENCE.md"
echo ""

echo -e "${GREEN}✓${NC} Alert configuration test complete!"