8cf9437c78
- Restored S3 storage functionality with AWS SDK - Fixed field access issues (removed underscore prefixes) - Created Benzinga historical module - Initial SIMD optimization (needs consolidation) - Fixed multiple compilation errors PENDING: SIMD consolidation, config centralization, shared libraries
769 lines
21 KiB
Bash
Executable File
769 lines
21 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
#============================================================================
|
|
# FOXHUNT HFT TRADING SYSTEM - COMPREHENSIVE HEALTH CHECK SCRIPT
|
|
#============================================================================
|
|
# Performs detailed health checks on all system components
|
|
# Supports continuous monitoring and alerting integration
|
|
#
|
|
# Usage:
|
|
# ./health_check.sh [OPTIONS]
|
|
#
|
|
# Options:
|
|
# --mode MODE Check mode: quick, comprehensive, continuous (default: quick)
|
|
# --format FORMAT Output format: text, json, prometheus (default: text)
|
|
# --timeout SECONDS Health check timeout per service (default: 10)
|
|
# --alert-webhook URL Webhook URL for alerts
|
|
# --nagios Output in Nagios-compatible format
|
|
# --verbose Enable verbose output
|
|
# --help Show this help message
|
|
#============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
# Configuration
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
CHECK_MODE="${CHECK_MODE:-quick}"
|
|
OUTPUT_FORMAT="${OUTPUT_FORMAT:-text}"
|
|
TIMEOUT="${TIMEOUT:-10}"
|
|
ALERT_WEBHOOK="${ALERT_WEBHOOK:-}"
|
|
NAGIOS_MODE="${NAGIOS_MODE:-false}"
|
|
VERBOSE="${VERBOSE:-false}"
|
|
|
|
# Service endpoints
|
|
declare -A SERVICES=(
|
|
["trading"]="8080:50051:Trading Service"
|
|
["ml-training"]="8082:50052:ML Training Service"
|
|
["backtesting"]="8083:50053:Backtesting Service"
|
|
["tli"]="8081:50054:Terminal Line Interface"
|
|
)
|
|
|
|
# Database endpoints
|
|
declare -A DATABASES=(
|
|
["postgresql"]="5432:PostgreSQL Database"
|
|
["redis"]="6379:Redis Cache"
|
|
["influxdb"]="8086:InfluxDB Time Series"
|
|
)
|
|
|
|
# External services
|
|
declare -A EXTERNAL=(
|
|
["vault"]="8200:HashiCorp Vault"
|
|
["prometheus"]="9090:Prometheus Metrics"
|
|
["grafana"]="3000:Grafana Dashboard"
|
|
)
|
|
|
|
# Health check results
|
|
declare -A HEALTH_RESULTS=()
|
|
declare -A PERFORMANCE_METRICS=()
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m'
|
|
|
|
log_info() {
|
|
if [[ "$VERBOSE" == "true" ]] || [[ "$OUTPUT_FORMAT" == "text" ]]; then
|
|
echo -e "${BLUE}[INFO]${NC} $1"
|
|
fi
|
|
}
|
|
|
|
log_success() {
|
|
if [[ "$OUTPUT_FORMAT" == "text" ]]; then
|
|
echo -e "${GREEN}[OK]${NC} $1"
|
|
fi
|
|
}
|
|
|
|
log_warning() {
|
|
if [[ "$OUTPUT_FORMAT" == "text" ]]; then
|
|
echo -e "${YELLOW}[WARNING]${NC} $1"
|
|
fi
|
|
}
|
|
|
|
log_error() {
|
|
if [[ "$OUTPUT_FORMAT" == "text" ]]; then
|
|
echo -e "${RED}[ERROR]${NC} $1"
|
|
fi
|
|
}
|
|
|
|
show_help() {
|
|
cat << EOF
|
|
Foxhunt HFT Trading System - Health Check Script
|
|
|
|
Usage: $0 [OPTIONS]
|
|
|
|
OPTIONS:
|
|
--mode MODE Check mode (quick, comprehensive, continuous)
|
|
--format FORMAT Output format (text, json, prometheus)
|
|
--timeout SECONDS Health check timeout per service (default: 10)
|
|
--alert-webhook URL Webhook URL for critical alerts
|
|
--nagios Output in Nagios-compatible format
|
|
--verbose Enable verbose output
|
|
--help Show this help message
|
|
|
|
CHECK MODES:
|
|
quick Basic health checks (HTTP endpoints, process status)
|
|
comprehensive Deep health checks (performance, dependencies, security)
|
|
continuous Continuous monitoring with alerting
|
|
|
|
OUTPUT FORMATS:
|
|
text Human-readable console output
|
|
json JSON format for programmatic parsing
|
|
prometheus Prometheus metrics format
|
|
|
|
EXAMPLES:
|
|
$0 # Quick health check
|
|
$0 --mode comprehensive --verbose # Comprehensive check with details
|
|
$0 --format json > health.json # Export health status to JSON
|
|
$0 --mode continuous --timeout 5 # Continuous monitoring
|
|
|
|
EOF
|
|
}
|
|
|
|
# Parse arguments
|
|
parse_args() {
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
--mode)
|
|
CHECK_MODE="$2"
|
|
shift 2
|
|
;;
|
|
--format)
|
|
OUTPUT_FORMAT="$2"
|
|
shift 2
|
|
;;
|
|
--timeout)
|
|
TIMEOUT="$2"
|
|
shift 2
|
|
;;
|
|
--alert-webhook)
|
|
ALERT_WEBHOOK="$2"
|
|
shift 2
|
|
;;
|
|
--nagios)
|
|
NAGIOS_MODE=true
|
|
OUTPUT_FORMAT="nagios"
|
|
shift
|
|
;;
|
|
--verbose)
|
|
VERBOSE=true
|
|
shift
|
|
;;
|
|
--help)
|
|
show_help
|
|
exit 0
|
|
;;
|
|
*)
|
|
log_error "Unknown option: $1"
|
|
show_help
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
# Check if command exists
|
|
command_exists() {
|
|
command -v "$1" >/dev/null 2>&1
|
|
}
|
|
|
|
# HTTP health check
|
|
check_http_endpoint() {
|
|
local service="$1"
|
|
local port="$2"
|
|
local endpoint="${3:-/health}"
|
|
|
|
if command_exists curl; then
|
|
local response_time start_time end_time http_code
|
|
start_time=$(date +%s%N)
|
|
|
|
if http_code=$(curl -w "%{http_code}" -s -o /dev/null --max-time "$TIMEOUT" "http://localhost:$port$endpoint"); then
|
|
end_time=$(date +%s%N)
|
|
response_time=$(( (end_time - start_time) / 1000000 )) # Convert to milliseconds
|
|
|
|
PERFORMANCE_METRICS["${service}_response_time"]="$response_time"
|
|
|
|
if [[ "$http_code" == "200" ]]; then
|
|
HEALTH_RESULTS["${service}_http"]="OK"
|
|
return 0
|
|
else
|
|
HEALTH_RESULTS["${service}_http"]="FAIL:HTTP_$http_code"
|
|
return 1
|
|
fi
|
|
else
|
|
HEALTH_RESULTS["${service}_http"]="FAIL:TIMEOUT"
|
|
return 1
|
|
fi
|
|
else
|
|
HEALTH_RESULTS["${service}_http"]="FAIL:NO_CURL"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# gRPC health check
|
|
check_grpc_endpoint() {
|
|
local service="$1"
|
|
local port="$2"
|
|
|
|
if command_exists grpcurl; then
|
|
if timeout "$TIMEOUT" grpcurl -plaintext "localhost:$port" grpc.health.v1.Health/Check >/dev/null 2>&1; then
|
|
HEALTH_RESULTS["${service}_grpc"]="OK"
|
|
return 0
|
|
else
|
|
HEALTH_RESULTS["${service}_grpc"]="FAIL:UNREACHABLE"
|
|
return 1
|
|
fi
|
|
else
|
|
# Fallback to TCP connection test
|
|
if timeout "$TIMEOUT" nc -z localhost "$port" >/dev/null 2>&1; then
|
|
HEALTH_RESULTS["${service}_grpc"]="OK"
|
|
return 0
|
|
else
|
|
HEALTH_RESULTS["${service}_grpc"]="FAIL:TCP_FAIL"
|
|
return 1
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# Process health check
|
|
check_process_status() {
|
|
local service="$1"
|
|
local binary_name="$2"
|
|
|
|
if systemctl is-active "foxhunt-$service" >/dev/null 2>&1; then
|
|
HEALTH_RESULTS["${service}_process"]="OK"
|
|
|
|
# Get process metrics
|
|
local pid cpu_usage memory_usage
|
|
if pid=$(systemctl show "foxhunt-$service" --property=MainPID --value) && [[ "$pid" != "0" ]]; then
|
|
if command_exists ps; then
|
|
cpu_usage=$(ps -p "$pid" -o %cpu --no-headers | tr -d ' ' || echo "0.0")
|
|
memory_usage=$(ps -p "$pid" -o rss --no-headers | tr -d ' ' || echo "0")
|
|
|
|
PERFORMANCE_METRICS["${service}_cpu_percent"]="$cpu_usage"
|
|
PERFORMANCE_METRICS["${service}_memory_kb"]="$memory_usage"
|
|
fi
|
|
fi
|
|
return 0
|
|
else
|
|
HEALTH_RESULTS["${service}_process"]="FAIL:NOT_RUNNING"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Database connectivity check
|
|
check_database() {
|
|
local db_name="$1"
|
|
local port="$2"
|
|
|
|
case "$db_name" in
|
|
"postgresql")
|
|
if command_exists pg_isready; then
|
|
if pg_isready -h localhost -p "$port" >/dev/null 2>&1; then
|
|
HEALTH_RESULTS["${db_name}_connectivity"]="OK"
|
|
return 0
|
|
fi
|
|
fi
|
|
;;
|
|
"redis")
|
|
if command_exists redis-cli; then
|
|
if timeout "$TIMEOUT" redis-cli -p "$port" ping | grep -q "PONG"; then
|
|
HEALTH_RESULTS["${db_name}_connectivity"]="OK"
|
|
return 0
|
|
fi
|
|
fi
|
|
;;
|
|
"influxdb")
|
|
if check_http_endpoint "$db_name" "$port" "/health"; then
|
|
HEALTH_RESULTS["${db_name}_connectivity"]="OK"
|
|
return 0
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
# Fallback to TCP connection test
|
|
if timeout "$TIMEOUT" nc -z localhost "$port" >/dev/null 2>&1; then
|
|
HEALTH_RESULTS["${db_name}_connectivity"]="OK"
|
|
return 0
|
|
else
|
|
HEALTH_RESULTS["${db_name}_connectivity"]="FAIL:UNREACHABLE"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# System resource check
|
|
check_system_resources() {
|
|
log_info "Checking system resources..."
|
|
|
|
# CPU usage
|
|
if command_exists top; then
|
|
local cpu_usage
|
|
cpu_usage=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}' | cut -d'%' -f1 | cut -d'u' -f1)
|
|
PERFORMANCE_METRICS["system_cpu_percent"]="$cpu_usage"
|
|
fi
|
|
|
|
# Memory usage
|
|
if command_exists free; then
|
|
local mem_total mem_used mem_percent
|
|
mem_total=$(free -m | awk 'NR==2{print $2}')
|
|
mem_used=$(free -m | awk 'NR==2{print $3}')
|
|
if [[ "$mem_total" -gt 0 ]]; then
|
|
mem_percent=$((mem_used * 100 / mem_total))
|
|
PERFORMANCE_METRICS["system_memory_percent"]="$mem_percent"
|
|
fi
|
|
fi
|
|
|
|
# Disk usage
|
|
if command_exists df; then
|
|
local disk_usage
|
|
disk_usage=$(df / | tail -1 | awk '{print $5}' | cut -d'%' -f1)
|
|
PERFORMANCE_METRICS["system_disk_percent"]="$disk_usage"
|
|
fi
|
|
|
|
# Load average
|
|
if [[ -f /proc/loadavg ]]; then
|
|
local load_1min
|
|
load_1min=$(cat /proc/loadavg | awk '{print $1}')
|
|
PERFORMANCE_METRICS["system_load_1min"]="$load_1min"
|
|
fi
|
|
|
|
HEALTH_RESULTS["system_resources"]="OK"
|
|
}
|
|
|
|
# Network connectivity check
|
|
check_network_connectivity() {
|
|
log_info "Checking network connectivity..."
|
|
|
|
local dns_check=false
|
|
local internet_check=false
|
|
|
|
# DNS resolution check
|
|
if command_exists nslookup; then
|
|
if nslookup google.com >/dev/null 2>&1; then
|
|
dns_check=true
|
|
fi
|
|
fi
|
|
|
|
# Internet connectivity check
|
|
if command_exists ping; then
|
|
if ping -c 1 -W 5 8.8.8.8 >/dev/null 2>&1; then
|
|
internet_check=true
|
|
fi
|
|
fi
|
|
|
|
if [[ "$dns_check" == "true" ]] && [[ "$internet_check" == "true" ]]; then
|
|
HEALTH_RESULTS["network_connectivity"]="OK"
|
|
else
|
|
HEALTH_RESULTS["network_connectivity"]="FAIL:LIMITED"
|
|
fi
|
|
}
|
|
|
|
# Security check
|
|
check_security_status() {
|
|
log_info "Checking security status..."
|
|
|
|
local security_issues=()
|
|
|
|
# Check if services are running as non-root
|
|
for service in "${!SERVICES[@]}"; do
|
|
if systemctl is-active "foxhunt-$service" >/dev/null 2>&1; then
|
|
local service_user
|
|
service_user=$(systemctl show "foxhunt-$service" --property=User --value)
|
|
if [[ "$service_user" == "root" ]]; then
|
|
security_issues+=("$service running as root")
|
|
fi
|
|
fi
|
|
done
|
|
|
|
# Check file permissions
|
|
local data_dir="/opt/foxhunt"
|
|
if [[ -d "$data_dir" ]]; then
|
|
local perms
|
|
perms=$(stat -c "%a" "$data_dir" 2>/dev/null || echo "000")
|
|
if [[ "$perms" == "777" ]]; then
|
|
security_issues+=("Data directory has overly permissive permissions")
|
|
fi
|
|
fi
|
|
|
|
if [[ ${#security_issues[@]} -eq 0 ]]; then
|
|
HEALTH_RESULTS["security_status"]="OK"
|
|
else
|
|
HEALTH_RESULTS["security_status"]="WARN:${security_issues[*]}"
|
|
fi
|
|
}
|
|
|
|
# Performance benchmark
|
|
check_performance_benchmark() {
|
|
log_info "Running performance benchmark..."
|
|
|
|
# CPU benchmark (simple calculation)
|
|
local start_time end_time duration
|
|
start_time=$(date +%s%N)
|
|
|
|
# Simple CPU-intensive calculation
|
|
local result=0
|
|
for ((i=1; i<=100000; i++)); do
|
|
result=$((result + i))
|
|
done
|
|
|
|
end_time=$(date +%s%N)
|
|
duration=$(( (end_time - start_time) / 1000000 )) # Convert to milliseconds
|
|
|
|
PERFORMANCE_METRICS["cpu_benchmark_ms"]="$duration"
|
|
|
|
# Memory allocation test
|
|
if command_exists free; then
|
|
local mem_before mem_after
|
|
mem_before=$(free -m | awk 'NR==2{print $3}')
|
|
|
|
# Allocate some memory (create temporary file)
|
|
dd if=/dev/zero of=/tmp/health_check_mem_test bs=1M count=10 >/dev/null 2>&1
|
|
|
|
mem_after=$(free -m | awk 'NR==2{print $3}')
|
|
rm -f /tmp/health_check_mem_test
|
|
|
|
local mem_delta=$((mem_after - mem_before))
|
|
PERFORMANCE_METRICS["memory_allocation_mb"]="$mem_delta"
|
|
fi
|
|
|
|
HEALTH_RESULTS["performance_benchmark"]="OK"
|
|
}
|
|
|
|
# Quick health check
|
|
perform_quick_check() {
|
|
log_info "Performing quick health check..."
|
|
|
|
# Check core services
|
|
for service in "${!SERVICES[@]}"; do
|
|
IFS=':' read -r http_port grpc_port description <<< "${SERVICES[$service]}"
|
|
|
|
log_info "Checking $service ($description)..."
|
|
|
|
check_process_status "$service" "$service"
|
|
check_http_endpoint "$service" "$http_port"
|
|
check_grpc_endpoint "$service" "$grpc_port"
|
|
done
|
|
|
|
# Check databases
|
|
for db in "${!DATABASES[@]}"; do
|
|
IFS=':' read -r port description <<< "${DATABASES[$db]}"
|
|
log_info "Checking $db ($description)..."
|
|
check_database "$db" "$port"
|
|
done
|
|
}
|
|
|
|
# Comprehensive health check
|
|
perform_comprehensive_check() {
|
|
log_info "Performing comprehensive health check..."
|
|
|
|
perform_quick_check
|
|
check_system_resources
|
|
check_network_connectivity
|
|
check_security_status
|
|
|
|
if [[ "$CHECK_MODE" == "comprehensive" ]]; then
|
|
check_performance_benchmark
|
|
fi
|
|
|
|
# Check external services
|
|
for service in "${!EXTERNAL[@]}"; do
|
|
IFS=':' read -r port description <<< "${EXTERNAL[$service]}"
|
|
log_info "Checking $service ($description)..."
|
|
check_http_endpoint "$service" "$port"
|
|
done
|
|
}
|
|
|
|
# Send alert
|
|
send_alert() {
|
|
local message="$1"
|
|
local severity="$2"
|
|
|
|
if [[ -n "$ALERT_WEBHOOK" ]]; then
|
|
local payload
|
|
payload=$(cat << EOF
|
|
{
|
|
"text": "Foxhunt Health Alert",
|
|
"attachments": [
|
|
{
|
|
"color": "$( [[ "$severity" == "critical" ]] && echo "danger" || echo "warning" )",
|
|
"fields": [
|
|
{
|
|
"title": "Severity",
|
|
"value": "$severity",
|
|
"short": true
|
|
},
|
|
{
|
|
"title": "Message",
|
|
"value": "$message",
|
|
"short": false
|
|
},
|
|
{
|
|
"title": "Timestamp",
|
|
"value": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
|
"short": true
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
EOF
|
|
)
|
|
|
|
if command_exists curl; then
|
|
curl -X POST -H "Content-Type: application/json" -d "$payload" "$ALERT_WEBHOOK" >/dev/null 2>&1 || true
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# Output results in text format
|
|
output_text() {
|
|
local failed_checks=()
|
|
local warning_checks=()
|
|
local total_checks=0
|
|
local passed_checks=0
|
|
|
|
echo
|
|
echo "=================================="
|
|
echo "FOXHUNT HEALTH CHECK RESULTS"
|
|
echo "=================================="
|
|
echo "Timestamp: $(date)"
|
|
echo "Mode: $CHECK_MODE"
|
|
echo
|
|
|
|
# Service status
|
|
echo "SERVICE STATUS:"
|
|
for check_name in "${!HEALTH_RESULTS[@]}"; do
|
|
total_checks=$((total_checks + 1))
|
|
local result="${HEALTH_RESULTS[$check_name]}"
|
|
|
|
if [[ "$result" == "OK" ]]; then
|
|
log_success "$check_name: $result"
|
|
passed_checks=$((passed_checks + 1))
|
|
elif [[ "$result" == WARN:* ]]; then
|
|
log_warning "$check_name: $result"
|
|
warning_checks+=("$check_name")
|
|
else
|
|
log_error "$check_name: $result"
|
|
failed_checks+=("$check_name")
|
|
fi
|
|
done
|
|
|
|
# Performance metrics
|
|
if [[ ${#PERFORMANCE_METRICS[@]} -gt 0 ]]; then
|
|
echo
|
|
echo "PERFORMANCE METRICS:"
|
|
for metric in "${!PERFORMANCE_METRICS[@]}"; do
|
|
echo " $metric: ${PERFORMANCE_METRICS[$metric]}"
|
|
done
|
|
fi
|
|
|
|
# Summary
|
|
echo
|
|
echo "SUMMARY:"
|
|
echo " Total Checks: $total_checks"
|
|
echo " Passed: $passed_checks"
|
|
echo " Warnings: ${#warning_checks[@]}"
|
|
echo " Failed: ${#failed_checks[@]}"
|
|
|
|
if [[ ${#failed_checks[@]} -gt 0 ]]; then
|
|
echo
|
|
echo "FAILED CHECKS:"
|
|
for check in "${failed_checks[@]}"; do
|
|
echo " • $check: ${HEALTH_RESULTS[$check]}"
|
|
done
|
|
|
|
send_alert "Health check failed: ${failed_checks[*]}" "critical"
|
|
return 2
|
|
elif [[ ${#warning_checks[@]} -gt 0 ]]; then
|
|
echo
|
|
echo "WARNING CHECKS:"
|
|
for check in "${warning_checks[@]}"; do
|
|
echo " • $check: ${HEALTH_RESULTS[$check]}"
|
|
done
|
|
|
|
send_alert "Health check warnings: ${warning_checks[*]}" "warning"
|
|
return 1
|
|
else
|
|
log_success "All health checks passed!"
|
|
return 0
|
|
fi
|
|
}
|
|
|
|
# Output results in JSON format
|
|
output_json() {
|
|
local timestamp=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
|
|
|
echo "{"
|
|
echo " \"timestamp\": \"$timestamp\","
|
|
echo " \"mode\": \"$CHECK_MODE\","
|
|
echo " \"status\": {"
|
|
|
|
local first_result=true
|
|
for check_name in "${!HEALTH_RESULTS[@]}"; do
|
|
if [[ "$first_result" == "false" ]]; then
|
|
echo ","
|
|
fi
|
|
echo -n " \"$check_name\": \"${HEALTH_RESULTS[$check_name]}\""
|
|
first_result=false
|
|
done
|
|
|
|
echo
|
|
echo " },"
|
|
echo " \"metrics\": {"
|
|
|
|
local first_metric=true
|
|
for metric in "${!PERFORMANCE_METRICS[@]}"; do
|
|
if [[ "$first_metric" == "false" ]]; then
|
|
echo ","
|
|
fi
|
|
echo -n " \"$metric\": ${PERFORMANCE_METRICS[$metric]}"
|
|
first_metric=false
|
|
done
|
|
|
|
echo
|
|
echo " }"
|
|
echo "}"
|
|
}
|
|
|
|
# Output results in Prometheus format
|
|
output_prometheus() {
|
|
local timestamp=$(date +%s)
|
|
|
|
# Health check results (0 = OK, 1 = WARNING, 2 = CRITICAL)
|
|
for check_name in "${!HEALTH_RESULTS[@]}"; do
|
|
local result="${HEALTH_RESULTS[$check_name]}"
|
|
local value=0
|
|
|
|
if [[ "$result" == WARN:* ]]; then
|
|
value=1
|
|
elif [[ "$result" != "OK" ]]; then
|
|
value=2
|
|
fi
|
|
|
|
echo "foxhunt_health_status{check=\"$check_name\"} $value $timestamp"
|
|
done
|
|
|
|
# Performance metrics
|
|
for metric in "${!PERFORMANCE_METRICS[@]}"; do
|
|
echo "foxhunt_${metric} ${PERFORMANCE_METRICS[$metric]} $timestamp"
|
|
done
|
|
}
|
|
|
|
# Output results in Nagios format
|
|
output_nagios() {
|
|
local failed_checks=()
|
|
local warning_checks=()
|
|
|
|
for check_name in "${!HEALTH_RESULTS[@]}"; do
|
|
local result="${HEALTH_RESULTS[$check_name]}"
|
|
|
|
if [[ "$result" == WARN:* ]]; then
|
|
warning_checks+=("$check_name")
|
|
elif [[ "$result" != "OK" ]]; then
|
|
failed_checks+=("$check_name")
|
|
fi
|
|
done
|
|
|
|
if [[ ${#failed_checks[@]} -gt 0 ]]; then
|
|
echo "CRITICAL - Health check failed: ${failed_checks[*]}"
|
|
exit 2
|
|
elif [[ ${#warning_checks[@]} -gt 0 ]]; then
|
|
echo "WARNING - Health check warnings: ${warning_checks[*]}"
|
|
exit 1
|
|
else
|
|
echo "OK - All health checks passed"
|
|
exit 0
|
|
fi
|
|
}
|
|
|
|
# Continuous monitoring mode
|
|
run_continuous_monitoring() {
|
|
log_info "Starting continuous monitoring mode..."
|
|
|
|
local check_interval=60 # seconds
|
|
local consecutive_failures=0
|
|
local max_consecutive_failures=3
|
|
|
|
while true; do
|
|
log_info "Running health check cycle..."
|
|
|
|
# Clear previous results
|
|
HEALTH_RESULTS=()
|
|
PERFORMANCE_METRICS=()
|
|
|
|
perform_comprehensive_check
|
|
|
|
local exit_code=0
|
|
case "$OUTPUT_FORMAT" in
|
|
"json")
|
|
output_json > /tmp/foxhunt_health.json
|
|
;;
|
|
"prometheus")
|
|
output_prometheus > /tmp/foxhunt_health.prom
|
|
;;
|
|
*)
|
|
if ! output_text >/dev/null; then
|
|
exit_code=$?
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
if [[ $exit_code -eq 0 ]]; then
|
|
consecutive_failures=0
|
|
else
|
|
consecutive_failures=$((consecutive_failures + 1))
|
|
|
|
if [[ $consecutive_failures -ge $max_consecutive_failures ]]; then
|
|
send_alert "Critical: $consecutive_failures consecutive health check failures" "critical"
|
|
fi
|
|
fi
|
|
|
|
log_info "Health check cycle complete. Waiting ${check_interval}s..."
|
|
sleep $check_interval
|
|
done
|
|
}
|
|
|
|
# Main execution
|
|
main() {
|
|
parse_args "$@"
|
|
|
|
case "$CHECK_MODE" in
|
|
"quick")
|
|
perform_quick_check
|
|
;;
|
|
"comprehensive")
|
|
perform_comprehensive_check
|
|
;;
|
|
"continuous")
|
|
run_continuous_monitoring
|
|
return 0
|
|
;;
|
|
*)
|
|
log_error "Unknown check mode: $CHECK_MODE"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
# Output results
|
|
case "$OUTPUT_FORMAT" in
|
|
"json")
|
|
output_json
|
|
;;
|
|
"prometheus")
|
|
output_prometheus
|
|
;;
|
|
"nagios")
|
|
output_nagios
|
|
;;
|
|
*)
|
|
output_text
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Handle interruption
|
|
trap 'log_error "Health check interrupted"; exit 130' INT TERM
|
|
|
|
# Run main function
|
|
main "$@" |