8d89fe80ff
- Archive: 85 agent .txt files → docs/archive/agents/legacy_txt/ - Scripts: Move 110 shell scripts → scripts/ (keep deploy.sh in root) - Models: Move 18 .safetensors → ml/models/checkpoints/training_artifacts/ - Delete: 34 directories (~33GB freed) - target/, coverage_*, test artifacts - Build: Clean 14 build artifacts (.rlib, .o, .pid, binaries) - Tests: Move 14 .rs files → tests/standalone/ - SQL: Move 5 files → sql/ (keep init-db*.sql for Docker) - Wave 153: Archive to docs/archive/historical/wave153/ - Docs: Archive 9 markdown files to wave_d/reports/ and historical/ Total impact: ~34GB freed (both waves), root directory cleaned from 583 to ~40 essential files Directory count reduced from 65 to 31 (52% reduction) All historical data preserved in organized archive structure
474 lines
16 KiB
Bash
Executable File
474 lines
16 KiB
Bash
Executable File
#!/bin/bash
|
|
# Foxhunt HFT System - Comprehensive Health Check Script
|
|
# Wave 75 Agent 6 - Service Health Validation
|
|
#
|
|
# This script validates:
|
|
# - 4 gRPC Application Services (ports 50050-50053)
|
|
# - 6 Infrastructure Services (PostgreSQL, Redis, Vault, InfluxDB, Prometheus, Grafana)
|
|
# - Inter-service communication
|
|
# - Resource usage
|
|
# - Hot-reload functionality
|
|
|
|
set -e
|
|
|
|
# Color codes for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Results tracking
|
|
TOTAL_CHECKS=0
|
|
PASSED_CHECKS=0
|
|
FAILED_CHECKS=0
|
|
WARNING_CHECKS=0
|
|
|
|
# Logging
|
|
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
|
|
LOG_DIR="./logs"
|
|
HEALTH_LOG="${LOG_DIR}/health_check_${TIMESTAMP}.log"
|
|
|
|
mkdir -p "$LOG_DIR"
|
|
|
|
# Logging functions
|
|
log_info() {
|
|
echo -e "${BLUE}[INFO]${NC} $1" | tee -a "$HEALTH_LOG"
|
|
}
|
|
|
|
log_success() {
|
|
echo -e "${GREEN}[PASS]${NC} $1" | tee -a "$HEALTH_LOG"
|
|
((PASSED_CHECKS++))
|
|
}
|
|
|
|
log_error() {
|
|
echo -e "${RED}[FAIL]${NC} $1" | tee -a "$HEALTH_LOG"
|
|
((FAILED_CHECKS++))
|
|
}
|
|
|
|
log_warning() {
|
|
echo -e "${YELLOW}[WARN]${NC} $1" | tee -a "$HEALTH_LOG"
|
|
((WARNING_CHECKS++))
|
|
}
|
|
|
|
log_header() {
|
|
echo -e "\n${BLUE}========================================${NC}" | tee -a "$HEALTH_LOG"
|
|
echo -e "${BLUE}$1${NC}" | tee -a "$HEALTH_LOG"
|
|
echo -e "${BLUE}========================================${NC}\n" | tee -a "$HEALTH_LOG"
|
|
}
|
|
|
|
# Check if required tools are installed
|
|
check_prerequisites() {
|
|
log_header "Checking Prerequisites"
|
|
|
|
local tools=("grpcurl" "psql" "curl" "jq" "docker")
|
|
local missing_tools=()
|
|
|
|
# redis-cli is optional (can use docker exec)
|
|
|
|
for tool in "${tools[@]}"; do
|
|
((TOTAL_CHECKS++))
|
|
if command -v "$tool" &> /dev/null; then
|
|
log_success "$tool is installed"
|
|
else
|
|
log_error "$tool is NOT installed"
|
|
missing_tools+=("$tool")
|
|
fi
|
|
done
|
|
|
|
if [ ${#missing_tools[@]} -gt 0 ]; then
|
|
log_error "Missing required tools: ${missing_tools[*]}"
|
|
log_info "Install missing tools before proceeding"
|
|
return 1
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
# Check gRPC service health
|
|
check_grpc_service() {
|
|
local service_name=$1
|
|
local port=$2
|
|
local package=$3
|
|
local service=$4
|
|
|
|
log_info "Checking $service_name on port $port..."
|
|
((TOTAL_CHECKS++))
|
|
|
|
# Check if port is listening
|
|
if ! netstat -tuln 2>/dev/null | grep -q ":$port "; then
|
|
if ! ss -tuln 2>/dev/null | grep -q ":$port "; then
|
|
log_error "$service_name: Port $port is NOT listening"
|
|
return 1
|
|
fi
|
|
fi
|
|
log_success "$service_name: Port $port is listening"
|
|
|
|
# List available services
|
|
((TOTAL_CHECKS++))
|
|
if grpcurl -plaintext localhost:$port list > /dev/null 2>&1; then
|
|
log_success "$service_name: gRPC server responding"
|
|
|
|
# Get service list
|
|
local services=$(grpcurl -plaintext localhost:$port list 2>/dev/null)
|
|
echo " Available services:" >> "$HEALTH_LOG"
|
|
echo "$services" | sed 's/^/ /' >> "$HEALTH_LOG"
|
|
else
|
|
log_error "$service_name: gRPC server NOT responding"
|
|
return 1
|
|
fi
|
|
|
|
# Check health endpoint
|
|
((TOTAL_CHECKS++))
|
|
if grpcurl -plaintext localhost:$port grpc.health.v1.Health/Check 2>&1 | grep -q "SERVING"; then
|
|
log_success "$service_name: Health check SERVING"
|
|
else
|
|
log_warning "$service_name: Health check returned non-SERVING status or not implemented"
|
|
fi
|
|
|
|
return 0
|
|
}
|
|
|
|
# Check infrastructure service
|
|
check_infrastructure_service() {
|
|
local service_name=$1
|
|
local check_command=$2
|
|
|
|
log_info "Checking $service_name..."
|
|
((TOTAL_CHECKS++))
|
|
|
|
if eval "$check_command" > /dev/null 2>&1; then
|
|
log_success "$service_name is healthy"
|
|
return 0
|
|
else
|
|
log_error "$service_name is NOT healthy"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Check gRPC application services
|
|
check_grpc_services() {
|
|
log_header "Checking gRPC Application Services"
|
|
|
|
# API Gateway (port 50050)
|
|
check_grpc_service "API Gateway" 50050 "foxhunt" "ApiGateway"
|
|
|
|
# Trading Service (port 50051)
|
|
check_grpc_service "Trading Service" 50051 "trading" "TradingService"
|
|
|
|
# Backtesting Service (port 50052)
|
|
check_grpc_service "Backtesting Service" 50052 "backtesting" "BacktestingService"
|
|
|
|
# ML Training Service (port 50053)
|
|
check_grpc_service "ML Training Service" 50053 "ml_training" "MLTrainingService"
|
|
}
|
|
|
|
# Check infrastructure services
|
|
check_infrastructure_services() {
|
|
log_header "Checking Infrastructure Services"
|
|
|
|
# PostgreSQL (port 5433)
|
|
log_info "Checking PostgreSQL on port 5433..."
|
|
((TOTAL_CHECKS++))
|
|
# Try test credentials first (from docker-compose)
|
|
if PGPASSWORD=test_password psql -h localhost -p 5433 -U foxhunt_test -d foxhunt_test -c "SELECT 1;" > /dev/null 2>&1; then
|
|
log_success "PostgreSQL is healthy (test database)"
|
|
|
|
# Check database exists and has tables
|
|
((TOTAL_CHECKS++))
|
|
local table_count=$(PGPASSWORD=test_password psql -h localhost -p 5433 -U foxhunt_test -d foxhunt_test -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public';" 2>/dev/null | tr -d ' ')
|
|
if [ "$table_count" -gt 0 ]; then
|
|
log_success "PostgreSQL has $table_count tables"
|
|
else
|
|
log_warning "PostgreSQL database exists but has no tables"
|
|
fi
|
|
elif PGPASSWORD=postgres psql -h localhost -p 5433 -U postgres -d foxhunt -c "SELECT 1;" > /dev/null 2>&1; then
|
|
log_success "PostgreSQL is healthy (production database)"
|
|
|
|
# Check database exists and has tables
|
|
((TOTAL_CHECKS++))
|
|
local table_count=$(PGPASSWORD=postgres psql -h localhost -p 5433 -U postgres -d foxhunt -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public';" 2>/dev/null | tr -d ' ')
|
|
if [ "$table_count" -gt 0 ]; then
|
|
log_success "PostgreSQL has $table_count tables"
|
|
else
|
|
log_warning "PostgreSQL database exists but has no tables"
|
|
fi
|
|
else
|
|
log_error "PostgreSQL is NOT healthy (tried both test and production credentials)"
|
|
fi
|
|
|
|
# Redis (port 6380)
|
|
log_info "Checking Redis on port 6380..."
|
|
((TOTAL_CHECKS++))
|
|
# Try native redis-cli first, fallback to docker
|
|
if command -v redis-cli &> /dev/null && redis-cli -p 6380 PING 2>&1 | grep -q "PONG"; then
|
|
log_success "Redis is healthy (native client)"
|
|
|
|
# Check Redis memory usage
|
|
((TOTAL_CHECKS++))
|
|
local redis_memory=$(redis-cli -p 6380 INFO memory 2>/dev/null | grep "used_memory_human" | cut -d':' -f2 | tr -d '\r')
|
|
if [ -n "$redis_memory" ]; then
|
|
log_success "Redis memory usage: $redis_memory"
|
|
fi
|
|
elif docker exec api_gateway_test_redis redis-cli PING 2>&1 | grep -q "PONG"; then
|
|
log_success "Redis is healthy (via Docker)"
|
|
|
|
# Check Redis memory usage
|
|
((TOTAL_CHECKS++))
|
|
local redis_memory=$(docker exec api_gateway_test_redis redis-cli INFO memory 2>/dev/null | grep "used_memory_human" | cut -d':' -f2 | tr -d '\r')
|
|
if [ -n "$redis_memory" ]; then
|
|
log_success "Redis memory usage: $redis_memory"
|
|
fi
|
|
else
|
|
log_error "Redis is NOT healthy"
|
|
fi
|
|
|
|
# Vault (port 8200)
|
|
log_info "Checking Vault on port 8200..."
|
|
((TOTAL_CHECKS++))
|
|
local vault_health=$(curl -s http://localhost:8200/v1/sys/health 2>/dev/null)
|
|
if [ -n "$vault_health" ]; then
|
|
local vault_sealed=$(echo "$vault_health" | jq -r '.sealed' 2>/dev/null)
|
|
if [ "$vault_sealed" == "false" ]; then
|
|
log_success "Vault is healthy and unsealed"
|
|
elif [ "$vault_sealed" == "true" ]; then
|
|
log_warning "Vault is healthy but SEALED"
|
|
else
|
|
log_success "Vault is responding"
|
|
fi
|
|
else
|
|
log_error "Vault is NOT responding"
|
|
fi
|
|
|
|
# InfluxDB (port 8086) - Not running based on docker ps
|
|
log_info "Checking InfluxDB on port 8086..."
|
|
((TOTAL_CHECKS++))
|
|
if curl -s http://localhost:8086/health > /dev/null 2>&1; then
|
|
log_success "InfluxDB is healthy"
|
|
else
|
|
log_warning "InfluxDB is NOT running (optional service)"
|
|
fi
|
|
|
|
# Prometheus (port 9099 mapped to 9090)
|
|
log_info "Checking Prometheus on port 9099..."
|
|
((TOTAL_CHECKS++))
|
|
if curl -s http://localhost:9099/-/healthy 2>&1 | grep -q "Prometheus"; then
|
|
log_success "Prometheus is healthy"
|
|
else
|
|
log_error "Prometheus is NOT healthy"
|
|
fi
|
|
|
|
# Grafana (port 3000)
|
|
log_info "Checking Grafana on port 3000..."
|
|
((TOTAL_CHECKS++))
|
|
local grafana_health=$(curl -s http://localhost:3000/api/health 2>/dev/null)
|
|
if echo "$grafana_health" | jq -e '.database == "ok"' > /dev/null 2>&1; then
|
|
log_success "Grafana is healthy"
|
|
else
|
|
log_warning "Grafana is responding but may have issues"
|
|
fi
|
|
}
|
|
|
|
# Check Docker containers
|
|
check_docker_containers() {
|
|
log_header "Checking Docker Containers"
|
|
|
|
log_info "Running Docker containers:"
|
|
docker ps --format "table {{.Names}}\t{{.Status}}\t{{.Ports}}" | tee -a "$HEALTH_LOG"
|
|
|
|
# Check for unhealthy containers
|
|
((TOTAL_CHECKS++))
|
|
local unhealthy=$(docker ps --filter "health=unhealthy" --format "{{.Names}}" 2>/dev/null)
|
|
if [ -z "$unhealthy" ]; then
|
|
log_success "No unhealthy containers detected"
|
|
else
|
|
log_error "Unhealthy containers detected: $unhealthy"
|
|
fi
|
|
}
|
|
|
|
# Check service processes
|
|
check_service_processes() {
|
|
log_header "Checking Service Processes"
|
|
|
|
local services=("trading_service" "backtesting_service" "ml_training_service" "api_gateway")
|
|
|
|
for service in "${services[@]}"; do
|
|
((TOTAL_CHECKS++))
|
|
if pgrep -f "$service" > /dev/null; then
|
|
local pid=$(pgrep -f "$service")
|
|
local mem_usage=$(ps -p $pid -o %mem --no-headers 2>/dev/null | tr -d ' ')
|
|
local cpu_usage=$(ps -p $pid -o %cpu --no-headers 2>/dev/null | tr -d ' ')
|
|
log_success "$service is running (PID: $pid, CPU: ${cpu_usage}%, MEM: ${mem_usage}%)"
|
|
else
|
|
log_error "$service is NOT running"
|
|
fi
|
|
done
|
|
}
|
|
|
|
# Check resource usage
|
|
check_resource_usage() {
|
|
log_header "Checking System Resource Usage"
|
|
|
|
# CPU usage
|
|
((TOTAL_CHECKS++))
|
|
local cpu_usage=$(top -bn1 | grep "Cpu(s)" | sed "s/.*, *\([0-9.]*\)%* id.*/\1/" | awk '{print 100 - $1}')
|
|
if (( $(echo "$cpu_usage < 80" | bc -l) )); then
|
|
log_success "CPU usage: ${cpu_usage}% (healthy)"
|
|
else
|
|
log_warning "CPU usage: ${cpu_usage}% (high)"
|
|
fi
|
|
|
|
# Memory usage
|
|
((TOTAL_CHECKS++))
|
|
local mem_total=$(free -g | awk '/^Mem:/{print $2}')
|
|
local mem_used=$(free -g | awk '/^Mem:/{print $3}')
|
|
local mem_percent=$(awk "BEGIN {printf \"%.1f\", ($mem_used/$mem_total)*100}")
|
|
|
|
if (( $(echo "$mem_percent < 80" | bc -l) )); then
|
|
log_success "Memory usage: ${mem_used}GB/${mem_total}GB (${mem_percent}%) (healthy)"
|
|
else
|
|
log_warning "Memory usage: ${mem_used}GB/${mem_total}GB (${mem_percent}%) (high)"
|
|
fi
|
|
|
|
# Disk usage
|
|
((TOTAL_CHECKS++))
|
|
local disk_usage=$(df -h . | awk 'NR==2 {print $5}' | sed 's/%//')
|
|
if [ "$disk_usage" -lt 80 ]; then
|
|
log_success "Disk usage: ${disk_usage}% (healthy)"
|
|
else
|
|
log_warning "Disk usage: ${disk_usage}% (high)"
|
|
fi
|
|
}
|
|
|
|
# Test inter-service communication
|
|
test_inter_service_communication() {
|
|
log_header "Testing Inter-Service Communication"
|
|
|
|
log_info "Testing API Gateway routing to Trading Service..."
|
|
((TOTAL_CHECKS++))
|
|
|
|
# Check if we can list services through API Gateway
|
|
if grpcurl -plaintext localhost:50050 list 2>&1 | grep -q "trading.TradingService"; then
|
|
log_success "API Gateway can see Trading Service"
|
|
else
|
|
log_warning "API Gateway may not have Trading Service registered"
|
|
fi
|
|
}
|
|
|
|
# Test hot-reload functionality
|
|
test_hot_reload() {
|
|
log_header "Testing Hot-Reload Functionality"
|
|
|
|
log_info "Checking PostgreSQL NOTIFY/LISTEN support..."
|
|
((TOTAL_CHECKS++))
|
|
|
|
# Check if config_settings table exists (try both databases)
|
|
if PGPASSWORD=test_password psql -h localhost -p 5433 -U foxhunt_test -d foxhunt_test -c "\d config_settings" > /dev/null 2>&1; then
|
|
log_success "config_settings table exists (test database)"
|
|
|
|
# Count configuration entries
|
|
((TOTAL_CHECKS++))
|
|
local config_count=$(PGPASSWORD=test_password psql -h localhost -p 5433 -U foxhunt_test -d foxhunt_test -t -c "SELECT COUNT(*) FROM config_settings;" 2>/dev/null | tr -d ' ')
|
|
if [ "$config_count" -gt 0 ]; then
|
|
log_success "Found $config_count configuration entries"
|
|
else
|
|
log_warning "config_settings table exists but is empty"
|
|
fi
|
|
elif PGPASSWORD=postgres psql -h localhost -p 5433 -U postgres -d foxhunt -c "\d config_settings" > /dev/null 2>&1; then
|
|
log_success "config_settings table exists (production database)"
|
|
|
|
# Count configuration entries
|
|
((TOTAL_CHECKS++))
|
|
local config_count=$(PGPASSWORD=postgres psql -h localhost -p 5433 -U postgres -d foxhunt -t -c "SELECT COUNT(*) FROM config_settings;" 2>/dev/null | tr -d ' ')
|
|
if [ "$config_count" -gt 0 ]; then
|
|
log_success "Found $config_count configuration entries"
|
|
else
|
|
log_warning "config_settings table exists but is empty"
|
|
fi
|
|
else
|
|
log_warning "config_settings table does not exist (hot-reload may not be configured)"
|
|
fi
|
|
}
|
|
|
|
# Check service logs for errors
|
|
check_service_logs() {
|
|
log_header "Checking Service Logs for Recent Errors"
|
|
|
|
local log_files=("api_gateway.log" "trading_service.log" "backtesting_service.log" "ml_training_service.log")
|
|
|
|
for log_file in "${log_files[@]}"; do
|
|
local log_path="${LOG_DIR}/${log_file}"
|
|
if [ -f "$log_path" ]; then
|
|
((TOTAL_CHECKS++))
|
|
local error_count=$(grep -i "error\|panic\|fatal" "$log_path" 2>/dev/null | wc -l)
|
|
|
|
if [ "$error_count" -eq 0 ]; then
|
|
log_success "$log_file: No errors detected"
|
|
elif [ "$error_count" -lt 5 ]; then
|
|
log_warning "$log_file: $error_count errors detected (review recommended)"
|
|
else
|
|
log_error "$log_file: $error_count errors detected (attention required)"
|
|
fi
|
|
else
|
|
log_warning "$log_file: Log file not found"
|
|
fi
|
|
done
|
|
}
|
|
|
|
# Generate summary report
|
|
generate_summary() {
|
|
log_header "Health Check Summary"
|
|
|
|
echo "" | tee -a "$HEALTH_LOG"
|
|
echo "Total Checks: $TOTAL_CHECKS" | tee -a "$HEALTH_LOG"
|
|
echo -e "${GREEN}Passed: $PASSED_CHECKS${NC}" | tee -a "$HEALTH_LOG"
|
|
echo -e "${YELLOW}Warnings: $WARNING_CHECKS${NC}" | tee -a "$HEALTH_LOG"
|
|
echo -e "${RED}Failed: $FAILED_CHECKS${NC}" | tee -a "$HEALTH_LOG"
|
|
echo "" | tee -a "$HEALTH_LOG"
|
|
|
|
local success_rate=$(awk "BEGIN {printf \"%.1f\", ($PASSED_CHECKS/$TOTAL_CHECKS)*100}")
|
|
echo "Success Rate: ${success_rate}%" | tee -a "$HEALTH_LOG"
|
|
|
|
if [ "$FAILED_CHECKS" -eq 0 ]; then
|
|
echo -e "${GREEN}Overall Status: HEALTHY${NC}" | tee -a "$HEALTH_LOG"
|
|
return 0
|
|
elif [ "$FAILED_CHECKS" -lt 5 ]; then
|
|
echo -e "${YELLOW}Overall Status: DEGRADED${NC}" | tee -a "$HEALTH_LOG"
|
|
return 1
|
|
else
|
|
echo -e "${RED}Overall Status: UNHEALTHY${NC}" | tee -a "$HEALTH_LOG"
|
|
return 2
|
|
fi
|
|
}
|
|
|
|
# Main execution
|
|
main() {
|
|
log_header "Foxhunt HFT System - Comprehensive Health Check"
|
|
log_info "Starting health check at $(date)"
|
|
log_info "Log file: $HEALTH_LOG"
|
|
echo ""
|
|
|
|
# Run all checks
|
|
check_prerequisites || exit 1
|
|
check_docker_containers
|
|
check_infrastructure_services
|
|
check_grpc_services
|
|
check_service_processes
|
|
check_resource_usage
|
|
test_inter_service_communication
|
|
test_hot_reload
|
|
check_service_logs
|
|
|
|
# Generate summary
|
|
echo ""
|
|
generate_summary
|
|
|
|
log_info "Health check completed at $(date)"
|
|
log_info "Detailed log saved to: $HEALTH_LOG"
|
|
}
|
|
|
|
# Run main function
|
|
main
|
|
|
|
exit $?
|