569 lines
15 KiB
Bash
Executable File
569 lines
15 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
#============================================================================
|
|
# FOXHUNT HFT TRADING SYSTEM - PRODUCTION DEPLOYMENT SCRIPT
|
|
#============================================================================
|
|
# Deploys the complete Foxhunt HFT system to production environment
|
|
# Supports bare metal, Docker, and Kubernetes deployments
|
|
#
|
|
# Usage:
|
|
# ./deploy_production.sh [OPTIONS]
|
|
#
|
|
# Options:
|
|
# --mode MODE Deployment mode: bare-metal, docker, k8s (default: bare-metal)
|
|
# --env ENV Environment: production, staging, testing (default: production)
|
|
# --config-dir DIR Configuration directory (default: ./config)
|
|
# --data-dir DIR Data directory (default: /opt/foxhunt/data)
|
|
# --skip-health Skip health checks after deployment
|
|
# --rollback Rollback to previous deployment
|
|
# --dry-run Show what would be deployed without executing
|
|
# --help Show this help message
|
|
#============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
# Configuration
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
DEPLOY_MODE="${DEPLOY_MODE:-bare-metal}"
|
|
ENVIRONMENT="${ENVIRONMENT:-production}"
|
|
CONFIG_DIR="${CONFIG_DIR:-$PROJECT_ROOT/config}"
|
|
DATA_DIR="${DATA_DIR:-/opt/foxhunt}"
|
|
SKIP_HEALTH="${SKIP_HEALTH:-false}"
|
|
DRY_RUN="${DRY_RUN:-false}"
|
|
ROLLBACK="${ROLLBACK:-false}"
|
|
|
|
# Service configuration
|
|
SERVICES=(
|
|
"trading-service:8080:50051"
|
|
"ml-training-service:8082:50052"
|
|
"backtesting-service:8083:50053"
|
|
"tli:8081:50054"
|
|
)
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m'
|
|
|
|
log_info() {
|
|
echo -e "${BLUE}[INFO]${NC} $1"
|
|
}
|
|
|
|
log_success() {
|
|
echo -e "${GREEN}[SUCCESS]${NC} $1"
|
|
}
|
|
|
|
log_warning() {
|
|
echo -e "${YELLOW}[WARNING]${NC} $1"
|
|
}
|
|
|
|
log_error() {
|
|
echo -e "${RED}[ERROR]${NC} $1"
|
|
}
|
|
|
|
show_help() {
|
|
cat << EOF
|
|
Foxhunt HFT Trading System - Production Deployment
|
|
|
|
Usage: $0 [OPTIONS]
|
|
|
|
OPTIONS:
|
|
--mode MODE Deployment mode (bare-metal, docker, k8s)
|
|
--env ENV Environment (production, staging, testing)
|
|
--config-dir DIR Configuration directory
|
|
--data-dir DIR Data directory
|
|
--skip-health Skip health checks after deployment
|
|
--rollback Rollback to previous deployment
|
|
--dry-run Show what would be deployed without executing
|
|
--help Show this help message
|
|
|
|
DEPLOYMENT MODES:
|
|
bare-metal Deploy as SystemD services (recommended for HFT)
|
|
docker Deploy using Docker Compose
|
|
k8s Deploy to Kubernetes cluster
|
|
|
|
EXAMPLES:
|
|
$0 # Deploy to production bare-metal
|
|
$0 --mode docker --env staging # Deploy to staging with Docker
|
|
$0 --rollback # Rollback to previous deployment
|
|
$0 --dry-run # Preview deployment
|
|
|
|
EOF
|
|
}
|
|
|
|
# Parse arguments
|
|
parse_args() {
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
--mode)
|
|
DEPLOY_MODE="$2"
|
|
shift 2
|
|
;;
|
|
--env)
|
|
ENVIRONMENT="$2"
|
|
shift 2
|
|
;;
|
|
--config-dir)
|
|
CONFIG_DIR="$2"
|
|
shift 2
|
|
;;
|
|
--data-dir)
|
|
DATA_DIR="$2"
|
|
shift 2
|
|
;;
|
|
--skip-health)
|
|
SKIP_HEALTH=true
|
|
shift
|
|
;;
|
|
--rollback)
|
|
ROLLBACK=true
|
|
shift
|
|
;;
|
|
--dry-run)
|
|
DRY_RUN=true
|
|
shift
|
|
;;
|
|
--help)
|
|
show_help
|
|
exit 0
|
|
;;
|
|
*)
|
|
log_error "Unknown option: $1"
|
|
show_help
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
# Execute command with dry-run support
|
|
execute() {
|
|
if [[ "$DRY_RUN" == "true" ]]; then
|
|
log_info "[DRY-RUN] Would execute: $*"
|
|
else
|
|
"$@"
|
|
fi
|
|
}
|
|
|
|
# Check prerequisites
|
|
check_prerequisites() {
|
|
log_info "Checking deployment prerequisites..."
|
|
|
|
# Check if running as root for bare-metal deployment
|
|
if [[ "$DEPLOY_MODE" == "bare-metal" ]] && [[ $EUID -ne 0 ]] && [[ "$DRY_RUN" == "false" ]]; then
|
|
log_error "Bare-metal deployment requires root privileges"
|
|
log_info "Run with: sudo $0"
|
|
exit 1
|
|
fi
|
|
|
|
# Check deployment mode specific requirements
|
|
case "$DEPLOY_MODE" in
|
|
bare-metal)
|
|
check_systemd
|
|
;;
|
|
docker)
|
|
check_docker
|
|
;;
|
|
k8s)
|
|
check_kubernetes
|
|
;;
|
|
*)
|
|
log_error "Unknown deployment mode: $DEPLOY_MODE"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
# Check if binaries exist
|
|
local binary_dir="$PROJECT_ROOT/target/release"
|
|
local required_binaries=("trading_service" "ml_training_service" "backtesting_service" "tli")
|
|
|
|
for binary in "${required_binaries[@]}"; do
|
|
if [[ ! -f "$binary_dir/$binary" ]]; then
|
|
log_error "Required binary not found: $binary_dir/$binary"
|
|
log_info "Run: ./deployment/build_release.sh"
|
|
exit 1
|
|
fi
|
|
done
|
|
|
|
log_success "Prerequisites check completed"
|
|
}
|
|
|
|
# Check SystemD availability
|
|
check_systemd() {
|
|
if ! command -v systemctl &> /dev/null; then
|
|
log_error "SystemD is not available - required for bare-metal deployment"
|
|
exit 1
|
|
fi
|
|
|
|
log_info "SystemD available for bare-metal deployment"
|
|
}
|
|
|
|
# Check Docker availability
|
|
check_docker() {
|
|
if ! command -v docker &> /dev/null; then
|
|
log_error "Docker is not installed"
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v docker-compose &> /dev/null; then
|
|
log_error "Docker Compose is not installed"
|
|
exit 1
|
|
fi
|
|
|
|
log_info "Docker environment ready"
|
|
}
|
|
|
|
# Check Kubernetes availability
|
|
check_kubernetes() {
|
|
if ! command -v kubectl &> /dev/null; then
|
|
log_error "kubectl is not installed"
|
|
exit 1
|
|
fi
|
|
|
|
if ! kubectl cluster-info &> /dev/null; then
|
|
log_error "Kubernetes cluster is not accessible"
|
|
exit 1
|
|
fi
|
|
|
|
log_info "Kubernetes cluster ready"
|
|
}
|
|
|
|
# Setup deployment directories
|
|
setup_directories() {
|
|
log_info "Setting up deployment directories..."
|
|
|
|
local dirs=(
|
|
"$DATA_DIR"
|
|
"$DATA_DIR/config"
|
|
"$DATA_DIR/data"
|
|
"$DATA_DIR/models"
|
|
"$DATA_DIR/backtests"
|
|
"$DATA_DIR/logs"
|
|
"$DATA_DIR/bin"
|
|
"$DATA_DIR/systemd"
|
|
"/var/log/foxhunt"
|
|
"/etc/systemd/system"
|
|
)
|
|
|
|
for dir in "${dirs[@]}"; do
|
|
execute mkdir -p "$dir"
|
|
if [[ "$DEPLOY_MODE" == "bare-metal" ]]; then
|
|
execute chown -R foxhunt:foxhunt "$dir" 2>/dev/null || true
|
|
fi
|
|
done
|
|
|
|
log_success "Deployment directories created"
|
|
}
|
|
|
|
# Create system user for bare-metal deployment
|
|
create_system_user() {
|
|
if [[ "$DEPLOY_MODE" != "bare-metal" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
log_info "Creating system user for Foxhunt services..."
|
|
|
|
if ! id -u foxhunt &>/dev/null; then
|
|
execute useradd -r -s /bin/false -d "$DATA_DIR" -c "Foxhunt HFT Trading System" foxhunt
|
|
log_success "Created system user: foxhunt"
|
|
else
|
|
log_info "System user already exists: foxhunt"
|
|
fi
|
|
}
|
|
|
|
# Deploy binaries
|
|
deploy_binaries() {
|
|
log_info "Deploying application binaries..."
|
|
|
|
local binary_dir="$PROJECT_ROOT/target/release"
|
|
local binaries=("trading_service" "ml_training_service" "backtesting_service" "tli")
|
|
|
|
for binary in "${binaries[@]}"; do
|
|
local src="$binary_dir/$binary"
|
|
local dst="$DATA_DIR/bin/$binary"
|
|
|
|
execute cp "$src" "$dst"
|
|
execute chmod +x "$dst"
|
|
|
|
if [[ "$DEPLOY_MODE" == "bare-metal" ]]; then
|
|
execute chown foxhunt:foxhunt "$dst"
|
|
fi
|
|
|
|
log_success "Deployed binary: $binary"
|
|
done
|
|
}
|
|
|
|
# Deploy configuration
|
|
deploy_configuration() {
|
|
log_info "Deploying configuration files..."
|
|
|
|
if [[ -d "$CONFIG_DIR" ]]; then
|
|
execute cp -r "$CONFIG_DIR"/* "$DATA_DIR/config/"
|
|
|
|
if [[ "$DEPLOY_MODE" == "bare-metal" ]]; then
|
|
execute chown -R foxhunt:foxhunt "$DATA_DIR/config"
|
|
execute chmod -R 640 "$DATA_DIR/config"
|
|
fi
|
|
|
|
log_success "Configuration deployed"
|
|
else
|
|
log_warning "Configuration directory not found: $CONFIG_DIR"
|
|
fi
|
|
}
|
|
|
|
# Deploy SystemD services
|
|
deploy_systemd_services() {
|
|
if [[ "$DEPLOY_MODE" != "bare-metal" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
log_info "Deploying SystemD services..."
|
|
|
|
# Generate service files
|
|
"$SCRIPT_DIR/create_systemd_services.sh" --output-dir "$DATA_DIR/systemd"
|
|
|
|
# Install service files
|
|
for service_file in "$DATA_DIR/systemd"/*.service; do
|
|
local service_name=$(basename "$service_file")
|
|
execute cp "$service_file" "/etc/systemd/system/"
|
|
log_info "Installed service: $service_name"
|
|
done
|
|
|
|
execute systemctl daemon-reload
|
|
log_success "SystemD services deployed"
|
|
}
|
|
|
|
# Start services based on deployment mode
|
|
start_services() {
|
|
log_info "Starting Foxhunt services..."
|
|
|
|
case "$DEPLOY_MODE" in
|
|
bare-metal)
|
|
start_systemd_services
|
|
;;
|
|
docker)
|
|
start_docker_services
|
|
;;
|
|
k8s)
|
|
start_kubernetes_services
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Start SystemD services
|
|
start_systemd_services() {
|
|
local services=("foxhunt-trading" "foxhunt-ml-training" "foxhunt-backtesting" "foxhunt-tli")
|
|
|
|
for service in "${services[@]}"; do
|
|
execute systemctl enable "$service"
|
|
execute systemctl start "$service"
|
|
log_success "Started service: $service"
|
|
done
|
|
}
|
|
|
|
# Start Docker services
|
|
start_docker_services() {
|
|
execute docker-compose -f "$PROJECT_ROOT/docker-compose.production.yml" up -d
|
|
log_success "Started Docker services"
|
|
}
|
|
|
|
# Start Kubernetes services
|
|
start_kubernetes_services() {
|
|
execute kubectl apply -f "$PROJECT_ROOT/k8s/"
|
|
log_success "Deployed to Kubernetes"
|
|
}
|
|
|
|
# Health check
|
|
perform_health_check() {
|
|
if [[ "$SKIP_HEALTH" == "true" ]]; then
|
|
log_warning "Skipping health checks as requested"
|
|
return 0
|
|
fi
|
|
|
|
log_info "Performing deployment health check..."
|
|
|
|
local failed_services=()
|
|
|
|
for service_config in "${SERVICES[@]}"; do
|
|
IFS=':' read -r service_name http_port grpc_port <<< "$service_config"
|
|
|
|
log_info "Checking $service_name..."
|
|
|
|
# Check HTTP health endpoint
|
|
if curl -f -s "http://localhost:$http_port/health" >/dev/null; then
|
|
log_success "$service_name HTTP endpoint is healthy"
|
|
else
|
|
log_warning "$service_name HTTP endpoint is not responding"
|
|
failed_services+=("$service_name-http")
|
|
fi
|
|
|
|
# Check gRPC health endpoint (if available)
|
|
if command -v grpcurl &> /dev/null; then
|
|
if grpcurl -plaintext "localhost:$grpc_port" grpc.health.v1.Health/Check >/dev/null 2>&1; then
|
|
log_success "$service_name gRPC endpoint is healthy"
|
|
else
|
|
log_warning "$service_name gRPC endpoint is not responding"
|
|
failed_services+=("$service_name-grpc")
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if [[ ${#failed_services[@]} -eq 0 ]]; then
|
|
log_success "All services are healthy"
|
|
else
|
|
log_warning "Some services failed health check: ${failed_services[*]}"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Rollback deployment
|
|
perform_rollback() {
|
|
if [[ "$ROLLBACK" != "true" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
log_info "Performing deployment rollback..."
|
|
|
|
case "$DEPLOY_MODE" in
|
|
bare-metal)
|
|
rollback_systemd
|
|
;;
|
|
docker)
|
|
rollback_docker
|
|
;;
|
|
k8s)
|
|
rollback_kubernetes
|
|
;;
|
|
esac
|
|
|
|
log_success "Rollback completed"
|
|
}
|
|
|
|
# Rollback SystemD services
|
|
rollback_systemd() {
|
|
local backup_dir="$DATA_DIR/backup/$(date +%Y%m%d-%H%M%S)"
|
|
|
|
if [[ -d "$backup_dir" ]]; then
|
|
log_info "Restoring from backup: $backup_dir"
|
|
execute cp -r "$backup_dir"/* "$DATA_DIR/"
|
|
execute systemctl daemon-reload
|
|
execute systemctl restart foxhunt-*
|
|
else
|
|
log_error "No backup found for rollback"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# Generate deployment summary
|
|
generate_summary() {
|
|
log_info "Generating deployment summary..."
|
|
|
|
cat << EOF
|
|
|
|
${GREEN}=============================================================================
|
|
FOXHUNT HFT TRADING SYSTEM - DEPLOYMENT SUMMARY
|
|
=============================================================================${NC}
|
|
|
|
${BLUE}Deployment Details:${NC}
|
|
• Mode: $DEPLOY_MODE
|
|
• Environment: $ENVIRONMENT
|
|
• Data Directory: $DATA_DIR
|
|
• Deployment Time: $(date)
|
|
|
|
${BLUE}Services Deployed:${NC}
|
|
EOF
|
|
|
|
for service_config in "${SERVICES[@]}"; do
|
|
IFS=':' read -r service_name http_port grpc_port <<< "$service_config"
|
|
echo " • $service_name http://localhost:$http_port (gRPC: $grpc_port)"
|
|
done
|
|
|
|
cat << EOF
|
|
|
|
${BLUE}Management Commands:${NC}
|
|
EOF
|
|
|
|
case "$DEPLOY_MODE" in
|
|
bare-metal)
|
|
cat << EOF
|
|
• View logs: journalctl -f -u foxhunt-*
|
|
• Restart service: systemctl restart foxhunt-<service>
|
|
• Stop all: systemctl stop foxhunt-*
|
|
• Service status: systemctl status foxhunt-*
|
|
EOF
|
|
;;
|
|
docker)
|
|
cat << EOF
|
|
• View logs: docker-compose logs -f
|
|
• Restart service: docker-compose restart <service>
|
|
• Stop all: docker-compose down
|
|
• Service status: docker-compose ps
|
|
EOF
|
|
;;
|
|
k8s)
|
|
cat << EOF
|
|
• View logs: kubectl logs -f deployment/<service>
|
|
• Restart service: kubectl rollout restart deployment/<service>
|
|
• Stop all: kubectl delete -f k8s/
|
|
• Service status: kubectl get pods
|
|
EOF
|
|
;;
|
|
esac
|
|
|
|
cat << EOF
|
|
|
|
${BLUE}Monitoring:${NC}
|
|
• Prometheus: http://localhost:9090
|
|
• Grafana: http://localhost:3000
|
|
• Health Checks: ./deployment/health_check.sh
|
|
|
|
${GREEN}Deployment completed successfully!${NC}
|
|
|
|
EOF
|
|
}
|
|
|
|
# Main deployment logic
|
|
main() {
|
|
parse_args "$@"
|
|
|
|
log_info "Starting Foxhunt HFT Trading System deployment..."
|
|
log_info "Mode: $DEPLOY_MODE, Environment: $ENVIRONMENT"
|
|
|
|
if [[ "$DRY_RUN" == "true" ]]; then
|
|
log_warning "DRY-RUN MODE - No changes will be made"
|
|
fi
|
|
|
|
check_prerequisites
|
|
|
|
if [[ "$ROLLBACK" == "true" ]]; then
|
|
perform_rollback
|
|
exit 0
|
|
fi
|
|
|
|
setup_directories
|
|
create_system_user
|
|
deploy_binaries
|
|
deploy_configuration
|
|
deploy_systemd_services
|
|
start_services
|
|
|
|
# Wait for services to initialize
|
|
if [[ "$SKIP_HEALTH" == "false" ]]; then
|
|
log_info "Waiting for services to initialize..."
|
|
sleep 30
|
|
perform_health_check
|
|
fi
|
|
|
|
generate_summary
|
|
|
|
log_success "Foxhunt HFT Trading System deployment completed!"
|
|
}
|
|
|
|
# Handle interruption
|
|
trap 'log_error "Deployment interrupted"; exit 1' INT TERM
|
|
|
|
# Run main function
|
|
main "$@" |