foxhunt/deployment/create_systemd_services.sh

#!/bin/bash

#============================================================================
# FOXHUNT HFT TRADING SYSTEM - SYSTEMD SERVICE GENERATOR
#============================================================================
# Generates optimized SystemD service files for all Foxhunt services
# Configured for high-performance, low-latency trading operations
#
# Usage:
#   ./create_systemd_services.sh [OPTIONS]
#
# Options:
#   --output-dir DIR     Output directory for service files (default: ./systemd)
#   --user USER          Service user (default: foxhunt)
#   --data-dir DIR       Data directory (default: /opt/foxhunt)
#   --enable-gpu        Enable GPU support for ML services
#   --cpu-affinity      Enable CPU affinity for performance
#   --help              Show this help message
#============================================================================

set -euo pipefail

# Configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
OUTPUT_DIR="${OUTPUT_DIR:-$SCRIPT_DIR/systemd}"
SERVICE_USER="${SERVICE_USER:-foxhunt}"
DATA_DIR="${DATA_DIR:-/opt/foxhunt}"
ENABLE_GPU="${ENABLE_GPU:-false}"
CPU_AFFINITY="${CPU_AFFINITY:-true}"

# Service configurations
declare -A SERVICES=(
    ["trading"]="trading_service:8080:50051:\"Core trading engine with ultra-low latency execution\""
    ["ml-training"]="ml_training_service:8082:50052:\"Machine learning model training and inference service\""
    ["backtesting"]="backtesting_service:8083:50053:\"Strategy backtesting and historical analysis service\""
    ["tli"]="tli:8081:50054:\"Terminal Line Interface for system management and monitoring\""
)

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_info() {
    echo -e "${BLUE}[INFO]${NC} $1"
}

log_success() {
    echo -e "${GREEN}[SUCCESS]${NC} $1"
}

log_warning() {
    echo -e "${YELLOW}[WARNING]${NC} $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

show_help() {
    cat << EOF
Foxhunt HFT Trading System - SystemD Service Generator

Usage: $0 [OPTIONS]

OPTIONS:
    --output-dir DIR     Output directory for service files (default: ./systemd)
    --user USER          Service user (default: foxhunt)
    --data-dir DIR       Data directory (default: /opt/foxhunt)
    --enable-gpu        Enable GPU support for ML services
    --cpu-affinity      Enable CPU affinity for performance
    --help              Show this help message

SERVICES GENERATED:
    foxhunt-trading      Core trading engine service
    foxhunt-ml-training  ML training and inference service  
    foxhunt-backtesting  Backtesting and analysis service
    foxhunt-tli          Terminal Line Interface service

EXAMPLES:
    $0                                    # Generate all service files
    $0 --output-dir /etc/systemd/system   # Generate directly to system directory
    $0 --enable-gpu --cpu-affinity        # Generate with GPU and CPU optimizations

EOF
}

# Parse arguments
parse_args() {
    while [[ $# -gt 0 ]]; do
        case $1 in
            --output-dir)
                OUTPUT_DIR="$2"
                shift 2
                ;;
            --user)
                SERVICE_USER="$2"
                shift 2
                ;;
            --data-dir)
                DATA_DIR="$2"
                shift 2
                ;;
            --enable-gpu)
                ENABLE_GPU=true
                shift
                ;;
            --cpu-affinity)
                CPU_AFFINITY=true
                shift
                ;;
            --help)
                show_help
                exit 0
                ;;
            *)
                log_error "Unknown option: $1"
                show_help
                exit 1
                ;;
        esac
    done
}

# Create output directory
setup_output_directory() {
    log_info "Setting up output directory: $OUTPUT_DIR"
    
    mkdir -p "$OUTPUT_DIR"
    
    if [[ ! -w "$OUTPUT_DIR" ]]; then
        log_error "Output directory is not writable: $OUTPUT_DIR"
        exit 1
    fi
    
    log_success "Output directory ready: $OUTPUT_DIR"
}

# Generate common service environment
generate_common_environment() {
    local service_name="$1"
    
    cat << EOF
# Production environment configuration
Environment=RUST_LOG=info
Environment=RUST_BACKTRACE=0
Environment=RUST_LOG_STYLE=never

# Application-specific environment
Environment=FOXHUNT_ENV=production
Environment=FOXHUNT_SERVICE_HOST=0.0.0.0
Environment=FOXHUNT_CONFIG_PATH=$DATA_DIR/config
Environment=FOXHUNT_DATA_PATH=$DATA_DIR/data
Environment=FOXHUNT_MODELS_PATH=$DATA_DIR/models
Environment=FOXHUNT_CACHE_PATH=$DATA_DIR/cache
Environment=FOXHUNT_LOG_LEVEL=info
Environment=FOXHUNT_LOG_FORMAT=json

# Security environment
Environment=FOXHUNT_SECURITY_ENABLED=true
Environment=FOXHUNT_TLS_ENABLED=true

# Performance environment
Environment=FOXHUNT_THREAD_POOL_SIZE=0
Environment=FOXHUNT_MAX_CONNECTIONS=1000
Environment=FOXHUNT_LATENCY_TARGET_MS=50

# Database connections
Environment=DATABASE_URL=postgresql://foxhunt:password@localhost:5432/foxhunt
Environment=REDIS_URL=redis://localhost:6379/0
Environment=INFLUXDB_URL=http://localhost:8086

# Vault integration
Environment=VAULT_ADDR=http://localhost:8200
Environment=VAULT_TOKEN_FILE=$DATA_DIR/config/vault-token
EOF

    if [[ "$ENABLE_GPU" == "true" ]] && [[ "$service_name" == "ml-training" ]]; then
        cat << EOF

# GPU configuration
Environment=CUDA_VISIBLE_DEVICES=0
Environment=FOXHUNT_GPU_ENABLED=true
Environment=FOXHUNT_GPU_MEMORY_FRACTION=0.8
EOF
    fi
}

# Generate performance optimizations
generate_performance_config() {
    local service_name="$1"
    local cpu_cores=""
    
    # CPU affinity configuration
    if [[ "$CPU_AFFINITY" == "true" ]]; then
        case "$service_name" in
            "trading")
                cpu_cores="0-3"  # Reserve first 4 cores for trading
                ;;
            "ml-training")
                cpu_cores="4-7"  # Use cores 4-7 for ML
                ;;
            "backtesting")
                cpu_cores="8-11" # Use cores 8-11 for backtesting
                ;;
            "tli")
                cpu_cores="12-15" # Use remaining cores for TLI
                ;;
        esac
    fi
    
    cat << EOF

# Performance optimizations
Nice=-10
IOSchedulingClass=1
IOSchedulingPriority=4
CPUSchedulingPolicy=1
CPUSchedulingPriority=50

# Resource limits
LimitNOFILE=65536
LimitNPROC=32768
LimitCORE=infinity
LimitMEMLOCK=infinity

# Memory management
MemoryAccounting=yes
MemoryMax=8G
MemorySwapMax=0

# Process management
TasksMax=4096
EOF

    if [[ -n "$cpu_cores" ]]; then
        cat << EOF
CPUAffinity=$cpu_cores
EOF
    fi

    # Service-specific optimizations
    case "$service_name" in
        "trading")
            cat << EOF

# Trading-specific optimizations
OOMScoreAdjust=-500
Environment=FOXHUNT_LATENCY_TARGET_MS=1
Environment=FOXHUNT_HIGH_PRIORITY=true
EOF
            ;;
        "ml-training")
            cat << EOF

# ML training optimizations
MemoryMax=16G
Environment=FOXHUNT_ML_BATCH_SIZE=1000
Environment=FOXHUNT_ML_WORKERS=4
EOF
            ;;
    esac
}

# Generate security configuration
generate_security_config() {
    cat << EOF

# Security hardening
NoNewPrivileges=yes
ProtectSystem=strict
ProtectHome=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
ProtectControlGroups=yes
RestrictRealtime=yes
RestrictNamespaces=yes
LockPersonality=yes
MemoryDenyWriteExecute=yes
RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
SystemCallFilter=@system-service
SystemCallErrorNumber=EPERM

# Filesystem isolation
ReadWritePaths=$DATA_DIR
ReadOnlyPaths=/etc /usr
TemporaryFileSystem=/tmp
PrivateTmp=yes
PrivateDevices=yes

# Network isolation
IPAddressDeny=any
IPAddressAllow=localhost
IPAddressAllow=10.0.0.0/8
IPAddressAllow=172.16.0.0/12
IPAddressAllow=192.168.0.0/16
EOF
}

# Generate service file
generate_service_file() {
    local service_key="$1"
    local service_config="${SERVICES[$service_key]}"
    
    IFS=':' read -r binary_name http_port grpc_port description <<< "$service_config"
    description=$(echo "$description" | sed 's/"//g')
    
    local service_name="foxhunt-$service_key"
    local service_file="$OUTPUT_DIR/$service_name.service"
    
    log_info "Generating service file: $service_name.service"
    
    cat > "$service_file" << EOF
# ============================================================================
# Foxhunt HFT Trading System - $service_name Service
# ============================================================================
# SystemD service file for ${description}
# 
# Generated: $(date)
# Service: $service_key
# Binary: $binary_name
# HTTP Port: $http_port
# gRPC Port: $grpc_port
# ============================================================================

[Unit]
Description=$description
Documentation=https://github.com/user/foxhunt
After=network.target postgresql.service redis.service
Wants=postgresql.service redis.service
Requires=network.target

# Service dependencies
EOF

    # Add service-specific dependencies
    case "$service_key" in
        "ml-training")
            echo "After=foxhunt-trading.service" >> "$service_file"
            ;;
        "backtesting")
            echo "After=foxhunt-trading.service" >> "$service_file"
            ;;
        "tli")
            echo "After=foxhunt-trading.service foxhunt-ml-training.service" >> "$service_file"
            ;;
    esac

    # Determine if service needs subcommand
    local exec_command="$DATA_DIR/bin/$binary_name"
    if [[ "$binary_name" == "ml_training_service" ]]; then
        exec_command="$DATA_DIR/bin/$binary_name serve"
    fi

    cat >> "$service_file" << EOF

[Service]
Type=exec
ExecStart=$exec_command
ExecReload=/bin/kill -HUP \$MAINPID
Restart=always
RestartSec=5
TimeoutStartSec=60
TimeoutStopSec=30
KillMode=mixed
KillSignal=SIGTERM

# User and group
User=$SERVICE_USER
Group=$SERVICE_USER
WorkingDirectory=$DATA_DIR

# Logging
StandardOutput=journal
StandardError=journal
SyslogIdentifier=$service_name

EOF

    # Add common environment
    generate_common_environment "$service_key" >> "$service_file"
    
    # Add service-specific environment
    case "$service_key" in
        "trading")
            cat >> "$service_file" << EOF

# Trading service specific environment
Environment=FOXHUNT_HTTP_PORT=$http_port
Environment=FOXHUNT_GRPC_PORT=$grpc_port
Environment=FOXHUNT_METRICS_PORT=9090
Environment=FOXHUNT_TRADING_MODE=live
EOF
            ;;
        "ml-training")
            cat >> "$service_file" << EOF

# ML training service specific environment
Environment=FOXHUNT_HTTP_PORT=$http_port
Environment=FOXHUNT_GRPC_PORT=$grpc_port
Environment=FOXHUNT_METRICS_PORT=9091
Environment=FOXHUNT_MODEL_PATH=$DATA_DIR/models
Environment=FOXHUNT_CHECKPOINT_PATH=$DATA_DIR/checkpoints
EOF
            ;;
        "backtesting")
            cat >> "$service_file" << EOF

# Backtesting service specific environment
Environment=FOXHUNT_HTTP_PORT=$http_port
Environment=FOXHUNT_GRPC_PORT=$grpc_port
Environment=FOXHUNT_METRICS_PORT=9092
Environment=FOXHUNT_BACKTEST_PATH=$DATA_DIR/backtests
EOF
            ;;
        "tli")
            cat >> "$service_file" << EOF

# TLI service specific environment
Environment=FOXHUNT_HTTP_PORT=$http_port
Environment=FOXHUNT_GRPC_PORT=$grpc_port
Environment=FOXHUNT_METRICS_PORT=9093
Environment=FOXHUNT_WEB_ROOT=$DATA_DIR/web
EOF
            ;;
    esac
    
    # Add performance configuration
    generate_performance_config "$service_key" >> "$service_file"
    
    # Add security configuration
    generate_security_config >> "$service_file"
    
    cat >> "$service_file" << EOF

[Install]
WantedBy=multi-user.target
Alias=$binary_name.service

# ============================================================================
# Service Management Commands:
#
# Enable service:    systemctl enable $service_name
# Start service:     systemctl start $service_name
# Stop service:      systemctl stop $service_name
# Restart service:   systemctl restart $service_name
# View status:       systemctl status $service_name
# View logs:         journalctl -f -u $service_name
# ============================================================================
EOF
    
    log_success "Generated service file: $service_file"
}

# Generate target file for managing all services
generate_target_file() {
    local target_file="$OUTPUT_DIR/foxhunt.target"
    
    log_info "Generating target file: foxhunt.target"
    
    cat > "$target_file" << EOF
# ============================================================================
# Foxhunt HFT Trading System - Master Target
# ============================================================================
# SystemD target for managing all Foxhunt services as a unit
# 
# Generated: $(date)
# ============================================================================

[Unit]
Description=Foxhunt HFT Trading System
Documentation=https://github.com/user/foxhunt
After=network.target postgresql.service redis.service
Wants=foxhunt-trading.service foxhunt-ml-training.service foxhunt-backtesting.service foxhunt-tli.service

[Install]
WantedBy=multi-user.target

# ============================================================================
# Target Management Commands:
#
# Enable all services:  systemctl enable foxhunt.target
# Start all services:   systemctl start foxhunt.target  
# Stop all services:    systemctl stop foxhunt.target
# View status:          systemctl status foxhunt.target
# ============================================================================
EOF
    
    log_success "Generated target file: $target_file"
}

# Generate installation script
generate_install_script() {
    local install_script="$OUTPUT_DIR/install_services.sh"
    
    log_info "Generating installation script: install_services.sh"
    
    cat > "$install_script" << EOF
#!/bin/bash

#============================================================================
# FOXHUNT SYSTEMD SERVICES - INSTALLATION SCRIPT
#============================================================================
# Installs SystemD service files and enables them
#============================================================================

set -euo pipefail

# Check if running as root
if [[ \$EUID -ne 0 ]]; then
    echo "This script must be run as root"
    exit 1
fi

SCRIPT_DIR="\$(cd "\$(dirname "\${BASH_SOURCE[0]}")" && pwd)"

echo "Installing Foxhunt SystemD services..."

# Copy service files to systemd directory
cp "\$SCRIPT_DIR"/*.service /etc/systemd/system/
cp "\$SCRIPT_DIR"/*.target /etc/systemd/system/

# Reload systemd configuration
systemctl daemon-reload

# Enable services
systemctl enable foxhunt-trading.service
systemctl enable foxhunt-ml-training.service
systemctl enable foxhunt-backtesting.service
systemctl enable foxhunt-tli.service
systemctl enable foxhunt.target

echo "SystemD services installed and enabled successfully!"
echo "Start all services with: systemctl start foxhunt.target"
EOF
    
    chmod +x "$install_script"
    log_success "Generated installation script: $install_script"
}

# Generate uninstall script
generate_uninstall_script() {
    local uninstall_script="$OUTPUT_DIR/uninstall_services.sh"
    
    log_info "Generating uninstall script: uninstall_services.sh"
    
    cat > "$uninstall_script" << EOF
#!/bin/bash

#============================================================================
# FOXHUNT SYSTEMD SERVICES - UNINSTALLATION SCRIPT
#============================================================================
# Removes SystemD service files and disables them
#============================================================================

set -euo pipefail

# Check if running as root
if [[ \$EUID -ne 0 ]]; then
    echo "This script must be run as root"
    exit 1
fi

echo "Uninstalling Foxhunt SystemD services..."

# Stop and disable services
systemctl stop foxhunt.target 2>/dev/null || true
systemctl disable foxhunt.target 2>/dev/null || true
systemctl disable foxhunt-trading.service 2>/dev/null || true
systemctl disable foxhunt-ml-training.service 2>/dev/null || true
systemctl disable foxhunt-backtesting.service 2>/dev/null || true
systemctl disable foxhunt-tli.service 2>/dev/null || true

# Remove service files
rm -f /etc/systemd/system/foxhunt*.service
rm -f /etc/systemd/system/foxhunt.target

# Reload systemd configuration
systemctl daemon-reload

echo "SystemD services uninstalled successfully!"
EOF
    
    chmod +x "$uninstall_script"
    log_success "Generated uninstall script: $uninstall_script"
}

# Generate summary
generate_summary() {
    cat << EOF

${GREEN}=============================================================================
FOXHUNT HFT TRADING SYSTEM - SYSTEMD SERVICES GENERATED
=============================================================================${NC}

${BLUE}Generated Files:${NC}
$(ls -la "$OUTPUT_DIR" | tail -n +2 | awk '{print "  • " $9 " (" $5 " bytes)"}')

${BLUE}Service Configuration:${NC}
  • User:               $SERVICE_USER
  • Data Directory:     $DATA_DIR
  • GPU Support:        $ENABLE_GPU
  • CPU Affinity:       $CPU_AFFINITY

${BLUE}Installation Commands:${NC}
  • Install services:   sudo $OUTPUT_DIR/install_services.sh
  • Start all services: sudo systemctl start foxhunt.target
  • Check status:       systemctl status foxhunt.target
  • View logs:          journalctl -f -u foxhunt-*

${BLUE}Individual Service Commands:${NC}
EOF

    for service in "${!SERVICES[@]}"; do
        echo "  • foxhunt-$service:     systemctl start foxhunt-$service"
    done

    cat << EOF

${GREEN}SystemD service files generated successfully!${NC}

EOF
}

# Main generation logic
main() {
    parse_args "$@"
    
    log_info "Generating Foxhunt SystemD service files..."
    log_info "Output directory: $OUTPUT_DIR"
    log_info "Service user: $SERVICE_USER"
    log_info "Data directory: $DATA_DIR"
    
    setup_output_directory
    
    # Generate service files
    for service in "${!SERVICES[@]}"; do
        generate_service_file "$service"
    done
    
    # Generate target and utility files
    generate_target_file
    generate_install_script
    generate_uninstall_script
    
    generate_summary
    
    log_success "Foxhunt SystemD service generation completed!"
}

# Handle interruption
trap 'log_error "Generation interrupted"; exit 1' INT TERM

# Run main function
main "$@"