diff --git a/deployment/DEPLOYMENT_CHECKLIST.md b/deployment/DEPLOYMENT_CHECKLIST.md new file mode 100644 index 000000000..8618046b4 --- /dev/null +++ b/deployment/DEPLOYMENT_CHECKLIST.md @@ -0,0 +1,457 @@ +# FOXHUNT HFT TRADING SYSTEM - PRODUCTION DEPLOYMENT CHECKLIST + +## 📋 Pre-Deployment Phase + +### ✅ Infrastructure Preparation + +#### Hardware Requirements +- [ ] **Server Hardware Validated** + - [ ] CPU: Intel Xeon or AMD EPYC with AVX2 support + - [ ] RAM: Minimum 32GB, recommended 64GB+ for ML models + - [ ] Storage: NVMe SSD with 10GB+ free space + - [ ] Network: Low-latency connection to exchanges (sub-10ms preferred) + - [ ] GPU (optional): NVIDIA GPU with CUDA 12.0+ for ML acceleration + +#### Operating System +- [ ] **Linux Environment Ready** + - [ ] Ubuntu 22.04 LTS or RHEL 8.x/9.x installed + - [ ] Kernel version 5.15+ for optimal performance + - [ ] CPU frequency scaling disabled (performance governor) + - [ ] Transparent huge pages disabled + - [ ] NUMA topology optimized if multi-socket + +#### Network Configuration +- [ ] **Network Optimized** + - [ ] Firewall rules configured for service ports + - [ ] Time synchronization (NTP/PTP) configured with < 1ms accuracy + - [ ] Network interface MTU optimized (9000 for jumbo frames) + - [ ] TCP/UDP buffer sizes tuned for high frequency + - [ ] Exchange connectivity tested (FIX/WebSocket endpoints) + +### ✅ Software Dependencies + +#### System Packages +- [ ] **Required Packages Installed** + - [ ] `build-essential`, `cmake`, `pkg-config` + - [ ] `libssl-dev`, `libpq-dev`, `libc6-dev` + - [ ] `curl`, `wget`, `git`, `htop`, `systemd` + - [ ] NVIDIA drivers and CUDA toolkit (if using GPU) + +#### Runtime Dependencies +- [ ] **Runtime Libraries Available** + - [ ] `libssl3`, `libpq5`, `ca-certificates` + - [ ] CUDA runtime libraries (if GPU enabled) + - [ ] Database client libraries (PostgreSQL, Redis) + +#### Database Systems +- [ ] **Database Infrastructure Ready** + - [ ] PostgreSQL 15+ installed and configured + - [ ] Redis 7+ installed with persistence enabled + - [ ] InfluxDB 2.x installed for time-series data + - [ ] Database schemas created and migrated + - [ ] Connection pools configured with appropriate limits + +#### Security Infrastructure +- [ ] **Security Components Deployed** + - [ ] HashiCorp Vault installed and initialized + - [ ] SSL/TLS certificates generated and installed + - [ ] SSH key-based authentication configured + - [ ] Fail2ban or similar intrusion prevention active + - [ ] Log aggregation system (ELK/Grafana Loki) ready + +### ✅ Configuration Management + +#### Environment Configuration +- [ ] **Environment Files Created** + - [ ] `.env.production` configured with production values + - [ ] Database connection strings encrypted in Vault + - [ ] API keys and secrets stored securely in Vault + - [ ] Exchange credentials configured and tested + +#### Service Configuration +- [ ] **Service Configs Validated** + - [ ] Trading service configuration reviewed + - [ ] ML model paths and parameters configured + - [ ] Risk management parameters set appropriately + - [ ] Monitoring and alerting thresholds defined + +#### Compliance Configuration +- [ ] **Regulatory Compliance Ready** + - [ ] SOX compliance logging enabled + - [ ] MiFID II transaction reporting configured + - [ ] Best execution tracking parameters set + - [ ] Audit trail configuration verified + +--- + +## 🚀 Deployment Phase + +### ✅ Build and Package + +#### Binary Compilation +- [ ] **Release Binaries Built** + ```bash + ./deployment/build_release.sh --cpu-target native + ``` + - [ ] Trading service binary compiled and optimized + - [ ] ML training service binary ready + - [ ] Backtesting service binary compiled + - [ ] TLI (Terminal Line Interface) binary ready + - [ ] All binaries stripped and checksummed + +#### Docker Images (if using containers) +- [ ] **Docker Images Built** + ```bash + ./deployment/build_docker_images.sh --service all --enable-gpu + ``` + - [ ] Multi-stage builds completed successfully + - [ ] Image sizes optimized (< 200MB per service) + - [ ] GPU support enabled if required + - [ ] Images tagged with version and commit hash + +#### Deployment Artifacts +- [ ] **Deployment Package Ready** + - [ ] SystemD service files generated + - [ ] Configuration files bundled + - [ ] Deployment scripts executable + - [ ] Health check scripts prepared + +### ✅ Deployment Execution + +#### Pre-Deployment Validation +- [ ] **System Readiness Check** + ```bash + ./deployment/deploy_production.sh --dry-run + ``` + - [ ] All prerequisites satisfied + - [ ] No conflicting processes on target ports + - [ ] Sufficient disk space available + - [ ] Network connectivity to all endpoints verified + +#### Service Deployment +- [ ] **Services Deployed Successfully** + ```bash + sudo ./deployment/deploy_production.sh --mode bare-metal + ``` + - [ ] System user 'foxhunt' created + - [ ] Binaries installed in `/opt/foxhunt/bin/` + - [ ] Configuration files deployed + - [ ] SystemD services installed and enabled + +#### Database Initialization +- [ ] **Database Setup Complete** + - [ ] PostgreSQL schemas created + - [ ] Initial configuration data loaded + - [ ] Redis configuration cache populated + - [ ] InfluxDB retention policies configured + +### ✅ Service Startup + +#### Core Services +- [ ] **Trading Service Started** + ```bash + systemctl start foxhunt-trading + systemctl status foxhunt-trading + ``` + - [ ] Service started without errors + - [ ] HTTP health endpoint responding (port 8080) + - [ ] gRPC endpoint accessible (port 50051) + - [ ] Prometheus metrics available (port 9090) + +- [ ] **ML Training Service Started** + ```bash + systemctl start foxhunt-ml-training + systemctl status foxhunt-ml-training + ``` + - [ ] Service started without errors + - [ ] HTTP health endpoint responding (port 8082) + - [ ] gRPC endpoint accessible (port 50052) + - [ ] GPU utilization visible (if GPU enabled) + +- [ ] **Backtesting Service Started** + ```bash + systemctl start foxhunt-backtesting + systemctl status foxhunt-backtesting + ``` + - [ ] Service started without errors + - [ ] HTTP health endpoint responding (port 8083) + - [ ] gRPC endpoint accessible (port 50053) + - [ ] Historical data access confirmed + +- [ ] **TLI Interface Started** + ```bash + systemctl start foxhunt-tli + systemctl status foxhunt-tli + ``` + - [ ] Service started without errors + - [ ] Web interface accessible (port 8081) + - [ ] gRPC client connections working + - [ ] Real-time data updates functioning + +--- + +## 🔍 Post-Deployment Validation + +### ✅ Health Checks + +#### Service Health Verification +- [ ] **All Services Healthy** + ```bash + ./deployment/health_check.sh --comprehensive + ``` + - [ ] HTTP endpoints returning 200 OK + - [ ] gRPC health checks passing + - [ ] Service dependencies connected + - [ ] Memory usage within normal ranges + +#### Performance Validation +- [ ] **Performance Metrics Verified** + - [ ] Trading service latency < 50Ξs (target: 14ns) + - [ ] ML inference time < 100ms + - [ ] Database query response < 10ms + - [ ] Network round-trip time optimized + +#### Database Connectivity +- [ ] **Database Operations Verified** + - [ ] PostgreSQL connections established + - [ ] Redis cache operations working + - [ ] InfluxDB time-series writes successful + - [ ] Configuration hot-reload tested + +### ✅ Integration Testing + +#### Service Communication +- [ ] **Inter-Service Communication Tested** + - [ ] TLI → Trading Service gRPC calls working + - [ ] Trading Service → ML Service requests successful + - [ ] Backtesting Service → Data Service integration verified + - [ ] Configuration updates propagating across services + +#### External Integrations +- [ ] **Exchange Connectivity Verified** + - [ ] Market data feeds connected and streaming + - [ ] Order placement and execution working + - [ ] WebSocket connections stable + - [ ] FIX protocol handshake successful + +#### Security Integration +- [ ] **Security Controls Active** + - [ ] Vault secrets accessible to services + - [ ] SSL/TLS certificates valid and trusted + - [ ] Authentication/authorization working + - [ ] Audit logging functional + +### ✅ Monitoring and Alerting + +#### Monitoring Setup +- [ ] **Monitoring Stack Active** + - [ ] Prometheus scraping all service metrics + - [ ] Grafana dashboards displaying real-time data + - [ ] Log aggregation collecting from all services + - [ ] Application performance monitoring active + +#### Alerting Configuration +- [ ] **Alerts Configured and Tested** + - [ ] High latency alerts (> 100Ξs) + - [ ] Error rate alerts (> 1%) + - [ ] Memory usage alerts (> 80%) + - [ ] Disk space alerts (> 90%) + - [ ] Service downtime alerts + - [ ] Exchange connectivity alerts + +#### Compliance Monitoring +- [ ] **Regulatory Monitoring Active** + - [ ] SOX audit trails being generated + - [ ] MiFID II transaction reporting working + - [ ] Best execution monitoring active + - [ ] Risk limit monitoring functional + +--- + +## 📊 Production Readiness Validation + +### ✅ Load Testing + +#### Performance Testing +- [ ] **Load Tests Executed** + ```bash + ./tests/load_test.sh --scenario production + ``` + - [ ] High-frequency trading simulation passed + - [ ] Concurrent user load testing completed + - [ ] ML model inference under load verified + - [ ] Database performance under load confirmed + +#### Stress Testing +- [ ] **Stress Tests Passed** + - [ ] Memory leak detection completed + - [ ] CPU usage spike testing passed + - [ ] Network congestion handling verified + - [ ] Recovery from resource exhaustion tested + +### ✅ Disaster Recovery + +#### Backup Verification +- [ ] **Backup Systems Tested** + - [ ] Database backups completing successfully + - [ ] Configuration backups automated + - [ ] Model checkpoints being saved + - [ ] Log rotation and archival working + +#### Recovery Procedures +- [ ] **Recovery Plans Tested** + - [ ] Service restart procedures documented + - [ ] Database recovery tested + - [ ] Configuration rollback procedures verified + - [ ] Failover scenarios documented + +### ✅ Security Validation + +#### Security Testing +- [ ] **Security Scan Completed** + - [ ] Vulnerability scan passed + - [ ] Port scan results reviewed + - [ ] SSL/TLS configuration verified + - [ ] Access control testing completed + +#### Penetration Testing +- [ ] **Security Penetration Tests** + - [ ] Web interface security tested + - [ ] API endpoint security verified + - [ ] Network security validated + - [ ] Social engineering resistance confirmed + +--- + +## ðŸŽŊ Go-Live Checklist + +### ✅ Final Preparation + +#### Documentation +- [ ] **Documentation Complete** + - [ ] Deployment runbook updated + - [ ] Troubleshooting guide available + - [ ] User manuals distributed + - [ ] Change management procedures documented + +#### Team Readiness +- [ ] **Team Prepared** + - [ ] Operations team trained on new deployment + - [ ] Support team familiar with troubleshooting + - [ ] Development team on standby for issues + - [ ] Emergency contacts list distributed + +#### Communication +- [ ] **Stakeholders Informed** + - [ ] Go-live schedule communicated + - [ ] Risk assessment shared + - [ ] Rollback plan communicated + - [ ] Success criteria defined + +### ✅ Go-Live Execution + +#### Production Cutover +- [ ] **Live Trading Enabled** + - [ ] Paper trading mode disabled + - [ ] Real exchange connections activated + - [ ] Live order routing enabled + - [ ] Risk limits set to production values + +#### Monitoring Activation +- [ ] **Production Monitoring Active** + - [ ] Real-time alerts enabled + - [ ] Dashboard monitoring started + - [ ] Log analysis automated + - [ ] Performance baselines established + +#### Final Verification +- [ ] **Production Verification Complete** + - [ ] First live trades executed successfully + - [ ] All services responding normally + - [ ] No critical alerts triggered + - [ ] Performance within expected ranges + +--- + +## 📈 Post Go-Live + +### ✅ Immediate Post-Launch (0-24 hours) + +#### Intensive Monitoring +- [ ] **24x7 Monitoring Active** + - [ ] System performance trending normally + - [ ] No memory leaks detected + - [ ] Trading latency within targets + - [ ] All scheduled jobs completing successfully + +#### Issue Response +- [ ] **Incident Response Ready** + - [ ] No P1/P2 incidents reported + - [ ] Minor issues documented and tracked + - [ ] Performance tuning opportunities identified + - [ ] User feedback collected and analyzed + +### ✅ Stabilization Period (1-7 days) + +#### Performance Optimization +- [ ] **System Optimization Completed** + - [ ] Performance bottlenecks identified and resolved + - [ ] Resource utilization optimized + - [ ] Alert thresholds fine-tuned + - [ ] Capacity planning validated + +#### Operational Handoff +- [ ] **Operations Team Transition** + - [ ] 24x7 support schedule activated + - [ ] Escalation procedures tested + - [ ] Knowledge transfer completed + - [ ] Runbook validated through actual use + +--- + +## 🎉 Deployment Success Criteria + +### Key Performance Indicators (KPIs) +- [ ] **Performance Targets Met** + - [ ] Trading latency: < 50Ξs average, < 100Ξs 99th percentile + - [ ] System availability: > 99.95% + - [ ] Order success rate: > 99.9% + - [ ] ML model accuracy: > 85% on validation set + +### Business Metrics +- [ ] **Business Objectives Achieved** + - [ ] Daily trading volume targets met + - [ ] Profit/loss within expected ranges + - [ ] Risk limits respected and effective + - [ ] Regulatory compliance maintained + +### Operational Metrics +- [ ] **Operational Excellence Demonstrated** + - [ ] Zero unplanned downtime in first 48 hours + - [ ] All scheduled maintenance windows respected + - [ ] Support ticket volume within normal ranges + - [ ] User satisfaction scores above threshold + +--- + +## 📞 Emergency Contacts + +### Critical Escalation +- **System Admin**: [Name] - [Phone] - [Email] +- **Lead Developer**: [Name] - [Phone] - [Email] +- **DevOps Engineer**: [Name] - [Phone] - [Email] +- **Database Administrator**: [Name] - [Phone] - [Email] + +### Business Contacts +- **Trading Operations**: [Name] - [Phone] - [Email] +- **Risk Management**: [Name] - [Phone] - [Email] +- **Compliance Officer**: [Name] - [Phone] - [Email] +- **Executive Sponsor**: [Name] - [Phone] - [Email] + +--- + +**Deployment Checklist Version**: 1.0.0 +**Last Updated**: $(date) +**Next Review**: 30 days post go-live + +*This checklist ensures comprehensive validation of the Foxhunt HFT Trading System deployment from infrastructure preparation through production stabilization.* \ No newline at end of file diff --git a/deployment/create_systemd_services.sh b/deployment/create_systemd_services.sh new file mode 100755 index 000000000..c71c9be49 --- /dev/null +++ b/deployment/create_systemd_services.sh @@ -0,0 +1,651 @@ +#!/bin/bash + +#============================================================================ +# FOXHUNT HFT TRADING SYSTEM - SYSTEMD SERVICE GENERATOR +#============================================================================ +# Generates optimized SystemD service files for all Foxhunt services +# Configured for high-performance, low-latency trading operations +# +# Usage: +# ./create_systemd_services.sh [OPTIONS] +# +# Options: +# --output-dir DIR Output directory for service files (default: ./systemd) +# --user USER Service user (default: foxhunt) +# --data-dir DIR Data directory (default: /opt/foxhunt) +# --enable-gpu Enable GPU support for ML services +# --cpu-affinity Enable CPU affinity for performance +# --help Show this help message +#============================================================================ + +set -euo pipefail + +# Configuration +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUTPUT_DIR="${OUTPUT_DIR:-$SCRIPT_DIR/systemd}" +SERVICE_USER="${SERVICE_USER:-foxhunt}" +DATA_DIR="${DATA_DIR:-/opt/foxhunt}" +ENABLE_GPU="${ENABLE_GPU:-false}" +CPU_AFFINITY="${CPU_AFFINITY:-true}" + +# Service configurations +declare -A SERVICES=( + ["trading"]="trading_service:8080:50051:\"Core trading engine with ultra-low latency execution\"" + ["ml-training"]="ml_training_service:8082:50052:\"Machine learning model training and inference service\"" + ["backtesting"]="backtesting_service:8083:50053:\"Strategy backtesting and historical analysis service\"" + ["tli"]="tli:8081:50054:\"Terminal Line Interface for system management and monitoring\"" +) + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +show_help() { + cat << EOF +Foxhunt HFT Trading System - SystemD Service Generator + +Usage: $0 [OPTIONS] + +OPTIONS: + --output-dir DIR Output directory for service files (default: ./systemd) + --user USER Service user (default: foxhunt) + --data-dir DIR Data directory (default: /opt/foxhunt) + --enable-gpu Enable GPU support for ML services + --cpu-affinity Enable CPU affinity for performance + --help Show this help message + +SERVICES GENERATED: + foxhunt-trading Core trading engine service + foxhunt-ml-training ML training and inference service + foxhunt-backtesting Backtesting and analysis service + foxhunt-tli Terminal Line Interface service + +EXAMPLES: + $0 # Generate all service files + $0 --output-dir /etc/systemd/system # Generate directly to system directory + $0 --enable-gpu --cpu-affinity # Generate with GPU and CPU optimizations + +EOF +} + +# Parse arguments +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + --output-dir) + OUTPUT_DIR="$2" + shift 2 + ;; + --user) + SERVICE_USER="$2" + shift 2 + ;; + --data-dir) + DATA_DIR="$2" + shift 2 + ;; + --enable-gpu) + ENABLE_GPU=true + shift + ;; + --cpu-affinity) + CPU_AFFINITY=true + shift + ;; + --help) + show_help + exit 0 + ;; + *) + log_error "Unknown option: $1" + show_help + exit 1 + ;; + esac + done +} + +# Create output directory +setup_output_directory() { + log_info "Setting up output directory: $OUTPUT_DIR" + + mkdir -p "$OUTPUT_DIR" + + if [[ ! -w "$OUTPUT_DIR" ]]; then + log_error "Output directory is not writable: $OUTPUT_DIR" + exit 1 + fi + + log_success "Output directory ready: $OUTPUT_DIR" +} + +# Generate common service environment +generate_common_environment() { + local service_name="$1" + + cat << EOF +# Production environment configuration +Environment=RUST_LOG=info +Environment=RUST_BACKTRACE=0 +Environment=RUST_LOG_STYLE=never + +# Application-specific environment +Environment=FOXHUNT_ENV=production +Environment=FOXHUNT_SERVICE_HOST=0.0.0.0 +Environment=FOXHUNT_CONFIG_PATH=$DATA_DIR/config +Environment=FOXHUNT_DATA_PATH=$DATA_DIR/data +Environment=FOXHUNT_MODELS_PATH=$DATA_DIR/models +Environment=FOXHUNT_CACHE_PATH=$DATA_DIR/cache +Environment=FOXHUNT_LOG_LEVEL=info +Environment=FOXHUNT_LOG_FORMAT=json + +# Security environment +Environment=FOXHUNT_SECURITY_ENABLED=true +Environment=FOXHUNT_TLS_ENABLED=true + +# Performance environment +Environment=FOXHUNT_THREAD_POOL_SIZE=0 +Environment=FOXHUNT_MAX_CONNECTIONS=1000 +Environment=FOXHUNT_LATENCY_TARGET_MS=50 + +# Database connections +Environment=DATABASE_URL=postgresql://foxhunt:password@localhost:5432/foxhunt +Environment=REDIS_URL=redis://localhost:6379/0 +Environment=INFLUXDB_URL=http://localhost:8086 + +# Vault integration +Environment=VAULT_ADDR=http://localhost:8200 +Environment=VAULT_TOKEN_FILE=$DATA_DIR/config/vault-token +EOF + + if [[ "$ENABLE_GPU" == "true" ]] && [[ "$service_name" == "ml-training" ]]; then + cat << EOF + +# GPU configuration +Environment=CUDA_VISIBLE_DEVICES=0 +Environment=FOXHUNT_GPU_ENABLED=true +Environment=FOXHUNT_GPU_MEMORY_FRACTION=0.8 +EOF + fi +} + +# Generate performance optimizations +generate_performance_config() { + local service_name="$1" + local cpu_cores="" + + # CPU affinity configuration + if [[ "$CPU_AFFINITY" == "true" ]]; then + case "$service_name" in + "trading") + cpu_cores="0-3" # Reserve first 4 cores for trading + ;; + "ml-training") + cpu_cores="4-7" # Use cores 4-7 for ML + ;; + "backtesting") + cpu_cores="8-11" # Use cores 8-11 for backtesting + ;; + "tli") + cpu_cores="12-15" # Use remaining cores for TLI + ;; + esac + fi + + cat << EOF + +# Performance optimizations +Nice=-10 +IOSchedulingClass=1 +IOSchedulingPriority=4 +CPUSchedulingPolicy=1 +CPUSchedulingPriority=50 + +# Resource limits +LimitNOFILE=65536 +LimitNPROC=32768 +LimitCORE=infinity +LimitMEMLOCK=infinity + +# Memory management +MemoryAccounting=yes +MemoryMax=8G +MemorySwapMax=0 + +# Process management +TasksMax=4096 +EOF + + if [[ -n "$cpu_cores" ]]; then + cat << EOF +CPUAffinity=$cpu_cores +EOF + fi + + # Service-specific optimizations + case "$service_name" in + "trading") + cat << EOF + +# Trading-specific optimizations +OOMScoreAdjust=-500 +Environment=FOXHUNT_LATENCY_TARGET_MS=1 +Environment=FOXHUNT_HIGH_PRIORITY=true +EOF + ;; + "ml-training") + cat << EOF + +# ML training optimizations +MemoryMax=16G +Environment=FOXHUNT_ML_BATCH_SIZE=1000 +Environment=FOXHUNT_ML_WORKERS=4 +EOF + ;; + esac +} + +# Generate security configuration +generate_security_config() { + cat << EOF + +# Security hardening +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +ProtectKernelTunables=yes +ProtectKernelModules=yes +ProtectControlGroups=yes +RestrictRealtime=yes +RestrictNamespaces=yes +LockPersonality=yes +MemoryDenyWriteExecute=yes +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +SystemCallFilter=@system-service +SystemCallErrorNumber=EPERM + +# Filesystem isolation +ReadWritePaths=$DATA_DIR +ReadOnlyPaths=/etc /usr +TemporaryFileSystem=/tmp +PrivateTmp=yes +PrivateDevices=yes + +# Network isolation +IPAddressDeny=any +IPAddressAllow=localhost +IPAddressAllow=10.0.0.0/8 +IPAddressAllow=172.16.0.0/12 +IPAddressAllow=192.168.0.0/16 +EOF +} + +# Generate service file +generate_service_file() { + local service_key="$1" + local service_config="${SERVICES[$service_key]}" + + IFS=':' read -r binary_name http_port grpc_port description <<< "$service_config" + description=$(echo "$description" | sed 's/"//g') + + local service_name="foxhunt-$service_key" + local service_file="$OUTPUT_DIR/$service_name.service" + + log_info "Generating service file: $service_name.service" + + cat > "$service_file" << EOF +# ============================================================================ +# Foxhunt HFT Trading System - $service_name Service +# ============================================================================ +# SystemD service file for ${description} +# +# Generated: $(date) +# Service: $service_key +# Binary: $binary_name +# HTTP Port: $http_port +# gRPC Port: $grpc_port +# ============================================================================ + +[Unit] +Description=$description +Documentation=https://github.com/user/foxhunt +After=network.target postgresql.service redis.service +Wants=postgresql.service redis.service +Requires=network.target + +# Service dependencies +EOF + + # Add service-specific dependencies + case "$service_key" in + "ml-training") + echo "After=foxhunt-trading.service" >> "$service_file" + ;; + "backtesting") + echo "After=foxhunt-trading.service" >> "$service_file" + ;; + "tli") + echo "After=foxhunt-trading.service foxhunt-ml-training.service" >> "$service_file" + ;; + esac + + cat >> "$service_file" << EOF + +[Service] +Type=exec +ExecStart=$DATA_DIR/bin/$binary_name +ExecReload=/bin/kill -HUP \$MAINPID +Restart=always +RestartSec=5 +TimeoutStartSec=60 +TimeoutStopSec=30 +KillMode=mixed +KillSignal=SIGTERM + +# User and group +User=$SERVICE_USER +Group=$SERVICE_USER +WorkingDirectory=$DATA_DIR + +# Logging +StandardOutput=journal +StandardError=journal +SyslogIdentifier=$service_name + +EOF + + # Add common environment + generate_common_environment "$service_key" >> "$service_file" + + # Add service-specific environment + case "$service_key" in + "trading") + cat >> "$service_file" << EOF + +# Trading service specific environment +Environment=FOXHUNT_HTTP_PORT=$http_port +Environment=FOXHUNT_GRPC_PORT=$grpc_port +Environment=FOXHUNT_METRICS_PORT=9090 +Environment=FOXHUNT_TRADING_MODE=live +EOF + ;; + "ml-training") + cat >> "$service_file" << EOF + +# ML training service specific environment +Environment=FOXHUNT_HTTP_PORT=$http_port +Environment=FOXHUNT_GRPC_PORT=$grpc_port +Environment=FOXHUNT_METRICS_PORT=9091 +Environment=FOXHUNT_MODEL_PATH=$DATA_DIR/models +Environment=FOXHUNT_CHECKPOINT_PATH=$DATA_DIR/checkpoints +EOF + ;; + "backtesting") + cat >> "$service_file" << EOF + +# Backtesting service specific environment +Environment=FOXHUNT_HTTP_PORT=$http_port +Environment=FOXHUNT_GRPC_PORT=$grpc_port +Environment=FOXHUNT_METRICS_PORT=9092 +Environment=FOXHUNT_BACKTEST_PATH=$DATA_DIR/backtests +EOF + ;; + "tli") + cat >> "$service_file" << EOF + +# TLI service specific environment +Environment=FOXHUNT_HTTP_PORT=$http_port +Environment=FOXHUNT_GRPC_PORT=$grpc_port +Environment=FOXHUNT_METRICS_PORT=9093 +Environment=FOXHUNT_WEB_ROOT=$DATA_DIR/web +EOF + ;; + esac + + # Add performance configuration + generate_performance_config "$service_key" >> "$service_file" + + # Add security configuration + generate_security_config >> "$service_file" + + cat >> "$service_file" << EOF + +[Install] +WantedBy=multi-user.target +Alias=$binary_name.service + +# ============================================================================ +# Service Management Commands: +# +# Enable service: systemctl enable $service_name +# Start service: systemctl start $service_name +# Stop service: systemctl stop $service_name +# Restart service: systemctl restart $service_name +# View status: systemctl status $service_name +# View logs: journalctl -f -u $service_name +# ============================================================================ +EOF + + log_success "Generated service file: $service_file" +} + +# Generate target file for managing all services +generate_target_file() { + local target_file="$OUTPUT_DIR/foxhunt.target" + + log_info "Generating target file: foxhunt.target" + + cat > "$target_file" << EOF +# ============================================================================ +# Foxhunt HFT Trading System - Master Target +# ============================================================================ +# SystemD target for managing all Foxhunt services as a unit +# +# Generated: $(date) +# ============================================================================ + +[Unit] +Description=Foxhunt HFT Trading System +Documentation=https://github.com/user/foxhunt +After=network.target postgresql.service redis.service +Wants=foxhunt-trading.service foxhunt-ml-training.service foxhunt-backtesting.service foxhunt-tli.service + +[Install] +WantedBy=multi-user.target + +# ============================================================================ +# Target Management Commands: +# +# Enable all services: systemctl enable foxhunt.target +# Start all services: systemctl start foxhunt.target +# Stop all services: systemctl stop foxhunt.target +# View status: systemctl status foxhunt.target +# ============================================================================ +EOF + + log_success "Generated target file: $target_file" +} + +# Generate installation script +generate_install_script() { + local install_script="$OUTPUT_DIR/install_services.sh" + + log_info "Generating installation script: install_services.sh" + + cat > "$install_script" << EOF +#!/bin/bash + +#============================================================================ +# FOXHUNT SYSTEMD SERVICES - INSTALLATION SCRIPT +#============================================================================ +# Installs SystemD service files and enables them +#============================================================================ + +set -euo pipefail + +# Check if running as root +if [[ \$EUID -ne 0 ]]; then + echo "This script must be run as root" + exit 1 +fi + +SCRIPT_DIR="\$(cd "\$(dirname "\${BASH_SOURCE[0]}")" && pwd)" + +echo "Installing Foxhunt SystemD services..." + +# Copy service files to systemd directory +cp "\$SCRIPT_DIR"/*.service /etc/systemd/system/ +cp "\$SCRIPT_DIR"/*.target /etc/systemd/system/ + +# Reload systemd configuration +systemctl daemon-reload + +# Enable services +systemctl enable foxhunt-trading.service +systemctl enable foxhunt-ml-training.service +systemctl enable foxhunt-backtesting.service +systemctl enable foxhunt-tli.service +systemctl enable foxhunt.target + +echo "SystemD services installed and enabled successfully!" +echo "Start all services with: systemctl start foxhunt.target" +EOF + + chmod +x "$install_script" + log_success "Generated installation script: $install_script" +} + +# Generate uninstall script +generate_uninstall_script() { + local uninstall_script="$OUTPUT_DIR/uninstall_services.sh" + + log_info "Generating uninstall script: uninstall_services.sh" + + cat > "$uninstall_script" << EOF +#!/bin/bash + +#============================================================================ +# FOXHUNT SYSTEMD SERVICES - UNINSTALLATION SCRIPT +#============================================================================ +# Removes SystemD service files and disables them +#============================================================================ + +set -euo pipefail + +# Check if running as root +if [[ \$EUID -ne 0 ]]; then + echo "This script must be run as root" + exit 1 +fi + +echo "Uninstalling Foxhunt SystemD services..." + +# Stop and disable services +systemctl stop foxhunt.target 2>/dev/null || true +systemctl disable foxhunt.target 2>/dev/null || true +systemctl disable foxhunt-trading.service 2>/dev/null || true +systemctl disable foxhunt-ml-training.service 2>/dev/null || true +systemctl disable foxhunt-backtesting.service 2>/dev/null || true +systemctl disable foxhunt-tli.service 2>/dev/null || true + +# Remove service files +rm -f /etc/systemd/system/foxhunt*.service +rm -f /etc/systemd/system/foxhunt.target + +# Reload systemd configuration +systemctl daemon-reload + +echo "SystemD services uninstalled successfully!" +EOF + + chmod +x "$uninstall_script" + log_success "Generated uninstall script: $uninstall_script" +} + +# Generate summary +generate_summary() { + cat << EOF + +${GREEN}============================================================================= +FOXHUNT HFT TRADING SYSTEM - SYSTEMD SERVICES GENERATED +=============================================================================${NC} + +${BLUE}Generated Files:${NC} +$(ls -la "$OUTPUT_DIR" | tail -n +2 | awk '{print " â€Ē " $9 " (" $5 " bytes)"}') + +${BLUE}Service Configuration:${NC} + â€Ē User: $SERVICE_USER + â€Ē Data Directory: $DATA_DIR + â€Ē GPU Support: $ENABLE_GPU + â€Ē CPU Affinity: $CPU_AFFINITY + +${BLUE}Installation Commands:${NC} + â€Ē Install services: sudo $OUTPUT_DIR/install_services.sh + â€Ē Start all services: sudo systemctl start foxhunt.target + â€Ē Check status: systemctl status foxhunt.target + â€Ē View logs: journalctl -f -u foxhunt-* + +${BLUE}Individual Service Commands:${NC} +EOF + + for service in "${!SERVICES[@]}"; do + echo " â€Ē foxhunt-$service: systemctl start foxhunt-$service" + done + + cat << EOF + +${GREEN}SystemD service files generated successfully!${NC} + +EOF +} + +# Main generation logic +main() { + parse_args "$@" + + log_info "Generating Foxhunt SystemD service files..." + log_info "Output directory: $OUTPUT_DIR" + log_info "Service user: $SERVICE_USER" + log_info "Data directory: $DATA_DIR" + + setup_output_directory + + # Generate service files + for service in "${!SERVICES[@]}"; do + generate_service_file "$service" + done + + # Generate target and utility files + generate_target_file + generate_install_script + generate_uninstall_script + + generate_summary + + log_success "Foxhunt SystemD service generation completed!" +} + +# Handle interruption +trap 'log_error "Generation interrupted"; exit 1' INT TERM + +# Run main function +main "$@" \ No newline at end of file