From eed2808b86745b2cbcbeb7228647a10e14178df2 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Thu, 25 Sep 2025 00:02:30 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=80=20ULTIMATE:=20Complete=20deploymen?= =?UTF-8?q?t=20scripts=20and=20production=20finalization?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deployment/Dockerfile.production | 358 +++++++++++++++++++ deployment/build_docker_images.sh | 506 ++++++++++++++++++++++++++ deployment/build_release.sh | 0 deployment/deploy_production.sh | 569 ++++++++++++++++++++++++++++++ 4 files changed, 1433 insertions(+) create mode 100644 deployment/Dockerfile.production create mode 100644 deployment/build_docker_images.sh mode change 100644 => 100755 deployment/build_release.sh create mode 100755 deployment/deploy_production.sh diff --git a/deployment/Dockerfile.production b/deployment/Dockerfile.production new file mode 100644 index 000000000..9d112dc30 --- /dev/null +++ b/deployment/Dockerfile.production @@ -0,0 +1,358 @@ +# ============================================================================ +# FOXHUNT HFT TRADING SYSTEM - PRODUCTION MULTI-STAGE DOCKERFILE +# ============================================================================ +# Optimized multi-stage build for production deployment +# Supports all services: trading, ml-training, backtesting, tli +# +# Build Args: +# SERVICE_NAME - Service to build (trading_service, ml_training_service, backtesting_service, tli) +# TARGET_ARCH - Target architecture (x86_64, aarch64) +# ENABLE_GPU - Enable CUDA/GPU support (true, false) +# BUILD_MODE - Build mode (release, debug) +# ============================================================================ + +################################################################# +# Stage 1: Dependencies and Build Environment +################################################################# +FROM rust:1.75-slim-bookworm AS dependencies + +# Build arguments +ARG TARGET_ARCH=x86_64 +ARG ENABLE_GPU=false + +# Install system dependencies based on architecture and GPU support +RUN apt-get update && apt-get install -y \ + pkg-config \ + libssl-dev \ + libpq-dev \ + ca-certificates \ + curl \ + build-essential \ + cmake \ + git \ + clang \ + llvm \ + libc6-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install CUDA toolkit if GPU support is enabled +RUN if [ "$ENABLE_GPU" = "true" ] && [ "$TARGET_ARCH" = "x86_64" ]; then \ + wget https://developer.download.nvidia.com/compute/cuda/repos/debian11/x86_64/cuda-keyring_1.1-1_all.deb && \ + dpkg -i cuda-keyring_1.1-1_all.deb && \ + apt-get update && \ + apt-get install -y cuda-toolkit-12-0 cuda-runtime-12-0 && \ + rm -rf /var/lib/apt/lists/* && \ + rm cuda-keyring_1.1-1_all.deb; \ + fi + +# Setup Rust toolchain for target architecture +RUN case "$TARGET_ARCH" in \ + "x86_64") RUST_TARGET=x86_64-unknown-linux-gnu ;; \ + "aarch64") RUST_TARGET=aarch64-unknown-linux-gnu ;; \ + *) echo "Unsupported architecture: $TARGET_ARCH" && exit 1 ;; \ + esac && \ + rustup target add $RUST_TARGET && \ + echo "RUST_TARGET=$RUST_TARGET" > /tmp/rust_target + +# Install additional cross-compilation tools if needed +RUN if [ "$TARGET_ARCH" = "aarch64" ]; then \ + apt-get update && apt-get install -y gcc-aarch64-linux-gnu && \ + rm -rf /var/lib/apt/lists/*; \ + fi + +################################################################# +# Stage 2: Dependency Pre-compilation +################################################################# +FROM dependencies AS deps-builder + +WORKDIR /app + +# Copy workspace and dependency manifests +COPY Cargo.toml Cargo.lock ./ +COPY core/Cargo.toml ./core/ +COPY ml/Cargo.toml ./ml/ +COPY risk/Cargo.toml ./risk/ +COPY data/Cargo.toml ./data/ +COPY tli/Cargo.toml ./tli/ +COPY backtesting/Cargo.toml ./backtesting/ +COPY adaptive-strategy/Cargo.toml ./adaptive-strategy/ +COPY services/trading_service/Cargo.toml ./services/trading_service/ +COPY services/ml_training_service/Cargo.toml ./services/ml_training_service/ +COPY services/backtesting_service/Cargo.toml ./services/backtesting_service/ + +# Create dummy source files for dependency pre-compilation +RUN mkdir -p \ + core/src \ + ml/src \ + risk/src \ + data/src \ + tli/src \ + backtesting/src \ + adaptive-strategy/src \ + services/trading_service/src \ + services/ml_training_service/src \ + services/backtesting_service/src \ + src + +# Create minimal lib.rs files +RUN echo "pub fn main() {}" > core/src/lib.rs && \ + echo "pub fn main() {}" > ml/src/lib.rs && \ + echo "pub fn main() {}" > risk/src/lib.rs && \ + echo "pub fn main() {}" > data/src/lib.rs && \ + echo "pub fn main() {}" > backtesting/src/lib.rs && \ + echo "pub fn main() {}" > adaptive-strategy/src/lib.rs && \ + echo "fn main() {}" > tli/src/main.rs && \ + echo "fn main() {}" > services/trading_service/src/main.rs && \ + echo "fn main() {}" > services/ml_training_service/src/main.rs && \ + echo "fn main() {}" > services/backtesting_service/src/main.rs && \ + echo "fn main() {}" > src/lib.rs + +# Set environment variables for compilation +ENV RUST_BACKTRACE=0 +ENV CARGO_INCREMENTAL=0 + +# Build dependencies only (cached layer) +RUN . /tmp/rust_target && \ + if [ "$ENABLE_GPU" = "true" ]; then \ + FEATURES="cuda"; \ + else \ + FEATURES=""; \ + fi && \ + cargo build --release --target="$RUST_TARGET" --features="$FEATURES" + +# Clean up dummy source files and their build artifacts +RUN rm -rf \ + core/src \ + ml/src \ + risk/src \ + data/src \ + tli/src \ + backtesting/src \ + adaptive-strategy/src \ + services/*/src \ + src \ + target/*/release/deps/*core* \ + target/*/release/deps/*ml* \ + target/*/release/deps/*risk* \ + target/*/release/deps/*data* \ + target/*/release/deps/*tli* \ + target/*/release/deps/*backtesting* \ + target/*/release/deps/*adaptive* \ + target/*/release/deps/*trading* \ + target/*/release/deps/*foxhunt* + +################################################################# +# Stage 3: Application Build +################################################################# +FROM deps-builder AS app-builder + +# Build arguments for service selection +ARG SERVICE_NAME=trading_service +ARG BUILD_MODE=release + +# Copy all source code +COPY . . + +# Set build optimizations +ENV RUSTFLAGS="-C target-cpu=native -C opt-level=3 -C lto=fat -C codegen-units=1 -C panic=abort -C prefer-dynamic=no" + +# Build the specified service +RUN . /tmp/rust_target && \ + if [ "$ENABLE_GPU" = "true" ]; then \ + FEATURES="cuda"; \ + else \ + FEATURES=""; \ + fi && \ + case "$SERVICE_NAME" in \ + "trading_service") \ + cargo build --$BUILD_MODE --target="$RUST_TARGET" --features="$FEATURES" --bin trading_service ;; \ + "ml_training_service") \ + cargo build --$BUILD_MODE --target="$RUST_TARGET" --features="$FEATURES" --bin ml_training_service ;; \ + "backtesting_service") \ + cargo build --$BUILD_MODE --target="$RUST_TARGET" --features="$FEATURES" --bin backtesting_service ;; \ + "tli") \ + cargo build --$BUILD_MODE --target="$RUST_TARGET" --features="$FEATURES" --bin tli ;; \ + *) \ + echo "Unknown service: $SERVICE_NAME" && exit 1 ;; \ + esac + +# Strip debug symbols for smaller binary +RUN . /tmp/rust_target && \ + strip "target/$RUST_TARGET/$BUILD_MODE/$SERVICE_NAME" 2>/dev/null || true + +################################################################# +# Stage 4: Runtime Base Image +################################################################# +FROM debian:bookworm-slim AS runtime-base + +# Install runtime dependencies +RUN apt-get update && apt-get install -y \ + ca-certificates \ + libssl3 \ + libpq5 \ + curl \ + htop \ + procps \ + net-tools \ + dumb-init \ + && rm -rf /var/lib/apt/lists/* + +# Install CUDA runtime if GPU support was enabled +ARG ENABLE_GPU=false +RUN if [ "$ENABLE_GPU" = "true" ]; then \ + apt-get update && \ + apt-get install -y \ + libnvidia-compute-535 \ + libnvidia-ml1 \ + libcudart12 \ + libcublas12 \ + && rm -rf /var/lib/apt/lists/*; \ + fi + +# Create application user and directories +RUN groupadd -g 1000 foxhunt && \ + useradd -r -u 1000 -g foxhunt -d /app -s /bin/bash -c "Foxhunt Service" foxhunt + +WORKDIR /app + +# Create required directories with proper permissions +RUN mkdir -p \ + /app/config \ + /app/data \ + /app/models \ + /app/cache \ + /app/logs \ + /app/certificates \ + /app/tmp \ + && chown -R foxhunt:foxhunt /app \ + && chmod -R 755 /app + +################################################################# +# Stage 5: Service-Specific Runtime Images +################################################################# + +# Trading Service Runtime +FROM runtime-base AS trading-service + +ARG SERVICE_NAME=trading_service +ARG TARGET_ARCH=x86_64 +ARG BUILD_MODE=release + +# Copy binary from builder +COPY --from=app-builder /app/target/*/release/trading_service /app/foxhunt-trading + +# Copy service-specific configuration +COPY --chown=foxhunt:foxhunt config/trading/ /app/config/ +COPY --chown=foxhunt:foxhunt deployment/entrypoints/trading-entrypoint.sh /app/entrypoint.sh + +# Set binary permissions +RUN chmod +x /app/foxhunt-trading /app/entrypoint.sh + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ + CMD curl -f http://localhost:8080/health || exit 1 + +# Expose ports +EXPOSE 8080 50051 9090 + +USER foxhunt +ENTRYPOINT ["/usr/bin/dumb-init", "--", "/app/entrypoint.sh"] +CMD ["/app/foxhunt-trading"] + +# ML Training Service Runtime +FROM runtime-base AS ml-training-service + +ARG SERVICE_NAME=ml_training_service +ARG TARGET_ARCH=x86_64 +ARG BUILD_MODE=release + +# Copy binary from builder +COPY --from=app-builder /app/target/*/release/ml_training_service /app/foxhunt-ml + +# Copy service-specific configuration +COPY --chown=foxhunt:foxhunt config/ml/ /app/config/ +COPY --chown=foxhunt:foxhunt deployment/entrypoints/ml-entrypoint.sh /app/entrypoint.sh + +# Set binary permissions +RUN chmod +x /app/foxhunt-ml /app/entrypoint.sh + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=120s --retries=3 \ + CMD curl -f http://localhost:8082/health || exit 1 + +# Expose ports +EXPOSE 8082 50052 9091 + +USER foxhunt +ENTRYPOINT ["/usr/bin/dumb-init", "--", "/app/entrypoint.sh"] +CMD ["/app/foxhunt-ml"] + +# Backtesting Service Runtime +FROM runtime-base AS backtesting-service + +ARG SERVICE_NAME=backtesting_service +ARG TARGET_ARCH=x86_64 +ARG BUILD_MODE=release + +# Copy binary from builder +COPY --from=app-builder /app/target/*/release/backtesting_service /app/foxhunt-backtest + +# Copy service-specific configuration +COPY --chown=foxhunt:foxhunt config/backtesting/ /app/config/ +COPY --chown=foxhunt:foxhunt deployment/entrypoints/backtest-entrypoint.sh /app/entrypoint.sh + +# Set binary permissions +RUN chmod +x /app/foxhunt-backtest /app/entrypoint.sh + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ + CMD curl -f http://localhost:8083/health || exit 1 + +# Expose ports +EXPOSE 8083 50053 9092 + +USER foxhunt +ENTRYPOINT ["/usr/bin/dumb-init", "--", "/app/entrypoint.sh"] +CMD ["/app/foxhunt-backtest"] + +# TLI (Terminal Line Interface) Runtime +FROM runtime-base AS tli + +ARG SERVICE_NAME=tli +ARG TARGET_ARCH=x86_64 +ARG BUILD_MODE=release + +# Copy binary from builder +COPY --from=app-builder /app/target/*/release/tli /app/foxhunt-tli + +# Copy service-specific configuration +COPY --chown=foxhunt:foxhunt config/tli/ /app/config/ +COPY --chown=foxhunt:foxhunt deployment/entrypoints/tli-entrypoint.sh /app/entrypoint.sh + +# Set binary permissions +RUN chmod +x /app/foxhunt-tli /app/entrypoint.sh + +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ + CMD curl -f http://localhost:8081/health || exit 1 + +# Expose ports +EXPOSE 8081 50054 9093 + +USER foxhunt +ENTRYPOINT ["/usr/bin/dumb-init", "--", "/app/entrypoint.sh"] +CMD ["/app/foxhunt-tli"] + +################################################################# +# Labels for Container Metadata +################################################################# +LABEL maintainer="Foxhunt Development Team" +LABEL version="1.0.0" +LABEL description="Foxhunt HFT Trading System - Production Service" +LABEL vendor="Foxhunt" +LABEL org.opencontainers.image.title="Foxhunt HFT Service" +LABEL org.opencontainers.image.description="High-frequency trading system service" +LABEL org.opencontainers.image.version="1.0.0" +LABEL org.opencontainers.image.vendor="Foxhunt" +LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0" +LABEL org.opencontainers.image.source="https://github.com/user/foxhunt" \ No newline at end of file diff --git a/deployment/build_docker_images.sh b/deployment/build_docker_images.sh new file mode 100644 index 000000000..c824e8315 --- /dev/null +++ b/deployment/build_docker_images.sh @@ -0,0 +1,506 @@ +#!/bin/bash + +#============================================================================ +# FOXHUNT HFT TRADING SYSTEM - DOCKER IMAGE BUILD SCRIPT +#============================================================================ +# Builds optimized Docker images for all services using multi-stage builds +# +# Usage: +# ./build_docker_images.sh [OPTIONS] +# +# Options: +# --service SERVICE Build specific service (trading, ml, backtesting, tli, all) +# --tag-prefix PREFIX Docker image tag prefix (default: foxhunt) +# --version VERSION Version tag (default: latest) +# --enable-gpu Enable CUDA/GPU support +# --target-arch ARCH Target architecture (x86_64, aarch64) +# --build-mode MODE Build mode (release, debug) +# --push Push images to registry after building +# --registry URL Docker registry URL +# --parallel Build images in parallel +# --help Show this help message +#============================================================================ + +set -euo pipefail + +# Configuration +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +TAG_PREFIX="${TAG_PREFIX:-foxhunt}" +VERSION="${VERSION:-latest}" +ENABLE_GPU="${ENABLE_GPU:-false}" +TARGET_ARCH="${TARGET_ARCH:-x86_64}" +BUILD_MODE="${BUILD_MODE:-release}" +PUSH_IMAGES="${PUSH_IMAGES:-false}" +REGISTRY_URL="${REGISTRY_URL:-}" +BUILD_PARALLEL="${BUILD_PARALLEL:-false}" +SERVICE="${SERVICE:-all}" + +# Service configurations +declare -A SERVICES=( + ["trading"]="trading_service:8080" + ["ml"]="ml_training_service:8082" + ["backtesting"]="backtesting_service:8083" + ["tli"]="tli:8081" +) + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +show_help() { + cat << EOF +Foxhunt HFT Trading System - Docker Image Builder + +Usage: $0 [OPTIONS] + +OPTIONS: + --service SERVICE Build specific service (trading, ml, backtesting, tli, all) + --tag-prefix PREFIX Docker image tag prefix (default: foxhunt) + --version VERSION Version tag (default: latest) + --enable-gpu Enable CUDA/GPU support + --target-arch ARCH Target architecture (x86_64, aarch64) + --build-mode MODE Build mode (release, debug) + --push Push images to registry after building + --registry URL Docker registry URL + --parallel Build images in parallel + --help Show this help message + +SERVICES: + trading Trading Service + ml ML Training Service + backtesting Backtesting Service + tli Terminal Line Interface + all Build all services + +EXAMPLES: + $0 # Build all services + $0 --service trading --push # Build and push trading service + $0 --enable-gpu --parallel # Build all with GPU support in parallel + $0 --target-arch aarch64 # Build for ARM64 architecture + +EOF +} + +# Parse arguments +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + --service) + SERVICE="$2" + shift 2 + ;; + --tag-prefix) + TAG_PREFIX="$2" + shift 2 + ;; + --version) + VERSION="$2" + shift 2 + ;; + --enable-gpu) + ENABLE_GPU=true + shift + ;; + --target-arch) + TARGET_ARCH="$2" + shift 2 + ;; + --build-mode) + BUILD_MODE="$2" + shift 2 + ;; + --push) + PUSH_IMAGES=true + shift + ;; + --registry) + REGISTRY_URL="$2" + shift 2 + ;; + --parallel) + BUILD_PARALLEL=true + shift + ;; + --help) + show_help + exit 0 + ;; + *) + log_error "Unknown option: $1" + show_help + exit 1 + ;; + esac + done +} + +# Check prerequisites +check_prerequisites() { + log_info "Checking Docker build prerequisites..." + + # Check Docker + if ! command -v docker &> /dev/null; then + log_error "Docker is not installed" + exit 1 + fi + + # Check Docker version for multi-stage builds + local docker_version=$(docker --version | grep -oE '[0-9]+\.[0-9]+' | head -1) + local major_version=$(echo "$docker_version" | cut -d. -f1) + local minor_version=$(echo "$docker_version" | cut -d. -f2) + + if [[ $major_version -lt 17 ]] || [[ $major_version -eq 17 && $minor_version -lt 5 ]]; then + log_error "Docker version 17.05 or higher is required for multi-stage builds" + exit 1 + fi + + # Check if Docker daemon is running + if ! docker info &> /dev/null; then + log_error "Docker daemon is not running" + exit 1 + fi + + # Check Dockerfile exists + local dockerfile="$SCRIPT_DIR/Dockerfile.production" + if [[ ! -f "$dockerfile" ]]; then + log_error "Production Dockerfile not found: $dockerfile" + exit 1 + fi + + # Check if buildx is available for cross-platform builds + if [[ "$TARGET_ARCH" != "x86_64" ]]; then + if ! docker buildx version &> /dev/null; then + log_warning "Docker Buildx not available - cross-platform builds may not work" + fi + fi + + log_success "Prerequisites check completed" +} + +# Create Docker builder for multi-platform builds +create_builder() { + if [[ "$TARGET_ARCH" == "x86_64" ]]; then + return 0 + fi + + log_info "Creating Docker builder for multi-platform builds..." + + local builder_name="foxhunt-builder" + + # Check if builder already exists + if ! docker buildx inspect "$builder_name" &> /dev/null; then + docker buildx create --name "$builder_name" --driver docker-container --bootstrap + log_success "Created Docker builder: $builder_name" + else + log_info "Using existing Docker builder: $builder_name" + fi + + docker buildx use "$builder_name" +} + +# Build Docker image for a service +build_service_image() { + local service_name="$1" + local service_config="${SERVICES[$service_name]}" + + if [[ -z "$service_config" ]]; then + log_error "Unknown service: $service_name" + return 1 + fi + + IFS=':' read -r binary_name port <<< "$service_config" + + local image_name="$TAG_PREFIX/$service_name" + local full_tag="$image_name:$VERSION" + + # Add registry prefix if specified + if [[ -n "$REGISTRY_URL" ]]; then + full_tag="$REGISTRY_URL/$full_tag" + fi + + log_info "Building Docker image: $full_tag" + log_info "Service: $service_name, Binary: $binary_name, Target: $service_name" + + local build_start=$(date +%s) + + # Prepare build arguments + local build_args=( + "--build-arg" "SERVICE_NAME=$binary_name" + "--build-arg" "TARGET_ARCH=$TARGET_ARCH" + "--build-arg" "ENABLE_GPU=$ENABLE_GPU" + "--build-arg" "BUILD_MODE=$BUILD_MODE" + "--target" "$service_name" + "--tag" "$full_tag" + "--file" "$SCRIPT_DIR/Dockerfile.production" + ) + + # Add platform specification for cross-platform builds + if [[ "$TARGET_ARCH" != "x86_64" ]]; then + case "$TARGET_ARCH" in + "aarch64") + build_args+=("--platform" "linux/arm64") + ;; + *) + log_error "Unsupported target architecture: $TARGET_ARCH" + return 1 + ;; + esac + fi + + # Add context + build_args+=("$PROJECT_ROOT") + + # Execute build + if docker build "${build_args[@]}"; then + local build_end=$(date +%s) + local build_duration=$((build_end - build_start)) + + # Get image size + local image_size=$(docker images "$full_tag" --format "table {{.Size}}" | tail -n 1) + + log_success "$service_name image built successfully (${build_duration}s, ${image_size})" + + # Tag with additional tags if needed + tag_additional_versions "$full_tag" "$service_name" + + # Push image if requested + if [[ "$PUSH_IMAGES" == "true" ]]; then + push_image "$full_tag" + fi + + return 0 + else + log_error "Failed to build $service_name image" + return 1 + fi +} + +# Tag image with additional versions +tag_additional_versions() { + local full_tag="$1" + local service_name="$2" + + local base_name="${full_tag%:*}" + + # Add git commit tag if in git repository + if git rev-parse HEAD &>/dev/null; then + local git_commit=$(git rev-parse --short HEAD) + local commit_tag="$base_name:$git_commit" + docker tag "$full_tag" "$commit_tag" + log_info "Tagged with commit: $commit_tag" + fi + + # Add architecture-specific tag + local arch_tag="$base_name:$VERSION-$TARGET_ARCH" + docker tag "$full_tag" "$arch_tag" + log_info "Tagged with architecture: $arch_tag" + + # Add GPU-specific tag if enabled + if [[ "$ENABLE_GPU" == "true" ]]; then + local gpu_tag="$base_name:$VERSION-gpu" + docker tag "$full_tag" "$gpu_tag" + log_info "Tagged with GPU support: $gpu_tag" + fi +} + +# Push image to registry +push_image() { + local image_tag="$1" + + log_info "Pushing image: $image_tag" + + if docker push "$image_tag"; then + log_success "Successfully pushed: $image_tag" + + # Push additional tags + local base_name="${image_tag%:*}" + + # Push commit tag + if git rev-parse HEAD &>/dev/null; then + local git_commit=$(git rev-parse --short HEAD) + docker push "$base_name:$git_commit" 2>/dev/null || true + fi + + # Push architecture tag + docker push "$base_name:$VERSION-$TARGET_ARCH" 2>/dev/null || true + + # Push GPU tag if enabled + if [[ "$ENABLE_GPU" == "true" ]]; then + docker push "$base_name:$VERSION-gpu" 2>/dev/null || true + fi + else + log_error "Failed to push: $image_tag" + return 1 + fi +} + +# Build all services in parallel +build_all_parallel() { + log_info "Building all services in parallel..." + + local pids=() + local failed_services=() + + for service in "${!SERVICES[@]}"; do + ( + if build_service_image "$service"; then + echo "$service: SUCCESS" + else + echo "$service: FAILED" + fi + ) & + pids+=($!) + done + + # Wait for all builds to complete + for i in "${!pids[@]}"; do + local pid=${pids[$i]} + local service=${!SERVICES[@]:$i:1} + + if wait $pid; then + log_success "$service build completed" + else + log_error "$service build failed" + failed_services+=("$service") + fi + done + + if [[ ${#failed_services[@]} -eq 0 ]]; then + log_success "All services built successfully" + return 0 + else + log_error "Failed to build services: ${failed_services[*]}" + return 1 + fi +} + +# Build all services sequentially +build_all_sequential() { + log_info "Building all services sequentially..." + + local failed_services=() + + for service in "${!SERVICES[@]}"; do + if build_service_image "$service"; then + log_success "$service build completed" + else + log_error "$service build failed" + failed_services+=("$service") + fi + done + + if [[ ${#failed_services[@]} -eq 0 ]]; then + log_success "All services built successfully" + return 0 + else + log_error "Failed to build services: ${failed_services[*]}" + return 1 + fi +} + +# Generate build summary +generate_summary() { + log_info "Generating build summary..." + + local total_size=0 + + cat << EOF + +${GREEN}============================================================================= +FOXHUNT HFT TRADING SYSTEM - DOCKER BUILD SUMMARY +=============================================================================${NC} + +${BLUE}Build Configuration:${NC} + • Tag Prefix: $TAG_PREFIX + • Version: $VERSION + • Target Architecture: $TARGET_ARCH + • GPU Support: $ENABLE_GPU + • Build Mode: $BUILD_MODE + • Registry: ${REGISTRY_URL:-"(local)"} + +${BLUE}Built Images:${NC} +EOF + + for service in "${!SERVICES[@]}"; do + local image_name="$TAG_PREFIX/$service:$VERSION" + if [[ -n "$REGISTRY_URL" ]]; then + image_name="$REGISTRY_URL/$image_name" + fi + + if docker images "$image_name" --format "table {{.Repository}}:{{.Tag}} {{.Size}}" | tail -n 1 | grep -q "$service"; then + local size=$(docker images "$image_name" --format "table {{.Size}}" | tail -n 1) + echo " • $service: $image_name ($size)" + fi + done + + cat << EOF + +${BLUE}Management Commands:${NC} + • Run service: docker run -d $TAG_PREFIX/:$VERSION + • View images: docker images | grep $TAG_PREFIX + • Remove images: docker rmi $TAG_PREFIX/:$VERSION + • Deploy: docker-compose -f docker-compose.production.yml up -d + +${GREEN}Docker image build completed successfully!${NC} + +EOF +} + +# Main build logic +main() { + parse_args "$@" + + log_info "Starting Foxhunt Docker image build..." + log_info "Service: $SERVICE, Version: $VERSION, Architecture: $TARGET_ARCH" + + check_prerequisites + create_builder + + case "$SERVICE" in + "all") + if [[ "$BUILD_PARALLEL" == "true" ]]; then + build_all_parallel + else + build_all_sequential + fi + ;; + "trading"|"ml"|"backtesting"|"tli") + build_service_image "$SERVICE" + ;; + *) + log_error "Unknown service: $SERVICE" + log_info "Available services: ${!SERVICES[*]} all" + exit 1 + ;; + esac + + generate_summary + + log_success "Foxhunt Docker image build completed!" +} + +# Handle interruption +trap 'log_error "Build interrupted"; exit 1' INT TERM + +# Run main function +main "$@" \ No newline at end of file diff --git a/deployment/build_release.sh b/deployment/build_release.sh old mode 100644 new mode 100755 diff --git a/deployment/deploy_production.sh b/deployment/deploy_production.sh new file mode 100755 index 000000000..6ae5eb6c3 --- /dev/null +++ b/deployment/deploy_production.sh @@ -0,0 +1,569 @@ +#!/bin/bash + +#============================================================================ +# FOXHUNT HFT TRADING SYSTEM - PRODUCTION DEPLOYMENT SCRIPT +#============================================================================ +# Deploys the complete Foxhunt HFT system to production environment +# Supports bare metal, Docker, and Kubernetes deployments +# +# Usage: +# ./deploy_production.sh [OPTIONS] +# +# Options: +# --mode MODE Deployment mode: bare-metal, docker, k8s (default: bare-metal) +# --env ENV Environment: production, staging, testing (default: production) +# --config-dir DIR Configuration directory (default: ./config) +# --data-dir DIR Data directory (default: /opt/foxhunt/data) +# --skip-health Skip health checks after deployment +# --rollback Rollback to previous deployment +# --dry-run Show what would be deployed without executing +# --help Show this help message +#============================================================================ + +set -euo pipefail + +# Configuration +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +DEPLOY_MODE="${DEPLOY_MODE:-bare-metal}" +ENVIRONMENT="${ENVIRONMENT:-production}" +CONFIG_DIR="${CONFIG_DIR:-$PROJECT_ROOT/config}" +DATA_DIR="${DATA_DIR:-/opt/foxhunt}" +SKIP_HEALTH="${SKIP_HEALTH:-false}" +DRY_RUN="${DRY_RUN:-false}" +ROLLBACK="${ROLLBACK:-false}" + +# Service configuration +SERVICES=( + "trading-service:8080:50051" + "ml-training-service:8082:50052" + "backtesting-service:8083:50053" + "tli:8081:50054" +) + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +log_info() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +show_help() { + cat << EOF +Foxhunt HFT Trading System - Production Deployment + +Usage: $0 [OPTIONS] + +OPTIONS: + --mode MODE Deployment mode (bare-metal, docker, k8s) + --env ENV Environment (production, staging, testing) + --config-dir DIR Configuration directory + --data-dir DIR Data directory + --skip-health Skip health checks after deployment + --rollback Rollback to previous deployment + --dry-run Show what would be deployed without executing + --help Show this help message + +DEPLOYMENT MODES: + bare-metal Deploy as SystemD services (recommended for HFT) + docker Deploy using Docker Compose + k8s Deploy to Kubernetes cluster + +EXAMPLES: + $0 # Deploy to production bare-metal + $0 --mode docker --env staging # Deploy to staging with Docker + $0 --rollback # Rollback to previous deployment + $0 --dry-run # Preview deployment + +EOF +} + +# Parse arguments +parse_args() { + while [[ $# -gt 0 ]]; do + case $1 in + --mode) + DEPLOY_MODE="$2" + shift 2 + ;; + --env) + ENVIRONMENT="$2" + shift 2 + ;; + --config-dir) + CONFIG_DIR="$2" + shift 2 + ;; + --data-dir) + DATA_DIR="$2" + shift 2 + ;; + --skip-health) + SKIP_HEALTH=true + shift + ;; + --rollback) + ROLLBACK=true + shift + ;; + --dry-run) + DRY_RUN=true + shift + ;; + --help) + show_help + exit 0 + ;; + *) + log_error "Unknown option: $1" + show_help + exit 1 + ;; + esac + done +} + +# Execute command with dry-run support +execute() { + if [[ "$DRY_RUN" == "true" ]]; then + log_info "[DRY-RUN] Would execute: $*" + else + "$@" + fi +} + +# Check prerequisites +check_prerequisites() { + log_info "Checking deployment prerequisites..." + + # Check if running as root for bare-metal deployment + if [[ "$DEPLOY_MODE" == "bare-metal" ]] && [[ $EUID -ne 0 ]] && [[ "$DRY_RUN" == "false" ]]; then + log_error "Bare-metal deployment requires root privileges" + log_info "Run with: sudo $0" + exit 1 + fi + + # Check deployment mode specific requirements + case "$DEPLOY_MODE" in + bare-metal) + check_systemd + ;; + docker) + check_docker + ;; + k8s) + check_kubernetes + ;; + *) + log_error "Unknown deployment mode: $DEPLOY_MODE" + exit 1 + ;; + esac + + # Check if binaries exist + local binary_dir="$PROJECT_ROOT/target/release" + local required_binaries=("trading_service" "ml_training_service" "backtesting_service" "tli") + + for binary in "${required_binaries[@]}"; do + if [[ ! -f "$binary_dir/$binary" ]]; then + log_error "Required binary not found: $binary_dir/$binary" + log_info "Run: ./deployment/build_release.sh" + exit 1 + fi + done + + log_success "Prerequisites check completed" +} + +# Check SystemD availability +check_systemd() { + if ! command -v systemctl &> /dev/null; then + log_error "SystemD is not available - required for bare-metal deployment" + exit 1 + fi + + log_info "SystemD available for bare-metal deployment" +} + +# Check Docker availability +check_docker() { + if ! command -v docker &> /dev/null; then + log_error "Docker is not installed" + exit 1 + fi + + if ! command -v docker-compose &> /dev/null; then + log_error "Docker Compose is not installed" + exit 1 + fi + + log_info "Docker environment ready" +} + +# Check Kubernetes availability +check_kubernetes() { + if ! command -v kubectl &> /dev/null; then + log_error "kubectl is not installed" + exit 1 + fi + + if ! kubectl cluster-info &> /dev/null; then + log_error "Kubernetes cluster is not accessible" + exit 1 + fi + + log_info "Kubernetes cluster ready" +} + +# Setup deployment directories +setup_directories() { + log_info "Setting up deployment directories..." + + local dirs=( + "$DATA_DIR" + "$DATA_DIR/config" + "$DATA_DIR/data" + "$DATA_DIR/models" + "$DATA_DIR/backtests" + "$DATA_DIR/logs" + "$DATA_DIR/bin" + "$DATA_DIR/systemd" + "/var/log/foxhunt" + "/etc/systemd/system" + ) + + for dir in "${dirs[@]}"; do + execute mkdir -p "$dir" + if [[ "$DEPLOY_MODE" == "bare-metal" ]]; then + execute chown -R foxhunt:foxhunt "$dir" 2>/dev/null || true + fi + done + + log_success "Deployment directories created" +} + +# Create system user for bare-metal deployment +create_system_user() { + if [[ "$DEPLOY_MODE" != "bare-metal" ]]; then + return 0 + fi + + log_info "Creating system user for Foxhunt services..." + + if ! id -u foxhunt &>/dev/null; then + execute useradd -r -s /bin/false -d "$DATA_DIR" -c "Foxhunt HFT Trading System" foxhunt + log_success "Created system user: foxhunt" + else + log_info "System user already exists: foxhunt" + fi +} + +# Deploy binaries +deploy_binaries() { + log_info "Deploying application binaries..." + + local binary_dir="$PROJECT_ROOT/target/release" + local binaries=("trading_service" "ml_training_service" "backtesting_service" "tli") + + for binary in "${binaries[@]}"; do + local src="$binary_dir/$binary" + local dst="$DATA_DIR/bin/$binary" + + execute cp "$src" "$dst" + execute chmod +x "$dst" + + if [[ "$DEPLOY_MODE" == "bare-metal" ]]; then + execute chown foxhunt:foxhunt "$dst" + fi + + log_success "Deployed binary: $binary" + done +} + +# Deploy configuration +deploy_configuration() { + log_info "Deploying configuration files..." + + if [[ -d "$CONFIG_DIR" ]]; then + execute cp -r "$CONFIG_DIR"/* "$DATA_DIR/config/" + + if [[ "$DEPLOY_MODE" == "bare-metal" ]]; then + execute chown -R foxhunt:foxhunt "$DATA_DIR/config" + execute chmod -R 640 "$DATA_DIR/config" + fi + + log_success "Configuration deployed" + else + log_warning "Configuration directory not found: $CONFIG_DIR" + fi +} + +# Deploy SystemD services +deploy_systemd_services() { + if [[ "$DEPLOY_MODE" != "bare-metal" ]]; then + return 0 + fi + + log_info "Deploying SystemD services..." + + # Generate service files + "$SCRIPT_DIR/create_systemd_services.sh" --output-dir "$DATA_DIR/systemd" + + # Install service files + for service_file in "$DATA_DIR/systemd"/*.service; do + local service_name=$(basename "$service_file") + execute cp "$service_file" "/etc/systemd/system/" + log_info "Installed service: $service_name" + done + + execute systemctl daemon-reload + log_success "SystemD services deployed" +} + +# Start services based on deployment mode +start_services() { + log_info "Starting Foxhunt services..." + + case "$DEPLOY_MODE" in + bare-metal) + start_systemd_services + ;; + docker) + start_docker_services + ;; + k8s) + start_kubernetes_services + ;; + esac +} + +# Start SystemD services +start_systemd_services() { + local services=("foxhunt-trading" "foxhunt-ml-training" "foxhunt-backtesting" "foxhunt-tli") + + for service in "${services[@]}"; do + execute systemctl enable "$service" + execute systemctl start "$service" + log_success "Started service: $service" + done +} + +# Start Docker services +start_docker_services() { + execute docker-compose -f "$PROJECT_ROOT/docker-compose.production.yml" up -d + log_success "Started Docker services" +} + +# Start Kubernetes services +start_kubernetes_services() { + execute kubectl apply -f "$PROJECT_ROOT/k8s/" + log_success "Deployed to Kubernetes" +} + +# Health check +perform_health_check() { + if [[ "$SKIP_HEALTH" == "true" ]]; then + log_warning "Skipping health checks as requested" + return 0 + fi + + log_info "Performing deployment health check..." + + local failed_services=() + + for service_config in "${SERVICES[@]}"; do + IFS=':' read -r service_name http_port grpc_port <<< "$service_config" + + log_info "Checking $service_name..." + + # Check HTTP health endpoint + if curl -f -s "http://localhost:$http_port/health" >/dev/null; then + log_success "$service_name HTTP endpoint is healthy" + else + log_warning "$service_name HTTP endpoint is not responding" + failed_services+=("$service_name-http") + fi + + # Check gRPC health endpoint (if available) + if command -v grpcurl &> /dev/null; then + if grpcurl -plaintext "localhost:$grpc_port" grpc.health.v1.Health/Check >/dev/null 2>&1; then + log_success "$service_name gRPC endpoint is healthy" + else + log_warning "$service_name gRPC endpoint is not responding" + failed_services+=("$service_name-grpc") + fi + fi + done + + if [[ ${#failed_services[@]} -eq 0 ]]; then + log_success "All services are healthy" + else + log_warning "Some services failed health check: ${failed_services[*]}" + return 1 + fi +} + +# Rollback deployment +perform_rollback() { + if [[ "$ROLLBACK" != "true" ]]; then + return 0 + fi + + log_info "Performing deployment rollback..." + + case "$DEPLOY_MODE" in + bare-metal) + rollback_systemd + ;; + docker) + rollback_docker + ;; + k8s) + rollback_kubernetes + ;; + esac + + log_success "Rollback completed" +} + +# Rollback SystemD services +rollback_systemd() { + local backup_dir="$DATA_DIR/backup/$(date +%Y%m%d-%H%M%S)" + + if [[ -d "$backup_dir" ]]; then + log_info "Restoring from backup: $backup_dir" + execute cp -r "$backup_dir"/* "$DATA_DIR/" + execute systemctl daemon-reload + execute systemctl restart foxhunt-* + else + log_error "No backup found for rollback" + exit 1 + fi +} + +# Generate deployment summary +generate_summary() { + log_info "Generating deployment summary..." + + cat << EOF + +${GREEN}============================================================================= +FOXHUNT HFT TRADING SYSTEM - DEPLOYMENT SUMMARY +=============================================================================${NC} + +${BLUE}Deployment Details:${NC} + • Mode: $DEPLOY_MODE + • Environment: $ENVIRONMENT + • Data Directory: $DATA_DIR + • Deployment Time: $(date) + +${BLUE}Services Deployed:${NC} +EOF + + for service_config in "${SERVICES[@]}"; do + IFS=':' read -r service_name http_port grpc_port <<< "$service_config" + echo " • $service_name http://localhost:$http_port (gRPC: $grpc_port)" + done + + cat << EOF + +${BLUE}Management Commands:${NC} +EOF + + case "$DEPLOY_MODE" in + bare-metal) + cat << EOF + • View logs: journalctl -f -u foxhunt-* + • Restart service: systemctl restart foxhunt- + • Stop all: systemctl stop foxhunt-* + • Service status: systemctl status foxhunt-* +EOF + ;; + docker) + cat << EOF + • View logs: docker-compose logs -f + • Restart service: docker-compose restart + • Stop all: docker-compose down + • Service status: docker-compose ps +EOF + ;; + k8s) + cat << EOF + • View logs: kubectl logs -f deployment/ + • Restart service: kubectl rollout restart deployment/ + • Stop all: kubectl delete -f k8s/ + • Service status: kubectl get pods +EOF + ;; + esac + + cat << EOF + +${BLUE}Monitoring:${NC} + • Prometheus: http://localhost:9090 + • Grafana: http://localhost:3000 + • Health Checks: ./deployment/health_check.sh + +${GREEN}Deployment completed successfully!${NC} + +EOF +} + +# Main deployment logic +main() { + parse_args "$@" + + log_info "Starting Foxhunt HFT Trading System deployment..." + log_info "Mode: $DEPLOY_MODE, Environment: $ENVIRONMENT" + + if [[ "$DRY_RUN" == "true" ]]; then + log_warning "DRY-RUN MODE - No changes will be made" + fi + + check_prerequisites + + if [[ "$ROLLBACK" == "true" ]]; then + perform_rollback + exit 0 + fi + + setup_directories + create_system_user + deploy_binaries + deploy_configuration + deploy_systemd_services + start_services + + # Wait for services to initialize + if [[ "$SKIP_HEALTH" == "false" ]]; then + log_info "Waiting for services to initialize..." + sleep 30 + perform_health_check + fi + + generate_summary + + log_success "Foxhunt HFT Trading System deployment completed!" +} + +# Handle interruption +trap 'log_error "Deployment interrupted"; exit 1' INT TERM + +# Run main function +main "$@" \ No newline at end of file