foxhunt/backup.sh

#!/bin/bash

#============================================================================
# FOXHUNT HFT TRADING SYSTEM - BACKUP SCRIPT
#============================================================================
# Automated backup solution for all data stores and configurations
#
# Usage: ./backup.sh [OPTIONS]
#
# Options:
#   --full             Perform full backup (default)
#   --incremental      Perform incremental backup
#   --config-only      Backup only configuration
#   --data-only        Backup only data stores
#   --restore FILE     Restore from backup file
#   --list            List available backups
#   --help            Show this help message
#============================================================================

set -euo pipefail

# Configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BACKUP_DIR="/opt/foxhunt/backups"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
BACKUP_PREFIX="foxhunt_backup"

# Retention settings
DAILY_RETENTION=7
WEEKLY_RETENTION=4
MONTHLY_RETENTION=12

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Logging functions
log_info() {
    echo -e "${BLUE}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') $1"
}

log_success() {
    echo -e "${GREEN}[SUCCESS]${NC} $(date '+%Y-%m-%d %H:%M:%S') $1"
}

log_warning() {
    echo -e "${YELLOW}[WARNING]${NC} $(date '+%Y-%m-%d %H:%M:%S') $1"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') $1"
}

# Show help
show_help() {
    cat << EOF
Foxhunt HFT Trading System - Backup Management

Usage: $0 [OPTIONS]

OPTIONS:
    --full             Perform full backup (default)
    --incremental      Perform incremental backup
    --config-only      Backup only configuration files
    --data-only        Backup only data stores (databases)
    --restore FILE     Restore from backup file
    --list            List available backups
    --cleanup         Clean up old backups according to retention policy
    --help            Show this help message

BACKUP TYPES:
    Full:              Complete backup of all data and configurations
    Incremental:       Only changed files since last backup
    Config-only:       Configuration files, Vault data, and schemas
    Data-only:         Database dumps and data volumes

EXAMPLES:
    $0                                # Full backup
    $0 --incremental                  # Incremental backup
    $0 --config-only                  # Configuration backup only
    $0 --restore backup_20240924.tar.gz  # Restore from backup
    $0 --list                         # List all backups

EOF
}

# Create backup directory
create_backup_dir() {
    if [[ ! -d "$BACKUP_DIR" ]]; then
        sudo mkdir -p "$BACKUP_DIR"
        sudo chown $(id -u):$(id -g) "$BACKUP_DIR"
        log_info "Created backup directory: $BACKUP_DIR"
    fi
}

# Backup PostgreSQL
backup_postgresql() {
    local backup_file="$1/postgresql_${TIMESTAMP}.sql"

    log_info "Backing up PostgreSQL database..."

    if docker exec foxhunt-postgres-prod pg_dumpall -U foxhunt > "$backup_file"; then
        gzip "$backup_file"
        log_success "PostgreSQL backup completed: ${backup_file}.gz"
    else
        log_error "PostgreSQL backup failed"
        return 1
    fi
}

# Backup Redis
backup_redis() {
    local backup_file="$1/redis_${TIMESTAMP}.rdb"

    log_info "Backing up Redis data..."

    # Force Redis to save current state
    if docker exec foxhunt-redis-prod redis-cli BGSAVE; then
        # Wait for background save to complete
        sleep 5

        # Copy the dump file
        if docker cp foxhunt-redis-prod:/data/dump.rdb "$backup_file"; then
            gzip "$backup_file"
            log_success "Redis backup completed: ${backup_file}.gz"
        else
            log_error "Failed to copy Redis dump file"
            return 1
        fi
    else
        log_error "Redis backup failed"
        return 1
    fi
}

# Backup InfluxDB
backup_influxdb() {
    local backup_file="$1/influxdb_${TIMESTAMP}.tar.gz"

    log_info "Backing up InfluxDB data..."

    # Create InfluxDB backup
    if docker exec foxhunt-influxdb-prod influx backup /tmp/backup_${TIMESTAMP}; then
        # Copy backup from container
        if docker cp foxhunt-influxdb-prod:/tmp/backup_${TIMESTAMP} "$1/influxdb_${TIMESTAMP}"; then
            tar -czf "$backup_file" -C "$1" "influxdb_${TIMESTAMP}"
            rm -rf "$1/influxdb_${TIMESTAMP}"
            log_success "InfluxDB backup completed: $backup_file"
        else
            log_error "Failed to copy InfluxDB backup"
            return 1
        fi
    else
        log_error "InfluxDB backup failed"
        return 1
    fi
}

# Backup Vault
backup_vault() {
    local backup_file="$1/vault_${TIMESTAMP}.tar.gz"

    log_info "Backing up Vault data..."

    # Create snapshot (requires Vault Enterprise or manual file copy)
    if docker exec foxhunt-vault-prod vault operator raft snapshot save /vault/data/snapshot_${TIMESTAMP}; then
        # Copy vault data directory
        if docker cp foxhunt-vault-prod:/vault/data "$1/vault_${TIMESTAMP}"; then
            tar -czf "$backup_file" -C "$1" "vault_${TIMESTAMP}"
            rm -rf "$1/vault_${TIMESTAMP}"
            log_success "Vault backup completed: $backup_file"
        else
            log_error "Failed to copy Vault data"
            return 1
        fi
    else
        log_warning "Vault snapshot failed, copying data directory instead"
        # Fallback: copy data directory
        if docker cp foxhunt-vault-prod:/vault/data "$1/vault_${TIMESTAMP}"; then
            tar -czf "$backup_file" -C "$1" "vault_${TIMESTAMP}"
            rm -rf "$1/vault_${TIMESTAMP}"
            log_success "Vault backup completed: $backup_file"
        else
            log_error "Vault backup failed"
            return 1
        fi
    fi
}

# Backup configuration files
backup_configurations() {
    local backup_file="$1/configurations_${TIMESTAMP}.tar.gz"

    log_info "Backing up configuration files..."

    local config_paths=(
        "/opt/foxhunt/config"
        "$SCRIPT_DIR/deployment"
        "$SCRIPT_DIR/.env.production"
        "$SCRIPT_DIR/docker-compose.production.yml"
        "$SCRIPT_DIR/docker-compose.infrastructure.yml"
        "$SCRIPT_DIR/docker-compose.monitoring.yml"
    )

    local existing_paths=()
    for path in "${config_paths[@]}"; do
        if [[ -e "$path" ]]; then
            existing_paths+=("$path")
        fi
    done

    if [[ ${#existing_paths[@]} -gt 0 ]]; then
        if tar -czf "$backup_file" "${existing_paths[@]}"; then
            log_success "Configuration backup completed: $backup_file"
        else
            log_error "Configuration backup failed"
            return 1
        fi
    else
        log_warning "No configuration files found to backup"
    fi
}

# Backup application data
backup_application_data() {
    local backup_file="$1/application_data_${TIMESTAMP}.tar.gz"

    log_info "Backing up application data..."

    local data_paths=(
        "/opt/foxhunt/models"
        "/opt/foxhunt/data"
        "/opt/foxhunt/backtests"
        "/opt/foxhunt/checkpoints"
        "/var/log/foxhunt"
    )

    local existing_paths=()
    for path in "${data_paths[@]}"; do
        if [[ -d "$path" ]]; then
            existing_paths+=("$path")
        fi
    done

    if [[ ${#existing_paths[@]} -gt 0 ]]; then
        if tar -czf "$backup_file" "${existing_paths[@]}"; then
            log_success "Application data backup completed: $backup_file"
        else
            log_error "Application data backup failed"
            return 1
        fi
    else
        log_warning "No application data found to backup"
    fi
}

# Create backup manifest
create_manifest() {
    local backup_dir="$1"
    local manifest_file="$backup_dir/manifest.json"

    cat > "$manifest_file" << EOF
{
    "backup_timestamp": "$TIMESTAMP",
    "backup_type": "$BACKUP_TYPE",
    "system_info": {
        "hostname": "$(hostname)",
        "os": "$(uname -s)",
        "kernel": "$(uname -r)",
        "docker_version": "$(docker --version 2>/dev/null || echo 'unknown')"
    },
    "services": {
        "postgresql": "$(docker exec foxhunt-postgres-prod psql -U foxhunt -d foxhunt -c 'SELECT version();' -t 2>/dev/null | head -1 || echo 'unknown')",
        "redis": "$(docker exec foxhunt-redis-prod redis-cli INFO server | grep redis_version 2>/dev/null || echo 'unknown')",
        "vault": "$(docker exec foxhunt-vault-prod vault version 2>/dev/null || echo 'unknown')"
    },
    "files": [
$(find "$backup_dir" -type f -name "*.gz" -o -name "*.sql" | sed 's/.*/"&",/' | sed '$ s/,$//')
    ]
}
EOF

    log_info "Backup manifest created: $manifest_file"
}

# Full backup
perform_full_backup() {
    local backup_name="${BACKUP_PREFIX}_full_${TIMESTAMP}"
    local backup_path="$BACKUP_DIR/$backup_name"

    log_info "Starting full backup: $backup_name"

    mkdir -p "$backup_path"

    # Backup all components
    backup_postgresql "$backup_path" || true
    backup_redis "$backup_path" || true
    backup_influxdb "$backup_path" || true
    backup_vault "$backup_path" || true
    backup_configurations "$backup_path" || true
    backup_application_data "$backup_path" || true

    # Create manifest
    BACKUP_TYPE="full"
    create_manifest "$backup_path"

    # Create archive
    local archive_name="${backup_name}.tar.gz"
    if tar -czf "$BACKUP_DIR/$archive_name" -C "$BACKUP_DIR" "$backup_name"; then
        rm -rf "$backup_path"
        log_success "Full backup completed: $BACKUP_DIR/$archive_name"
    else
        log_error "Failed to create backup archive"
        return 1
    fi
}

# Incremental backup
perform_incremental_backup() {
    local backup_name="${BACKUP_PREFIX}_incremental_${TIMESTAMP}"
    local backup_path="$BACKUP_DIR/$backup_name"
    local last_backup_file="$BACKUP_DIR/.last_backup_timestamp"

    log_info "Starting incremental backup: $backup_name"

    # Find last backup timestamp
    local last_backup_time=""
    if [[ -f "$last_backup_file" ]]; then
        last_backup_time=$(cat "$last_backup_file")
        log_info "Last backup: $last_backup_time"
    else
        log_warning "No previous backup found, performing full backup instead"
        perform_full_backup
        return $?
    fi

    mkdir -p "$backup_path"

    # Backup only changed files
    local data_paths=(
        "/opt/foxhunt/config"
        "/opt/foxhunt/models"
        "/opt/foxhunt/data"
        "/var/log/foxhunt"
    )

    local changed_files=()
    for path in "${data_paths[@]}"; do
        if [[ -d "$path" ]]; then
            while IFS= read -r -d '' file; do
                changed_files+=("$file")
            done < <(find "$path" -newer "$last_backup_file" -type f -print0 2>/dev/null || true)
        fi
    done

    if [[ ${#changed_files[@]} -gt 0 ]]; then
        local incremental_file="$backup_path/changed_files_${TIMESTAMP}.tar.gz"
        if tar -czf "$incremental_file" "${changed_files[@]}"; then
            log_success "Incremental file backup completed: $incremental_file"
        else
            log_error "Incremental file backup failed"
        fi
    else
        log_info "No changed files found for incremental backup"
    fi

    # Always backup databases (they change frequently)
    backup_postgresql "$backup_path" || true
    backup_redis "$backup_path" || true

    # Create manifest
    BACKUP_TYPE="incremental"
    create_manifest "$backup_path"

    # Create archive
    local archive_name="${backup_name}.tar.gz"
    if tar -czf "$BACKUP_DIR/$archive_name" -C "$BACKUP_DIR" "$backup_name"; then
        rm -rf "$backup_path"
        echo "$TIMESTAMP" > "$last_backup_file"
        log_success "Incremental backup completed: $BACKUP_DIR/$archive_name"
    else
        log_error "Failed to create incremental backup archive"
        return 1
    fi
}

# Configuration-only backup
perform_config_backup() {
    local backup_name="${BACKUP_PREFIX}_config_${TIMESTAMP}"
    local backup_path="$BACKUP_DIR/$backup_name"

    log_info "Starting configuration backup: $backup_name"

    mkdir -p "$backup_path"

    backup_configurations "$backup_path"
    backup_vault "$backup_path"

    # Backup database schemas (without data)
    log_info "Backing up database schemas..."
    local schema_file="$backup_path/postgresql_schema_${TIMESTAMP}.sql"
    if docker exec foxhunt-postgres-prod pg_dump -U foxhunt -d foxhunt --schema-only > "$schema_file"; then
        gzip "$schema_file"
        log_success "Database schema backup completed: ${schema_file}.gz"
    else
        log_error "Database schema backup failed"
    fi

    # Create manifest
    BACKUP_TYPE="config"
    create_manifest "$backup_path"

    # Create archive
    local archive_name="${backup_name}.tar.gz"
    if tar -czf "$BACKUP_DIR/$archive_name" -C "$BACKUP_DIR" "$backup_name"; then
        rm -rf "$backup_path"
        log_success "Configuration backup completed: $BACKUP_DIR/$archive_name"
    else
        log_error "Failed to create configuration backup archive"
        return 1
    fi
}

# Data-only backup
perform_data_backup() {
    local backup_name="${BACKUP_PREFIX}_data_${TIMESTAMP}"
    local backup_path="$BACKUP_DIR/$backup_name"

    log_info "Starting data backup: $backup_name"

    mkdir -p "$backup_path"

    backup_postgresql "$backup_path"
    backup_redis "$backup_path"
    backup_influxdb "$backup_path"
    backup_application_data "$backup_path"

    # Create manifest
    BACKUP_TYPE="data"
    create_manifest "$backup_path"

    # Create archive
    local archive_name="${backup_name}.tar.gz"
    if tar -czf "$BACKUP_DIR/$archive_name" -C "$BACKUP_DIR" "$backup_name"; then
        rm -rf "$backup_path"
        log_success "Data backup completed: $BACKUP_DIR/$archive_name"
    else
        log_error "Failed to create data backup archive"
        return 1
    fi
}

# List available backups
list_backups() {
    log_info "Available backups:"

    if [[ ! -d "$BACKUP_DIR" ]]; then
        log_warning "No backup directory found"
        return
    fi

    local backups=($(ls -1 "$BACKUP_DIR"/*.tar.gz 2>/dev/null | sort -r || true))

    if [[ ${#backups[@]} -eq 0 ]]; then
        log_warning "No backups found"
        return
    fi

    printf "%-40s %-15s %-10s\n" "Backup File" "Date" "Size"
    printf "%-40s %-15s %-10s\n" "----------------------------------------" "---------------" "----------"

    for backup in "${backups[@]}"; do
        local filename=$(basename "$backup")
        local date_created=$(stat -c %y "$backup" | cut -d' ' -f1)
        local size=$(du -h "$backup" | cut -f1)

        printf "%-40s %-15s %-10s\n" "$filename" "$date_created" "$size"
    done
}

# Clean up old backups
cleanup_backups() {
    log_info "Cleaning up old backups..."

    if [[ ! -d "$BACKUP_DIR" ]]; then
        log_warning "No backup directory found"
        return
    fi

    # Clean up daily backups (keep last 7 days)
    find "$BACKUP_DIR" -name "${BACKUP_PREFIX}_*_*.tar.gz" -mtime +${DAILY_RETENTION} -delete

    log_success "Backup cleanup completed"
}

# Restore from backup
restore_backup() {
    local backup_file="$1"

    if [[ ! -f "$backup_file" ]]; then
        log_error "Backup file not found: $backup_file"
        return 1
    fi

    log_warning "RESTORE OPERATION - This will overwrite existing data!"
    read -p "Are you sure you want to continue? (y/N): " -n 1 -r
    echo
    if [[ ! $REPLY =~ ^[Yy]$ ]]; then
        log_info "Restore cancelled"
        return 0
    fi

    local restore_dir="$BACKUP_DIR/restore_${TIMESTAMP}"
    mkdir -p "$restore_dir"

    log_info "Extracting backup: $backup_file"
    if tar -xzf "$backup_file" -C "$restore_dir"; then
        log_success "Backup extracted to: $restore_dir"
        log_info "Manual restore steps required:"
        log_info "1. Stop services: docker-compose -f docker-compose.production.yml down"
        log_info "2. Restore data from: $restore_dir"
        log_info "3. Restart services: docker-compose -f docker-compose.production.yml up -d"
    else
        log_error "Failed to extract backup"
        return 1
    fi
}

# Main function
main() {
    local backup_type="full"
    local restore_file=""

    # Parse arguments
    while [[ $# -gt 0 ]]; do
        case $1 in
            --full)
                backup_type="full"
                shift
                ;;
            --incremental)
                backup_type="incremental"
                shift
                ;;
            --config-only)
                backup_type="config"
                shift
                ;;
            --data-only)
                backup_type="data"
                shift
                ;;
            --restore)
                restore_file="$2"
                shift 2
                ;;
            --list)
                list_backups
                exit 0
                ;;
            --cleanup)
                cleanup_backups
                exit 0
                ;;
            --help)
                show_help
                exit 0
                ;;
            *)
                log_error "Unknown option: $1"
                show_help
                exit 1
                ;;
        esac
    done

    create_backup_dir

    if [[ -n "$restore_file" ]]; then
        restore_backup "$restore_file"
    else
        case "$backup_type" in
            "full")
                perform_full_backup
                ;;
            "incremental")
                perform_incremental_backup
                ;;
            "config")
                perform_config_backup
                ;;
            "data")
                perform_data_backup
                ;;
        esac

        # Update last backup timestamp
        echo "$TIMESTAMP" > "$BACKUP_DIR/.last_backup_timestamp"
    fi
}

# Run main function
main "$@"