foxhunt/Dockerfile.runpod.s3

# =============================================================================
# FOXHUNT HFT - RUNPOD S3-BASED DEPLOYMENT (GENERIC DOCKER IMAGE)
# =============================================================================
# Generic Docker image that downloads binaries and data from Runpod S3 at runtime
# This eliminates the need to rebuild Docker images for every code change
#
# Build Time: ~5 minutes (no Rust compilation - downloads at runtime)
# Image Size: ~2GB (base CUDA + HTTP client only)
# CUDA Support: CUDA 12.1 + cuDNN 8
#
# Architecture:
#   1. Build generic Docker image once (push to Docker Hub)
#   2. Upload binaries/data to Runpod Network Volume via S3-compatible API
#   3. Container downloads from Runpod S3 at startup and executes training
#
# IMPORTANT: This uses Runpod Network Volume S3 API (NOT AWS S3)
#   - Storage: 100% Runpod infrastructure ($0.10/GB/month)
#   - Credentials: Runpod User ID + Runpod API Key (NOT AWS credentials)
#   - Endpoint: https://s3api-<datacenter>.runpod.io/ (Runpod's S3-compatible API)
#   - Tool: AWS CLI (because Runpod API is S3-compatible, but all storage is Runpod)
#
# Advantages:
#   - No rebuilds: Update binaries by uploading to Runpod S3
#   - Fast startup: ~30 seconds download time for 23MB binary
#   - Reusable: Same image works for all training jobs
#   - Cost-effective: $0.10/GB/month Runpod storage (~$0.002/month for 23MB binary)
#
# Usage:
#   docker build -f Dockerfile.runpod.s3 -t foxhunt-runpod-s3:latest .
#   docker push jgrusewski/foxhunt-runpod-s3:latest
#
# Runpod Deployment:
#   1. Create Runpod Network Volume (get volume ID from Runpod console)
#   2. Get Runpod S3 credentials (User ID + API Key from Runpod settings)
#   3. Upload binaries to Runpod: ./upload_to_runpod_s3.sh
#   4. Deploy pod with env vars (all Runpod values, zero AWS infrastructure)
# =============================================================================

FROM nvidia/cuda:12.1.0-cudnn8-runtime-ubuntu22.04

# Prevent interactive prompts
ENV DEBIAN_FRONTEND=noninteractive

# Install runtime dependencies + AWS CLI (for Runpod S3-compatible API access)
# NOTE: AWS CLI is just a tool to access Runpod's S3-compatible API
#       All storage is on Runpod infrastructure, NOT AWS
RUN apt-get update && apt-get install -y \
    # Core runtime libraries
    ca-certificates \
    libssl3 \
    libpq5 \
    curl \
    wget \
    unzip \
    # AWS CLI v2 (for accessing Runpod S3-compatible API, NOT AWS S3)
    && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
    && unzip awscliv2.zip \
    && ./aws/install \
    && rm -rf aws awscliv2.zip \
    # Cleanup
    && rm -rf /var/lib/apt/lists/* \
    && apt-get clean

# Verify AWS CLI installation (used for Runpod S3 API, not AWS)
RUN aws --version

# Create app user for security
RUN groupadd -r foxhunt && useradd -r -g foxhunt -m -d /home/foxhunt foxhunt

# Configure CUDA runtime environment
ENV CUDA_HOME=/usr/local/cuda
ENV PATH="${CUDA_HOME}/bin:${PATH}"
ENV LD_LIBRARY_PATH="${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}"
ENV NVIDIA_VISIBLE_DEVICES=all
ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
ENV CUDA_VISIBLE_DEVICES=0

# Memory allocation optimization for ML training
ENV MALLOC_CONF="background_thread:false,dirty_decay_ms:0,muzzy_decay_ms:0"

# Rust runtime optimizations
ENV RUST_BACKTRACE=1
ENV RUST_LOG=info

# Set working directory
WORKDIR /workspace

# Create directories for models, checkpoints, data, and downloaded binaries
RUN mkdir -p \
    /workspace/models \
    /workspace/checkpoints \
    /workspace/test_data \
    /workspace/logs \
    /workspace/bin \
    && chown -R foxhunt:foxhunt /workspace

# Copy entrypoint script that handles Runpod S3 downloads
COPY <<'EOF' /usr/local/bin/entrypoint.sh
#!/bin/bash
set -e

# =============================================================================
# RUNPOD S3 ENTRYPOINT SCRIPT
# =============================================================================
# Downloads training binary and data from Runpod S3 at container startup
# Validates downloads and executes training with passed arguments
#
# IMPORTANT: This uses Runpod Network Volume S3 API (NOT AWS S3)
#   - All storage is on Runpod infrastructure
#   - Credentials are Runpod User ID + Runpod API Key (NOT AWS credentials)
#   - Endpoint is Runpod's S3-compatible API (https://s3api-<datacenter>.runpod.io/)

echo "=========================================="
echo "Foxhunt HFT - Runpod S3 Training Setup"
echo "=========================================="

# Validate required environment variables
required_vars=("S3_ENDPOINT" "S3_BUCKET" "AWS_ACCESS_KEY_ID" "AWS_SECRET_ACCESS_KEY" "BINARY_NAME")
for var in "${required_vars[@]}"; do
    if [ -z "${!var}" ]; then
        echo "ERROR: Required environment variable $var is not set"
        exit 1
    fi
done

echo "Configuration:"
echo "  S3 Endpoint: $S3_ENDPOINT (Runpod S3-compatible API)"
echo "  S3 Bucket: $S3_BUCKET (Runpod Network Volume ID)"
echo "  Binary: $BINARY_NAME"
echo "  Data: ${DATA_NAME:-not specified}"

# Configure AWS CLI for Runpod S3-compatible API
# NOTE: AWS CLI is just a tool - all storage is on Runpod infrastructure
export AWS_REGION="${AWS_REGION:-us-east-1}"
aws configure set default.s3.signature_version s3v4
aws configure set default.s3.addressing_style path

echo ""
echo "Downloading training binary from Runpod S3..."
aws s3 cp \
    "s3://${S3_BUCKET}/binaries/${BINARY_NAME}" \
    "/workspace/bin/${BINARY_NAME}" \
    --endpoint-url="${S3_ENDPOINT}" \
    --region="${AWS_REGION}"

# Verify download and make executable
if [ ! -f "/workspace/bin/${BINARY_NAME}" ]; then
    echo "ERROR: Failed to download binary from Runpod S3"
    exit 1
fi

chmod +x "/workspace/bin/${BINARY_NAME}"
echo "✓ Binary downloaded successfully ($(stat -f%z "/workspace/bin/${BINARY_NAME}" 2>/dev/null || stat -c%s "/workspace/bin/${BINARY_NAME}") bytes)"

# Download training data if specified
if [ -n "${DATA_NAME}" ]; then
    echo ""
    echo "Downloading training data from Runpod S3..."
    aws s3 cp \
        "s3://${S3_BUCKET}/data/${DATA_NAME}" \
        "/workspace/test_data/${DATA_NAME}" \
        --endpoint-url="${S3_ENDPOINT}" \
        --region="${AWS_REGION}"

    if [ ! -f "/workspace/test_data/${DATA_NAME}" ]; then
        echo "ERROR: Failed to download data from Runpod S3"
        exit 1
    fi

    echo "✓ Data downloaded successfully ($(stat -f%z "/workspace/test_data/${DATA_NAME}" 2>/dev/null || stat -c%s "/workspace/test_data/${DATA_NAME}") bytes)"
fi

# Verify GPU is accessible
echo ""
echo "GPU Status:"
nvidia-smi --query-gpu=name,memory.total,memory.free --format=csv,noheader || {
    echo "WARNING: nvidia-smi failed - GPU may not be accessible"
}

echo ""
echo "=========================================="
echo "Starting Training..."
echo "=========================================="

# Execute training binary with all passed arguments
exec "/workspace/bin/${BINARY_NAME}" "$@"
EOF

RUN chmod +x /usr/local/bin/entrypoint.sh

# Volume mounts for external data and model storage
VOLUME ["/workspace/test_data", "/workspace/models", "/workspace/checkpoints"]

# Switch to non-root user
USER foxhunt

# Health check to verify GPU is accessible
HEALTHCHECK --interval=60s --timeout=10s --start-period=30s --retries=3 \
    CMD nvidia-smi || exit 1

# Entry point downloads from Runpod S3 and executes training
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

# Default arguments (override with docker run command)
# Example: docker run --env-file .env.runpod foxhunt-runpod-s3:latest --epochs 50 --batch-size 32
CMD ["--parquet-file", "/workspace/test_data/ES_FUT_180d.parquet", \
     "--epochs", "50", \
     "--batch-size", "32"]

# =============================================================================
# BUILD INSTRUCTIONS
# =============================================================================
#
# 1. Build generic image (one-time):
#    docker build -f Dockerfile.runpod.s3 -t foxhunt-runpod-s3:latest .
#
# 2. Push to Docker Hub (one-time):
#    docker tag foxhunt-runpod-s3:latest jgrusewski/foxhunt-runpod-s3:latest
#    docker push jgrusewski/foxhunt-runpod-s3:latest
#
# 3. Upload binaries and data to Runpod S3:
#    ./upload_to_runpod_s3.sh
#
# 4. Deploy on Runpod with environment variables (all Runpod values):
#    - S3_ENDPOINT=https://s3api-DATACENTER.runpod.io (Runpod S3-compatible API)
#    - S3_BUCKET=your-network-volume-id (from Runpod console)
#    - AWS_ACCESS_KEY_ID=your-runpod-user-id (Runpod User ID, NOT AWS)
#    - AWS_SECRET_ACCESS_KEY=your-runpod-api-key (Runpod API Key, NOT AWS)
#    - BINARY_NAME=train_tft_parquet
#    - DATA_NAME=ES_FUT_180d.parquet (optional)
#
# =============================================================================