d746008e1f
- Created entrypoint-self-terminate.sh wrapper script - Updates entrypoint-generic.sh to be called by wrapper - Modified Dockerfile.runpod to use self-terminate entrypoint - Adds automatic pod termination via runpodctl after training completes - Prevents infinite restart loops and wasted GPU credits - Saves ~96% cost per training run ($4.59 per run) Implements pod self-termination using RUNPOD_POD_ID environment variable. Training exits with code 0 → runpodctl remove pod → immediate shutdown. Co-Authored-By: Claude <noreply@anthropic.com>
86 lines
2.8 KiB
Docker
86 lines
2.8 KiB
Docker
# =============================================================================
|
|
# RUNPOD DEBUG DOCKERFILE - ENHANCED ERROR DIAGNOSTICS
|
|
# =============================================================================
|
|
# Purpose: Debug version of production Dockerfile with enhanced error capture
|
|
# Changes from Dockerfile.runpod:
|
|
# 1. Uses entrypoint_debug.sh instead of entrypoint.sh
|
|
# 2. Installs strace for system call tracing
|
|
# 3. Adds additional debugging tools (ldd, file, etc.)
|
|
# 4. Enables RUST_BACKTRACE=full for verbose panic messages
|
|
# =============================================================================
|
|
|
|
FROM nvidia/cuda:13.0.0-devel-ubuntu24.04
|
|
|
|
ENV DEBIAN_FRONTEND=noninteractive
|
|
|
|
# Install runtime dependencies + debugging tools
|
|
RUN apt-get update && apt-get install -y \
|
|
ca-certificates \
|
|
wget \
|
|
strace \
|
|
file \
|
|
binutils \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
# Install cuDNN 9 for CUDA 13.0
|
|
RUN apt-get update && apt-get install -y \
|
|
libcudnn9-cuda-13 \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
|
|
# Set CUDA environment variables
|
|
ENV CUDA_HOME=/usr/local/cuda
|
|
ENV PATH="${CUDA_HOME}/bin:${PATH}"
|
|
ENV LD_LIBRARY_PATH="${CUDA_HOME}/lib64:${LD_LIBRARY_PATH}"
|
|
|
|
# NVIDIA runtime configuration
|
|
ENV NVIDIA_VISIBLE_DEVICES=all
|
|
ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
|
|
ENV CUDA_VISIBLE_DEVICES=0
|
|
|
|
# Rust runtime environment (FULL backtrace for debugging)
|
|
ENV RUST_BACKTRACE=full
|
|
ENV RUST_LOG=debug
|
|
|
|
# Copy DEBUG entrypoint script
|
|
COPY entrypoint_debug.sh /entrypoint.sh
|
|
RUN chmod +x /entrypoint.sh
|
|
|
|
# Create workspace directory
|
|
WORKDIR /workspace
|
|
|
|
# Health check to verify GPU is accessible
|
|
HEALTHCHECK --interval=60s --timeout=10s --start-period=30s --retries=3 \
|
|
CMD nvidia-smi || exit 1
|
|
|
|
# Entrypoint executes training binary with enhanced error capture
|
|
ENTRYPOINT ["/entrypoint.sh"]
|
|
|
|
# Default CMD shows help (deployment script overrides this with dockerStartCmd)
|
|
CMD ["--help"]
|
|
|
|
# =============================================================================
|
|
# BUILD AND DEPLOY INSTRUCTIONS - DEBUG VERSION
|
|
# =============================================================================
|
|
#
|
|
# 1. BUILD DEBUG IMAGE:
|
|
# docker build -f Dockerfile.runpod.debug -t jgrusewski/foxhunt:debug .
|
|
#
|
|
# 2. PUSH TO DOCKER HUB:
|
|
# docker login
|
|
# docker push jgrusewski/foxhunt:debug
|
|
#
|
|
# 3. DEPLOY ON RUNPOD:
|
|
# - Use image: jgrusewski/foxhunt:debug
|
|
# - All other settings same as production deployment
|
|
# - Check logs for detailed error diagnostics
|
|
#
|
|
# 4. RETRIEVE ERROR LOGS:
|
|
# - View RunPod pod logs (console output)
|
|
# - Or connect via SSH and check:
|
|
# * /tmp/ldd_output.log (library dependencies)
|
|
# * /tmp/help_stderr.log (--help test errors)
|
|
# * /tmp/strace.log (system call trace)
|
|
# * /tmp/training_error.log (stderr capture)
|
|
#
|
|
# =============================================================================
|