d746008e1f
- Created entrypoint-self-terminate.sh wrapper script - Updates entrypoint-generic.sh to be called by wrapper - Modified Dockerfile.runpod to use self-terminate entrypoint - Adds automatic pod termination via runpodctl after training completes - Prevents infinite restart loops and wasted GPU credits - Saves ~96% cost per training run ($4.59 per run) Implements pod self-termination using RUNPOD_POD_ID environment variable. Training exits with code 0 → runpodctl remove pod → immediate shutdown. Co-Authored-By: Claude <noreply@anthropic.com>
81 lines
2.2 KiB
Bash
Executable File
81 lines
2.2 KiB
Bash
Executable File
#!/bin/bash
|
|
# DEBUG VERSION - Does NOT exit on error, prints everything
|
|
|
|
echo "=========================================="
|
|
echo "Foxhunt HFT - DEBUGGING VERSION"
|
|
echo "=========================================="
|
|
echo "Date: $(date)"
|
|
echo "Hostname: $(hostname)"
|
|
echo "User: $(whoami)"
|
|
echo "PWD: $(pwd)"
|
|
echo ""
|
|
|
|
# Show environment variables
|
|
echo "=========================================="
|
|
echo "ENVIRONMENT VARIABLES"
|
|
echo "=========================================="
|
|
env | sort
|
|
echo ""
|
|
|
|
# Show filesystem root
|
|
echo "=========================================="
|
|
echo "FILESYSTEM ROOT"
|
|
echo "=========================================="
|
|
ls -la / | head -30
|
|
echo ""
|
|
|
|
# Check if /runpod-volume exists
|
|
echo "=========================================="
|
|
echo "CHECKING /runpod-volume"
|
|
echo "=========================================="
|
|
if [ -d "/runpod-volume" ]; then
|
|
echo "✓ /runpod-volume directory EXISTS"
|
|
echo ""
|
|
echo "Directory contents:"
|
|
ls -laR /runpod-volume 2>&1 | head -50
|
|
else
|
|
echo "✗ /runpod-volume directory DOES NOT EXIST"
|
|
echo ""
|
|
echo "Mount points:"
|
|
mount | grep runpod || echo " (no runpod mounts found)"
|
|
fi
|
|
echo ""
|
|
|
|
# Check GPU
|
|
echo "=========================================="
|
|
echo "GPU STATUS"
|
|
echo "=========================================="
|
|
if command -v nvidia-smi &> /dev/null; then
|
|
nvidia-smi
|
|
else
|
|
echo "nvidia-smi not found"
|
|
fi
|
|
echo ""
|
|
|
|
# Check CUDA libraries
|
|
echo "=========================================="
|
|
echo "CUDA LIBRARIES"
|
|
echo "=========================================="
|
|
echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
|
|
echo ""
|
|
echo "libcudnn search:"
|
|
ldconfig -p | grep cudnn || echo " (no cudnn found)"
|
|
echo ""
|
|
echo "libcuda search:"
|
|
ldconfig -p | grep libcuda || echo " (no libcuda found)"
|
|
echo ""
|
|
echo "libcublas search:"
|
|
ldconfig -p | grep libcublas || echo " (no libcublas found)"
|
|
echo ""
|
|
|
|
# Keep container alive for inspection
|
|
echo "=========================================="
|
|
echo "CONTAINER READY FOR INSPECTION"
|
|
echo "=========================================="
|
|
echo "Container will stay alive for 1 hour for debugging"
|
|
echo "SSH into pod and run: docker ps, docker logs <container_id>"
|
|
echo ""
|
|
|
|
# Sleep for 1 hour to allow inspection
|
|
sleep 3600
|