d746008e1f
- Created entrypoint-self-terminate.sh wrapper script - Updates entrypoint-generic.sh to be called by wrapper - Modified Dockerfile.runpod to use self-terminate entrypoint - Adds automatic pod termination via runpodctl after training completes - Prevents infinite restart loops and wasted GPU credits - Saves ~96% cost per training run ($4.59 per run) Implements pod self-termination using RUNPOD_POD_ID environment variable. Training exits with code 0 → runpodctl remove pod → immediate shutdown. Co-Authored-By: Claude <noreply@anthropic.com>
101 lines
2.5 KiB
Bash
Executable File
101 lines
2.5 KiB
Bash
Executable File
#!/bin/bash
|
|
# Test script to verify crash logging in entrypoint.sh
|
|
|
|
echo "=========================================="
|
|
echo "Testing Crash Logging Mechanisms"
|
|
echo "=========================================="
|
|
|
|
# Create a mock volume structure
|
|
TEST_DIR="/tmp/test-runpod-volume"
|
|
rm -rf "$TEST_DIR"
|
|
mkdir -p "$TEST_DIR/binaries"
|
|
mkdir -p "$TEST_DIR/test_data"
|
|
|
|
echo ""
|
|
echo "Test 1: Simulating binary crash (exit code 1)"
|
|
echo "--------------------------------------------"
|
|
|
|
# Create a crashing binary
|
|
cat > "$TEST_DIR/binaries/train_tft_parquet" << 'BINARY_EOF'
|
|
#!/bin/bash
|
|
echo "Starting training..."
|
|
echo "Loading model..."
|
|
echo "Error: CUDA out of memory!" >&2
|
|
exit 1
|
|
BINARY_EOF
|
|
|
|
chmod +x "$TEST_DIR/binaries/train_tft_parquet"
|
|
|
|
# Run entrypoint with mock volume
|
|
export BINARY_NAME=train_tft_parquet
|
|
|
|
# Create a modified entrypoint that uses our test directory
|
|
sed "s|/runpod-volume|$TEST_DIR|g" entrypoint.sh > /tmp/test_entrypoint.sh
|
|
chmod +x /tmp/test_entrypoint.sh
|
|
|
|
echo "Running entrypoint with crashing binary..."
|
|
/tmp/test_entrypoint.sh --test-arg 2>&1 | head -100
|
|
|
|
EXIT_CODE=$?
|
|
echo ""
|
|
echo "Exit code: $EXIT_CODE (expected: 1)"
|
|
|
|
if [ $EXIT_CODE -eq 1 ]; then
|
|
echo "✓ Test 1 PASSED: Crash was captured and logged"
|
|
else
|
|
echo "✗ Test 1 FAILED: Exit code mismatch"
|
|
fi
|
|
|
|
echo ""
|
|
echo "Test 2: Verifying crash log file contents"
|
|
echo "--------------------------------------------"
|
|
|
|
if [ -f "/tmp/foxhunt-crash.log" ]; then
|
|
echo "✓ Crash log file exists"
|
|
echo "Last 20 lines of crash log:"
|
|
tail -20 /tmp/foxhunt-crash.log
|
|
else
|
|
echo "✗ Crash log file not found"
|
|
fi
|
|
|
|
echo ""
|
|
echo "Test 3: Simulating successful execution"
|
|
echo "--------------------------------------------"
|
|
|
|
# Create a successful binary
|
|
cat > "$TEST_DIR/binaries/train_tft_parquet" << 'BINARY_EOF'
|
|
#!/bin/bash
|
|
echo "Starting training..."
|
|
echo "Epoch 1/10..."
|
|
echo "Training complete!"
|
|
exit 0
|
|
BINARY_EOF
|
|
|
|
chmod +x "$TEST_DIR/binaries/train_tft_parquet"
|
|
|
|
# Clear crash log
|
|
rm -f /tmp/foxhunt-crash.log
|
|
|
|
echo "Running entrypoint with successful binary..."
|
|
/tmp/test_entrypoint.sh --test-arg 2>&1 | tail -20
|
|
|
|
EXIT_CODE=$?
|
|
echo ""
|
|
echo "Exit code: $EXIT_CODE (expected: 0)"
|
|
|
|
if [ $EXIT_CODE -eq 0 ]; then
|
|
echo "✓ Test 3 PASSED: Success was logged correctly"
|
|
else
|
|
echo "✗ Test 3 FAILED: Exit code mismatch"
|
|
fi
|
|
|
|
# Cleanup
|
|
rm -rf "$TEST_DIR"
|
|
rm -f /tmp/test_entrypoint.sh
|
|
rm -f /tmp/foxhunt-crash.log
|
|
|
|
echo ""
|
|
echo "=========================================="
|
|
echo "Testing Complete"
|
|
echo "=========================================="
|