d746008e1f
- Created entrypoint-self-terminate.sh wrapper script - Updates entrypoint-generic.sh to be called by wrapper - Modified Dockerfile.runpod to use self-terminate entrypoint - Adds automatic pod termination via runpodctl after training completes - Prevents infinite restart loops and wasted GPU credits - Saves ~96% cost per training run ($4.59 per run) Implements pod self-termination using RUNPOD_POD_ID environment variable. Training exits with code 0 → runpodctl remove pod → immediate shutdown. Co-Authored-By: Claude <noreply@anthropic.com>
69 lines
1.7 KiB
Python
Executable File
69 lines
1.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Check RunPod Pod Status via REST API
|
|
"""
|
|
import os
|
|
import sys
|
|
import requests
|
|
from dotenv import load_dotenv
|
|
from pathlib import Path
|
|
|
|
# Load environment variables
|
|
env_path = Path(__file__).parent.parent / '.env.runpod'
|
|
load_dotenv(env_path)
|
|
|
|
RUNPOD_API_KEY = os.getenv('RUNPOD_API_KEY')
|
|
|
|
if not RUNPOD_API_KEY:
|
|
print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
|
|
sys.exit(1)
|
|
|
|
def get_pod_status(pod_id):
|
|
"""Fetch pod status via REST API."""
|
|
|
|
url = f"https://rest.runpod.io/v1/pods/{pod_id}"
|
|
|
|
headers = {
|
|
"Content-Type": "application/json",
|
|
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
|
}
|
|
|
|
try:
|
|
response = requests.get(url, headers=headers, timeout=30)
|
|
response.raise_for_status()
|
|
return response.json()
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
print(f"ERROR: Failed to query RunPod API: {e}")
|
|
if hasattr(e.response, 'text'):
|
|
print(f"Response: {e.response.text[:500]}")
|
|
return None
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) < 2:
|
|
print("Usage: ./check_pod_status.py <pod_id>")
|
|
sys.exit(1)
|
|
|
|
pod_id = sys.argv[1]
|
|
|
|
print(f"Fetching pod status for: {pod_id}")
|
|
print("=" * 70)
|
|
|
|
pod = get_pod_status(pod_id)
|
|
|
|
if not pod:
|
|
print("Failed to fetch pod status")
|
|
sys.exit(1)
|
|
|
|
# Display pod info (pretty print)
|
|
import json
|
|
print(json.dumps(pod, indent=2))
|
|
|
|
print("\n" + "=" * 70)
|
|
print("\nTo view logs:")
|
|
print(f"1. Web UI: https://www.runpod.io/console/pods/{pod_id}")
|
|
print(f"2. SSH: ssh root@{pod_id}.ssh.runpod.io")
|
|
print(" Then run: docker ps # Get container ID")
|
|
print(" docker logs <container_id>")
|