d746008e1f
- Created entrypoint-self-terminate.sh wrapper script - Updates entrypoint-generic.sh to be called by wrapper - Modified Dockerfile.runpod to use self-terminate entrypoint - Adds automatic pod termination via runpodctl after training completes - Prevents infinite restart loops and wasted GPU credits - Saves ~96% cost per training run ($4.59 per run) Implements pod self-termination using RUNPOD_POD_ID environment variable. Training exits with code 0 → runpodctl remove pod → immediate shutdown. Co-Authored-By: Claude <noreply@anthropic.com>
128 lines
3.8 KiB
Python
Executable File
128 lines
3.8 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Terminate failing RunPod pod - non-interactive version
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import requests
|
|
from dotenv import load_dotenv
|
|
|
|
# Load environment variables
|
|
env_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.env.runpod')
|
|
load_dotenv(env_path)
|
|
|
|
RUNPOD_API_KEY = os.getenv('RUNPOD_API_KEY')
|
|
|
|
if not RUNPOD_API_KEY:
|
|
print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
|
|
sys.exit(1)
|
|
|
|
REST_API_URL = "https://rest.runpod.io/v1/pods"
|
|
|
|
def terminate_pod(pod_id):
|
|
"""Terminate a RunPod pod."""
|
|
print(f"\n🗑️ Terminating pod {pod_id}...")
|
|
|
|
headers = {
|
|
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
|
}
|
|
|
|
try:
|
|
response = requests.delete(
|
|
f"{REST_API_URL}/{pod_id}",
|
|
headers=headers,
|
|
timeout=30
|
|
)
|
|
|
|
if response.status_code in [200, 204]:
|
|
print(f" ✅ Pod {pod_id} terminated successfully")
|
|
return True
|
|
else:
|
|
print(f" ⚠️ Termination failed (status {response.status_code}): {response.text[:200]}")
|
|
return False
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
print(f" ❌ Failed to terminate pod: {e}")
|
|
return False
|
|
|
|
|
|
def get_pod_status(pod_id):
|
|
"""Get current status of a pod."""
|
|
print(f"\n🔍 Checking pod {pod_id} status...")
|
|
|
|
headers = {
|
|
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
|
}
|
|
|
|
try:
|
|
response = requests.get(
|
|
f"{REST_API_URL}/{pod_id}",
|
|
headers=headers,
|
|
timeout=30
|
|
)
|
|
|
|
if response.status_code == 200:
|
|
pod_data = response.json()
|
|
print(f" Status: {pod_data.get('desiredStatus', 'UNKNOWN')}")
|
|
print(f" Runtime Status: {pod_data.get('runtime', {}).get('status', 'UNKNOWN')}")
|
|
|
|
machine = pod_data.get('machine', {})
|
|
datacenter = machine.get('dataCenterId', 'N/A')
|
|
print(f" Datacenter: {datacenter}")
|
|
|
|
gpu_info = machine.get('gpuType', {})
|
|
print(f" GPU: {gpu_info.get('displayName', 'N/A')}")
|
|
|
|
return pod_data
|
|
elif response.status_code == 404:
|
|
print(f" ⚠️ Pod not found (may already be terminated)")
|
|
return None
|
|
else:
|
|
print(f" ⚠️ Failed to get pod status (status {response.status_code}): {response.text[:200]}")
|
|
return None
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
print(f" ❌ Failed to check pod status: {e}")
|
|
return None
|
|
|
|
|
|
def main():
|
|
print("="*70)
|
|
print("TERMINATE FAILING RUNPOD POD")
|
|
print("="*70)
|
|
|
|
failing_pod_id = "3518shhoirbof4"
|
|
|
|
# Check pod status first
|
|
pod_data = get_pod_status(failing_pod_id)
|
|
|
|
if pod_data:
|
|
# Terminate the pod
|
|
if terminate_pod(failing_pod_id):
|
|
print("\n✅ Pod terminated successfully!")
|
|
print("\n📋 NEXT STEPS:")
|
|
print("="*70)
|
|
print("1. Deployment script has been fixed (EUR-IS-1 only)")
|
|
print("2. Wait 30 seconds for pod cleanup to complete")
|
|
print("3. Deploy new pod with:")
|
|
print()
|
|
print(" cd /home/jgrusewski/Work/foxhunt")
|
|
print(" ./scripts/runpod_deploy.py \\")
|
|
print(" --image jgrusewski/foxhunt:debug \\")
|
|
print(" --command \"--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 50 --learning-rate 0.001\"")
|
|
print()
|
|
print("4. Verify new pod is in EUR-IS-1 datacenter")
|
|
print("5. Check logs show volume mounted successfully")
|
|
print("="*70)
|
|
else:
|
|
print("\n❌ Failed to terminate pod")
|
|
sys.exit(1)
|
|
else:
|
|
print(f"\n⚠️ Pod {failing_pod_id} not found or already terminated")
|
|
print("\n✅ Ready to deploy new pod!")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|