d746008e1f
- Created entrypoint-self-terminate.sh wrapper script - Updates entrypoint-generic.sh to be called by wrapper - Modified Dockerfile.runpod to use self-terminate entrypoint - Adds automatic pod termination via runpodctl after training completes - Prevents infinite restart loops and wasted GPU credits - Saves ~96% cost per training run ($4.59 per run) Implements pod self-termination using RUNPOD_POD_ID environment variable. Training exits with code 0 → runpodctl remove pod → immediate shutdown. Co-Authored-By: Claude <noreply@anthropic.com>
256 lines
8.2 KiB
Python
Executable File
256 lines
8.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Fix RunPod Deployment - Terminate failing pod and redeploy in EUR-IS-1
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import time
|
|
import requests
|
|
import json
|
|
from dotenv import load_dotenv
|
|
|
|
# Load environment variables
|
|
env_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.env.runpod')
|
|
load_dotenv(env_path)
|
|
|
|
RUNPOD_API_KEY = os.getenv('RUNPOD_API_KEY')
|
|
|
|
if not RUNPOD_API_KEY:
|
|
print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
|
|
sys.exit(1)
|
|
|
|
# API endpoints
|
|
REST_API_URL = "https://rest.runpod.io/v1/pods"
|
|
|
|
def terminate_pod(pod_id):
|
|
"""Terminate a RunPod pod."""
|
|
print(f"\n🗑️ Terminating pod {pod_id}...")
|
|
|
|
headers = {
|
|
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
|
}
|
|
|
|
try:
|
|
response = requests.delete(
|
|
f"{REST_API_URL}/{pod_id}",
|
|
headers=headers,
|
|
timeout=30
|
|
)
|
|
|
|
if response.status_code in [200, 204]:
|
|
print(f" ✅ Pod {pod_id} terminated successfully")
|
|
return True
|
|
else:
|
|
print(f" ⚠️ Termination failed (status {response.status_code}): {response.text[:200]}")
|
|
return False
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
print(f" ❌ Failed to terminate pod: {e}")
|
|
return False
|
|
|
|
|
|
def get_pod_status(pod_id):
|
|
"""Get current status of a pod."""
|
|
print(f"\n🔍 Checking pod {pod_id} status...")
|
|
|
|
headers = {
|
|
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
|
}
|
|
|
|
try:
|
|
response = requests.get(
|
|
f"{REST_API_URL}/{pod_id}",
|
|
headers=headers,
|
|
timeout=30
|
|
)
|
|
|
|
if response.status_code == 200:
|
|
pod_data = response.json()
|
|
print(f" Status: {pod_data.get('desiredStatus', 'UNKNOWN')}")
|
|
print(f" Runtime Status: {pod_data.get('runtime', {}).get('status', 'UNKNOWN')}")
|
|
|
|
machine = pod_data.get('machine', {})
|
|
datacenter = machine.get('dataCenterId', 'N/A')
|
|
print(f" Datacenter: {datacenter}")
|
|
|
|
gpu_info = machine.get('gpuType', {})
|
|
print(f" GPU: {gpu_info.get('displayName', 'N/A')}")
|
|
|
|
return pod_data
|
|
elif response.status_code == 404:
|
|
print(f" ⚠️ Pod not found (may already be terminated)")
|
|
return None
|
|
else:
|
|
print(f" ⚠️ Failed to get pod status (status {response.status_code}): {response.text[:200]}")
|
|
return None
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
print(f" ❌ Failed to check pod status: {e}")
|
|
return None
|
|
|
|
|
|
def list_all_pods():
|
|
"""List all active pods."""
|
|
print("\n📋 Listing all active pods...")
|
|
|
|
headers = {
|
|
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
|
}
|
|
|
|
try:
|
|
response = requests.get(
|
|
REST_API_URL,
|
|
headers=headers,
|
|
timeout=30
|
|
)
|
|
|
|
if response.status_code == 200:
|
|
pods_data = response.json()
|
|
|
|
# Handle both array response and object with 'pods' key
|
|
if isinstance(pods_data, list):
|
|
pods = pods_data
|
|
elif isinstance(pods_data, dict) and 'pods' in pods_data:
|
|
pods = pods_data['pods']
|
|
else:
|
|
print(f" ⚠️ Unexpected response format: {type(pods_data)}")
|
|
return []
|
|
|
|
print(f" Found {len(pods)} active pod(s)")
|
|
|
|
for pod in pods:
|
|
pod_id = pod.get('id', 'N/A')
|
|
pod_name = pod.get('name', 'N/A')
|
|
status = pod.get('desiredStatus', 'UNKNOWN')
|
|
machine = pod.get('machine', {})
|
|
datacenter = machine.get('dataCenterId', 'N/A')
|
|
gpu_info = machine.get('gpuType', {})
|
|
gpu_name = gpu_info.get('displayName', 'N/A')
|
|
|
|
print(f"\n Pod: {pod_id}")
|
|
print(f" Name: {pod_name}")
|
|
print(f" Status: {status}")
|
|
print(f" Datacenter: {datacenter}")
|
|
print(f" GPU: {gpu_name}")
|
|
|
|
return pods
|
|
else:
|
|
print(f" ⚠️ Failed to list pods (status {response.status_code}): {response.text[:200]}")
|
|
return []
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
print(f" ❌ Failed to list pods: {e}")
|
|
return []
|
|
|
|
|
|
def get_pod_logs(pod_id):
|
|
"""Get logs from a pod."""
|
|
print(f"\n📝 Fetching logs for pod {pod_id}...")
|
|
|
|
headers = {
|
|
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
|
}
|
|
|
|
try:
|
|
response = requests.get(
|
|
f"{REST_API_URL}/{pod_id}/logs",
|
|
headers=headers,
|
|
timeout=30
|
|
)
|
|
|
|
if response.status_code == 200:
|
|
logs_data = response.json()
|
|
|
|
# Display logs
|
|
if isinstance(logs_data, list):
|
|
print("\n" + "="*70)
|
|
print("CONTAINER LOGS")
|
|
print("="*70)
|
|
for log_entry in logs_data[-50:]: # Last 50 lines
|
|
timestamp = log_entry.get('timestamp', '')
|
|
message = log_entry.get('message', '')
|
|
print(f"{timestamp} {message}")
|
|
print("="*70)
|
|
elif isinstance(logs_data, dict) and 'logs' in logs_data:
|
|
print(logs_data['logs'])
|
|
else:
|
|
print(f" Raw logs response: {logs_data}")
|
|
|
|
return logs_data
|
|
else:
|
|
print(f" ⚠️ Failed to get logs (status {response.status_code}): {response.text[:200]}")
|
|
return None
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
print(f" ❌ Failed to fetch logs: {e}")
|
|
return None
|
|
|
|
|
|
def main():
|
|
print("="*70)
|
|
print("RUNPOD DEPLOYMENT FIX SCRIPT")
|
|
print("="*70)
|
|
print("\nThis script will:")
|
|
print("1. List all active pods")
|
|
print("2. Check status of pod 3518shhoirbof4 (if it exists)")
|
|
print("3. Get logs from the pod (to debug volume mounting)")
|
|
print("4. Terminate the failing pod")
|
|
print("5. Instructions for redeploying with fixed script")
|
|
print("="*70)
|
|
|
|
# List all active pods first
|
|
pods = list_all_pods()
|
|
|
|
# Check specific failing pod
|
|
failing_pod_id = "3518shhoirbof4"
|
|
pod_data = get_pod_status(failing_pod_id)
|
|
|
|
if pod_data:
|
|
# Get logs before terminating
|
|
get_pod_logs(failing_pod_id)
|
|
|
|
# Ask user confirmation
|
|
print(f"\n⚠️ Ready to terminate pod {failing_pod_id}")
|
|
print(" This will stop the restart loop and prepare for clean redeployment")
|
|
|
|
confirm = input("\nContinue with termination? [y/N]: ").strip().lower()
|
|
|
|
if confirm == 'y':
|
|
if terminate_pod(failing_pod_id):
|
|
print("\n✅ Pod terminated successfully!")
|
|
print("\n📋 NEXT STEPS:")
|
|
print("="*70)
|
|
print("1. Deployment script has been fixed (EUR-IS-1 only)")
|
|
print("2. Wait 30 seconds for pod cleanup to complete")
|
|
print("3. Deploy new pod with:")
|
|
print()
|
|
print(" cd /home/jgrusewski/Work/foxhunt")
|
|
print(" ./scripts/runpod_deploy.py \\")
|
|
print(" --image jgrusewski/foxhunt:debug \\")
|
|
print(" --command \"--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 50 --learning-rate 0.001\"")
|
|
print()
|
|
print("4. Verify new pod is in EUR-IS-1 datacenter")
|
|
print("5. Check logs show volume mounted successfully")
|
|
print("="*70)
|
|
else:
|
|
print("\n❌ Failed to terminate pod")
|
|
sys.exit(1)
|
|
else:
|
|
print("\n❌ Termination cancelled by user")
|
|
sys.exit(0)
|
|
else:
|
|
print(f"\n⚠️ Pod {failing_pod_id} not found or already terminated")
|
|
print("\n✅ Ready to deploy new pod!")
|
|
print("\n📋 DEPLOYMENT COMMAND:")
|
|
print("="*70)
|
|
print("cd /home/jgrusewski/Work/foxhunt")
|
|
print("./scripts/runpod_deploy.py \\")
|
|
print(" --image jgrusewski/foxhunt:debug \\")
|
|
print(" --command \"--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 50 --learning-rate 0.001\"")
|
|
print("="*70)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|