#!/usr/bin/env python3 """ Fix RunPod Deployment - Terminate failing pod and redeploy in EUR-IS-1 """ import os import sys import time import requests import json from dotenv import load_dotenv # Load environment variables env_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.env.runpod') load_dotenv(env_path) RUNPOD_API_KEY = os.getenv('RUNPOD_API_KEY') if not RUNPOD_API_KEY: print("ERROR: RUNPOD_API_KEY not found in .env.runpod") sys.exit(1) # API endpoints REST_API_URL = "https://rest.runpod.io/v1/pods" def terminate_pod(pod_id): """Terminate a RunPod pod.""" print(f"\n🗑️ Terminating pod {pod_id}...") headers = { "Authorization": f"Bearer {RUNPOD_API_KEY}" } try: response = requests.delete( f"{REST_API_URL}/{pod_id}", headers=headers, timeout=30 ) if response.status_code in [200, 204]: print(f" ✅ Pod {pod_id} terminated successfully") return True else: print(f" ⚠️ Termination failed (status {response.status_code}): {response.text[:200]}") return False except requests.exceptions.RequestException as e: print(f" ❌ Failed to terminate pod: {e}") return False def get_pod_status(pod_id): """Get current status of a pod.""" print(f"\n🔍 Checking pod {pod_id} status...") headers = { "Authorization": f"Bearer {RUNPOD_API_KEY}" } try: response = requests.get( f"{REST_API_URL}/{pod_id}", headers=headers, timeout=30 ) if response.status_code == 200: pod_data = response.json() print(f" Status: {pod_data.get('desiredStatus', 'UNKNOWN')}") print(f" Runtime Status: {pod_data.get('runtime', {}).get('status', 'UNKNOWN')}") machine = pod_data.get('machine', {}) datacenter = machine.get('dataCenterId', 'N/A') print(f" Datacenter: {datacenter}") gpu_info = machine.get('gpuType', {}) print(f" GPU: {gpu_info.get('displayName', 'N/A')}") return pod_data elif response.status_code == 404: print(f" ⚠️ Pod not found (may already be terminated)") return None else: print(f" ⚠️ Failed to get pod status (status {response.status_code}): {response.text[:200]}") return None except requests.exceptions.RequestException as e: print(f" ❌ Failed to check pod status: {e}") return None def list_all_pods(): """List all active pods.""" print("\n📋 Listing all active pods...") headers = { "Authorization": f"Bearer {RUNPOD_API_KEY}" } try: response = requests.get( REST_API_URL, headers=headers, timeout=30 ) if response.status_code == 200: pods_data = response.json() # Handle both array response and object with 'pods' key if isinstance(pods_data, list): pods = pods_data elif isinstance(pods_data, dict) and 'pods' in pods_data: pods = pods_data['pods'] else: print(f" ⚠️ Unexpected response format: {type(pods_data)}") return [] print(f" Found {len(pods)} active pod(s)") for pod in pods: pod_id = pod.get('id', 'N/A') pod_name = pod.get('name', 'N/A') status = pod.get('desiredStatus', 'UNKNOWN') machine = pod.get('machine', {}) datacenter = machine.get('dataCenterId', 'N/A') gpu_info = machine.get('gpuType', {}) gpu_name = gpu_info.get('displayName', 'N/A') print(f"\n Pod: {pod_id}") print(f" Name: {pod_name}") print(f" Status: {status}") print(f" Datacenter: {datacenter}") print(f" GPU: {gpu_name}") return pods else: print(f" ⚠️ Failed to list pods (status {response.status_code}): {response.text[:200]}") return [] except requests.exceptions.RequestException as e: print(f" ❌ Failed to list pods: {e}") return [] def get_pod_logs(pod_id): """Get logs from a pod.""" print(f"\n📝 Fetching logs for pod {pod_id}...") headers = { "Authorization": f"Bearer {RUNPOD_API_KEY}" } try: response = requests.get( f"{REST_API_URL}/{pod_id}/logs", headers=headers, timeout=30 ) if response.status_code == 200: logs_data = response.json() # Display logs if isinstance(logs_data, list): print("\n" + "="*70) print("CONTAINER LOGS") print("="*70) for log_entry in logs_data[-50:]: # Last 50 lines timestamp = log_entry.get('timestamp', '') message = log_entry.get('message', '') print(f"{timestamp} {message}") print("="*70) elif isinstance(logs_data, dict) and 'logs' in logs_data: print(logs_data['logs']) else: print(f" Raw logs response: {logs_data}") return logs_data else: print(f" ⚠️ Failed to get logs (status {response.status_code}): {response.text[:200]}") return None except requests.exceptions.RequestException as e: print(f" ❌ Failed to fetch logs: {e}") return None def main(): print("="*70) print("RUNPOD DEPLOYMENT FIX SCRIPT") print("="*70) print("\nThis script will:") print("1. List all active pods") print("2. Check status of pod 3518shhoirbof4 (if it exists)") print("3. Get logs from the pod (to debug volume mounting)") print("4. Terminate the failing pod") print("5. Instructions for redeploying with fixed script") print("="*70) # List all active pods first pods = list_all_pods() # Check specific failing pod failing_pod_id = "3518shhoirbof4" pod_data = get_pod_status(failing_pod_id) if pod_data: # Get logs before terminating get_pod_logs(failing_pod_id) # Ask user confirmation print(f"\n⚠️ Ready to terminate pod {failing_pod_id}") print(" This will stop the restart loop and prepare for clean redeployment") confirm = input("\nContinue with termination? [y/N]: ").strip().lower() if confirm == 'y': if terminate_pod(failing_pod_id): print("\n✅ Pod terminated successfully!") print("\n📋 NEXT STEPS:") print("="*70) print("1. Deployment script has been fixed (EUR-IS-1 only)") print("2. Wait 30 seconds for pod cleanup to complete") print("3. Deploy new pod with:") print() print(" cd /home/jgrusewski/Work/foxhunt") print(" ./scripts/runpod_deploy.py \\") print(" --image jgrusewski/foxhunt:debug \\") print(" --command \"--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 50 --learning-rate 0.001\"") print() print("4. Verify new pod is in EUR-IS-1 datacenter") print("5. Check logs show volume mounted successfully") print("="*70) else: print("\n❌ Failed to terminate pod") sys.exit(1) else: print("\n❌ Termination cancelled by user") sys.exit(0) else: print(f"\n⚠️ Pod {failing_pod_id} not found or already terminated") print("\n✅ Ready to deploy new pod!") print("\n📋 DEPLOYMENT COMMAND:") print("="*70) print("cd /home/jgrusewski/Work/foxhunt") print("./scripts/runpod_deploy.py \\") print(" --image jgrusewski/foxhunt:debug \\") print(" --command \"--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 50 --learning-rate 0.001\"") print("="*70) if __name__ == "__main__": main()