#!/usr/bin/env python3 """ Terminate failing RunPod pod - non-interactive version """ import os import sys import requests from dotenv import load_dotenv # Load environment variables env_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.env.runpod') load_dotenv(env_path) RUNPOD_API_KEY = os.getenv('RUNPOD_API_KEY') if not RUNPOD_API_KEY: print("ERROR: RUNPOD_API_KEY not found in .env.runpod") sys.exit(1) REST_API_URL = "https://rest.runpod.io/v1/pods" def terminate_pod(pod_id): """Terminate a RunPod pod.""" print(f"\n🗑️ Terminating pod {pod_id}...") headers = { "Authorization": f"Bearer {RUNPOD_API_KEY}" } try: response = requests.delete( f"{REST_API_URL}/{pod_id}", headers=headers, timeout=30 ) if response.status_code in [200, 204]: print(f" ✅ Pod {pod_id} terminated successfully") return True else: print(f" ⚠️ Termination failed (status {response.status_code}): {response.text[:200]}") return False except requests.exceptions.RequestException as e: print(f" ❌ Failed to terminate pod: {e}") return False def get_pod_status(pod_id): """Get current status of a pod.""" print(f"\n🔍 Checking pod {pod_id} status...") headers = { "Authorization": f"Bearer {RUNPOD_API_KEY}" } try: response = requests.get( f"{REST_API_URL}/{pod_id}", headers=headers, timeout=30 ) if response.status_code == 200: pod_data = response.json() print(f" Status: {pod_data.get('desiredStatus', 'UNKNOWN')}") print(f" Runtime Status: {pod_data.get('runtime', {}).get('status', 'UNKNOWN')}") machine = pod_data.get('machine', {}) datacenter = machine.get('dataCenterId', 'N/A') print(f" Datacenter: {datacenter}") gpu_info = machine.get('gpuType', {}) print(f" GPU: {gpu_info.get('displayName', 'N/A')}") return pod_data elif response.status_code == 404: print(f" ⚠️ Pod not found (may already be terminated)") return None else: print(f" ⚠️ Failed to get pod status (status {response.status_code}): {response.text[:200]}") return None except requests.exceptions.RequestException as e: print(f" ❌ Failed to check pod status: {e}") return None def main(): print("="*70) print("TERMINATE FAILING RUNPOD POD") print("="*70) failing_pod_id = "3518shhoirbof4" # Check pod status first pod_data = get_pod_status(failing_pod_id) if pod_data: # Terminate the pod if terminate_pod(failing_pod_id): print("\n✅ Pod terminated successfully!") print("\n📋 NEXT STEPS:") print("="*70) print("1. Deployment script has been fixed (EUR-IS-1 only)") print("2. Wait 30 seconds for pod cleanup to complete") print("3. Deploy new pod with:") print() print(" cd /home/jgrusewski/Work/foxhunt") print(" ./scripts/runpod_deploy.py \\") print(" --image jgrusewski/foxhunt:debug \\") print(" --command \"--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 50 --learning-rate 0.001\"") print() print("4. Verify new pod is in EUR-IS-1 datacenter") print("5. Check logs show volume mounted successfully") print("="*70) else: print("\n❌ Failed to terminate pod") sys.exit(1) else: print(f"\n⚠️ Pod {failing_pod_id} not found or already terminated") print("\n✅ Ready to deploy new pod!") if __name__ == "__main__": main()