foxhunt/scripts/archive/terminate_failing_pod.py

#!/usr/bin/env python3
"""
Terminate failing RunPod pod - non-interactive version
"""

import os
import sys
import requests
from dotenv import load_dotenv

# Load environment variables
env_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.env.runpod')
load_dotenv(env_path)

RUNPOD_API_KEY = os.getenv('RUNPOD_API_KEY')

if not RUNPOD_API_KEY:
    print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
    sys.exit(1)

REST_API_URL = "https://rest.runpod.io/v1/pods"

def terminate_pod(pod_id):
    """Terminate a RunPod pod."""
    print(f"\n🗑️  Terminating pod {pod_id}...")

    headers = {
        "Authorization": f"Bearer {RUNPOD_API_KEY}"
    }

    try:
        response = requests.delete(
            f"{REST_API_URL}/{pod_id}",
            headers=headers,
            timeout=30
        )

        if response.status_code in [200, 204]:
            print(f"  ✅ Pod {pod_id} terminated successfully")
            return True
        else:
            print(f"  ⚠️  Termination failed (status {response.status_code}): {response.text[:200]}")
            return False

    except requests.exceptions.RequestException as e:
        print(f"  ❌ Failed to terminate pod: {e}")
        return False


def get_pod_status(pod_id):
    """Get current status of a pod."""
    print(f"\n🔍 Checking pod {pod_id} status...")

    headers = {
        "Authorization": f"Bearer {RUNPOD_API_KEY}"
    }

    try:
        response = requests.get(
            f"{REST_API_URL}/{pod_id}",
            headers=headers,
            timeout=30
        )

        if response.status_code == 200:
            pod_data = response.json()
            print(f"  Status: {pod_data.get('desiredStatus', 'UNKNOWN')}")
            print(f"  Runtime Status: {pod_data.get('runtime', {}).get('status', 'UNKNOWN')}")

            machine = pod_data.get('machine', {})
            datacenter = machine.get('dataCenterId', 'N/A')
            print(f"  Datacenter: {datacenter}")

            gpu_info = machine.get('gpuType', {})
            print(f"  GPU: {gpu_info.get('displayName', 'N/A')}")

            return pod_data
        elif response.status_code == 404:
            print(f"  ⚠️  Pod not found (may already be terminated)")
            return None
        else:
            print(f"  ⚠️  Failed to get pod status (status {response.status_code}): {response.text[:200]}")
            return None

    except requests.exceptions.RequestException as e:
        print(f"  ❌ Failed to check pod status: {e}")
        return None


def main():
    print("="*70)
    print("TERMINATE FAILING RUNPOD POD")
    print("="*70)

    failing_pod_id = "3518shhoirbof4"

    # Check pod status first
    pod_data = get_pod_status(failing_pod_id)

    if pod_data:
        # Terminate the pod
        if terminate_pod(failing_pod_id):
            print("\n✅ Pod terminated successfully!")
            print("\n📋 NEXT STEPS:")
            print("="*70)
            print("1. Deployment script has been fixed (EUR-IS-1 only)")
            print("2. Wait 30 seconds for pod cleanup to complete")
            print("3. Deploy new pod with:")
            print()
            print("   cd /home/jgrusewski/Work/foxhunt")
            print("   ./scripts/runpod_deploy.py \\")
            print("       --image jgrusewski/foxhunt:debug \\")
            print("       --command \"--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 50 --learning-rate 0.001\"")
            print()
            print("4. Verify new pod is in EUR-IS-1 datacenter")
            print("5. Check logs show volume mounted successfully")
            print("="*70)
        else:
            print("\n❌ Failed to terminate pod")
            sys.exit(1)
    else:
        print(f"\n⚠️  Pod {failing_pod_id} not found or already terminated")
        print("\n✅ Ready to deploy new pod!")


if __name__ == "__main__":
    main()