foxhunt/scripts/archive/fix_runpod_deployment.py

#!/usr/bin/env python3
"""
Fix RunPod Deployment - Terminate failing pod and redeploy in EUR-IS-1
"""

import os
import sys
import time
import requests
import json
from dotenv import load_dotenv

# Load environment variables
env_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.env.runpod')
load_dotenv(env_path)

RUNPOD_API_KEY = os.getenv('RUNPOD_API_KEY')

if not RUNPOD_API_KEY:
    print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
    sys.exit(1)

# API endpoints
REST_API_URL = "https://rest.runpod.io/v1/pods"

def terminate_pod(pod_id):
    """Terminate a RunPod pod."""
    print(f"\n🗑️  Terminating pod {pod_id}...")

    headers = {
        "Authorization": f"Bearer {RUNPOD_API_KEY}"
    }

    try:
        response = requests.delete(
            f"{REST_API_URL}/{pod_id}",
            headers=headers,
            timeout=30
        )

        if response.status_code in [200, 204]:
            print(f"  ✅ Pod {pod_id} terminated successfully")
            return True
        else:
            print(f"  ⚠️  Termination failed (status {response.status_code}): {response.text[:200]}")
            return False

    except requests.exceptions.RequestException as e:
        print(f"  ❌ Failed to terminate pod: {e}")
        return False


def get_pod_status(pod_id):
    """Get current status of a pod."""
    print(f"\n🔍 Checking pod {pod_id} status...")

    headers = {
        "Authorization": f"Bearer {RUNPOD_API_KEY}"
    }

    try:
        response = requests.get(
            f"{REST_API_URL}/{pod_id}",
            headers=headers,
            timeout=30
        )

        if response.status_code == 200:
            pod_data = response.json()
            print(f"  Status: {pod_data.get('desiredStatus', 'UNKNOWN')}")
            print(f"  Runtime Status: {pod_data.get('runtime', {}).get('status', 'UNKNOWN')}")

            machine = pod_data.get('machine', {})
            datacenter = machine.get('dataCenterId', 'N/A')
            print(f"  Datacenter: {datacenter}")

            gpu_info = machine.get('gpuType', {})
            print(f"  GPU: {gpu_info.get('displayName', 'N/A')}")

            return pod_data
        elif response.status_code == 404:
            print(f"  ⚠️  Pod not found (may already be terminated)")
            return None
        else:
            print(f"  ⚠️  Failed to get pod status (status {response.status_code}): {response.text[:200]}")
            return None

    except requests.exceptions.RequestException as e:
        print(f"  ❌ Failed to check pod status: {e}")
        return None


def list_all_pods():
    """List all active pods."""
    print("\n📋 Listing all active pods...")

    headers = {
        "Authorization": f"Bearer {RUNPOD_API_KEY}"
    }

    try:
        response = requests.get(
            REST_API_URL,
            headers=headers,
            timeout=30
        )

        if response.status_code == 200:
            pods_data = response.json()

            # Handle both array response and object with 'pods' key
            if isinstance(pods_data, list):
                pods = pods_data
            elif isinstance(pods_data, dict) and 'pods' in pods_data:
                pods = pods_data['pods']
            else:
                print(f"  ⚠️  Unexpected response format: {type(pods_data)}")
                return []

            print(f"  Found {len(pods)} active pod(s)")

            for pod in pods:
                pod_id = pod.get('id', 'N/A')
                pod_name = pod.get('name', 'N/A')
                status = pod.get('desiredStatus', 'UNKNOWN')
                machine = pod.get('machine', {})
                datacenter = machine.get('dataCenterId', 'N/A')
                gpu_info = machine.get('gpuType', {})
                gpu_name = gpu_info.get('displayName', 'N/A')

                print(f"\n  Pod: {pod_id}")
                print(f"    Name: {pod_name}")
                print(f"    Status: {status}")
                print(f"    Datacenter: {datacenter}")
                print(f"    GPU: {gpu_name}")

            return pods
        else:
            print(f"  ⚠️  Failed to list pods (status {response.status_code}): {response.text[:200]}")
            return []

    except requests.exceptions.RequestException as e:
        print(f"  ❌ Failed to list pods: {e}")
        return []


def get_pod_logs(pod_id):
    """Get logs from a pod."""
    print(f"\n📝 Fetching logs for pod {pod_id}...")

    headers = {
        "Authorization": f"Bearer {RUNPOD_API_KEY}"
    }

    try:
        response = requests.get(
            f"{REST_API_URL}/{pod_id}/logs",
            headers=headers,
            timeout=30
        )

        if response.status_code == 200:
            logs_data = response.json()

            # Display logs
            if isinstance(logs_data, list):
                print("\n" + "="*70)
                print("CONTAINER LOGS")
                print("="*70)
                for log_entry in logs_data[-50:]:  # Last 50 lines
                    timestamp = log_entry.get('timestamp', '')
                    message = log_entry.get('message', '')
                    print(f"{timestamp} {message}")
                print("="*70)
            elif isinstance(logs_data, dict) and 'logs' in logs_data:
                print(logs_data['logs'])
            else:
                print(f"  Raw logs response: {logs_data}")

            return logs_data
        else:
            print(f"  ⚠️  Failed to get logs (status {response.status_code}): {response.text[:200]}")
            return None

    except requests.exceptions.RequestException as e:
        print(f"  ❌ Failed to fetch logs: {e}")
        return None


def main():
    print("="*70)
    print("RUNPOD DEPLOYMENT FIX SCRIPT")
    print("="*70)
    print("\nThis script will:")
    print("1. List all active pods")
    print("2. Check status of pod 3518shhoirbof4 (if it exists)")
    print("3. Get logs from the pod (to debug volume mounting)")
    print("4. Terminate the failing pod")
    print("5. Instructions for redeploying with fixed script")
    print("="*70)

    # List all active pods first
    pods = list_all_pods()

    # Check specific failing pod
    failing_pod_id = "3518shhoirbof4"
    pod_data = get_pod_status(failing_pod_id)

    if pod_data:
        # Get logs before terminating
        get_pod_logs(failing_pod_id)

        # Ask user confirmation
        print(f"\n⚠️  Ready to terminate pod {failing_pod_id}")
        print("    This will stop the restart loop and prepare for clean redeployment")

        confirm = input("\nContinue with termination? [y/N]: ").strip().lower()

        if confirm == 'y':
            if terminate_pod(failing_pod_id):
                print("\n✅ Pod terminated successfully!")
                print("\n📋 NEXT STEPS:")
                print("="*70)
                print("1. Deployment script has been fixed (EUR-IS-1 only)")
                print("2. Wait 30 seconds for pod cleanup to complete")
                print("3. Deploy new pod with:")
                print()
                print("   cd /home/jgrusewski/Work/foxhunt")
                print("   ./scripts/runpod_deploy.py \\")
                print("       --image jgrusewski/foxhunt:debug \\")
                print("       --command \"--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 50 --learning-rate 0.001\"")
                print()
                print("4. Verify new pod is in EUR-IS-1 datacenter")
                print("5. Check logs show volume mounted successfully")
                print("="*70)
            else:
                print("\n❌ Failed to terminate pod")
                sys.exit(1)
        else:
            print("\n❌ Termination cancelled by user")
            sys.exit(0)
    else:
        print(f"\n⚠️  Pod {failing_pod_id} not found or already terminated")
        print("\n✅ Ready to deploy new pod!")
        print("\n📋 DEPLOYMENT COMMAND:")
        print("="*70)
        print("cd /home/jgrusewski/Work/foxhunt")
        print("./scripts/runpod_deploy.py \\")
        print("    --image jgrusewski/foxhunt:debug \\")
        print("    --command \"--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 50 --learning-rate 0.001\"")
        print("="*70)


if __name__ == "__main__":
    main()