Files
foxhunt/scripts/archive/get_pod_info.py
jgrusewski e61e8f54da feat(ml): Complete hyperopt infrastructure + documentation
Changes:
- CLAUDE.md: Update OOM fix validation status
- Add comprehensive documentation (30+ markdown reports)
- LSTM encoder varmap bug fix (tft/lstm_encoder.rs:290)
- Quantized LSTM layer matching fix (tft/quantized_lstm.rs)
- Hyperopt paths module (ml/src/hyperopt/paths.rs)
- Training path tests for all adapters (DQN, MAMBA-2, PPO, TFT)
- Checkpoint integrity tests
- Script cleanup: Remove 29 obsolete deployment scripts
- Archive old scripts to scripts/archive/
- New deployment utilities: check_gpu_availability.py, monitor_hyperopt.sh

Validation:
- OOM fixes validated: 5/5 trials successful (pod b6kc3mc5lbjiro)
- Batch-size-max 256 tested successfully
- All hyperopt adapters working correctly

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
2025-10-29 19:52:21 +01:00

201 lines
6.1 KiB
Python
Executable File

#!/usr/bin/env python3
"""
Get comprehensive pod information including datacenter and status
"""
import os
import sys
import time
import requests
from dotenv import load_dotenv
# Load environment variables
env_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.env.runpod')
load_dotenv(env_path)
RUNPOD_API_KEY = os.getenv('RUNPOD_API_KEY')
if not RUNPOD_API_KEY:
print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
sys.exit(1)
GRAPHQL_ENDPOINT = "https://api.runpod.io/graphql"
def get_pod_details_graphql(pod_id):
"""Get pod details using GraphQL."""
print(f"\n🔍 Fetching pod {pod_id} details via GraphQL...")
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {RUNPOD_API_KEY}"
}
# Updated query with correct fields
query = """
query GetPodDetails($podId: String!) {
pod(input: {podId: $podId}) {
id
name
desiredStatus
imageName
costPerHr
machine {
podHostId
dataCenterId
gpuTypeId
}
runtime {
uptimeInSeconds
ports {
ip
isIpPublic
privatePort
publicPort
type
}
gpus {
id
gpuUtilPercent
memoryUtilPercent
}
container {
cpuPercent
memoryPercent
}
}
}
}
"""
variables = {
"podId": pod_id
}
try:
response = requests.post(
GRAPHQL_ENDPOINT,
json={"query": query, "variables": variables},
headers=headers,
timeout=30
)
if response.status_code == 200:
result = response.json()
if 'errors' in result:
print(f" ⚠️ GraphQL errors: {result['errors']}")
return None
pod_data = result.get('data', {}).get('pod', {})
if pod_data:
print("\n" + "="*70)
print("POD DETAILS")
print("="*70)
print(f"Pod ID: {pod_data.get('id', 'N/A')}")
print(f"Name: {pod_data.get('name', 'N/A')}")
print(f"Status: {pod_data.get('desiredStatus', 'UNKNOWN')}")
machine = pod_data.get('machine', {})
if machine:
datacenter = machine.get('dataCenterId', 'N/A')
print(f"Datacenter: {datacenter}")
print(f"GPU Type ID: {machine.get('gpuTypeId', 'N/A')}")
# Check if datacenter is EUR-IS-1
if datacenter == 'EUR-IS-1':
print(f"\n✅ DATACENTER VERIFIED: Pod is in EUR-IS-1 (volume accessible)")
elif datacenter != 'N/A':
print(f"\n❌ DATACENTER MISMATCH: Pod is in {datacenter}, volume is in EUR-IS-1!")
else:
print(f"\n⚠️ Datacenter unknown - pod may still be initializing")
print(f"\nImage: {pod_data.get('imageName', 'N/A')}")
print(f"Cost: ${pod_data.get('costPerHr', 'N/A')}/hr")
runtime = pod_data.get('runtime', {})
if runtime:
uptime = runtime.get('uptimeInSeconds', 0)
print(f"\nRuntime Uptime: {uptime}s")
ports = runtime.get('ports', [])
if ports:
print("\nPorts:")
for port in ports:
print(f" {port.get('privatePort')} -> {port.get('publicPort')} ({port.get('type')})")
gpus = runtime.get('gpus', [])
if gpus:
print("\nGPU Status:")
for gpu in gpus:
gpu_util = gpu.get('gpuUtilPercent', 'N/A')
mem_util = gpu.get('memoryUtilPercent', 'N/A')
print(f" GPU {gpu.get('id')}: {gpu_util}% util, {mem_util}% memory")
container = runtime.get('container', {})
if container:
print(f"\nContainer:")
print(f" CPU: {container.get('cpuPercent', 'N/A')}%")
print(f" Memory: {container.get('memoryPercent', 'N/A')}%")
print("="*70)
return pod_data
else:
print(" ⚠️ No pod data returned")
return None
else:
print(f" ⚠️ Failed to get pod details (status {response.status_code}): {response.text[:200]}")
return None
except requests.exceptions.RequestException as e:
print(f" ❌ Failed to fetch pod details: {e}")
return None
def main():
print("="*70)
print("GET RUNPOD POD INFO")
print("="*70)
if len(sys.argv) < 2:
print("\nUsage: python3 get_pod_info.py <pod_id> [--watch]")
print("\nExample:")
print(" python3 get_pod_info.py b1m7v451nexg5r")
print(" python3 get_pod_info.py b1m7v451nexg5r --watch # Check every 30s")
sys.exit(1)
pod_id = sys.argv[1]
watch_mode = '--watch' in sys.argv
if watch_mode:
print("\n⏰ Watch mode enabled - will check every 30 seconds (Ctrl+C to stop)")
print("="*70)
while True:
pod_data = get_pod_details_graphql(pod_id)
if pod_data:
machine = pod_data.get('machine', {})
datacenter = machine.get('dataCenterId', 'N/A')
runtime = pod_data.get('runtime', {})
uptime = runtime.get('uptimeInSeconds', 0)
if datacenter == 'EUR-IS-1' and uptime > 0:
print("\n✅ Pod is ready in EUR-IS-1!")
break
print("\n⏳ Waiting 30 seconds before next check...")
time.sleep(30)
else:
get_pod_details_graphql(pod_id)
print("\n" + "="*70)
print("DONE")
print("="*70)
if __name__ == "__main__":
main()