e61e8f54da
Changes: - CLAUDE.md: Update OOM fix validation status - Add comprehensive documentation (30+ markdown reports) - LSTM encoder varmap bug fix (tft/lstm_encoder.rs:290) - Quantized LSTM layer matching fix (tft/quantized_lstm.rs) - Hyperopt paths module (ml/src/hyperopt/paths.rs) - Training path tests for all adapters (DQN, MAMBA-2, PPO, TFT) - Checkpoint integrity tests - Script cleanup: Remove 29 obsolete deployment scripts - Archive old scripts to scripts/archive/ - New deployment utilities: check_gpu_availability.py, monitor_hyperopt.sh Validation: - OOM fixes validated: 5/5 trials successful (pod b6kc3mc5lbjiro) - Batch-size-max 256 tested successfully - All hyperopt adapters working correctly 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
69 lines
1.7 KiB
Python
Executable File
69 lines
1.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Check RunPod Pod Status via REST API
|
|
"""
|
|
import os
|
|
import sys
|
|
import requests
|
|
from dotenv import load_dotenv
|
|
from pathlib import Path
|
|
|
|
# Load environment variables
|
|
env_path = Path(__file__).parent.parent / '.env.runpod'
|
|
load_dotenv(env_path)
|
|
|
|
RUNPOD_API_KEY = os.getenv('RUNPOD_API_KEY')
|
|
|
|
if not RUNPOD_API_KEY:
|
|
print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
|
|
sys.exit(1)
|
|
|
|
def get_pod_status(pod_id):
|
|
"""Fetch pod status via REST API."""
|
|
|
|
url = f"https://rest.runpod.io/v1/pods/{pod_id}"
|
|
|
|
headers = {
|
|
"Content-Type": "application/json",
|
|
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
|
}
|
|
|
|
try:
|
|
response = requests.get(url, headers=headers, timeout=30)
|
|
response.raise_for_status()
|
|
return response.json()
|
|
|
|
except requests.exceptions.RequestException as e:
|
|
print(f"ERROR: Failed to query RunPod API: {e}")
|
|
if hasattr(e.response, 'text'):
|
|
print(f"Response: {e.response.text[:500]}")
|
|
return None
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) < 2:
|
|
print("Usage: ./check_pod_status.py <pod_id>")
|
|
sys.exit(1)
|
|
|
|
pod_id = sys.argv[1]
|
|
|
|
print(f"Fetching pod status for: {pod_id}")
|
|
print("=" * 70)
|
|
|
|
pod = get_pod_status(pod_id)
|
|
|
|
if not pod:
|
|
print("Failed to fetch pod status")
|
|
sys.exit(1)
|
|
|
|
# Display pod info (pretty print)
|
|
import json
|
|
print(json.dumps(pod, indent=2))
|
|
|
|
print("\n" + "=" * 70)
|
|
print("\nTo view logs:")
|
|
print(f"1. Web UI: https://www.runpod.io/console/pods/{pod_id}")
|
|
print(f"2. SSH: ssh root@{pod_id}.ssh.runpod.io")
|
|
print(" Then run: docker ps # Get container ID")
|
|
print(" docker logs <container_id>")
|