e61e8f54da
Changes: - CLAUDE.md: Update OOM fix validation status - Add comprehensive documentation (30+ markdown reports) - LSTM encoder varmap bug fix (tft/lstm_encoder.rs:290) - Quantized LSTM layer matching fix (tft/quantized_lstm.rs) - Hyperopt paths module (ml/src/hyperopt/paths.rs) - Training path tests for all adapters (DQN, MAMBA-2, PPO, TFT) - Checkpoint integrity tests - Script cleanup: Remove 29 obsolete deployment scripts - Archive old scripts to scripts/archive/ - New deployment utilities: check_gpu_availability.py, monitor_hyperopt.sh Validation: - OOM fixes validated: 5/5 trials successful (pod b6kc3mc5lbjiro) - Batch-size-max 256 tested successfully - All hyperopt adapters working correctly 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
52 lines
1.8 KiB
Python
52 lines
1.8 KiB
Python
#!/usr/bin/env python3
|
|
import os
|
|
from pathlib import Path
|
|
from dotenv import load_dotenv
|
|
import runpod
|
|
|
|
# Load environment
|
|
env_path = Path.cwd().parent / '.env.runpod'
|
|
load_dotenv(env_path)
|
|
|
|
runpod.api_key = os.getenv('RUNPOD_API_KEY')
|
|
|
|
# Get all GPU types with any availability
|
|
gpu_types = runpod.get_gpus()
|
|
available_count = 0
|
|
|
|
print("ALL GPUs with ≥16GB VRAM and ANY availability:\n")
|
|
for gpu in gpu_types:
|
|
memory_gb = gpu.get('memoryInGb', 0)
|
|
secure = gpu.get('secureCloud', 0)
|
|
community = gpu.get('communityCloud', 0)
|
|
total = secure + community
|
|
name = gpu.get('displayName', 'Unknown')
|
|
|
|
if memory_gb >= 16 and total > 0:
|
|
price_info = gpu.get('lowestPrice', {})
|
|
price = price_info.get('uninterruptablePrice', 'N/A') if price_info else 'N/A'
|
|
|
|
print(f"{name}:")
|
|
print(f" VRAM: {memory_gb}GB")
|
|
print(f" Secure cloud: {secure}")
|
|
print(f" Community cloud: {community}")
|
|
print(f" Total: {total}")
|
|
print(f" Price: ${price}/hr" if price != 'N/A' else f" Price: {price}")
|
|
print()
|
|
available_count += 1
|
|
|
|
if available_count == 0:
|
|
print("NO GPUs with ≥16GB VRAM are available right now.")
|
|
print("\nMost common GPUs with ≥16GB VRAM (showing current status):")
|
|
|
|
common_gpus = ['RTX 4090', 'RTX 4000 Ada', 'RTX 5090', 'RTX A4000', 'A100', 'H100']
|
|
for target in common_gpus:
|
|
for gpu in gpu_types:
|
|
if target in gpu.get('displayName', ''):
|
|
memory_gb = gpu.get('memoryInGb', 0)
|
|
if memory_gb >= 16:
|
|
secure = gpu.get('secureCloud', 0)
|
|
community = gpu.get('communityCloud', 0)
|
|
print(f" {gpu['displayName']}: {secure + community} available")
|
|
break
|