d746008e1f
- Created entrypoint-self-terminate.sh wrapper script - Updates entrypoint-generic.sh to be called by wrapper - Modified Dockerfile.runpod to use self-terminate entrypoint - Adds automatic pod termination via runpodctl after training completes - Prevents infinite restart loops and wasted GPU credits - Saves ~96% cost per training run ($4.59 per run) Implements pod self-termination using RUNPOD_POD_ID environment variable. Training exits with code 0 → runpodctl remove pod → immediate shutdown. Co-Authored-By: Claude <noreply@anthropic.com>
405 lines
13 KiB
Python
Executable File
405 lines
13 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
RunPod Deployment Script
|
|
Scans for best value GPU in EUR-IS region and deploys a pod on SECURE cloud.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import argparse
|
|
import requests
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from dotenv import load_dotenv
|
|
|
|
# Load environment variables from .env.runpod
|
|
env_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.env.runpod')
|
|
load_dotenv(env_path)
|
|
|
|
RUNPOD_API_KEY = os.getenv('RUNPOD_API_KEY')
|
|
RUNPOD_VOLUME_ID = os.getenv('RUNPOD_VOLUME_ID')
|
|
RUNPOD_CONTAINER_REGISTRY_AUTH_ID = os.getenv('RUNPOD_CONTAINER_REGISTRY_AUTH_ID')
|
|
|
|
if not RUNPOD_API_KEY:
|
|
print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
|
|
sys.exit(1)
|
|
|
|
if not RUNPOD_VOLUME_ID:
|
|
print("ERROR: RUNPOD_VOLUME_ID not found in .env.runpod")
|
|
sys.exit(1)
|
|
|
|
GRAPHQL_ENDPOINT = "https://api.runpod.io/graphql"
|
|
|
|
# EUR-IS datacenters to scan
|
|
EUR_IS_DATACENTERS = ['EUR-IS-1', 'EUR-IS-2', 'EUR-IS-3']
|
|
|
|
# GraphQL query to fetch GPU types with availability
|
|
# Note: RunPod API doesn't support datacenter filtering in the query,
|
|
# so we query globally and then check per-datacenter availability
|
|
GPU_QUERY = """
|
|
{
|
|
gpuTypes {
|
|
id
|
|
displayName
|
|
memoryInGb
|
|
secureCloud
|
|
communityCloud
|
|
lowestPrice(input: {gpuCount: 1}) {
|
|
uninterruptablePrice
|
|
}
|
|
}
|
|
}
|
|
"""
|
|
|
|
# GraphQL mutation to deploy pod
|
|
# Note: dataCenterId is optional - if not specified, RunPod auto-selects
|
|
DEPLOY_MUTATION = """
|
|
mutation DeployPod($input: PodFindAndDeployOnDemandInput!) {
|
|
podFindAndDeployOnDemand(input: $input) {
|
|
id
|
|
imageName
|
|
gpuCount
|
|
costPerHr
|
|
containerDiskInGb
|
|
desiredStatus
|
|
machine {
|
|
gpuDisplayName
|
|
}
|
|
}
|
|
}
|
|
"""
|
|
|
|
def query_graphql(query, variables=None, raise_on_error=False):
|
|
"""Execute GraphQL query/mutation against RunPod API."""
|
|
headers = {
|
|
"Content-Type": "application/json",
|
|
"Authorization": f"Bearer {RUNPOD_API_KEY}"
|
|
}
|
|
|
|
payload = {"query": query}
|
|
if variables:
|
|
payload["variables"] = variables
|
|
|
|
try:
|
|
response = requests.post(GRAPHQL_ENDPOINT, json=payload, headers=headers, timeout=30)
|
|
response.raise_for_status()
|
|
result = response.json()
|
|
|
|
# Check for GraphQL errors
|
|
if 'errors' in result:
|
|
error_msg = result['errors'][0].get('message', 'Unknown error')
|
|
if raise_on_error:
|
|
raise RuntimeError(error_msg)
|
|
print(f"ERROR: GraphQL errors: {result['errors']}")
|
|
sys.exit(1)
|
|
|
|
return result
|
|
except requests.exceptions.RequestException as e:
|
|
if raise_on_error:
|
|
raise
|
|
print(f"ERROR: Failed to query RunPod API: {e}")
|
|
sys.exit(1)
|
|
|
|
def query_datacenter_gpus(datacenter):
|
|
"""
|
|
Query available GPUs in a specific EUR-IS datacenter.
|
|
Note: RunPod API doesn't support datacenter filtering in gpuTypes query,
|
|
so we query globally but will specify datacenter during deployment.
|
|
"""
|
|
try:
|
|
data = query_graphql(GPU_QUERY)
|
|
gpu_types = data.get('data', {}).get('gpuTypes', [])
|
|
|
|
# Filter criteria:
|
|
# 1. memoryInGb >= 16
|
|
# 2. secureCloud > 0 (available in secure cloud)
|
|
# 3. Has pricing information
|
|
datacenter_gpus = []
|
|
for gpu in gpu_types:
|
|
memory = gpu.get('memoryInGb', 0)
|
|
secure_count = gpu.get('secureCloud', 0)
|
|
lowest_price = gpu.get('lowestPrice', {})
|
|
price = lowest_price.get('uninterruptablePrice') if lowest_price else None
|
|
|
|
if memory >= 16 and secure_count > 0 and price is not None:
|
|
datacenter_gpus.append({
|
|
'id': gpu.get('id', ''),
|
|
'name': gpu.get('displayName', 'Unknown'),
|
|
'vram': memory,
|
|
'price': float(price),
|
|
'available': secure_count,
|
|
'datacenter': datacenter
|
|
})
|
|
|
|
return datacenter_gpus
|
|
|
|
except Exception as e:
|
|
print(f" ⚠️ {datacenter}: Query failed - {str(e)[:100]}")
|
|
return []
|
|
|
|
|
|
def get_available_gpus():
|
|
"""
|
|
Query available GPUs with ≥16GB VRAM in SECURE cloud across all EUR-IS datacenters.
|
|
Uses parallel queries to check all 3 datacenters simultaneously.
|
|
"""
|
|
print(" Scanning EUR-IS-1, EUR-IS-2, EUR-IS-3 in parallel...")
|
|
|
|
all_gpus = []
|
|
with ThreadPoolExecutor(max_workers=3) as executor:
|
|
futures = {executor.submit(query_datacenter_gpus, dc): dc for dc in EUR_IS_DATACENTERS}
|
|
|
|
for future in as_completed(futures):
|
|
datacenter = futures[future]
|
|
try:
|
|
gpus = future.result()
|
|
if gpus:
|
|
all_gpus.extend(gpus)
|
|
cheapest = min(gpus, key=lambda x: x['price'])
|
|
print(f" ✅ {datacenter}: {len(gpus)} GPU(s) - cheapest: {cheapest['name']} @ ${cheapest['price']:.3f}/hr")
|
|
else:
|
|
print(f" ⚠️ {datacenter}: No GPUs available")
|
|
except Exception as e:
|
|
print(f" ❌ {datacenter}: {str(e)[:100]}")
|
|
|
|
if not all_gpus:
|
|
return []
|
|
|
|
# Remove duplicates (same GPU type may appear in multiple datacenters)
|
|
# Keep cheapest instance of each GPU type
|
|
unique_gpus = {}
|
|
for gpu in all_gpus:
|
|
gpu_key = gpu['id']
|
|
if gpu_key not in unique_gpus or gpu['price'] < unique_gpus[gpu_key]['price']:
|
|
unique_gpus[gpu_key] = gpu
|
|
|
|
# Sort by price (cheapest first)
|
|
filtered_gpus = sorted(unique_gpus.values(), key=lambda x: x['price'])
|
|
|
|
return filtered_gpus
|
|
|
|
def select_best_gpu(gpus, preferred_gpu=None):
|
|
"""
|
|
Select best value GPU.
|
|
- If preferred_gpu specified and available, use it
|
|
- Otherwise, prefer RTX 4090 if available
|
|
- Fallback to cheapest available GPU
|
|
"""
|
|
if not gpus:
|
|
print("ERROR: No GPUs available matching criteria")
|
|
sys.exit(1)
|
|
|
|
# If user specified a GPU type
|
|
if preferred_gpu:
|
|
for gpu in gpus:
|
|
if preferred_gpu.lower() in gpu['name'].lower():
|
|
return gpu
|
|
print(f"WARNING: Preferred GPU '{preferred_gpu}' not available, selecting best value...")
|
|
|
|
# Prefer RTX 4090 if available (best performance/price)
|
|
for gpu in gpus:
|
|
if '4090' in gpu['name']:
|
|
return gpu
|
|
|
|
# Fallback to cheapest
|
|
return gpus[0]
|
|
|
|
def deploy_pod(gpu, image, command, container_disk, dry_run=False):
|
|
"""Deploy a pod on RunPod using the selected GPU."""
|
|
pod_name = "foxhunt-training"
|
|
|
|
deployment_input = {
|
|
"cloudType": "SECURE",
|
|
"gpuTypeId": gpu['id'],
|
|
"name": pod_name,
|
|
"imageName": image,
|
|
"dockerArgs": command if command else "",
|
|
"containerDiskInGb": container_disk,
|
|
"volumeInGb": 0,
|
|
"networkVolumeId": RUNPOD_VOLUME_ID,
|
|
"ports": "8888/http",
|
|
"env": []
|
|
}
|
|
|
|
# Add datacenter specification if available (EUR-IS specific)
|
|
if 'datacenter' in gpu:
|
|
deployment_input["dataCenterId"] = gpu['datacenter']
|
|
|
|
# Add Docker registry authentication if configured
|
|
if RUNPOD_CONTAINER_REGISTRY_AUTH_ID:
|
|
deployment_input["containerRegistryAuthId"] = RUNPOD_CONTAINER_REGISTRY_AUTH_ID
|
|
|
|
print("\n" + "="*70)
|
|
print("DEPLOYMENT PLAN")
|
|
print("="*70)
|
|
print(f"Pod Name: {pod_name}")
|
|
print(f"GPU: {gpu['name']} ({gpu['vram']}GB VRAM)")
|
|
print(f"Datacenter: {gpu.get('datacenter', 'Auto-select')}")
|
|
print(f"Price: ${gpu['price']:.3f}/hr")
|
|
print(f"Docker Image: {image}")
|
|
print(f"Container Disk: {container_disk}GB")
|
|
print(f"Network Volume: {RUNPOD_VOLUME_ID}")
|
|
print(f"Ports: 8888/http (Jupyter)")
|
|
if command:
|
|
print(f"Command: {command}")
|
|
print("="*70)
|
|
|
|
if dry_run:
|
|
print("\nDRY RUN: Skipping actual deployment")
|
|
return None
|
|
|
|
print("\nDeploying pod...")
|
|
try:
|
|
result = query_graphql(DEPLOY_MUTATION, {"input": deployment_input}, raise_on_error=True)
|
|
pod_data = result.get('data', {}).get('podFindAndDeployOnDemand')
|
|
|
|
if not pod_data:
|
|
return None
|
|
|
|
return pod_data
|
|
except (RuntimeError, requests.exceptions.RequestException) as e:
|
|
# GPU unavailable or other error - return None to allow fallback
|
|
print(f" ⚠️ Deployment failed: {str(e)[:100]}")
|
|
return None
|
|
|
|
def display_deployment_result(pod_data):
|
|
"""Display deployment results and connection info."""
|
|
print("\n" + "="*70)
|
|
print("✅ POD DEPLOYED SUCCESSFULLY")
|
|
print("="*70)
|
|
print(f"Pod ID: {pod_data['id']}")
|
|
print(f"GPU: {pod_data.get('machine', {}).get('gpuDisplayName', 'N/A')}")
|
|
print(f"GPU Count: {pod_data['gpuCount']}")
|
|
print(f"Cost: ${pod_data['costPerHr']:.3f}/hr")
|
|
print(f"Image: {pod_data['imageName']}")
|
|
print(f"Container Disk: {pod_data['containerDiskInGb']}GB")
|
|
print(f"Status: {pod_data.get('desiredStatus', 'RUNNING')}")
|
|
print("="*70)
|
|
print("\n📝 NEXT STEPS:")
|
|
print("1. Wait 2-3 minutes for pod to initialize")
|
|
print("2. Access Jupyter at: https://{pod_id}-8888.proxy.runpod.net")
|
|
print("3. SSH access: ssh root@{pod_id}.ssh.runpod.io")
|
|
print("4. Monitor pod: https://www.runpod.io/console/pods")
|
|
print(f"\n⚠️ IMPORTANT: Pod will cost ${pod_data['costPerHr']:.3f}/hr - remember to stop when done!")
|
|
print()
|
|
|
|
def main():
|
|
"""Main execution function."""
|
|
parser = argparse.ArgumentParser(
|
|
description="Deploy a RunPod GPU pod in EUR-IS region (SECURE cloud)",
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog="""
|
|
Examples:
|
|
# Auto-select best value GPU (RTX 4090 preferred, then cheapest)
|
|
./runpod_deploy.py
|
|
|
|
# Deploy with specific GPU
|
|
./runpod_deploy.py --gpu-type "RTX 4090"
|
|
|
|
# Custom image and command
|
|
./runpod_deploy.py --image runpod/pytorch:2.1.0 --command "jupyter lab"
|
|
|
|
# Dry run (show plan without deploying)
|
|
./runpod_deploy.py --dry-run
|
|
"""
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--gpu-type',
|
|
help='Preferred GPU type (e.g., "RTX 4090"). Auto-selects best value if not specified.'
|
|
)
|
|
parser.add_argument(
|
|
'--image',
|
|
default='jgrusewski/foxhunt:latest',
|
|
help='Docker image to use (default: jgrusewski/foxhunt:latest)'
|
|
)
|
|
parser.add_argument(
|
|
'--command',
|
|
default='',
|
|
help='Docker command to run (optional)'
|
|
)
|
|
parser.add_argument(
|
|
'--container-disk',
|
|
type=int,
|
|
default=50,
|
|
help='Container disk size in GB (default: 50)'
|
|
)
|
|
parser.add_argument(
|
|
'--dry-run',
|
|
action='store_true',
|
|
help='Show deployment plan without actually deploying'
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
print("🔍 Scanning available GPUs in EUR-IS region (SECURE cloud)...")
|
|
|
|
# Query available GPUs
|
|
gpus = get_available_gpus()
|
|
|
|
if not gpus:
|
|
print("ERROR: No GPUs available with ≥16GB VRAM in SECURE cloud")
|
|
sys.exit(1)
|
|
|
|
print(f"\n✅ Found {len(gpus)} available GPU(s)")
|
|
|
|
# Try to deploy with fallback logic
|
|
pod_data = None
|
|
attempted_gpus = []
|
|
|
|
# Create priority list: preferred GPU, RTX 4090, then sorted by price
|
|
priority_gpus = []
|
|
|
|
# Add user's preferred GPU if specified
|
|
if args.gpu_type:
|
|
for gpu in gpus:
|
|
if args.gpu_type.lower() in gpu['name'].lower():
|
|
priority_gpus.append(gpu)
|
|
break
|
|
|
|
# Add RTX 4090 if available and not already in list
|
|
for gpu in gpus:
|
|
if '4090' in gpu['name'] and gpu not in priority_gpus:
|
|
priority_gpus.append(gpu)
|
|
break
|
|
|
|
# Add remaining GPUs sorted by price
|
|
for gpu in gpus:
|
|
if gpu not in priority_gpus:
|
|
priority_gpus.append(gpu)
|
|
|
|
# Try each GPU until one succeeds
|
|
for gpu in priority_gpus:
|
|
if gpu in attempted_gpus:
|
|
continue
|
|
|
|
attempted_gpus.append(gpu)
|
|
|
|
print(f"\n🎯 Attempting deployment with {gpu['name']} (${gpu['price']:.3f}/hr)...")
|
|
|
|
pod_data = deploy_pod(
|
|
gpu,
|
|
args.image,
|
|
args.command,
|
|
args.container_disk,
|
|
args.dry_run
|
|
)
|
|
|
|
if pod_data:
|
|
break
|
|
else:
|
|
print(f"❌ {gpu['name']} unavailable, trying next option...")
|
|
|
|
# Display results
|
|
if pod_data:
|
|
display_deployment_result(pod_data)
|
|
else:
|
|
print("\n❌ ERROR: Failed to deploy pod on any available GPU")
|
|
print("\nAttempted GPUs:")
|
|
for gpu in attempted_gpus:
|
|
print(f" - {gpu['name']} ({gpu['vram']}GB VRAM) @ ${gpu['price']:.3f}/hr")
|
|
print("\n💡 TIP: RunPod availability changes frequently. Try again in a few minutes.")
|
|
sys.exit(1)
|
|
|
|
if __name__ == "__main__":
|
|
main()
|