220 lines
5.3 KiB
Python
Executable File
220 lines
5.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Example usage of foxhunt_runpod module.
|
|
|
|
Demonstrates:
|
|
- Pod deployment with GPU selection
|
|
- S3 log monitoring
|
|
- Auto-termination on completion
|
|
"""
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# Add parent directory to path
|
|
sys.path.insert(0, str(Path(__file__).parent.parent))
|
|
|
|
from foxhunt_runpod import RunPodClient, PodMonitor, S3Client
|
|
from foxhunt_runpod.errors import RunPodError
|
|
|
|
|
|
def example_1_deploy_and_monitor():
|
|
"""Example 1: Deploy pod and monitor training."""
|
|
print("\n" + "=" * 70)
|
|
print("EXAMPLE 1: Deploy and Monitor Training")
|
|
print("=" * 70 + "\n")
|
|
|
|
try:
|
|
# Initialize client
|
|
client = RunPodClient()
|
|
|
|
# Deploy pod with auto GPU selection
|
|
print("Deploying pod...")
|
|
pod = client.deploy_pod(
|
|
gpu_type="RTX A4000", # Preferred GPU (auto-selects if unavailable)
|
|
command="--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 50",
|
|
container_disk_gb=50,
|
|
)
|
|
|
|
pod_id = pod["id"]
|
|
print(f"\n✅ Pod deployed: {pod_id}")
|
|
|
|
# Monitor training
|
|
monitor = PodMonitor(pod_id)
|
|
|
|
# Wait for pod to start
|
|
print("\nWaiting for pod to start...")
|
|
monitor.wait_until_running(timeout=300)
|
|
|
|
# Stream logs until completion
|
|
print("\nStreaming training logs...")
|
|
monitor.stream_s3_logs(follow=True)
|
|
|
|
# Auto-terminate on completion
|
|
print("\nTerminating pod...")
|
|
monitor.auto_terminate()
|
|
|
|
except RunPodError as e:
|
|
print(f"\n❌ Error: {e}")
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def example_2_list_gpus():
|
|
"""Example 2: List available GPUs."""
|
|
print("\n" + "=" * 70)
|
|
print("EXAMPLE 2: List Available GPUs")
|
|
print("=" * 70 + "\n")
|
|
|
|
try:
|
|
client = RunPodClient()
|
|
|
|
# Query available GPUs
|
|
gpus = client.get_available_gpus(min_vram_gb=16)
|
|
|
|
print(f"Found {len(gpus)} GPU type(s):\n")
|
|
|
|
for i, gpu in enumerate(gpus, 1):
|
|
print(f"{i}. {gpu['name']}")
|
|
print(f" VRAM: {gpu['vram']}GB")
|
|
print(f" Price: ${gpu['price']:.3f}/hr")
|
|
print(f" Available: {gpu['global_available']} (global secure cloud)")
|
|
print()
|
|
|
|
except RunPodError as e:
|
|
print(f"\n❌ Error: {e}")
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def example_3_list_pods():
|
|
"""Example 3: List all pods."""
|
|
print("\n" + "=" * 70)
|
|
print("EXAMPLE 3: List All Pods")
|
|
print("=" * 70 + "\n")
|
|
|
|
try:
|
|
client = RunPodClient()
|
|
|
|
# List all pods
|
|
pods = client.list_pods()
|
|
|
|
if not pods:
|
|
print("No pods found")
|
|
return True
|
|
|
|
# Display in table format
|
|
client.display_pods_table(pods)
|
|
|
|
except RunPodError as e:
|
|
print(f"\n❌ Error: {e}")
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def example_4_s3_operations():
|
|
"""Example 4: S3 operations (list binaries, models)."""
|
|
print("\n" + "=" * 70)
|
|
print("EXAMPLE 4: S3 Operations")
|
|
print("=" * 70 + "\n")
|
|
|
|
try:
|
|
s3 = S3Client()
|
|
|
|
# List binaries
|
|
print("Binaries on S3:")
|
|
binaries = s3.list_binaries()
|
|
for binary in binaries:
|
|
size_mb = binary["size"] / (1024 * 1024)
|
|
print(f" - {binary['name']} ({size_mb:.1f} MB)")
|
|
|
|
print()
|
|
|
|
# List models
|
|
print("Models on S3:")
|
|
models = s3.list_models()
|
|
for model in models:
|
|
size_mb = model["size"] / (1024 * 1024)
|
|
print(f" - {model['name']} ({size_mb:.1f} MB)")
|
|
|
|
except RunPodError as e:
|
|
print(f"\n❌ Error: {e}")
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def example_5_monitor_existing_pod():
|
|
"""Example 5: Monitor an existing pod."""
|
|
print("\n" + "=" * 70)
|
|
print("EXAMPLE 5: Monitor Existing Pod")
|
|
print("=" * 70 + "\n")
|
|
|
|
pod_id = input("Enter pod ID to monitor: ").strip()
|
|
|
|
if not pod_id:
|
|
print("❌ No pod ID provided")
|
|
return False
|
|
|
|
try:
|
|
monitor = PodMonitor(pod_id)
|
|
|
|
# Display pod info
|
|
monitor.display_pod_info()
|
|
|
|
# Stream logs
|
|
print("\nStreaming logs (Ctrl+C to stop)...")
|
|
monitor.stream_s3_logs(follow=True)
|
|
|
|
except RunPodError as e:
|
|
print(f"\n❌ Error: {e}")
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def main():
|
|
"""Main menu."""
|
|
examples = {
|
|
"1": ("Deploy and Monitor Training", example_1_deploy_and_monitor),
|
|
"2": ("List Available GPUs", example_2_list_gpus),
|
|
"3": ("List All Pods", example_3_list_pods),
|
|
"4": ("S3 Operations", example_4_s3_operations),
|
|
"5": ("Monitor Existing Pod", example_5_monitor_existing_pod),
|
|
}
|
|
|
|
print("\n" + "=" * 70)
|
|
print("FOXHUNT RUNPOD MODULE - EXAMPLES")
|
|
print("=" * 70 + "\n")
|
|
|
|
for key, (name, _) in examples.items():
|
|
print(f"{key}. {name}")
|
|
|
|
print("q. Quit\n")
|
|
|
|
choice = input("Select example: ").strip().lower()
|
|
|
|
if choice == "q":
|
|
print("Goodbye!")
|
|
return
|
|
|
|
if choice not in examples:
|
|
print(f"❌ Invalid choice: {choice}")
|
|
return
|
|
|
|
# Run selected example
|
|
_, example_func = examples[choice]
|
|
success = example_func()
|
|
|
|
if success:
|
|
print("\n✅ Example completed successfully")
|
|
else:
|
|
print("\n❌ Example failed")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|