6da9d262db
CRITICAL P0 FIXES (Validated - Loss 0.87 → 0.07): - Add sigmoid activation to inference and training (ml/src/mamba/mod.rs:798, 1538) - Fix config.total_decay_steps (was hardcoded 10000) (ml/src/mamba/mod.rs:2271) - Update d_state: 16→64, 32→64 (Mamba-2 spec) (ml/src/mamba/mod.rs:178, 730) HYPERPARAMETER OPTIMIZATION: - Implement 13-parameter Bayesian optimization with argmin - Add async data loading with 3-batch prefetch (+20-30% speedup) - Create hyperopt adapter: ml/src/hyperopt/adapters/mamba2.rs - Add example: ml/examples/hyperopt_mamba2_demo.rs VALIDATION: - Local test: Loss 0.07 vs 0.87 (12× improvement) - Val loss: 0.04-0.14 vs 1.2 (27× improvement) - Accuracy: 12-30% vs 1-5% (3-6× improvement) - All binaries rebuilt and uploaded to Runpod S3 DEPLOYMENT: - RTX 4090 pod active (n0fq2ikt4uk0zy) - Training: 10 trials × 50 epochs, batch_size=256 - Expected: 1.3 days, $10.41 cost Fixes #P0-sigmoid #P0-decay-steps #hyperopt-mamba2
129 lines
3.7 KiB
Python
Executable File
129 lines
3.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
RunPod Pod Monitoring Script
|
|
Continuously monitors pod metrics and displays GPU utilization, memory usage, etc.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import time
|
|
import requests
|
|
from datetime import datetime
|
|
from dotenv import load_dotenv
|
|
|
|
# Load RunPod API key
|
|
env_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.env.runpod')
|
|
load_dotenv(env_path)
|
|
|
|
api_key = os.getenv('RUNPOD_API_KEY')
|
|
if not api_key:
|
|
print("ERROR: RUNPOD_API_KEY not found in .env.runpod")
|
|
sys.exit(1)
|
|
|
|
# Default pod ID (can be overridden via command line)
|
|
pod_id = 'k18xwnvja2mk1s'
|
|
if len(sys.argv) > 1:
|
|
pod_id = sys.argv[1]
|
|
|
|
query = """
|
|
query GetPodMetrics($podId: String!) {
|
|
pod(input: {podId: $podId}) {
|
|
id
|
|
name
|
|
desiredStatus
|
|
runtime {
|
|
uptimeInSeconds
|
|
container {
|
|
cpuPercent
|
|
memoryPercent
|
|
}
|
|
gpus {
|
|
gpuUtilPercent
|
|
memoryUtilPercent
|
|
}
|
|
}
|
|
}
|
|
}
|
|
"""
|
|
|
|
print(f"📊 Monitoring Pod: {pod_id}")
|
|
print(f"Press Ctrl+C to stop\n")
|
|
|
|
iteration = 0
|
|
while True:
|
|
try:
|
|
response = requests.post(
|
|
"https://api.runpod.io/graphql",
|
|
json={"query": query, "variables": {"podId": pod_id}},
|
|
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
|
timeout=30
|
|
)
|
|
|
|
if response.status_code != 200:
|
|
print(f"⚠️ API Error: HTTP {response.status_code}")
|
|
time.sleep(60)
|
|
continue
|
|
|
|
data = response.json()
|
|
|
|
if 'errors' in data:
|
|
print(f"⚠️ GraphQL Error: {data['errors']}")
|
|
time.sleep(60)
|
|
continue
|
|
|
|
pod = data.get('data', {}).get('pod', {})
|
|
if not pod:
|
|
print(f"⚠️ Pod {pod_id} not found")
|
|
time.sleep(60)
|
|
continue
|
|
|
|
status = pod.get('desiredStatus', 'UNKNOWN')
|
|
runtime = pod.get('runtime', {})
|
|
|
|
timestamp = datetime.now().strftime('%H:%M:%S')
|
|
|
|
if runtime:
|
|
uptime = runtime.get('uptimeInSeconds', 0)
|
|
container = runtime.get('container', {})
|
|
gpus = runtime.get('gpus', [{}])
|
|
|
|
cpu_pct = container.get('cpuPercent', 0)
|
|
mem_pct = container.get('memoryPercent', 0)
|
|
gpu_util = gpus[0].get('gpuUtilPercent', 0) if gpus else 0
|
|
gpu_mem = gpus[0].get('memoryUtilPercent', 0) if gpus else 0
|
|
|
|
# Format uptime
|
|
hours = uptime // 3600
|
|
minutes = (uptime % 3600) // 60
|
|
seconds = uptime % 60
|
|
uptime_str = f"{hours}h {minutes}m {seconds}s"
|
|
|
|
# Color coding for GPU utilization
|
|
gpu_indicator = "🟢" if gpu_util > 80 else ("🟡" if gpu_util > 50 else "🔴")
|
|
|
|
print(f"[{timestamp}] {gpu_indicator} Uptime: {uptime_str:>12} | "
|
|
f"CPU: {cpu_pct:5.1f}% | Mem: {mem_pct:5.1f}% | "
|
|
f"GPU: {gpu_util:5.1f}% | VRAM: {gpu_mem:5.1f}% | "
|
|
f"Status: {status}")
|
|
|
|
# Alert on low GPU utilization after 10 minutes
|
|
if uptime > 600 and gpu_util < 50:
|
|
print(f" ⚠️ WARNING: Low GPU utilization after {uptime}s - check if training started")
|
|
|
|
# Alert on high memory usage
|
|
if mem_pct > 90:
|
|
print(f" ⚠️ WARNING: High memory usage ({mem_pct:.1f}%) - possible OOM risk")
|
|
|
|
else:
|
|
print(f"[{timestamp}] ⏳ Pod initializing... (Status: {status})")
|
|
|
|
iteration += 1
|
|
time.sleep(60) # Check every 60 seconds
|
|
|
|
except KeyboardInterrupt:
|
|
print("\n\n✅ Monitoring stopped")
|
|
sys.exit(0)
|
|
except Exception as e:
|
|
print(f"⚠️ Error: {e}")
|
|
time.sleep(60)
|