#!/bin/bash
# Quick status check for MAMBA-2 hyperopt on RunPod
# Usage: ./check_hyperopt_status.sh [pod_id]

POD_ID="${1:-nyt86m4i89106a}"
ENV_FILE="/home/jgrusewski/Work/foxhunt/.env.runpod"

if [ ! -f "$ENV_FILE" ]; then
    echo "ERROR: .env.runpod not found at $ENV_FILE"
    exit 1
fi

source "$ENV_FILE"

echo "======================================================================"
echo "MAMBA-2 Hyperopt Status Check"
echo "======================================================================"
echo "Pod ID: $POD_ID"
echo "Time: $(date)"
echo ""

# Check pod status via REST API
echo "Fetching pod info..."
RESPONSE=$(curl -s -H "Authorization: Bearer $RUNPOD_API_KEY" \
  "https://rest.runpod.io/v1/pods/$POD_ID")

STATUS=$(echo "$RESPONSE" | jq -r '.desiredStatus // "UNKNOWN"')
GPU=$(echo "$RESPONSE" | jq -r '.machine.gpuType.displayName // "N/A"')
DATACENTER=$(echo "$RESPONSE" | jq -r '.machine.dataCenterId // "N/A"')
COST=$(echo "$RESPONSE" | jq -r '.costPerHr // "N/A"')

echo "Status: $STATUS"
echo "GPU: $GPU"
echo "Datacenter: $DATACENTER"
echo "Cost: \$${COST}/hr"
echo ""

if [ "$STATUS" = "RUNNING" ]; then
    echo "✅ Pod is RUNNING"
    echo ""
    echo "Access logs at: https://www.runpod.io/console/pods"
    echo "SSH: ssh root@${POD_ID}.ssh.runpod.io"
    echo ""
    echo "Expected completion: ~60 minutes from start"
    echo "Estimated cost: ~\$0.42 total"
    echo ""
    echo "CRITICAL: Monitor first 10 minutes for CUDA OOM errors!"
    echo "If OOM occurs with RTX A4000, redeploy with --batch-size-max 96"
elif [ "$STATUS" = "TERMINATED" ] || [ "$STATUS" = "EXITED" ]; then
    echo "✅ Pod has TERMINATED (training likely complete)"
    echo ""
    echo "Download results:"
    echo "  aws s3 ls s3://se3zdnb5o4/models/ \\"
    echo "    --profile runpod \\"
    echo "    --endpoint-url https://s3api-eur-is-1.runpod.io \\"
    echo "    --recursive"
else
    echo "⏳ Pod status: $STATUS"
    echo ""
    echo "Wait 2-3 minutes for initialization, then check again"
fi

echo ""
echo "======================================================================"