e61e8f54da
Changes: - CLAUDE.md: Update OOM fix validation status - Add comprehensive documentation (30+ markdown reports) - LSTM encoder varmap bug fix (tft/lstm_encoder.rs:290) - Quantized LSTM layer matching fix (tft/quantized_lstm.rs) - Hyperopt paths module (ml/src/hyperopt/paths.rs) - Training path tests for all adapters (DQN, MAMBA-2, PPO, TFT) - Checkpoint integrity tests - Script cleanup: Remove 29 obsolete deployment scripts - Archive old scripts to scripts/archive/ - New deployment utilities: check_gpu_availability.py, monitor_hyperopt.sh Validation: - OOM fixes validated: 5/5 trials successful (pod b6kc3mc5lbjiro) - Batch-size-max 256 tested successfully - All hyperopt adapters working correctly 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
65 lines
2.1 KiB
Bash
Executable File
65 lines
2.1 KiB
Bash
Executable File
#!/bin/bash
|
|
# Quick status check for MAMBA-2 hyperopt on RunPod
|
|
# Usage: ./check_hyperopt_status.sh [pod_id]
|
|
|
|
POD_ID="${1:-nyt86m4i89106a}"
|
|
ENV_FILE="/home/jgrusewski/Work/foxhunt/.env.runpod"
|
|
|
|
if [ ! -f "$ENV_FILE" ]; then
|
|
echo "ERROR: .env.runpod not found at $ENV_FILE"
|
|
exit 1
|
|
fi
|
|
|
|
source "$ENV_FILE"
|
|
|
|
echo "======================================================================"
|
|
echo "MAMBA-2 Hyperopt Status Check"
|
|
echo "======================================================================"
|
|
echo "Pod ID: $POD_ID"
|
|
echo "Time: $(date)"
|
|
echo ""
|
|
|
|
# Check pod status via REST API
|
|
echo "Fetching pod info..."
|
|
RESPONSE=$(curl -s -H "Authorization: Bearer $RUNPOD_API_KEY" \
|
|
"https://rest.runpod.io/v1/pods/$POD_ID")
|
|
|
|
STATUS=$(echo "$RESPONSE" | jq -r '.desiredStatus // "UNKNOWN"')
|
|
GPU=$(echo "$RESPONSE" | jq -r '.machine.gpuType.displayName // "N/A"')
|
|
DATACENTER=$(echo "$RESPONSE" | jq -r '.machine.dataCenterId // "N/A"')
|
|
COST=$(echo "$RESPONSE" | jq -r '.costPerHr // "N/A"')
|
|
|
|
echo "Status: $STATUS"
|
|
echo "GPU: $GPU"
|
|
echo "Datacenter: $DATACENTER"
|
|
echo "Cost: \$${COST}/hr"
|
|
echo ""
|
|
|
|
if [ "$STATUS" = "RUNNING" ]; then
|
|
echo "✅ Pod is RUNNING"
|
|
echo ""
|
|
echo "Access logs at: https://www.runpod.io/console/pods"
|
|
echo "SSH: ssh root@${POD_ID}.ssh.runpod.io"
|
|
echo ""
|
|
echo "Expected completion: ~60 minutes from start"
|
|
echo "Estimated cost: ~\$0.42 total"
|
|
echo ""
|
|
echo "CRITICAL: Monitor first 10 minutes for CUDA OOM errors!"
|
|
echo "If OOM occurs with RTX A4000, redeploy with --batch-size-max 96"
|
|
elif [ "$STATUS" = "TERMINATED" ] || [ "$STATUS" = "EXITED" ]; then
|
|
echo "✅ Pod has TERMINATED (training likely complete)"
|
|
echo ""
|
|
echo "Download results:"
|
|
echo " aws s3 ls s3://se3zdnb5o4/models/ \\"
|
|
echo " --profile runpod \\"
|
|
echo " --endpoint-url https://s3api-eur-is-1.runpod.io \\"
|
|
echo " --recursive"
|
|
else
|
|
echo "⏳ Pod status: $STATUS"
|
|
echo ""
|
|
echo "Wait 2-3 minutes for initialization, then check again"
|
|
fi
|
|
|
|
echo ""
|
|
echo "======================================================================"
|