8d89fe80ff
- Archive: 85 agent .txt files → docs/archive/agents/legacy_txt/ - Scripts: Move 110 shell scripts → scripts/ (keep deploy.sh in root) - Models: Move 18 .safetensors → ml/models/checkpoints/training_artifacts/ - Delete: 34 directories (~33GB freed) - target/, coverage_*, test artifacts - Build: Clean 14 build artifacts (.rlib, .o, .pid, binaries) - Tests: Move 14 .rs files → tests/standalone/ - SQL: Move 5 files → sql/ (keep init-db*.sql for Docker) - Wave 153: Archive to docs/archive/historical/wave153/ - Docs: Archive 9 markdown files to wave_d/reports/ and historical/ Total impact: ~34GB freed (both waves), root directory cleaned from 583 to ~40 essential files Directory count reduced from 65 to 31 (52% reduction) All historical data preserved in organized archive structure
65 lines
2.1 KiB
Bash
Executable File
65 lines
2.1 KiB
Bash
Executable File
#!/bin/bash
|
|
# Quick status check for MAMBA-2 hyperopt on RunPod
|
|
# Usage: ./check_hyperopt_status.sh [pod_id]
|
|
|
|
POD_ID="${1:-nyt86m4i89106a}"
|
|
ENV_FILE="/home/jgrusewski/Work/foxhunt/.env.runpod"
|
|
|
|
if [ ! -f "$ENV_FILE" ]; then
|
|
echo "ERROR: .env.runpod not found at $ENV_FILE"
|
|
exit 1
|
|
fi
|
|
|
|
source "$ENV_FILE"
|
|
|
|
echo "======================================================================"
|
|
echo "MAMBA-2 Hyperopt Status Check"
|
|
echo "======================================================================"
|
|
echo "Pod ID: $POD_ID"
|
|
echo "Time: $(date)"
|
|
echo ""
|
|
|
|
# Check pod status via REST API
|
|
echo "Fetching pod info..."
|
|
RESPONSE=$(curl -s -H "Authorization: Bearer $RUNPOD_API_KEY" \
|
|
"https://rest.runpod.io/v1/pods/$POD_ID")
|
|
|
|
STATUS=$(echo "$RESPONSE" | jq -r '.desiredStatus // "UNKNOWN"')
|
|
GPU=$(echo "$RESPONSE" | jq -r '.machine.gpuType.displayName // "N/A"')
|
|
DATACENTER=$(echo "$RESPONSE" | jq -r '.machine.dataCenterId // "N/A"')
|
|
COST=$(echo "$RESPONSE" | jq -r '.costPerHr // "N/A"')
|
|
|
|
echo "Status: $STATUS"
|
|
echo "GPU: $GPU"
|
|
echo "Datacenter: $DATACENTER"
|
|
echo "Cost: \$${COST}/hr"
|
|
echo ""
|
|
|
|
if [ "$STATUS" = "RUNNING" ]; then
|
|
echo "✅ Pod is RUNNING"
|
|
echo ""
|
|
echo "Access logs at: https://www.runpod.io/console/pods"
|
|
echo "SSH: ssh root@${POD_ID}.ssh.runpod.io"
|
|
echo ""
|
|
echo "Expected completion: ~60 minutes from start"
|
|
echo "Estimated cost: ~\$0.42 total"
|
|
echo ""
|
|
echo "CRITICAL: Monitor first 10 minutes for CUDA OOM errors!"
|
|
echo "If OOM occurs with RTX A4000, redeploy with --batch-size-max 96"
|
|
elif [ "$STATUS" = "TERMINATED" ] || [ "$STATUS" = "EXITED" ]; then
|
|
echo "✅ Pod has TERMINATED (training likely complete)"
|
|
echo ""
|
|
echo "Download results:"
|
|
echo " aws s3 ls s3://se3zdnb5o4/models/ \\"
|
|
echo " --profile runpod \\"
|
|
echo " --endpoint-url https://s3api-eur-is-1.runpod.io \\"
|
|
echo " --recursive"
|
|
else
|
|
echo "⏳ Pod status: $STATUS"
|
|
echo ""
|
|
echo "Wait 2-3 minutes for initialization, then check again"
|
|
fi
|
|
|
|
echo ""
|
|
echo "======================================================================"
|