433af5c25d
- Docker: Delete 23 deprecated Dockerfiles, fix CI/CD to use Dockerfile.foxhunt-build - Config: Remove 36 .env files, keep 4 essential, delete config/environments/ - Docs: Archive 614 Wave D files to docs/archive/wave_d/, 95% reduction in root - Scripts: Delete 56 deprecated scripts, keep 58 production-critical (49% reduction) - Python: Organize 37 scripts into scripts/python/ subdirectories, delete ml/python/ - Build: Remove 1GB artifacts, delete old venvs, clean Python cache from git - Migrations: Delete deprecated directory (4,432 lines), remove duplicate database/migrations/ - Infrastructure: Delete deployment/ (61 files), docs/scripts/ (8 files) Total impact: ~2,500 files cleaned, 750MB+ space freed, zero production impact All deleted scripts backed up to archives. runpod/ and tests/runpod/ preserved. data_acquisition_service retained per user request.
137 lines
5.1 KiB
Bash
Executable File
137 lines
5.1 KiB
Bash
Executable File
#!/bin/bash
|
|
# Hyperparameter Tuning Dashboard Monitor
|
|
# Real-time status dashboard for all 5 model tuning jobs
|
|
|
|
clear
|
|
echo "=================================================================="
|
|
echo " HYPERPARAMETER TUNING DASHBOARD"
|
|
echo "=================================================================="
|
|
echo "Updated: $(date)"
|
|
echo ""
|
|
|
|
# Check GPU status
|
|
echo "--- GPU STATUS ---"
|
|
nvidia-smi --query-gpu=index,name,utilization.gpu,memory.used,memory.total,temperature.gpu --format=csv | tail -n +2
|
|
echo ""
|
|
|
|
# Check DQN status
|
|
echo "--- DQN TUNING STATUS ---"
|
|
if ps -p 3911478 > /dev/null 2>&1; then
|
|
DQN_TRIALS=$(grep -c "Trial .* completed" /tmp/tuning_run.log 2>/dev/null || echo "0")
|
|
DQN_RUNTIME=$(ps -p 3911478 -o etime --no-headers)
|
|
echo "Status: RUNNING (PID 3911478)"
|
|
echo "Runtime: $DQN_RUNTIME"
|
|
echo "Progress: $DQN_TRIALS/50 trials ($(echo "scale=1; $DQN_TRIALS*100/50" | bc)%)"
|
|
LAST_TRIAL=$(grep "Trial .* completed" /tmp/tuning_run.log | tail -1)
|
|
echo "Last: $LAST_TRIAL"
|
|
else
|
|
DQN_TRIALS=$(grep -c "Trial .* completed" /tmp/tuning_run.log 2>/dev/null || echo "0")
|
|
echo "Status: COMPLETE"
|
|
echo "Completed: $DQN_TRIALS/50 trials"
|
|
fi
|
|
echo ""
|
|
|
|
# Check PPO status
|
|
echo "--- PPO TUNING STATUS ---"
|
|
PPO_PID=$(cat /tmp/ppo_tuning.pid 2>/dev/null)
|
|
if [ -n "$PPO_PID" ] && ps -p $PPO_PID > /dev/null 2>&1; then
|
|
PPO_TRIALS=$(grep -c "Trial .* completed" /tmp/ppo_tuning_run.log 2>/dev/null || echo "0")
|
|
PPO_RUNTIME=$(ps -p $PPO_PID -o etime --no-headers)
|
|
echo "Status: RUNNING (PID $PPO_PID)"
|
|
echo "Runtime: $PPO_RUNTIME"
|
|
echo "Progress: $PPO_TRIALS/50 trials ($(echo "scale=1; $PPO_TRIALS*100/50" | bc)%)"
|
|
else
|
|
if [ -f /tmp/ppo_tuning_run.log ]; then
|
|
PPO_TRIALS=$(grep -c "Trial .* completed" /tmp/ppo_tuning_run.log 2>/dev/null || echo "0")
|
|
if [ "$PPO_TRIALS" -gt 0 ]; then
|
|
echo "Status: COMPLETE"
|
|
echo "Completed: $PPO_TRIALS/50 trials"
|
|
else
|
|
echo "Status: NOT STARTED"
|
|
fi
|
|
else
|
|
echo "Status: NOT STARTED"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Check TFT status
|
|
echo "--- TFT TUNING STATUS ---"
|
|
TFT_PID=$(cat /tmp/tft_tuning.pid 2>/dev/null)
|
|
if [ -n "$TFT_PID" ] && ps -p $TFT_PID > /dev/null 2>&1; then
|
|
TFT_TRIALS=$(grep -c "Trial .* completed" /tmp/tft_tuning_run.log 2>/dev/null || echo "0")
|
|
TFT_RUNTIME=$(ps -p $TFT_PID -o etime --no-headers)
|
|
echo "Status: RUNNING (PID $TFT_PID)"
|
|
echo "Runtime: $TFT_RUNTIME"
|
|
echo "Progress: $TFT_TRIALS/50 trials ($(echo "scale=1; $TFT_TRIALS*100/50" | bc)%)"
|
|
else
|
|
if [ -f /tmp/tft_tuning_run.log ]; then
|
|
TFT_TRIALS=$(grep -c "Trial .* completed" /tmp/tft_tuning_run.log 2>/dev/null || echo "0")
|
|
if [ "$TFT_TRIALS" -gt 0 ]; then
|
|
echo "Status: COMPLETE"
|
|
echo "Completed: $TFT_TRIALS/50 trials"
|
|
else
|
|
echo "Status: NOT STARTED"
|
|
fi
|
|
else
|
|
echo "Status: NOT STARTED"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Check MAMBA-2 status
|
|
echo "--- MAMBA-2 TUNING STATUS ---"
|
|
MAMBA2_PID=$(cat /tmp/mamba2_tuning.pid 2>/dev/null)
|
|
if [ -n "$MAMBA2_PID" ] && ps -p $MAMBA2_PID > /dev/null 2>&1; then
|
|
MAMBA2_TRIALS=$(grep -c "Trial .* completed" /tmp/mamba2_tuning_run.log 2>/dev/null || echo "0")
|
|
MAMBA2_RUNTIME=$(ps -p $MAMBA2_PID -o etime --no-headers)
|
|
echo "Status: RUNNING (PID $MAMBA2_PID)"
|
|
echo "Runtime: $MAMBA2_RUNTIME"
|
|
echo "Progress: $MAMBA2_TRIALS/50 trials ($(echo "scale=1; $MAMBA2_TRIALS*100/50" | bc)%)"
|
|
else
|
|
if [ -f /tmp/mamba2_tuning_run.log ]; then
|
|
MAMBA2_TRIALS=$(grep -c "Trial .* completed" /tmp/mamba2_tuning_run.log 2>/dev/null || echo "0")
|
|
if [ "$MAMBA2_TRIALS" -gt 0 ]; then
|
|
echo "Status: COMPLETE"
|
|
echo "Completed: $MAMBA2_TRIALS/50 trials"
|
|
else
|
|
echo "Status: NOT STARTED"
|
|
fi
|
|
else
|
|
echo "Status: NOT STARTED"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# Check Liquid status
|
|
echo "--- LIQUID TUNING STATUS ---"
|
|
LIQUID_PID=$(cat /tmp/liquid_tuning.pid 2>/dev/null)
|
|
if [ -n "$LIQUID_PID" ] && ps -p $LIQUID_PID > /dev/null 2>&1; then
|
|
LIQUID_TRIALS=$(grep -c "Trial .* completed" /tmp/liquid_tuning_run.log 2>/dev/null || echo "0")
|
|
LIQUID_RUNTIME=$(ps -p $LIQUID_PID -o etime --no-headers)
|
|
echo "Status: RUNNING (PID $LIQUID_PID)"
|
|
echo "Runtime: $LIQUID_RUNTIME"
|
|
echo "Progress: $LIQUID_TRIALS/50 trials ($(echo "scale=1; $LIQUID_TRIALS*100/50" | bc)%)"
|
|
else
|
|
if [ -f /tmp/liquid_tuning_run.log ]; then
|
|
LIQUID_TRIALS=$(grep -c "Trial .* completed" /tmp/liquid_tuning_run.log 2>/dev/null || echo "0")
|
|
if [ "$LIQUID_TRIALS" -gt 0 ]; then
|
|
echo "Status: COMPLETE"
|
|
echo "Completed: $LIQUID_TRIALS/50 trials"
|
|
else
|
|
echo "Status: NOT STARTED"
|
|
fi
|
|
else
|
|
echo "Status: NOT STARTED"
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
echo "=================================================================="
|
|
echo "Commands:"
|
|
echo " Watch dashboard: watch -n 30 /home/jgrusewski/Work/foxhunt/scripts/dashboard_monitor.sh"
|
|
echo " View DQN log: tail -f /tmp/tuning_run.log"
|
|
echo " View PPO log: tail -f /tmp/ppo_tuning_run.log"
|
|
echo " View status: cat /tmp/tuning_pipeline_status.txt"
|
|
echo "=================================================================="
|