foxhunt/scripts/dashboard_monitor.sh

#!/bin/bash
# Hyperparameter Tuning Dashboard Monitor
# Real-time status dashboard for all 5 model tuning jobs

clear
echo "=================================================================="
echo "           HYPERPARAMETER TUNING DASHBOARD"
echo "=================================================================="
echo "Updated: $(date)"
echo ""

# Check GPU status
echo "--- GPU STATUS ---"
nvidia-smi --query-gpu=index,name,utilization.gpu,memory.used,memory.total,temperature.gpu --format=csv | tail -n +2
echo ""

# Check DQN status
echo "--- DQN TUNING STATUS ---"
if ps -p 3911478 > /dev/null 2>&1; then
    DQN_TRIALS=$(grep -c "Trial .* completed" /tmp/tuning_run.log 2>/dev/null || echo "0")
    DQN_RUNTIME=$(ps -p 3911478 -o etime --no-headers)
    echo "Status: RUNNING (PID 3911478)"
    echo "Runtime: $DQN_RUNTIME"
    echo "Progress: $DQN_TRIALS/50 trials ($(echo "scale=1; $DQN_TRIALS*100/50" | bc)%)"
    LAST_TRIAL=$(grep "Trial .* completed" /tmp/tuning_run.log | tail -1)
    echo "Last: $LAST_TRIAL"
else
    DQN_TRIALS=$(grep -c "Trial .* completed" /tmp/tuning_run.log 2>/dev/null || echo "0")
    echo "Status: COMPLETE"
    echo "Completed: $DQN_TRIALS/50 trials"
fi
echo ""

# Check PPO status
echo "--- PPO TUNING STATUS ---"
PPO_PID=$(cat /tmp/ppo_tuning.pid 2>/dev/null)
if [ -n "$PPO_PID" ] && ps -p $PPO_PID > /dev/null 2>&1; then
    PPO_TRIALS=$(grep -c "Trial .* completed" /tmp/ppo_tuning_run.log 2>/dev/null || echo "0")
    PPO_RUNTIME=$(ps -p $PPO_PID -o etime --no-headers)
    echo "Status: RUNNING (PID $PPO_PID)"
    echo "Runtime: $PPO_RUNTIME"
    echo "Progress: $PPO_TRIALS/50 trials ($(echo "scale=1; $PPO_TRIALS*100/50" | bc)%)"
else
    if [ -f /tmp/ppo_tuning_run.log ]; then
        PPO_TRIALS=$(grep -c "Trial .* completed" /tmp/ppo_tuning_run.log 2>/dev/null || echo "0")
        if [ "$PPO_TRIALS" -gt 0 ]; then
            echo "Status: COMPLETE"
            echo "Completed: $PPO_TRIALS/50 trials"
        else
            echo "Status: NOT STARTED"
        fi
    else
        echo "Status: NOT STARTED"
    fi
fi
echo ""

# Check TFT status
echo "--- TFT TUNING STATUS ---"
TFT_PID=$(cat /tmp/tft_tuning.pid 2>/dev/null)
if [ -n "$TFT_PID" ] && ps -p $TFT_PID > /dev/null 2>&1; then
    TFT_TRIALS=$(grep -c "Trial .* completed" /tmp/tft_tuning_run.log 2>/dev/null || echo "0")
    TFT_RUNTIME=$(ps -p $TFT_PID -o etime --no-headers)
    echo "Status: RUNNING (PID $TFT_PID)"
    echo "Runtime: $TFT_RUNTIME"
    echo "Progress: $TFT_TRIALS/50 trials ($(echo "scale=1; $TFT_TRIALS*100/50" | bc)%)"
else
    if [ -f /tmp/tft_tuning_run.log ]; then
        TFT_TRIALS=$(grep -c "Trial .* completed" /tmp/tft_tuning_run.log 2>/dev/null || echo "0")
        if [ "$TFT_TRIALS" -gt 0 ]; then
            echo "Status: COMPLETE"
            echo "Completed: $TFT_TRIALS/50 trials"
        else
            echo "Status: NOT STARTED"
        fi
    else
        echo "Status: NOT STARTED"
    fi
fi
echo ""

# Check MAMBA-2 status
echo "--- MAMBA-2 TUNING STATUS ---"
MAMBA2_PID=$(cat /tmp/mamba2_tuning.pid 2>/dev/null)
if [ -n "$MAMBA2_PID" ] && ps -p $MAMBA2_PID > /dev/null 2>&1; then
    MAMBA2_TRIALS=$(grep -c "Trial .* completed" /tmp/mamba2_tuning_run.log 2>/dev/null || echo "0")
    MAMBA2_RUNTIME=$(ps -p $MAMBA2_PID -o etime --no-headers)
    echo "Status: RUNNING (PID $MAMBA2_PID)"
    echo "Runtime: $MAMBA2_RUNTIME"
    echo "Progress: $MAMBA2_TRIALS/50 trials ($(echo "scale=1; $MAMBA2_TRIALS*100/50" | bc)%)"
else
    if [ -f /tmp/mamba2_tuning_run.log ]; then
        MAMBA2_TRIALS=$(grep -c "Trial .* completed" /tmp/mamba2_tuning_run.log 2>/dev/null || echo "0")
        if [ "$MAMBA2_TRIALS" -gt 0 ]; then
            echo "Status: COMPLETE"
            echo "Completed: $MAMBA2_TRIALS/50 trials"
        else
            echo "Status: NOT STARTED"
        fi
    else
        echo "Status: NOT STARTED"
    fi
fi
echo ""

# Check Liquid status
echo "--- LIQUID TUNING STATUS ---"
LIQUID_PID=$(cat /tmp/liquid_tuning.pid 2>/dev/null)
if [ -n "$LIQUID_PID" ] && ps -p $LIQUID_PID > /dev/null 2>&1; then
    LIQUID_TRIALS=$(grep -c "Trial .* completed" /tmp/liquid_tuning_run.log 2>/dev/null || echo "0")
    LIQUID_RUNTIME=$(ps -p $LIQUID_PID -o etime --no-headers)
    echo "Status: RUNNING (PID $LIQUID_PID)"
    echo "Runtime: $LIQUID_RUNTIME"
    echo "Progress: $LIQUID_TRIALS/50 trials ($(echo "scale=1; $LIQUID_TRIALS*100/50" | bc)%)"
else
    if [ -f /tmp/liquid_tuning_run.log ]; then
        LIQUID_TRIALS=$(grep -c "Trial .* completed" /tmp/liquid_tuning_run.log 2>/dev/null || echo "0")
        if [ "$LIQUID_TRIALS" -gt 0 ]; then
            echo "Status: COMPLETE"
            echo "Completed: $LIQUID_TRIALS/50 trials"
        else
            echo "Status: NOT STARTED"
        fi
    else
        echo "Status: NOT STARTED"
    fi
fi
echo ""

echo "=================================================================="
echo "Commands:"
echo "  Watch dashboard: watch -n 30 /home/jgrusewski/Work/foxhunt/scripts/dashboard_monitor.sh"
echo "  View DQN log: tail -f /tmp/tuning_run.log"
echo "  View PPO log: tail -f /tmp/ppo_tuning_run.log"
echo "  View status: cat /tmp/tuning_pipeline_status.txt"
echo "=================================================================="