#!/bin/bash # Quick Status Checker - Run anytime to see current tuning status echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo " HYPERPARAMETER TUNING PIPELINE - QUICK STATUS" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "Timestamp: $(date)" echo "" # GPU Status echo "🎮 GPU:" nvidia-smi --query-gpu=utilization.gpu,memory.used,memory.total,temperature.gpu --format=csv,noheader | \ awk -F', ' '{printf " Utilization: %s | Memory: %s/%s | Temp: %s\n", $1, $2, $3, $4}' echo "" # DQN Status echo "🤖 DQN:" if ps -p 3911478 > /dev/null 2>&1; then DQN_TRIALS=$(grep -c "Trial .* completed" /tmp/tuning_run.log 2>/dev/null || echo "0") DQN_RUNTIME=$(ps -p 3911478 -o etime --no-headers | xargs) DQN_PCT=$(echo "scale=1; $DQN_TRIALS*100/50" | bc) echo " Status: ⏳ RUNNING | Progress: $DQN_TRIALS/50 ($DQN_PCT%) | Runtime: $DQN_RUNTIME" LAST=$(grep "Trial .* completed" /tmp/tuning_run.log 2>/dev/null | tail -1 | grep -oP "Trial \d+" | tail -1) SHARPE=$(grep "Trial .* completed" /tmp/tuning_run.log 2>/dev/null | tail -1 | grep -oP "Sharpe=[0-9.]+") echo " Last: $LAST | $SHARPE" else echo " Status: ✅ COMPLETE or ❌ STOPPED" fi echo "" # Other Models for MODEL in PPO TFT MAMBA2 Liquid; do MODEL_LOWER=$(echo $MODEL | tr '[:upper:]' '[:lower:]') PID_FILE="/tmp/${MODEL_LOWER}_tuning.pid" LOG_FILE="/tmp/${MODEL_LOWER}_tuning_run.log" echo "🤖 $MODEL:" if [ -f "$PID_FILE" ]; then PID=$(cat "$PID_FILE") if ps -p $PID > /dev/null 2>&1; then TRIALS=$(grep -c "Trial .* completed" "$LOG_FILE" 2>/dev/null || echo "0") RUNTIME=$(ps -p $PID -o etime --no-headers | xargs) PCT=$(echo "scale=1; $TRIALS*100/50" | bc) echo " Status: ⏳ RUNNING | Progress: $TRIALS/50 ($PCT%) | Runtime: $RUNTIME" else TRIALS=$(grep -c "Trial .* completed" "$LOG_FILE" 2>/dev/null || echo "0") if [ "$TRIALS" -gt 0 ]; then echo " Status: ✅ COMPLETE | Trials: $TRIALS/50" else echo " Status: ❌ STOPPED/FAILED" fi fi else echo " Status: ⏳ PENDING (not started)" fi echo "" done # Auto-Monitor Status echo "🔧 Auto-Monitor:" if ps aux | grep -v grep | grep "auto_monitor_and_launch.sh" > /dev/null; then echo " Status: ✅ RUNNING" else echo " Status: ❌ STOPPED" fi echo "" # Results Files echo "📊 Results Files:" for MODEL in dqn ppo tft mamba2 liquid; do FILE="results/${MODEL}_tuning_50trials.json" if [ -f "$FILE" ]; then SIZE=$(ls -lh "$FILE" | awk '{print $5}') echo " ✅ $FILE ($SIZE)" else echo " ⏳ $FILE (not created yet)" fi done echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "Commands:" echo " Full dashboard: watch -n 30 /home/jgrusewski/Work/foxhunt/scripts/dashboard_monitor.sh" echo " DQN log: tail -f /tmp/tuning_run.log" echo " Pipeline status: cat /tmp/tuning_pipeline_status.txt" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"