#!/bin/bash
# Real-time Hyperparameter Tuning Monitor
# Displays progress for all active tuning jobs

LOGS=(
  "/tmp/tuning_run.log:DQN:3911478"
  "/tmp/ppo_tuning_run.log:PPO:"
  "/tmp/tft_tuning_run.log:TFT:"
  "/tmp/mamba2_tuning_run.log:MAMBA-2:"
  "/tmp/liquid_tuning_run.log:Liquid:"
)

# Colors
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
RED='\033[0;31m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

clear

echo "==================================================================="
echo "Hyperparameter Tuning Monitor"
echo "==================================================================="
echo "Time: $(date)"
echo ""

# GPU status
echo -e "${BLUE}GPU Status:${NC}"
nvidia-smi --query-gpu=index,name,memory.used,memory.total,utilization.gpu --format=csv,noheader,nounits | while IFS=, read -r idx name mem_used mem_total util; do
  echo "  GPU $idx: $util% util, ${mem_used}MB / ${mem_total}MB"
done
echo ""

# Check each model
echo -e "${BLUE}Model Tuning Status:${NC}"
echo ""

for entry in "${LOGS[@]}"; do
  IFS=: read -r LOG MODEL PID <<< "$entry"

  if [ -f "$LOG" ]; then
    TRIALS=$(grep -c "Trial .* completed" "$LOG" 2>/dev/null || echo "0")
    LAST_SHARPE=$(grep "Trial .* completed" "$LOG" | tail -1 | grep -oP 'Sharpe=\K[0-9.]+' || echo "N/A")
    LAST_LOSS=$(grep "Trial .* completed" "$LOG" | tail -1 | grep -oP 'Loss=\K[0-9.]+' || echo "N/A")

    # Check if process is running
    if [ -n "$PID" ] && ps -p "$PID" > /dev/null 2>&1; then
      STATUS="${GREEN}✓ RUNNING${NC}"
      PROGRESS="$TRIALS/50 trials"
    elif [ "$TRIALS" -eq 50 ]; then
      STATUS="${GREEN}✓ COMPLETE${NC}"
      PROGRESS="50/50 trials"
    elif [ "$TRIALS" -gt 0 ]; then
      STATUS="${RED}✗ STOPPED${NC}"
      PROGRESS="$TRIALS/50 trials (incomplete)"
    else
      STATUS="${YELLOW}⏳ PENDING${NC}"
      PROGRESS="Waiting to start"
    fi

    echo -e "  ${MODEL}:"
    echo -e "    Status: $STATUS"
    echo "    Progress: $PROGRESS"
    echo "    Last Sharpe: $LAST_SHARPE"
    echo "    Last Loss: $LAST_LOSS"
    echo ""
  fi
done

# Overall progress
echo -e "${BLUE}Overall Progress:${NC}"
TOTAL_TRIALS=0
for entry in "${LOGS[@]}"; do
  IFS=: read -r LOG MODEL PID <<< "$entry"
  if [ -f "$LOG" ]; then
    TRIALS=$(grep -c "Trial .* completed" "$LOG" 2>/dev/null || echo "0")
    TOTAL_TRIALS=$((TOTAL_TRIALS + TRIALS))
  fi
done

TOTAL_EXPECTED=250  # 50 trials × 5 models
PERCENT=$((TOTAL_TRIALS * 100 / TOTAL_EXPECTED))

echo "  Completed: $TOTAL_TRIALS / $TOTAL_EXPECTED trials ($PERCENT%)"
echo ""

# Estimated completion
if [ "$TOTAL_TRIALS" -gt 0 ]; then
  DQN_START="2025-10-14 16:57:00"
  NOW=$(date +%s)
  START=$(date -d "$DQN_START" +%s 2>/dev/null || echo "$NOW")
  ELAPSED=$((NOW - START))

  if [ "$TOTAL_TRIALS" -gt 10 ]; then
    AVG_TIME=$((ELAPSED / TOTAL_TRIALS))
    REMAINING=$((TOTAL_EXPECTED - TOTAL_TRIALS))
    ETA_SECONDS=$((REMAINING * AVG_TIME))
    ETA_DATE=$(date -d "@$((NOW + ETA_SECONDS))" +"%Y-%m-%d %H:%M:%S" 2>/dev/null || echo "N/A")

    echo -e "${BLUE}Time Estimates:${NC}"
    echo "  Elapsed: $((ELAPSED / 3600))h $((ELAPSED % 3600 / 60))m"
    echo "  Avg time/trial: ${AVG_TIME}s"
    echo "  Estimated completion: $ETA_DATE"
  fi
fi

echo ""
echo "==================================================================="
echo "Press Ctrl+C to exit, or run with 'watch -n 30' for auto-refresh"
echo "==================================================================="