#!/bin/bash
# Verify hyperopt training is actually running (not just pod running)
source .env.runpod

DQN_POD_ID="dy2bn5ninzaxma"
PPO_POD_ID="dytpb1mcqwj54t"

echo "========================================="
echo "VERIFYING HYPEROPT TRAINING EXECUTION"
echo "========================================="
echo ""

# Wait for pods to initialize
echo "Waiting 30 seconds for pods to initialize..."
sleep 30

echo ""
echo "Checking DQN pod logs for training activity..."
echo "----------------------------------------"
DQN_LOGS=$(curl -s "https://www.runpod.io/console/pods/${DQN_POD_ID}/logs" 2>/dev/null || echo "Log API not accessible")

if echo "$DQN_LOGS" | grep -q "hyperopt_dqn_demo"; then
  echo "✅ DQN training command detected"
elif echo "$DQN_LOGS" | grep -q "Trial"; then
  echo "✅ DQN trials running"
else
  echo "⚠️  Cannot verify DQN training from logs (check dashboard)"
fi

echo ""
echo "Checking PPO pod logs for training activity..."
echo "----------------------------------------"
PPO_LOGS=$(curl -s "https://www.runpod.io/console/pods/${PPO_POD_ID}/logs" 2>/dev/null || echo "Log API not accessible")

if echo "$PPO_LOGS" | grep -q "hyperopt_ppo_demo"; then
  echo "✅ PPO training command detected"
elif echo "$PPO_LOGS" | grep -q "Trial"; then
  echo "✅ PPO trials running"
else
  echo "⚠️  Cannot verify PPO training from logs (check dashboard)"
fi

echo ""
echo "========================================="
echo "MANUAL VERIFICATION"
echo "========================================="
echo ""
echo "Check DQN logs manually:"
echo "  https://www.runpod.io/console/pods/${DQN_POD_ID}"
echo ""
echo "Check PPO logs manually:"
echo "  https://www.runpod.io/console/pods/${PPO_POD_ID}"
echo ""
echo "Look for:"
echo "  - 'Starting hyperopt_*_demo' messages"
echo "  - 'Trial 1/50' progress indicators"
echo "  - GPU memory allocation logs"
echo "  - No error messages (OOM, CUDA errors)"
echo ""