3853988af7
- Fixed PSO budget calculation bug in ml/src/hyperopt/optimizer.rs - Root cause: Division by n_particles in sequential execution - Now correctly calculates max_iters = remaining_trials (no division) - Result: 50 trials complete instead of 23 (100% vs 46%) - Added comprehensive DQN hyperopt results analysis - 39/50 trials analyzed across 2 RunPod deployments - Best hyperparameters identified: LR 4.89e-5 (ultra-low) - Created DQN_HYPEROPT_RESULTS_SUMMARY.md with expert validation - GitLab CI/CD pipeline operational (48 lines fixed) - Fixed YAML syntax errors (unquoted colons) - All 7 jobs validated and working - Warning cleanup complete (136 → 0 warnings) - Removed 143 lines dead code - Fixed visibility, unused imports, Debug traits - Archived Wave D reports to docs/archive/ - 8 early stopping reports moved - Root directory cleaned up 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
60 lines
1.9 KiB
Bash
Executable File
60 lines
1.9 KiB
Bash
Executable File
#!/bin/bash
|
|
# Verify hyperopt training is actually running (not just pod running)
|
|
source .env.runpod
|
|
|
|
DQN_POD_ID="dy2bn5ninzaxma"
|
|
PPO_POD_ID="dytpb1mcqwj54t"
|
|
|
|
echo "========================================="
|
|
echo "VERIFYING HYPEROPT TRAINING EXECUTION"
|
|
echo "========================================="
|
|
echo ""
|
|
|
|
# Wait for pods to initialize
|
|
echo "Waiting 30 seconds for pods to initialize..."
|
|
sleep 30
|
|
|
|
echo ""
|
|
echo "Checking DQN pod logs for training activity..."
|
|
echo "----------------------------------------"
|
|
DQN_LOGS=$(curl -s "https://www.runpod.io/console/pods/${DQN_POD_ID}/logs" 2>/dev/null || echo "Log API not accessible")
|
|
|
|
if echo "$DQN_LOGS" | grep -q "hyperopt_dqn_demo"; then
|
|
echo "✅ DQN training command detected"
|
|
elif echo "$DQN_LOGS" | grep -q "Trial"; then
|
|
echo "✅ DQN trials running"
|
|
else
|
|
echo "⚠️ Cannot verify DQN training from logs (check dashboard)"
|
|
fi
|
|
|
|
echo ""
|
|
echo "Checking PPO pod logs for training activity..."
|
|
echo "----------------------------------------"
|
|
PPO_LOGS=$(curl -s "https://www.runpod.io/console/pods/${PPO_POD_ID}/logs" 2>/dev/null || echo "Log API not accessible")
|
|
|
|
if echo "$PPO_LOGS" | grep -q "hyperopt_ppo_demo"; then
|
|
echo "✅ PPO training command detected"
|
|
elif echo "$PPO_LOGS" | grep -q "Trial"; then
|
|
echo "✅ PPO trials running"
|
|
else
|
|
echo "⚠️ Cannot verify PPO training from logs (check dashboard)"
|
|
fi
|
|
|
|
echo ""
|
|
echo "========================================="
|
|
echo "MANUAL VERIFICATION"
|
|
echo "========================================="
|
|
echo ""
|
|
echo "Check DQN logs manually:"
|
|
echo " https://www.runpod.io/console/pods/${DQN_POD_ID}"
|
|
echo ""
|
|
echo "Check PPO logs manually:"
|
|
echo " https://www.runpod.io/console/pods/${PPO_POD_ID}"
|
|
echo ""
|
|
echo "Look for:"
|
|
echo " - 'Starting hyperopt_*_demo' messages"
|
|
echo " - 'Trial 1/50' progress indicators"
|
|
echo " - GPU memory allocation logs"
|
|
echo " - No error messages (OOM, CUDA errors)"
|
|
echo ""
|