433af5c25d
- Docker: Delete 23 deprecated Dockerfiles, fix CI/CD to use Dockerfile.foxhunt-build - Config: Remove 36 .env files, keep 4 essential, delete config/environments/ - Docs: Archive 614 Wave D files to docs/archive/wave_d/, 95% reduction in root - Scripts: Delete 56 deprecated scripts, keep 58 production-critical (49% reduction) - Python: Organize 37 scripts into scripts/python/ subdirectories, delete ml/python/ - Build: Remove 1GB artifacts, delete old venvs, clean Python cache from git - Migrations: Delete deprecated directory (4,432 lines), remove duplicate database/migrations/ - Infrastructure: Delete deployment/ (61 files), docs/scripts/ (8 files) Total impact: ~2,500 files cleaned, 750MB+ space freed, zero production impact All deleted scripts backed up to archives. runpod/ and tests/runpod/ preserved. data_acquisition_service retained per user request.
112 lines
3.4 KiB
Bash
Executable File
112 lines
3.4 KiB
Bash
Executable File
#!/bin/bash
|
||
# Real-time Hyperparameter Tuning Monitor
|
||
# Displays progress for all active tuning jobs
|
||
|
||
LOGS=(
|
||
"/tmp/tuning_run.log:DQN:3911478"
|
||
"/tmp/ppo_tuning_run.log:PPO:"
|
||
"/tmp/tft_tuning_run.log:TFT:"
|
||
"/tmp/mamba2_tuning_run.log:MAMBA-2:"
|
||
"/tmp/liquid_tuning_run.log:Liquid:"
|
||
)
|
||
|
||
# Colors
|
||
GREEN='\033[0;32m'
|
||
YELLOW='\033[1;33m'
|
||
RED='\033[0;31m'
|
||
BLUE='\033[0;34m'
|
||
NC='\033[0m' # No Color
|
||
|
||
clear
|
||
|
||
echo "==================================================================="
|
||
echo "Hyperparameter Tuning Monitor"
|
||
echo "==================================================================="
|
||
echo "Time: $(date)"
|
||
echo ""
|
||
|
||
# GPU status
|
||
echo -e "${BLUE}GPU Status:${NC}"
|
||
nvidia-smi --query-gpu=index,name,memory.used,memory.total,utilization.gpu --format=csv,noheader,nounits | while IFS=, read -r idx name mem_used mem_total util; do
|
||
echo " GPU $idx: $util% util, ${mem_used}MB / ${mem_total}MB"
|
||
done
|
||
echo ""
|
||
|
||
# Check each model
|
||
echo -e "${BLUE}Model Tuning Status:${NC}"
|
||
echo ""
|
||
|
||
for entry in "${LOGS[@]}"; do
|
||
IFS=: read -r LOG MODEL PID <<< "$entry"
|
||
|
||
if [ -f "$LOG" ]; then
|
||
TRIALS=$(grep -c "Trial .* completed" "$LOG" 2>/dev/null || echo "0")
|
||
LAST_SHARPE=$(grep "Trial .* completed" "$LOG" | tail -1 | grep -oP 'Sharpe=\K[0-9.]+' || echo "N/A")
|
||
LAST_LOSS=$(grep "Trial .* completed" "$LOG" | tail -1 | grep -oP 'Loss=\K[0-9.]+' || echo "N/A")
|
||
|
||
# Check if process is running
|
||
if [ -n "$PID" ] && ps -p "$PID" > /dev/null 2>&1; then
|
||
STATUS="${GREEN}✓ RUNNING${NC}"
|
||
PROGRESS="$TRIALS/50 trials"
|
||
elif [ "$TRIALS" -eq 50 ]; then
|
||
STATUS="${GREEN}✓ COMPLETE${NC}"
|
||
PROGRESS="50/50 trials"
|
||
elif [ "$TRIALS" -gt 0 ]; then
|
||
STATUS="${RED}✗ STOPPED${NC}"
|
||
PROGRESS="$TRIALS/50 trials (incomplete)"
|
||
else
|
||
STATUS="${YELLOW}⏳ PENDING${NC}"
|
||
PROGRESS="Waiting to start"
|
||
fi
|
||
|
||
echo -e " ${MODEL}:"
|
||
echo -e " Status: $STATUS"
|
||
echo " Progress: $PROGRESS"
|
||
echo " Last Sharpe: $LAST_SHARPE"
|
||
echo " Last Loss: $LAST_LOSS"
|
||
echo ""
|
||
fi
|
||
done
|
||
|
||
# Overall progress
|
||
echo -e "${BLUE}Overall Progress:${NC}"
|
||
TOTAL_TRIALS=0
|
||
for entry in "${LOGS[@]}"; do
|
||
IFS=: read -r LOG MODEL PID <<< "$entry"
|
||
if [ -f "$LOG" ]; then
|
||
TRIALS=$(grep -c "Trial .* completed" "$LOG" 2>/dev/null || echo "0")
|
||
TOTAL_TRIALS=$((TOTAL_TRIALS + TRIALS))
|
||
fi
|
||
done
|
||
|
||
TOTAL_EXPECTED=250 # 50 trials × 5 models
|
||
PERCENT=$((TOTAL_TRIALS * 100 / TOTAL_EXPECTED))
|
||
|
||
echo " Completed: $TOTAL_TRIALS / $TOTAL_EXPECTED trials ($PERCENT%)"
|
||
echo ""
|
||
|
||
# Estimated completion
|
||
if [ "$TOTAL_TRIALS" -gt 0 ]; then
|
||
DQN_START="2025-10-14 16:57:00"
|
||
NOW=$(date +%s)
|
||
START=$(date -d "$DQN_START" +%s 2>/dev/null || echo "$NOW")
|
||
ELAPSED=$((NOW - START))
|
||
|
||
if [ "$TOTAL_TRIALS" -gt 10 ]; then
|
||
AVG_TIME=$((ELAPSED / TOTAL_TRIALS))
|
||
REMAINING=$((TOTAL_EXPECTED - TOTAL_TRIALS))
|
||
ETA_SECONDS=$((REMAINING * AVG_TIME))
|
||
ETA_DATE=$(date -d "@$((NOW + ETA_SECONDS))" +"%Y-%m-%d %H:%M:%S" 2>/dev/null || echo "N/A")
|
||
|
||
echo -e "${BLUE}Time Estimates:${NC}"
|
||
echo " Elapsed: $((ELAPSED / 3600))h $((ELAPSED % 3600 / 60))m"
|
||
echo " Avg time/trial: ${AVG_TIME}s"
|
||
echo " Estimated completion: $ETA_DATE"
|
||
fi
|
||
fi
|
||
|
||
echo ""
|
||
echo "==================================================================="
|
||
echo "Press Ctrl+C to exit, or run with 'watch -n 30' for auto-refresh"
|
||
echo "==================================================================="
|