foxhunt/scripts/auto_monitor_and_launch.sh

#!/bin/bash
# Auto-Monitor DQN and Launch Sequential Tuning
# This script monitors DQN completion and automatically launches the remaining models

DQN_PID=3911478
DQN_LOG="/tmp/tuning_run.log"
SEQUENTIAL_LAUNCHER="/home/jgrusewski/Work/foxhunt/scripts/sequential_tuning_launcher.sh"
STATUS_FILE="/tmp/tuning_pipeline_status.txt"

echo "=================================================================="
echo "Automated Hyperparameter Tuning Pipeline Monitor"
echo "=================================================================="
echo "Started at: $(date)"
echo "DQN PID: $DQN_PID"
echo ""

# Create status file
cat > "$STATUS_FILE" << EOF
Hyperparameter Tuning Pipeline Status
Started: $(date)

Current Status: Monitoring DQN tuning (PID $DQN_PID)
Next Action: Launch sequential tuner when DQN completes

Expected Timeline:
  DQN:     Complete by ~19:28 (1.6h remaining)
  PPO:     21:01 - 00:13 (3.2h)
  TFT:     00:13 - 04:25 (4.2h)
  MAMBA-2: 04:25 - 06:31 (2.1h)
  Liquid:  06:31 - 08:13 (1.7h)

Pipeline completion ETA: 2025-10-15 08:13
EOF

echo "Status file created: $STATUS_FILE"
echo ""

# Monitor DQN completion
echo "Monitoring DQN tuning progress..."
while ps -p $DQN_PID > /dev/null 2>&1; do
    TRIALS=$(grep -c "Trial .* completed" "$DQN_LOG" 2>/dev/null || echo "0")
    CURRENT_TIME=$(date +"%H:%M:%S")
    echo "[$CURRENT_TIME] DQN still running... ($TRIALS/50 trials completed)"

    # Update status file
    cat > "$STATUS_FILE" << EOF
Hyperparameter Tuning Pipeline Status
Last Updated: $(date)

Current Status: DQN tuning in progress (PID $DQN_PID)
Trials Completed: $TRIALS/50
Next Action: Launch sequential tuner when DQN completes

Expected Timeline:
  DQN:     In Progress ($TRIALS/50 trials)
  PPO:     Waiting for DQN
  TFT:     Waiting for PPO
  MAMBA-2: Waiting for TFT
  Liquid:  Waiting for MAMBA-2

Pipeline completion ETA: 2025-10-15 08:13
EOF

    sleep 300  # Check every 5 minutes
done

echo ""
echo "=================================================================="
echo "DQN TUNING COMPLETE!"
echo "=================================================================="
TRIALS=$(grep -c "Trial .* completed" "$DQN_LOG" 2>/dev/null || echo "0")
echo "Completed $TRIALS trials"
echo "Finished at: $(date)"
echo ""

# Update status file
cat > "$STATUS_FILE" << EOF
Hyperparameter Tuning Pipeline Status
Last Updated: $(date)

Current Status: DQN complete, launching sequential tuner
Trials Completed: $TRIALS/50
Next Action: Starting PPO/TFT/MAMBA-2/Liquid sequential tuning

Expected Timeline:
  DQN:     COMPLETE
  PPO:     Starting now (3.2h)
  TFT:     After PPO (4.2h)
  MAMBA-2: After TFT (2.1h)
  Liquid:  After MAMBA-2 (1.7h)

Pipeline completion ETA: 2025-10-15 08:13
EOF

# Launch sequential tuning
echo "Launching sequential tuning for remaining models..."
echo "Command: $SEQUENTIAL_LAUNCHER"
echo ""

nohup "$SEQUENTIAL_LAUNCHER" > /tmp/sequential_tuning.log 2>&1 &
LAUNCHER_PID=$!
echo "$LAUNCHER_PID" > /tmp/sequential_launcher.pid

echo "Sequential tuning launched with PID: $LAUNCHER_PID"
echo "Monitor progress: tail -f /tmp/sequential_tuning.log"
echo ""

# Update final status
cat > "$STATUS_FILE" << EOF
Hyperparameter Tuning Pipeline Status
Last Updated: $(date)

Current Status: Sequential tuning in progress (PID $LAUNCHER_PID)
DQN: COMPLETE ($TRIALS/50 trials)
PPO: IN PROGRESS
TFT: PENDING
MAMBA-2: PENDING
Liquid: PENDING

Monitor:
  Sequential log: tail -f /tmp/sequential_tuning.log
  PPO log: tail -f /tmp/ppo_tuning_run.log
  Status: cat /tmp/tuning_pipeline_status.txt

Pipeline completion ETA: 2025-10-15 08:13
EOF

echo "=================================================================="
echo "Auto-monitor complete. Sequential tuning is now running."
echo "=================================================================="
echo "Status file: $STATUS_FILE"
echo "Sequential log: /tmp/sequential_tuning.log"
echo ""