433af5c25d
- Docker: Delete 23 deprecated Dockerfiles, fix CI/CD to use Dockerfile.foxhunt-build - Config: Remove 36 .env files, keep 4 essential, delete config/environments/ - Docs: Archive 614 Wave D files to docs/archive/wave_d/, 95% reduction in root - Scripts: Delete 56 deprecated scripts, keep 58 production-critical (49% reduction) - Python: Organize 37 scripts into scripts/python/ subdirectories, delete ml/python/ - Build: Remove 1GB artifacts, delete old venvs, clean Python cache from git - Migrations: Delete deprecated directory (4,432 lines), remove duplicate database/migrations/ - Infrastructure: Delete deployment/ (61 files), docs/scripts/ (8 files) Total impact: ~2,500 files cleaned, 750MB+ space freed, zero production impact All deleted scripts backed up to archives. runpod/ and tests/runpod/ preserved. data_acquisition_service retained per user request.
70 lines
2.1 KiB
Bash
Executable File
70 lines
2.1 KiB
Bash
Executable File
#!/bin/bash
|
|
# Auto-Launch PPO Hyperparameter Tuning
|
|
# Waits for DQN to complete, then launches PPO tuning automatically
|
|
|
|
DQN_PID=3911478
|
|
TUNING_BIN="/home/jgrusewski/Work/foxhunt/target/release/examples/tune_hyperparameters"
|
|
DATA_DIR="test_data/real/databento/ml_training"
|
|
|
|
echo "==================================================================="
|
|
echo "PPO Tuning Auto-Launcher"
|
|
echo "==================================================================="
|
|
echo "Waiting for DQN tuning (PID $DQN_PID) to complete..."
|
|
echo "Started at: $(date)"
|
|
echo ""
|
|
|
|
# Wait for DQN process to finish
|
|
while ps -p $DQN_PID > /dev/null 2>&1; do
|
|
TRIALS=$(grep -c "Trial .* completed" /tmp/tuning_run.log 2>/dev/null || echo "0")
|
|
echo "[$(date +%H:%M:%S)] DQN still running... ($TRIALS trials completed)"
|
|
sleep 60
|
|
done
|
|
|
|
echo ""
|
|
echo "==================================================================="
|
|
echo "DQN tuning completed at: $(date)"
|
|
echo "==================================================================="
|
|
echo ""
|
|
|
|
# Extract DQN results
|
|
TOTAL_TRIALS=$(grep -c "Trial .* completed" /tmp/tuning_run.log 2>/dev/null || echo "0")
|
|
echo "DQN completed $TOTAL_TRIALS trials"
|
|
echo ""
|
|
|
|
# Launch PPO tuning
|
|
echo "Launching PPO hyperparameter tuning..."
|
|
echo "Command: $TUNING_BIN --model PPO --num-trials 50 --epochs-per-trial 50"
|
|
echo ""
|
|
|
|
nohup $TUNING_BIN \
|
|
--model PPO \
|
|
--num-trials 50 \
|
|
--epochs-per-trial 50 \
|
|
--data-dir $DATA_DIR \
|
|
--output results/ppo_tuning_50trials.json \
|
|
> /tmp/ppo_tuning_run.log 2>&1 &
|
|
|
|
PPO_PID=$!
|
|
echo $PPO_PID > /tmp/ppo_tuning.pid
|
|
|
|
echo "==================================================================="
|
|
echo "PPO tuning started successfully!"
|
|
echo "PID: $PPO_PID"
|
|
echo "Log: /tmp/ppo_tuning_run.log"
|
|
echo "Started at: $(date)"
|
|
echo "==================================================================="
|
|
echo ""
|
|
|
|
# Monitor first 5 minutes
|
|
echo "Monitoring PPO startup (first 5 minutes)..."
|
|
sleep 300
|
|
|
|
if ps -p $PPO_PID > /dev/null 2>&1; then
|
|
echo "✅ PPO tuning running successfully (PID $PPO_PID)"
|
|
tail -20 /tmp/ppo_tuning_run.log
|
|
else
|
|
echo "❌ PPO tuning failed to start or crashed"
|
|
echo "Check /tmp/ppo_tuning_run.log for errors"
|
|
exit 1
|
|
fi
|