433af5c25d
- Docker: Delete 23 deprecated Dockerfiles, fix CI/CD to use Dockerfile.foxhunt-build - Config: Remove 36 .env files, keep 4 essential, delete config/environments/ - Docs: Archive 614 Wave D files to docs/archive/wave_d/, 95% reduction in root - Scripts: Delete 56 deprecated scripts, keep 58 production-critical (49% reduction) - Python: Organize 37 scripts into scripts/python/ subdirectories, delete ml/python/ - Build: Remove 1GB artifacts, delete old venvs, clean Python cache from git - Migrations: Delete deprecated directory (4,432 lines), remove duplicate database/migrations/ - Infrastructure: Delete deployment/ (61 files), docs/scripts/ (8 files) Total impact: ~2,500 files cleaned, 750MB+ space freed, zero production impact All deleted scripts backed up to archives. runpod/ and tests/runpod/ preserved. data_acquisition_service retained per user request.
109 lines
3.5 KiB
Bash
Executable File
109 lines
3.5 KiB
Bash
Executable File
#!/bin/bash
|
|
# Sequential Model Tuning Launcher
|
|
# Launches all 5 models sequentially: DQN → PPO → TFT → MAMBA-2 → Liquid
|
|
|
|
TUNING_BIN="/home/jgrusewski/Work/foxhunt/target/release/examples/tune_hyperparameters"
|
|
DATA_DIR="test_data/real/databento/ml_training"
|
|
NUM_TRIALS=50
|
|
EPOCHS=50
|
|
|
|
echo "==================================================================="
|
|
echo "Sequential Model Hyperparameter Tuning"
|
|
echo "==================================================================="
|
|
echo "Started at: $(date)"
|
|
echo "Models: DQN → PPO → TFT → MAMBA-2 → Liquid"
|
|
echo "Trials per model: $NUM_TRIALS"
|
|
echo "Epochs per trial: $EPOCHS"
|
|
echo "Estimated total time: 12-14 hours"
|
|
echo ""
|
|
|
|
# Function to run tuning and wait for completion
|
|
run_tuning() {
|
|
local MODEL=$1
|
|
local OUTPUT=$2
|
|
local LOG=$3
|
|
|
|
echo "==================================================================="
|
|
echo "Starting $MODEL tuning..."
|
|
echo "==================================================================="
|
|
echo "Output: $OUTPUT"
|
|
echo "Log: $LOG"
|
|
echo "Started at: $(date)"
|
|
echo ""
|
|
|
|
nohup $TUNING_BIN \
|
|
--model $MODEL \
|
|
--num-trials $NUM_TRIALS \
|
|
--epochs-per-trial $EPOCHS \
|
|
--data-dir $DATA_DIR \
|
|
--output $OUTPUT \
|
|
> $LOG 2>&1 &
|
|
|
|
local PID=$!
|
|
echo $PID > /tmp/${MODEL,,}_tuning.pid
|
|
echo "PID: $PID"
|
|
echo ""
|
|
|
|
# Wait for process to complete
|
|
while ps -p $PID > /dev/null 2>&1; do
|
|
local TRIALS=$(grep -c "Trial .* completed" $LOG 2>/dev/null || echo "0")
|
|
echo "[$(date +%H:%M:%S)] $MODEL tuning in progress... ($TRIALS/$NUM_TRIALS trials)"
|
|
sleep 300 # Check every 5 minutes
|
|
done
|
|
|
|
echo ""
|
|
echo "$MODEL tuning completed at: $(date)"
|
|
local TOTAL=$(grep -c "Trial .* completed" $LOG 2>/dev/null || echo "0")
|
|
echo "Completed $TOTAL trials"
|
|
echo ""
|
|
}
|
|
|
|
# Check if DQN is already running (PID 3911478)
|
|
if ps -p 3911478 > /dev/null 2>&1; then
|
|
echo "⚠️ DQN tuning already running (PID 3911478)"
|
|
echo "Waiting for DQN to complete before proceeding..."
|
|
echo ""
|
|
|
|
while ps -p 3911478 > /dev/null 2>&1; do
|
|
TRIALS=$(grep -c "Trial .* completed" /tmp/tuning_run.log 2>/dev/null || echo "0")
|
|
echo "[$(date +%H:%M:%S)] DQN still running... ($TRIALS trials completed)"
|
|
sleep 300
|
|
done
|
|
|
|
echo "DQN tuning completed!"
|
|
echo ""
|
|
else
|
|
# Run DQN tuning
|
|
run_tuning "DQN" "results/dqn_tuning_50trials.json" "/tmp/tuning_run.log"
|
|
fi
|
|
|
|
# Run PPO tuning
|
|
run_tuning "PPO" "results/ppo_tuning_50trials.json" "/tmp/ppo_tuning_run.log"
|
|
|
|
# Run TFT tuning
|
|
run_tuning "TFT" "results/tft_tuning_50trials.json" "/tmp/tft_tuning_run.log"
|
|
|
|
# Run MAMBA-2 tuning
|
|
run_tuning "MAMBA2" "results/mamba2_tuning_50trials.json" "/tmp/mamba2_tuning_run.log"
|
|
|
|
# Run Liquid tuning
|
|
run_tuning "Liquid" "results/liquid_tuning_50trials.json" "/tmp/liquid_tuning_run.log"
|
|
|
|
echo "==================================================================="
|
|
echo "ALL TUNING COMPLETE!"
|
|
echo "==================================================================="
|
|
echo "Completed at: $(date)"
|
|
echo ""
|
|
echo "Results:"
|
|
echo " - DQN: results/dqn_tuning_50trials.json"
|
|
echo " - PPO: results/ppo_tuning_50trials.json"
|
|
echo " - TFT: results/tft_tuning_50trials.json"
|
|
echo " - MAMBA-2: results/mamba2_tuning_50trials.json"
|
|
echo " - Liquid: results/liquid_tuning_50trials.json"
|
|
echo ""
|
|
echo "Next steps:"
|
|
echo " 1. Analyze results and extract best hyperparameters"
|
|
echo " 2. Update model configurations"
|
|
echo " 3. Run production training with optimized hyperparameters"
|
|
echo "==================================================================="
|