650b3894c6
## Executive Summary Deployed 27 parallel agents: all 6 models operational, ensemble working, adaptive strategy integrated, hyperparameter tuning automated, TFT fixed, critical blocker resolved (DbnSequenceLoader 99.85% memory reduction 40.6GB→61MB). ## Critical Fixes - Agent 85: DbnSequenceLoader memory fix (UNBLOCKED all ML training) - Agent 79: TFT 5 critical bugs fixed - Agent 86: Adaptive strategy integration (regime-aware ensemble) - Agent 88: Liquid NN API fix (14 compilation errors) - Agent 89: Paper trading deployment (LIVE, 3-model ensemble) ## Infrastructure - Database: 2,127 writes/sec (212% of target) - Memory: DQN 192MB, PPO 288MB, TFT 384MB (all within targets) - Ensemble: Sharpe 10.68, latency 35μs, throughput >20K/sec - Monitoring: 22 alerts, PagerDuty integration ## Files: 193 changed, +70,250 insertions, -414 deletions 🤖 Generated with Claude Code - Co-Authored-By: Claude <noreply@anthropic.com>
109 lines
3.5 KiB
Bash
Executable File
109 lines
3.5 KiB
Bash
Executable File
#!/bin/bash
|
|
# Sequential Model Tuning Launcher
|
|
# Launches all 5 models sequentially: DQN → PPO → TFT → MAMBA-2 → Liquid
|
|
|
|
TUNING_BIN="/home/jgrusewski/Work/foxhunt/target/release/examples/tune_hyperparameters"
|
|
DATA_DIR="test_data/real/databento/ml_training"
|
|
NUM_TRIALS=50
|
|
EPOCHS=50
|
|
|
|
echo "==================================================================="
|
|
echo "Sequential Model Hyperparameter Tuning"
|
|
echo "==================================================================="
|
|
echo "Started at: $(date)"
|
|
echo "Models: DQN → PPO → TFT → MAMBA-2 → Liquid"
|
|
echo "Trials per model: $NUM_TRIALS"
|
|
echo "Epochs per trial: $EPOCHS"
|
|
echo "Estimated total time: 12-14 hours"
|
|
echo ""
|
|
|
|
# Function to run tuning and wait for completion
|
|
run_tuning() {
|
|
local MODEL=$1
|
|
local OUTPUT=$2
|
|
local LOG=$3
|
|
|
|
echo "==================================================================="
|
|
echo "Starting $MODEL tuning..."
|
|
echo "==================================================================="
|
|
echo "Output: $OUTPUT"
|
|
echo "Log: $LOG"
|
|
echo "Started at: $(date)"
|
|
echo ""
|
|
|
|
nohup $TUNING_BIN \
|
|
--model $MODEL \
|
|
--num-trials $NUM_TRIALS \
|
|
--epochs-per-trial $EPOCHS \
|
|
--data-dir $DATA_DIR \
|
|
--output $OUTPUT \
|
|
> $LOG 2>&1 &
|
|
|
|
local PID=$!
|
|
echo $PID > /tmp/${MODEL,,}_tuning.pid
|
|
echo "PID: $PID"
|
|
echo ""
|
|
|
|
# Wait for process to complete
|
|
while ps -p $PID > /dev/null 2>&1; do
|
|
local TRIALS=$(grep -c "Trial .* completed" $LOG 2>/dev/null || echo "0")
|
|
echo "[$(date +%H:%M:%S)] $MODEL tuning in progress... ($TRIALS/$NUM_TRIALS trials)"
|
|
sleep 300 # Check every 5 minutes
|
|
done
|
|
|
|
echo ""
|
|
echo "$MODEL tuning completed at: $(date)"
|
|
local TOTAL=$(grep -c "Trial .* completed" $LOG 2>/dev/null || echo "0")
|
|
echo "Completed $TOTAL trials"
|
|
echo ""
|
|
}
|
|
|
|
# Check if DQN is already running (PID 3911478)
|
|
if ps -p 3911478 > /dev/null 2>&1; then
|
|
echo "⚠️ DQN tuning already running (PID 3911478)"
|
|
echo "Waiting for DQN to complete before proceeding..."
|
|
echo ""
|
|
|
|
while ps -p 3911478 > /dev/null 2>&1; do
|
|
TRIALS=$(grep -c "Trial .* completed" /tmp/tuning_run.log 2>/dev/null || echo "0")
|
|
echo "[$(date +%H:%M:%S)] DQN still running... ($TRIALS trials completed)"
|
|
sleep 300
|
|
done
|
|
|
|
echo "DQN tuning completed!"
|
|
echo ""
|
|
else
|
|
# Run DQN tuning
|
|
run_tuning "DQN" "results/dqn_tuning_50trials.json" "/tmp/tuning_run.log"
|
|
fi
|
|
|
|
# Run PPO tuning
|
|
run_tuning "PPO" "results/ppo_tuning_50trials.json" "/tmp/ppo_tuning_run.log"
|
|
|
|
# Run TFT tuning
|
|
run_tuning "TFT" "results/tft_tuning_50trials.json" "/tmp/tft_tuning_run.log"
|
|
|
|
# Run MAMBA-2 tuning
|
|
run_tuning "MAMBA2" "results/mamba2_tuning_50trials.json" "/tmp/mamba2_tuning_run.log"
|
|
|
|
# Run Liquid tuning
|
|
run_tuning "Liquid" "results/liquid_tuning_50trials.json" "/tmp/liquid_tuning_run.log"
|
|
|
|
echo "==================================================================="
|
|
echo "ALL TUNING COMPLETE!"
|
|
echo "==================================================================="
|
|
echo "Completed at: $(date)"
|
|
echo ""
|
|
echo "Results:"
|
|
echo " - DQN: results/dqn_tuning_50trials.json"
|
|
echo " - PPO: results/ppo_tuning_50trials.json"
|
|
echo " - TFT: results/tft_tuning_50trials.json"
|
|
echo " - MAMBA-2: results/mamba2_tuning_50trials.json"
|
|
echo " - Liquid: results/liquid_tuning_50trials.json"
|
|
echo ""
|
|
echo "Next steps:"
|
|
echo " 1. Analyze results and extract best hyperparameters"
|
|
echo " 2. Update model configurations"
|
|
echo " 3. Run production training with optimized hyperparameters"
|
|
echo "==================================================================="
|