e61e8f54da
Changes: - CLAUDE.md: Update OOM fix validation status - Add comprehensive documentation (30+ markdown reports) - LSTM encoder varmap bug fix (tft/lstm_encoder.rs:290) - Quantized LSTM layer matching fix (tft/quantized_lstm.rs) - Hyperopt paths module (ml/src/hyperopt/paths.rs) - Training path tests for all adapters (DQN, MAMBA-2, PPO, TFT) - Checkpoint integrity tests - Script cleanup: Remove 29 obsolete deployment scripts - Archive old scripts to scripts/archive/ - New deployment utilities: check_gpu_availability.py, monitor_hyperopt.sh Validation: - OOM fixes validated: 5/5 trials successful (pod b6kc3mc5lbjiro) - Batch-size-max 256 tested successfully - All hyperopt adapters working correctly 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
231 lines
8.2 KiB
Bash
Executable File
231 lines
8.2 KiB
Bash
Executable File
#!/bin/bash
|
||
# ================================================================================================
|
||
# Paper Trading Deployment Script
|
||
# ================================================================================================
|
||
# Purpose: Deploy 3-model ensemble (DQN + 2x PPO) to paper trading
|
||
# Usage: bash scripts/deploy_paper_trading.sh
|
||
# ================================================================================================
|
||
|
||
set -e # Exit on any error
|
||
|
||
# ================================================================================================
|
||
# CONFIGURATION
|
||
# ================================================================================================
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||
CONFIG_FILE="$PROJECT_ROOT/config/paper_trading_config.yaml"
|
||
CHECKPOINT_DIR="$PROJECT_ROOT/ml/trained_models/production"
|
||
|
||
# Service endpoints
|
||
API_GATEWAY="localhost:50051"
|
||
TRADING_SERVICE="localhost:50052"
|
||
ML_TRAINING_SERVICE="localhost:50054"
|
||
|
||
# Colors for output
|
||
RED='\033[0;31m'
|
||
GREEN='\033[0;32m'
|
||
YELLOW='\033[1;33m'
|
||
BLUE='\033[0;34m'
|
||
NC='\033[0m' # No Color
|
||
|
||
# ================================================================================================
|
||
# HELPER FUNCTIONS
|
||
# ================================================================================================
|
||
print_header() {
|
||
echo -e "${BLUE}========================================${NC}"
|
||
echo -e "${BLUE}$1${NC}"
|
||
echo -e "${BLUE}========================================${NC}"
|
||
}
|
||
|
||
print_success() {
|
||
echo -e "${GREEN}✅ $1${NC}"
|
||
}
|
||
|
||
print_error() {
|
||
echo -e "${RED}❌ $1${NC}"
|
||
}
|
||
|
||
print_warning() {
|
||
echo -e "${YELLOW}⚠️ $1${NC}"
|
||
}
|
||
|
||
print_info() {
|
||
echo -e "${BLUE}ℹ️ $1${NC}"
|
||
}
|
||
|
||
check_command() {
|
||
if ! command -v "$1" &> /dev/null; then
|
||
print_error "Required command '$1' not found. Please install it first."
|
||
exit 1
|
||
fi
|
||
}
|
||
|
||
# ================================================================================================
|
||
# PRE-FLIGHT CHECKS
|
||
# ================================================================================================
|
||
print_header "Pre-Flight Checks"
|
||
|
||
# Check required commands
|
||
print_info "Checking required commands..."
|
||
check_command docker
|
||
check_command docker-compose
|
||
check_command psql
|
||
check_command curl
|
||
print_success "All required commands available"
|
||
|
||
# Check configuration file exists
|
||
if [ ! -f "$CONFIG_FILE" ]; then
|
||
print_error "Configuration file not found: $CONFIG_FILE"
|
||
exit 1
|
||
fi
|
||
print_success "Configuration file found: $CONFIG_FILE"
|
||
|
||
# Check checkpoint directory exists
|
||
if [ ! -d "$CHECKPOINT_DIR" ]; then
|
||
print_error "Checkpoint directory not found: $CHECKPOINT_DIR"
|
||
exit 1
|
||
fi
|
||
print_success "Checkpoint directory found: $CHECKPOINT_DIR"
|
||
|
||
# ================================================================================================
|
||
# CHECKPOINT VERIFICATION
|
||
# ================================================================================================
|
||
print_header "Checkpoint Verification"
|
||
|
||
print_info "Verifying model checkpoints..."
|
||
|
||
# DQN epoch 30
|
||
DQN_CHECKPOINT="$CHECKPOINT_DIR/dqn/dqn_epoch_30.safetensors"
|
||
if [ -f "$DQN_CHECKPOINT" ]; then
|
||
SIZE=$(du -h "$DQN_CHECKPOINT" | cut -f1)
|
||
print_success "DQN epoch 30: $SIZE"
|
||
else
|
||
print_error "DQN checkpoint not found: $DQN_CHECKPOINT"
|
||
exit 1
|
||
fi
|
||
|
||
# PPO epoch 130 (actor + critic)
|
||
PPO_130_ACTOR="$CHECKPOINT_DIR/ppo/ppo_actor_epoch_130.safetensors"
|
||
PPO_130_CRITIC="$CHECKPOINT_DIR/ppo/ppo_critic_epoch_130.safetensors"
|
||
if [ -f "$PPO_130_ACTOR" ] && [ -f "$PPO_130_CRITIC" ]; then
|
||
SIZE_ACTOR=$(du -h "$PPO_130_ACTOR" | cut -f1)
|
||
SIZE_CRITIC=$(du -h "$PPO_130_CRITIC" | cut -f1)
|
||
print_success "PPO epoch 130: actor=$SIZE_ACTOR, critic=$SIZE_CRITIC"
|
||
else
|
||
print_error "PPO epoch 130 checkpoints not found"
|
||
exit 1
|
||
fi
|
||
|
||
# PPO epoch 420 (actor + critic)
|
||
PPO_420_ACTOR="$CHECKPOINT_DIR/ppo/ppo_actor_epoch_420.safetensors"
|
||
PPO_420_CRITIC="$CHECKPOINT_DIR/ppo/ppo_critic_epoch_420.safetensors"
|
||
if [ -f "$PPO_420_ACTOR" ] && [ -f "$PPO_420_CRITIC" ]; then
|
||
SIZE_ACTOR=$(du -h "$PPO_420_ACTOR" | cut -f1)
|
||
SIZE_CRITIC=$(du -h "$PPO_420_CRITIC" | cut -f1)
|
||
print_success "PPO epoch 420: actor=$SIZE_ACTOR, critic=$SIZE_CRITIC"
|
||
else
|
||
print_error "PPO epoch 420 checkpoints not found"
|
||
exit 1
|
||
fi
|
||
|
||
# ================================================================================================
|
||
# SERVICE HEALTH CHECKS
|
||
# ================================================================================================
|
||
print_header "Service Health Checks"
|
||
|
||
print_info "Checking Docker services..."
|
||
if ! docker-compose ps | grep -q "Up"; then
|
||
print_error "Docker services are not running. Start with: docker-compose up -d"
|
||
exit 1
|
||
fi
|
||
print_success "Docker services running"
|
||
|
||
# Trading Service HTTP health check (gRPC health probe may not be installed)
|
||
print_info "Checking Trading Service (port 8081 HTTP)..."
|
||
if curl -s http://localhost:8081/health > /dev/null 2>&1; then
|
||
print_success "Trading Service healthy (HTTP)"
|
||
else
|
||
print_warning "Trading Service HTTP health check failed (may be gRPC-only)"
|
||
fi
|
||
|
||
# PostgreSQL health check
|
||
print_info "Checking PostgreSQL..."
|
||
if psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -c "SELECT 1" > /dev/null 2>&1; then
|
||
print_success "PostgreSQL healthy"
|
||
else
|
||
print_error "PostgreSQL unhealthy or not responding"
|
||
exit 1
|
||
fi
|
||
|
||
# Redis health check
|
||
print_info "Checking Redis..."
|
||
if redis-cli -h localhost -p 6379 PING > /dev/null 2>&1; then
|
||
print_success "Redis healthy"
|
||
else
|
||
print_warning "Redis not responding (non-critical)"
|
||
fi
|
||
|
||
# Prometheus health check
|
||
print_info "Checking Prometheus..."
|
||
if curl -s http://localhost:9090/api/v1/targets > /dev/null 2>&1; then
|
||
print_success "Prometheus healthy"
|
||
else
|
||
print_warning "Prometheus not responding (non-critical)"
|
||
fi
|
||
|
||
# Grafana health check
|
||
print_info "Checking Grafana..."
|
||
if curl -s http://localhost:3000/api/health > /dev/null 2>&1; then
|
||
print_success "Grafana healthy"
|
||
else
|
||
print_warning "Grafana not responding (non-critical)"
|
||
fi
|
||
|
||
# ================================================================================================
|
||
# DATABASE TABLE VERIFICATION
|
||
# ================================================================================================
|
||
print_header "Database Table Verification"
|
||
|
||
print_info "Checking ensemble prediction tables..."
|
||
TABLE_COUNT=$(psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -t -c "SELECT COUNT(*) FROM pg_tables WHERE schemaname = 'public' AND tablename IN ('ensemble_predictions', 'model_performance_attribution', 'ab_test_experiments')" 2>/dev/null | tr -d ' ')
|
||
|
||
if [ "$TABLE_COUNT" = "3" ]; then
|
||
print_success "All ensemble tables exist (3/3)"
|
||
else
|
||
print_warning "Only $TABLE_COUNT/3 ensemble tables found"
|
||
print_info "Tables may need to be created manually from migration 022"
|
||
fi
|
||
|
||
# ================================================================================================
|
||
# DEPLOYMENT SUMMARY
|
||
# ================================================================================================
|
||
print_header "Deployment Summary"
|
||
|
||
echo ""
|
||
print_success "Paper trading infrastructure verified!"
|
||
echo ""
|
||
print_info "Configuration Details:"
|
||
echo " - Config file: $CONFIG_FILE"
|
||
echo " - Checkpoints: $CHECKPOINT_DIR"
|
||
echo " - Virtual capital: \$100,000"
|
||
echo " - Symbols: ES.FUT, NQ.FUT"
|
||
echo " - Models: DQN epoch 30, PPO epoch 130, PPO epoch 420"
|
||
echo " - Risk limits: Max position \$10K, max daily loss \$2K"
|
||
echo ""
|
||
print_info "Next Steps:"
|
||
echo " 1. Run smoke test: bash tests/paper_trading_smoke_test.sh"
|
||
echo " 2. Monitor Grafana: http://localhost:3000/d/ensemble-ml-prod"
|
||
echo " 3. Check predictions: psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt"
|
||
echo " SQL: SELECT COUNT(*) FROM ensemble_predictions WHERE timestamp > NOW() - INTERVAL '1 hour';"
|
||
echo " 4. Monitor logs: docker logs foxhunt-trading-service -f"
|
||
echo ""
|
||
print_info "Phase 1 Success Criteria (7 days):"
|
||
echo " - Sharpe ratio > 1.5"
|
||
echo " - Win rate > 52%"
|
||
echo " - Max drawdown < 10%"
|
||
echo " - Simulated P&L > \$10,000"
|
||
echo " - Zero model errors"
|
||
echo " - Latency P99 < 50μs"
|
||
echo ""
|
||
print_header "Deployment Ready"
|