foxhunt/scripts/runpod_deploy.sh

#!/bin/bash
# =============================================================================
# FOXHUNT RUNPOD MASTER DEPLOYMENT SCRIPT
# =============================================================================
# Orchestrates complete Runpod deployment workflow for FP32 model training
#
# Prerequisites:
#   - cargo (Rust toolchain)
#   - docker (Docker daemon running)
#   - aws cli (S3 API client for Runpod storage)
#   - RUNPOD_S3_ENDPOINT environment variable
#   - AWS_PROFILE=runpod (AWS CLI profile configured)
#   - DOCKER_USERNAME=jgrusewski
#
# What this script does:
#   1. Validates prerequisites
#   2. Builds release binaries (5-6 min)
#   3. Uploads binaries to Runpod S3 volume
#   4. Uploads test data to Runpod S3 volume
#   5. Prompts for .env upload (optional, secure confirmation)
#   6. Builds Docker image
#   7. Pushes to Docker Hub (jgrusewski/foxhunt:latest)
#   8. Prints deployment instructions
#
# Safety:
#   - Idempotent (safe to re-run)
#   - Progress indicators for long operations
#   - Validation at each step
#   - Confirmation prompts for sensitive operations
#
# Usage:
#   export RUNPOD_S3_ENDPOINT=https://s3api-eur-is-1.runpod.io  # Your datacenter
#   export AWS_PROFILE=runpod
#   export DOCKER_USERNAME=jgrusewski
#   ./scripts/runpod_deploy.sh
# =============================================================================

set -e  # Exit on error
set -u  # Exit on undefined variable

# Color codes for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m' # No Color

# Configuration
FOXHUNT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
DOCKER_IMAGE="${DOCKER_USERNAME:-jgrusewski}/foxhunt:latest"
S3_BUCKET="${S3_BUCKET:-se3zdnb5o4}"  # Override with your Network Volume ID

# Progress indicator
progress() {
    echo -e "${CYAN}▶${NC} $1"
}

success() {
    echo -e "${GREEN}✓${NC} $1"
}

warning() {
    echo -e "${YELLOW}⚠${NC} $1"
}

error() {
    echo -e "${RED}✗${NC} $1"
}

section() {
    echo ""
    echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
    echo -e "${BLUE}$1${NC}"
    echo -e "${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
}

# =============================================================================
# STEP 0: BANNER
# =============================================================================

echo -e "${CYAN}"
cat << "EOF"
╔═══════════════════════════════════════════════════════════════════════════╗
║                                                                           ║
║              FOXHUNT RUNPOD DEPLOYMENT ORCHESTRATOR                       ║
║                                                                           ║
║  FP32 Model Training - Production Ready                                  ║
║  Target GPU: Tesla V100 16GB / RTX 4090 / A4000+                         ║
║  Deployment Time: ~15-20 minutes                                         ║
║                                                                           ║
╚═══════════════════════════════════════════════════════════════════════════╝
EOF
echo -e "${NC}"

# =============================================================================
# STEP 1: VALIDATE PREREQUISITES
# =============================================================================

section "Step 1/8: Validating Prerequisites"

PREREQUISITES_OK=true

# Check cargo
progress "Checking Rust toolchain..."
if command -v cargo &> /dev/null; then
    RUST_VERSION=$(cargo --version | awk '{print $2}')
    success "Cargo installed: $RUST_VERSION"
else
    error "Cargo not found. Install Rust: https://rustup.rs/"
    PREREQUISITES_OK=false
fi

# Check docker
progress "Checking Docker..."
if command -v docker &> /dev/null; then
    if docker info &> /dev/null; then
        DOCKER_VERSION=$(docker --version | awk '{print $3}' | tr -d ',')
        success "Docker running: $DOCKER_VERSION"
    else
        error "Docker daemon not running. Start with: sudo systemctl start docker"
        PREREQUISITES_OK=false
    fi
else
    error "Docker not found. Install Docker: https://docs.docker.com/get-docker/"
    PREREQUISITES_OK=false
fi

# Check aws cli
progress "Checking AWS CLI (for Runpod S3)..."
if command -v aws &> /dev/null; then
    AWS_VERSION=$(aws --version 2>&1 | awk '{print $1}' | cut -d'/' -f2)
    success "AWS CLI installed: $AWS_VERSION"
else
    error "AWS CLI not found. Install: pip install awscli"
    PREREQUISITES_OK=false
fi

# Check RUNPOD_S3_ENDPOINT
progress "Checking RUNPOD_S3_ENDPOINT environment variable..."
if [ -n "${RUNPOD_S3_ENDPOINT:-}" ]; then
    success "RUNPOD_S3_ENDPOINT: $RUNPOD_S3_ENDPOINT"
else
    error "RUNPOD_S3_ENDPOINT not set. Example: export RUNPOD_S3_ENDPOINT=https://s3api-eur-is-1.runpod.io"
    PREREQUISITES_OK=false
fi

# Check AWS_PROFILE
progress "Checking AWS_PROFILE=runpod..."
if [ "${AWS_PROFILE:-}" = "runpod" ]; then
    success "AWS_PROFILE: runpod"

    # Validate profile exists
    if aws configure list --profile runpod &> /dev/null; then
        success "AWS profile 'runpod' configured"
    else
        error "AWS profile 'runpod' not configured. Run: aws configure --profile runpod"
        PREREQUISITES_OK=false
    fi
else
    error "AWS_PROFILE not set to 'runpod'. Run: export AWS_PROFILE=runpod"
    PREREQUISITES_OK=false
fi

# Check DOCKER_USERNAME
progress "Checking DOCKER_USERNAME..."
if [ -n "${DOCKER_USERNAME:-}" ]; then
    success "DOCKER_USERNAME: $DOCKER_USERNAME"

    # Check Docker Hub login
    if docker info 2>&1 | grep -q "Username: $DOCKER_USERNAME"; then
        success "Docker Hub authenticated as $DOCKER_USERNAME"
    else
        warning "Not logged into Docker Hub. Will prompt for login later."
    fi
else
    error "DOCKER_USERNAME not set. Run: export DOCKER_USERNAME=jgrusewski"
    PREREQUISITES_OK=false
fi

# Check workspace directory
progress "Checking workspace directory..."
if [ -d "$FOXHUNT_ROOT" ]; then
    success "Workspace: $FOXHUNT_ROOT"
else
    error "Workspace directory not found: $FOXHUNT_ROOT"
    PREREQUISITES_OK=false
fi

# Final validation
if [ "$PREREQUISITES_OK" = false ]; then
    error "Prerequisites check failed. Fix errors above and re-run."
    exit 1
fi

success "All prerequisites validated"

# =============================================================================
# STEP 2: BUILD RELEASE BINARIES
# =============================================================================

section "Step 2/8: Building Release Binaries"

cd "$FOXHUNT_ROOT"

progress "Building workspace with CUDA support..."
progress "This will take 5-6 minutes (grab coffee ☕)..."

BUILD_START=$(date +%s)

# Build with release profile and CUDA features
if cargo build --release --workspace --features cuda 2>&1 | tee /tmp/foxhunt_build.log | grep -E "(Compiling|Finished|error)"; then
    BUILD_END=$(date +%s)
    BUILD_DURATION=$((BUILD_END - BUILD_START))
    BUILD_MINUTES=$((BUILD_DURATION / 60))
    BUILD_SECONDS=$((BUILD_DURATION % 60))

    success "Build complete in ${BUILD_MINUTES}m ${BUILD_SECONDS}s"
else
    error "Build failed. See /tmp/foxhunt_build.log for details."
    exit 1
fi

# Verify critical binaries exist
progress "Verifying training binaries..."

BINARIES=(
    "train_tft_parquet"
    "train_mamba2_parquet"
    "train_dqn"
    "train_ppo"
)

TOTAL_BIN_SIZE=0
for binary in "${BINARIES[@]}"; do
    BINARY_PATH="$FOXHUNT_ROOT/target/release/examples/$binary"
    if [ -f "$BINARY_PATH" ]; then
        SIZE=$(stat -c%s "$BINARY_PATH" 2>/dev/null || stat -f%z "$BINARY_PATH")
        SIZE_MB=$(awk "BEGIN {printf \"%.2f\", $SIZE/1024/1024}")
        TOTAL_BIN_SIZE=$((TOTAL_BIN_SIZE + SIZE))
        success "$binary ($SIZE_MB MB)"
    else
        error "$binary not found at $BINARY_PATH"
        exit 1
    fi
done

TOTAL_BIN_MB=$(awk "BEGIN {printf \"%.2f\", $TOTAL_BIN_SIZE/1024/1024}")
success "Total binaries: $TOTAL_BIN_MB MB"

# =============================================================================
# STEP 3: UPLOAD BINARIES TO RUNPOD S3
# =============================================================================

section "Step 3/8: Uploading Binaries to Runpod S3"

progress "Uploading 4 training binaries to s3://${S3_BUCKET}/binaries/..."

UPLOAD_COUNT=0
for binary in "${BINARIES[@]}"; do
    BINARY_PATH="$FOXHUNT_ROOT/target/release/examples/$binary"
    SIZE_MB=$(stat -c%s "$BINARY_PATH" 2>/dev/null | awk '{printf "%.2f", $1/1024/1024}')

    progress "Uploading $binary ($SIZE_MB MB)..."

    if aws s3 cp "$BINARY_PATH" "s3://${S3_BUCKET}/binaries/$binary" \
        --endpoint-url "$RUNPOD_S3_ENDPOINT" \
        --profile runpod 2>&1 | grep -q "upload:"; then
        success "$binary uploaded"
        UPLOAD_COUNT=$((UPLOAD_COUNT + 1))
    else
        error "Failed to upload $binary"
        exit 1
    fi
done

success "Uploaded $UPLOAD_COUNT binaries to Runpod S3"

# =============================================================================
# STEP 4: UPLOAD TEST DATA TO RUNPOD S3
# =============================================================================

section "Step 4/8: Uploading Test Data to Runpod S3"

TEST_DATA_DIR="$FOXHUNT_ROOT/test_data"

if [ ! -d "$TEST_DATA_DIR" ]; then
    error "Test data directory not found: $TEST_DATA_DIR"
    exit 1
fi

# Count parquet files
PARQUET_COUNT=$(ls -1 "$TEST_DATA_DIR"/*.parquet 2>/dev/null | wc -l)

if [ "$PARQUET_COUNT" -eq 0 ]; then
    error "No parquet files found in $TEST_DATA_DIR"
    exit 1
fi

progress "Found $PARQUET_COUNT parquet files"

# Upload all parquet files
TOTAL_DATA_SIZE=0
DATA_UPLOAD_COUNT=0

for file in "$TEST_DATA_DIR"/*.parquet; do
    if [ -f "$file" ]; then
        filename=$(basename "$file")
        SIZE=$(stat -c%s "$file" 2>/dev/null || stat -f%z "$file")
        SIZE_MB=$(awk "BEGIN {printf \"%.2f\", $SIZE/1024/1024}")
        TOTAL_DATA_SIZE=$((TOTAL_DATA_SIZE + SIZE))

        progress "Uploading $filename ($SIZE_MB MB)..."

        if aws s3 cp "$file" "s3://${S3_BUCKET}/test_data/$filename" \
            --endpoint-url "$RUNPOD_S3_ENDPOINT" \
            --profile runpod 2>&1 | grep -q "upload:"; then
            success "$filename uploaded"
            DATA_UPLOAD_COUNT=$((DATA_UPLOAD_COUNT + 1))
        else
            error "Failed to upload $filename"
            exit 1
        fi
    fi
done

TOTAL_DATA_MB=$(awk "BEGIN {printf \"%.2f\", $TOTAL_DATA_SIZE/1024/1024}")
success "Uploaded $DATA_UPLOAD_COUNT data files ($TOTAL_DATA_MB MB)"

# =============================================================================
# STEP 5: UPLOAD .ENV FILE (OPTIONAL, WITH CONFIRMATION)
# =============================================================================

section "Step 5/8: Upload .env File (Optional)"

warning "The .env file may contain sensitive credentials (Vault tokens, API keys, etc.)"
warning "Only upload if you need these for training (usually not required for FP32)"

echo ""
echo -e "${YELLOW}Do you want to upload .env to Runpod S3?${NC}"
echo "  - Yes: Upload .env for pod configuration"
echo "  - No: Skip (recommended for FP32 training)"
echo ""

read -p "Upload .env? (y/N): " -n 1 -r
echo

if [[ $REPLY =~ ^[Yy]$ ]]; then
    ENV_FILE="$FOXHUNT_ROOT/.env"

    if [ ! -f "$ENV_FILE" ]; then
        error ".env file not found at $ENV_FILE"
        exit 1
    fi

    warning "Uploading .env to s3://${S3_BUCKET}/config/.env"
    warning "This file will be accessible on Runpod pods"

    echo ""
    read -p "Are you SURE? This may expose secrets. (y/N): " -n 1 -r
    echo

    if [[ $REPLY =~ ^[Yy]$ ]]; then
        if aws s3 cp "$ENV_FILE" "s3://${S3_BUCKET}/config/.env" \
            --endpoint-url "$RUNPOD_S3_ENDPOINT" \
            --profile runpod 2>&1 | grep -q "upload:"; then
            success ".env uploaded to Runpod S3"
        else
            error "Failed to upload .env"
            exit 1
        fi
    else
        warning ".env upload cancelled"
    fi
else
    success ".env upload skipped (recommended)"
fi

# =============================================================================
# STEP 6: BUILD DOCKER IMAGE
# =============================================================================

section "Step 6/8: Building Docker Image"

progress "Building Docker image: $DOCKER_IMAGE"
progress "This will take 2-3 minutes (multistage build with caching)..."

BUILD_START=$(date +%s)

if docker build -f "$FOXHUNT_ROOT/Dockerfile.runpod" -t "$DOCKER_IMAGE" "$FOXHUNT_ROOT" 2>&1 | \
    tee /tmp/foxhunt_docker_build.log | grep -E "(Step|Successfully built|error)"; then
    BUILD_END=$(date +%s)
    BUILD_DURATION=$((BUILD_END - BUILD_START))
    BUILD_MINUTES=$((BUILD_DURATION / 60))
    BUILD_SECONDS=$((BUILD_DURATION % 60))

    success "Docker image built in ${BUILD_MINUTES}m ${BUILD_SECONDS}s"
else
    error "Docker build failed. See /tmp/foxhunt_docker_build.log for details."
    exit 1
fi

# Check image size
IMAGE_SIZE=$(docker images "$DOCKER_IMAGE" --format "{{.Size}}")
success "Image size: $IMAGE_SIZE"

# =============================================================================
# STEP 7: PUSH TO DOCKER HUB
# =============================================================================

section "Step 7/8: Pushing to Docker Hub"

# Verify Docker Hub login
progress "Verifying Docker Hub authentication..."

if ! docker info 2>&1 | grep -q "Username:"; then
    warning "Not logged into Docker Hub. Please login now."
    docker login
fi

# Verify private repository
echo ""
warning "IMPORTANT: Verify your Docker Hub repository is PRIVATE"
warning "URL: https://hub.docker.com/repository/docker/${DOCKER_USERNAME}/foxhunt/general"
echo ""

read -p "Is your Docker Hub repo PRIVATE? (y/N): " -n 1 -r
echo

if [[ ! $REPLY =~ ^[Yy]$ ]]; then
    error "Make your Docker Hub repository PRIVATE before pushing!"
    error "Go to: https://hub.docker.com/repository/docker/${DOCKER_USERNAME}/foxhunt/settings"
    exit 1
fi

# Push image
progress "Pushing $DOCKER_IMAGE to Docker Hub..."
progress "This will take 3-5 minutes (uploading ~2GB image)..."

PUSH_START=$(date +%s)

if docker push "$DOCKER_IMAGE" 2>&1 | tee /tmp/foxhunt_docker_push.log | grep -E "(Pushed|Layer already exists|error)"; then
    PUSH_END=$(date +%s)
    PUSH_DURATION=$((PUSH_END - PUSH_START))
    PUSH_MINUTES=$((PUSH_DURATION / 60))
    PUSH_SECONDS=$((PUSH_DURATION % 60))

    success "Image pushed in ${PUSH_MINUTES}m ${PUSH_SECONDS}s"
else
    error "Docker push failed. See /tmp/foxhunt_docker_push.log for details."
    exit 1
fi

# =============================================================================
# STEP 8: PRINT DEPLOYMENT INSTRUCTIONS
# =============================================================================

section "Step 8/8: Deployment Instructions"

success "Deployment preparation complete! 🎉"

echo ""
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e "${GREEN}READY FOR RUNPOD DEPLOYMENT${NC}"
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo ""

echo -e "${CYAN}📦 Uploaded Resources:${NC}"
echo "  - Binaries: s3://${S3_BUCKET}/binaries/ ($TOTAL_BIN_MB MB)"
echo "  - Test Data: s3://${S3_BUCKET}/test_data/ ($TOTAL_DATA_MB MB)"
echo "  - Docker Image: $DOCKER_IMAGE ($IMAGE_SIZE)"
echo ""

echo -e "${CYAN}🚀 Deploy on Runpod Console:${NC}"
echo "  1. Go to: https://www.runpod.io/console/pods"
echo "  2. Click 'Deploy' and configure:"
echo ""

echo -e "${CYAN}   GPU Configuration:${NC}"
echo "     GPU Type: Tesla V100 16GB (\$0.14-0.39/hr) or RTX 4090 24GB (\$0.60/hr)"
echo "     vCPU: 6-8 cores (recommended)"
echo "     RAM: 30GB+ (recommended)"
echo "     Container Disk: 20GB minimum"
echo ""

echo -e "${CYAN}   Docker Configuration:${NC}"
echo "     Docker Image: $DOCKER_IMAGE"
echo "     Docker Hub Credentials: $DOCKER_USERNAME (required for private repo)"
echo ""

echo -e "${CYAN}   Volume Configuration:${NC}"
echo "     Volume Path: /runpod-volume"
echo "     Network Volume: $S3_BUCKET (select from dropdown)"
echo "     Access Mode: Read/Write"
echo ""

echo -e "${CYAN}   Environment Variables:${NC}"
echo "     BINARY_NAME=train_tft_parquet"
echo "     RUST_LOG=info"
echo ""

echo -e "${CYAN}   Container Arguments (override CMD):${NC}"
echo "     --parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet"
echo "     --epochs 50"
echo "     --batch-size 32"
echo "     --lookback-window 60"
echo "     --forecast-horizon 10"
echo ""

echo -e "${CYAN}📊 Expected Training Performance:${NC}"
echo "  - DQN: ~15-20 seconds (6MB memory)"
echo "  - PPO: ~7-10 seconds (145MB memory)"
echo "  - MAMBA-2: ~2-3 minutes (164MB memory)"
echo "  - TFT-FP32: ~3-5 minutes (500MB memory)"
echo "  - Total: ~10-15 minutes (815MB peak memory)"
echo ""

echo -e "${CYAN}💰 Cost Estimation:${NC}"
echo "  - Tesla V100 @ \$0.25/hr: ~\$0.06 per full training run"
echo "  - RTX 4090 @ \$0.60/hr: ~\$0.15 per full training run"
echo ""

echo -e "${CYAN}📈 Wave D Backtest Results (Expected):${NC}"
echo "  - Sharpe Ratio: 2.00 (≥2.0 target ✅)"
echo "  - Win Rate: 60% (≥60% target ✅)"
echo "  - Max Drawdown: 15% (≤15% target ✅)"
echo ""

echo -e "${CYAN}🔍 Monitor Training:${NC}"
echo "  1. Click 'Logs' tab in Runpod console"
echo "  2. Watch for training progress:"
echo "     - Epoch 1/50 [██████████] 100% | Loss: 0.0123"
echo "     - Validation RMSE: 0.0045"
echo "     - Training complete! Model saved."
echo ""

echo -e "${CYAN}📥 Download Trained Models:${NC}"
echo "  1. SSH into pod (get SSH command from Runpod console)"
echo "  2. Download models:"
echo "     scp root@<pod-ssh>:/workspace/models/*.pt ./models/"
echo "  Or:"
echo "     aws s3 sync s3://${S3_BUCKET}/models/ ./models/ \\"
echo "       --endpoint-url $RUNPOD_S3_ENDPOINT \\"
echo "       --profile runpod"
echo ""

echo -e "${CYAN}🔧 Troubleshooting:${NC}"
echo "  - Volume not mounted: Check pod settings → Volumes → /runpod-volume"
echo "  - Binary not found: Verify binaries uploaded to s3://${S3_BUCKET}/binaries/"
echo "  - Out of memory: Use smaller batch size (--batch-size 16)"
echo "  - CUDA not found: Verify GPU type is V100/4090 and CUDA runtime available"
echo ""

echo -e "${CYAN}📚 Documentation:${NC}"
echo "  - Full guide: $FOXHUNT_ROOT/RUNPOD_DEPLOYMENT_CHECKLIST.md"
echo "  - QAT blockers: $FOXHUNT_ROOT/QAT_BLOCKERS_ROOT_CAUSE_ANALYSIS.md"
echo "  - System status: $FOXHUNT_ROOT/CLAUDE.md"
echo ""

echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo -e "${GREEN}✅ DEPLOYMENT READY - GO TRAIN YOUR MODELS!${NC}"
echo -e "${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"
echo ""

# Log summary
TOTAL_TIME=$(($(date +%s) - BUILD_START))
TOTAL_MINUTES=$((TOTAL_TIME / 60))
TOTAL_SECONDS=$((TOTAL_TIME % 60))

echo -e "${CYAN}⏱️  Total deployment time: ${TOTAL_MINUTES}m ${TOTAL_SECONDS}s${NC}"
echo ""

exit 0