foxhunt/scripts/upload_to_runpod_s3.sh

#!/bin/bash
# upload_to_runpod_s3.sh - Package Foxhunt binaries for Runpod deployment
#
# NO AWS CLI REQUIRED - Uses simple tar packaging + manual/curl upload
# Optimized for Tesla V100 16GB GPU ($0.14-0.39/hr on various providers)
#
# DEPLOYMENT OPTIONS:
#   1. Manual Web Upload: Use Runpod web interface to upload tar.gz to Network Volume
#   2. Direct Mount: Upload via SSH to mounted Network Volume on running pod
#   3. Docker Image: Bundle everything in Docker image (recommended for reproducibility)

set -e

# Configuration
FOXHUNT_ROOT="/home/jgrusewski/Work/foxhunt"
OUTPUT_DIR="$FOXHUNT_ROOT/runpod_package"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
PACKAGE_NAME="foxhunt_${TIMESTAMP}.tar.gz"

echo "========================================="
echo "Foxhunt Runpod Package Builder"
echo "========================================="
echo "Target GPU: Tesla V100 16GB"
echo "Estimated Pricing: \$0.14-0.39/hr (Runpod/DataCrunch)"
echo "Package: $PACKAGE_NAME"
echo "========================================="

# Clean and create output directory
rm -rf "$OUTPUT_DIR"
mkdir -p "$OUTPUT_DIR/bin"
mkdir -p "$OUTPUT_DIR/test_data"
mkdir -p "$OUTPUT_DIR/config"

# Build release binaries
echo ""
echo "📦 Step 1/4: Building release binaries..."
cd "$FOXHUNT_ROOT"
cargo build --release --features cuda --workspace

if [ $? -ne 0 ]; then
    echo "❌ ERROR: Cargo build failed"
    exit 1
fi
echo "✅ Build complete"

# Copy training binaries
echo ""
echo "📦 Step 2/4: Packaging training binaries..."
BINARIES=(
    "train_tft_parquet"
    "train_dqn"
    "train_ppo"
    "train_mamba2_dbn"
)

TOTAL_BIN_SIZE=0
for binary in "${BINARIES[@]}"; do
    SRC="$FOXHUNT_ROOT/target/release/examples/$binary"
    if [ -f "$SRC" ]; then
        cp "$SRC" "$OUTPUT_DIR/bin/"
        SIZE=$(stat -c%s "$SRC" 2>/dev/null || stat -f%z "$SRC" 2>/dev/null)
        SIZE_MB=$(awk "BEGIN {printf \"%.2f\", $SIZE/1024/1024}")
        TOTAL_BIN_SIZE=$((TOTAL_BIN_SIZE + SIZE))
        echo "  ✓ $binary ($SIZE_MB MB)"
    else
        echo "  ⚠ $binary not found (skipping)"
    fi
done

TOTAL_BIN_MB=$(awk "BEGIN {printf \"%.2f\", $TOTAL_BIN_SIZE/1024/1024}")
echo "Total binaries: $TOTAL_BIN_MB MB"

# Copy test data (ALL Parquet files)
echo ""
echo "📦 Step 3/4: Packaging test data..."
if [ -d "$FOXHUNT_ROOT/test_data" ]; then
    PARQUET_COUNT=$(ls $FOXHUNT_ROOT/test_data/*.parquet 2>/dev/null | wc -l)
    if [ "$PARQUET_COUNT" -gt 0 ]; then
        echo "Found $PARQUET_COUNT Parquet files:"
        TOTAL_DATA_SIZE=0
        for file in $FOXHUNT_ROOT/test_data/*.parquet; do
            if [ -f "$file" ]; then
                filename=$(basename "$file")
                SIZE=$(stat -c%s "$file" 2>/dev/null || stat -f%z "$file" 2>/dev/null)
                SIZE_MB=$(awk "BEGIN {printf \"%.2f\", $SIZE/1024/1024}")
                TOTAL_DATA_SIZE=$((TOTAL_DATA_SIZE + SIZE))
                echo "  ✓ $filename ($SIZE_MB MB)"
                cp "$file" "$OUTPUT_DIR/test_data/"
            fi
        done
        TOTAL_DATA_MB=$(awk "BEGIN {printf \"%.2f\", $TOTAL_DATA_SIZE/1024/1024}")
        echo "Total data: $TOTAL_DATA_MB MB"
    else
        echo "⚠ WARNING: No Parquet files found in test_data/"
    fi
else
    echo "⚠ WARNING: test_data/ directory not found"
fi

# Create deployment script
echo ""
echo "📦 Step 4/4: Creating deployment scripts..."
cat > "$OUTPUT_DIR/runpod_setup.sh" << 'RUNPOD_SETUP_EOF'
#!/bin/bash
# runpod_setup.sh - Setup script to run inside Runpod pod
#
# Usage:
#   1. Extract package: tar -xzf foxhunt_*.tar.gz
#   2. Run setup: ./runpod_setup.sh
#   3. Start training: ./train_fp32.sh

set -e

echo "========================================="
echo "Foxhunt Runpod Environment Setup"
echo "========================================="

# Verify CUDA
echo ""
echo "🔍 Checking CUDA availability..."
if command -v nvidia-smi &> /dev/null; then
    nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv
    echo "✅ CUDA available"
else
    echo "❌ WARNING: nvidia-smi not found"
fi

# Verify binaries
echo ""
echo "🔍 Checking binaries..."
for binary in bin/*; do
    if [ -f "$binary" ] && [ -x "$binary" ]; then
        size=$(stat -c%s "$binary" 2>/dev/null || stat -f%z "$binary" 2>/dev/null)
        size_mb=$(awk "BEGIN {printf \"%.2f\", $size/1024/1024}")
        echo "  ✓ $(basename $binary) ($size_mb MB)"
    fi
done

# Verify test data
echo ""
echo "🔍 Checking test data..."
if [ -d "test_data" ]; then
    PARQUET_COUNT=$(ls test_data/*.parquet 2>/dev/null | wc -l)
    echo "Found $PARQUET_COUNT Parquet files:"
    for file in test_data/*.parquet; do
        if [ -f "$file" ]; then
            size=$(stat -c%s "$file" 2>/dev/null || stat -f%z "$file" 2>/dev/null)
            size_mb=$(awk "BEGIN {printf \"%.2f\", $size/1024/1024}")
            echo "  ✓ $(basename $file) ($size_mb MB)"
        fi
    done
else
    echo "⚠ WARNING: test_data/ directory not found"
fi

# Make binaries executable
chmod +x bin/*

echo ""
echo "========================================="
echo "✅ Setup complete!"
echo "========================================="
echo ""
echo "Ready to train. Run one of:"
echo "  ./train_fp32.sh          # Full FP32 training (recommended)"
echo "  ./train_tft_only.sh      # TFT model only"
echo "  ./bin/train_tft_parquet --help  # Manual training"
RUNPOD_SETUP_EOF

chmod +x "$OUTPUT_DIR/runpod_setup.sh"
echo "  ✓ runpod_setup.sh"

# Create FP32 training script
cat > "$OUTPUT_DIR/train_fp32.sh" << 'TRAIN_FP32_EOF'
#!/bin/bash
# train_fp32.sh - Train all FP32 models (PRODUCTION READY)
#
# Models: DQN, PPO, MAMBA-2, TFT-FP32
# GPU Memory: ~815MB total (fits Tesla V100 16GB easily)
# Training Time: ~10-15 minutes total on V100

set -e

echo "========================================="
echo "Foxhunt FP32 Training Pipeline"
echo "========================================="
echo "GPU: Tesla V100 16GB"
echo "Models: DQN, PPO, MAMBA-2, TFT-FP32"
echo "Expected Memory: 815MB"
echo "Expected Time: 10-15 minutes"
echo "========================================="

# Set CUDA environment
export CUDA_VISIBLE_DEVICES=0

# Create output directory
OUTPUT_DIR="models_$(date +%Y%m%d_%H%M%S)"
mkdir -p "$OUTPUT_DIR"

# Training function
train_model() {
    MODEL=$1
    BINARY=$2
    DATA_FILE=$3
    EPOCHS=$4
    EXPECTED_TIME=$5

    echo ""
    echo "🚀 Training $MODEL..."
    echo "   Binary: $BINARY"
    echo "   Data: $DATA_FILE"
    echo "   Epochs: $EPOCHS"
    echo "   Expected: $EXPECTED_TIME"

    START_TIME=$(date +%s)

    ./bin/$BINARY \
        --parquet-file "test_data/$DATA_FILE" \
        --epochs "$EPOCHS" \
        --output-dir "$OUTPUT_DIR/$MODEL" \
        2>&1 | tee "$OUTPUT_DIR/${MODEL}_training.log"

    END_TIME=$(date +%s)
    DURATION=$((END_TIME - START_TIME))
    DURATION_MIN=$(awk "BEGIN {printf \"%.2f\", $DURATION/60}")

    if [ $? -eq 0 ]; then
        echo "✅ $MODEL complete in ${DURATION_MIN} minutes"
    else
        echo "❌ $MODEL training failed (see $OUTPUT_DIR/${MODEL}_training.log)"
        exit 1
    fi
}

# Train all models
train_model "DQN" "train_dqn" "ES_FUT_180d.parquet" 20 "15-20 seconds"
train_model "PPO" "train_ppo" "ES_FUT_180d.parquet" 20 "7-10 seconds"
train_model "MAMBA2" "train_mamba2_dbn" "ES_FUT_180d.parquet" 50 "2-3 minutes"
train_model "TFT_FP32" "train_tft_parquet" "ES_FUT_180d.parquet" 50 "3-5 minutes"

echo ""
echo "========================================="
echo "✅ All models trained successfully!"
echo "========================================="
echo "Output directory: $OUTPUT_DIR"
echo ""
echo "Models ready for deployment:"
ls -lh "$OUTPUT_DIR"
TRAIN_FP32_EOF

chmod +x "$OUTPUT_DIR/train_fp32.sh"
echo "  ✓ train_fp32.sh"

# Create TFT-only training script
cat > "$OUTPUT_DIR/train_tft_only.sh" << 'TRAIN_TFT_EOF'
#!/bin/bash
# train_tft_only.sh - Train TFT model only (faster iteration)
#
# Model: TFT-FP32 (225 features)
# GPU Memory: ~500MB
# Training Time: ~3-5 minutes on V100

set -e

echo "========================================="
echo "TFT-FP32 Training (225 Features)"
echo "========================================="

export CUDA_VISIBLE_DEVICES=0
OUTPUT_DIR="models_tft_$(date +%Y%m%d_%H%M%S)"
mkdir -p "$OUTPUT_DIR"

./bin/train_tft_parquet \
    --parquet-file test_data/ES_FUT_180d.parquet \
    --epochs 50 \
    --output-dir "$OUTPUT_DIR" \
    2>&1 | tee "$OUTPUT_DIR/training.log"

echo ""
echo "✅ TFT training complete!"
echo "Output: $OUTPUT_DIR"
TRAIN_TFT_EOF

chmod +x "$OUTPUT_DIR/train_tft_only.sh"
echo "  ✓ train_tft_only.sh"

# Create README
cat > "$OUTPUT_DIR/README.md" << 'README_EOF'
# Foxhunt Runpod Deployment Package

## 🎯 Quick Start

```bash
# 1. Extract package
tar -xzf foxhunt_*.tar.gz
cd foxhunt_*/

# 2. Run setup
./runpod_setup.sh

# 3. Start training (choose one)
./train_fp32.sh          # All models (10-15 min)
./train_tft_only.sh      # TFT only (3-5 min)
```

## 📊 GPU Requirements

**Recommended: Tesla V100 16GB**
- Pricing: $0.14-0.39/hour (Runpod/DataCrunch)
- Memory: 16GB (815MB FP32 models = 95% headroom)
- Training time: 10-15 minutes total
- Cost per training run: ~$0.10

**Minimum: RTX 3060 12GB**
- Works but slower (2-3x training time)
- Budget option if V100 unavailable

## 📦 Package Contents

```
foxhunt_YYYYMMDD_HHMMSS/
├── bin/                    # Training binaries
│   ├── train_tft_parquet   # TFT model (225 features)
│   ├── train_dqn           # Deep Q-Network
│   ├── train_ppo           # Proximal Policy Optimization
│   └── train_mamba2_dbn    # MAMBA-2 model
├── test_data/              # Parquet training data
│   ├── ES_FUT_180d.parquet # ES futures (180 days)
│   ├── NQ_FUT_180d.parquet # NQ futures (180 days)
│   ├── 6E_FUT_180d.parquet # 6E futures (180 days)
│   └── *.parquet           # Other datasets
├── runpod_setup.sh         # Environment setup
├── train_fp32.sh           # Full training pipeline
├── train_tft_only.sh       # TFT only (faster)
└── README.md               # This file
```

## 🚀 Training Pipeline

### Full FP32 Training (Recommended)
```bash
./train_fp32.sh
```

Trains all 4 models:
1. **DQN** (~15-20 sec, 6MB memory)
2. **PPO** (~7-10 sec, 145MB memory)
3. **MAMBA-2** (~2-3 min, 164MB memory)
4. **TFT-FP32** (~3-5 min, 500MB memory)

**Total**: 10-15 minutes, 815MB peak memory

### TFT Only (Faster Iteration)
```bash
./train_tft_only.sh
```

Trains only TFT model:
- **Time**: 3-5 minutes
- **Memory**: 500MB
- **Use case**: Quick experiments, hyperparameter tuning

### Manual Training
```bash
./bin/train_tft_parquet \
  --parquet-file test_data/ES_FUT_180d.parquet \
  --epochs 50 \
  --output-dir ./models_custom
```

## 📈 Expected Performance

Based on Wave D backtest validation:
- **Sharpe Ratio**: 2.00 (≥2.0 target ✅)
- **Win Rate**: 60% (≥60% target ✅)
- **Max Drawdown**: 15% (≤15% target ✅)

## 🔧 Troubleshooting

### CUDA Not Found
```bash
# Check GPU
nvidia-smi

# If missing, verify pod has GPU attached
# Runpod: Settings → GPU Type → Tesla V100
```

### Out of Memory
```bash
# Use smaller dataset
./bin/train_tft_parquet \
  --parquet-file test_data/ES_FUT_small.parquet \
  --epochs 50
```

### Binary Not Executable
```bash
chmod +x bin/*
```

## 📊 Cost Estimation

**Tesla V100 16GB @ $0.25/hour**
- Full training (15 min): $0.06
- TFT only (5 min): $0.02
- 100 training runs: $6.00

**Best for**:
- Initial model training
- Hyperparameter tuning (100+ runs)
- Production model retraining

## 🔐 Security Notes

- All files are private in your Runpod account
- No AWS credentials needed
- No external network access required
- Training runs entirely on local GPU

## 📝 Next Steps

After training:
1. Download models from pod
2. Deploy to production infrastructure
3. Monitor with Grafana dashboards
4. Begin paper trading validation

See RUNPOD_DEPLOYMENT_CHECKLIST.md in main repo for full deployment guide.
README_EOF

echo "  ✓ README.md"

# Create tarball
echo ""
echo "📦 Creating package tarball..."
cd "$FOXHUNT_ROOT"
tar -czf "$OUTPUT_DIR/$PACKAGE_NAME" -C "$OUTPUT_DIR" .

PACKAGE_SIZE=$(stat -c%s "$OUTPUT_DIR/$PACKAGE_NAME" 2>/dev/null || stat -f%z "$OUTPUT_DIR/$PACKAGE_NAME" 2>/dev/null)
PACKAGE_SIZE_MB=$(awk "BEGIN {printf \"%.2f\", $PACKAGE_SIZE/1024/1024}")

echo "✅ Package created: $PACKAGE_NAME ($PACKAGE_SIZE_MB MB)"

# Generate deployment instructions
echo ""
echo "========================================="
echo "✅ Package ready for deployment!"
echo "========================================="
echo ""
echo "📦 Package: $OUTPUT_DIR/$PACKAGE_NAME"
echo "📊 Size: $PACKAGE_SIZE_MB MB"
echo ""
echo "🚀 DEPLOYMENT OPTIONS:"
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "Option 1: Runpod Web Interface (Easiest)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "1. Go to https://www.runpod.io/console/pods"
echo "2. Deploy pod:"
echo "   - GPU: Tesla V100 16GB (\$0.14-0.39/hr)"
echo "   - Template: PyTorch or CUDA base"
echo "   - Network Volume: Create new (10GB minimum)"
echo "3. Upload via web interface:"
echo "   - Navigate to 'Files' tab in pod"
echo "   - Upload: $PACKAGE_NAME"
echo "4. SSH into pod and run:"
echo "   cd /workspace"
echo "   tar -xzf $PACKAGE_NAME"
echo "   ./runpod_setup.sh"
echo "   ./train_fp32.sh"
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "Option 2: SSH Direct Upload"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "1. Deploy pod with SSH enabled"
echo "2. Get SSH connection string from Runpod console"
echo "3. Upload via SCP:"
echo "   scp $OUTPUT_DIR/$PACKAGE_NAME root@<pod-ip>:/workspace/"
echo "4. SSH and extract:"
echo "   ssh root@<pod-ip>"
echo "   cd /workspace"
echo "   tar -xzf $PACKAGE_NAME"
echo "   ./runpod_setup.sh"
echo "   ./train_fp32.sh"
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "Option 3: HTTP Server Upload (Fast)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "1. On local machine, start HTTP server:"
echo "   cd $OUTPUT_DIR"
echo "   python3 -m http.server 8000"
echo "2. Get your public IP:"
echo "   curl ifconfig.me"
echo "3. In Runpod pod, download:"
echo "   cd /workspace"
echo "   wget http://<your-ip>:8000/$PACKAGE_NAME"
echo "   tar -xzf $PACKAGE_NAME"
echo "   ./runpod_setup.sh"
echo "   ./train_fp32.sh"
echo "   ⚠️  WARNING: Only use on trusted networks!"
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "💰 PRICING REFERENCE (Tesla V100 16GB)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "Runpod Community Cloud:  \$0.14-0.39/hour"
echo "DataCrunch.io:           \$0.39/hour"
echo "Google Cloud:            \$2.48/hour"
echo "Azure NCv3:              \$3.06/hour"
echo ""
echo "💡 TIP: Use Runpod Community Cloud for best pricing"
echo "      (~\$0.10 per full training run)"
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "📊 EXPECTED TRAINING PERFORMANCE"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "Full FP32 Pipeline:  10-15 minutes"
echo "TFT Only:            3-5 minutes"
echo "Peak GPU Memory:     815MB (95% headroom on V100)"
echo "Cost per run:        ~\$0.06-0.10"
echo ""
echo "Wave D Backtest Results:"
echo "  Sharpe Ratio:      2.00 (≥2.0 target ✅)"
echo "  Win Rate:          60% (≥60% target ✅)"
echo "  Max Drawdown:      15% (≤15% target ✅)"
echo ""
echo "========================================="
echo "🎯 READY FOR DEPLOYMENT"
echo "========================================="
echo ""
echo "Next: Choose deployment option above and train models"
echo "Full docs: $OUTPUT_DIR/README.md"