foxhunt/scripts/archive/backtest_runpod_225.sh

#!/bin/bash
# Runpod 225-Feature Backtesting Script
# Tests trained TFT model against Wave D targets
# Targets: Sharpe ≥2.0, Win Rate ≥60%, Drawdown ≤15%

set -euo pipefail

# Color output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Configuration
MODEL_PATH="models/runpod_trained/tft_225_fp32.safetensors"
DATA_PATH="test_data/ES_FUT_180d.parquet"
INITIAL_CAPITAL=100000
SYMBOLS="ES.FUT,NQ.FUT"
STRATEGY="ml_adaptive_225"
START_DATE="2024-01-01"
END_DATE="2024-06-30"

echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Runpod 225-Feature Backtesting${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""

# Step 1: Verify model exists
echo -e "${YELLOW}Step 1/4: Verifying trained model...${NC}"

if [ ! -f "${MODEL_PATH}" ]; then
    # Try to find any safetensors file
    FOUND_MODEL=$(find models/runpod_trained -name "*.safetensors" -type f | head -1)

    if [ -z "$FOUND_MODEL" ]; then
        echo -e "${RED}❌ No trained model found in models/runpod_trained/${NC}"
        echo "Expected: ${MODEL_PATH}"
        echo ""
        echo "Available files:"
        ls -lh models/runpod_trained/ 2>/dev/null || echo "Directory does not exist"
        exit 1
    fi

    echo -e "${YELLOW}⚠️  Using found model: ${FOUND_MODEL}${NC}"
    MODEL_PATH="$FOUND_MODEL"
fi

MODEL_SIZE=$(du -h "${MODEL_PATH}" | cut -f1)
echo -e "${GREEN}✅ Model found: ${MODEL_SIZE}${NC}"

# Step 2: Verify data
echo -e "\n${YELLOW}Step 2/4: Verifying training data...${NC}"

if [ ! -f "${DATA_PATH}" ]; then
    echo -e "${RED}❌ Training data not found: ${DATA_PATH}${NC}"
    exit 1
fi

DATA_SIZE=$(du -h "${DATA_PATH}" | cut -f1)
ROW_COUNT=$(parquet-tools rowcount "${DATA_PATH}" 2>/dev/null || echo "unknown")
echo -e "${GREEN}✅ Data found: ${DATA_SIZE}, Rows: ${ROW_COUNT}${NC}"

# Step 3: Run backtest
echo -e "\n${YELLOW}Step 3/4: Running backtest...${NC}"
echo "Configuration:"
echo "  • Model: ${MODEL_PATH}"
echo "  • Data: ${DATA_PATH}"
echo "  • Initial Capital: \$${INITIAL_CAPITAL}"
echo "  • Symbols: ${SYMBOLS}"
echo "  • Strategy: ${STRATEGY}"
echo "  • Period: ${START_DATE} to ${END_DATE}"
echo ""

BACKTEST_START=$(date +%s)

# Run backtest (adjust command based on actual backtesting CLI)
cargo run -p backtesting --release --features cuda --example feature_comparison_backtest -- \
    --model-path "${MODEL_PATH}" \
    --parquet-file "${DATA_PATH}" \
    --initial-capital ${INITIAL_CAPITAL} \
    --symbols "${SYMBOLS}" \
    --strategy "${STRATEGY}" \
    --start-date "${START_DATE}" \
    --end-date "${END_DATE}" \
    2>&1 | tee backtest_225.log

BACKTEST_END=$(date +%s)
BACKTEST_DURATION=$((BACKTEST_END - BACKTEST_START))

echo -e "\n${GREEN}✅ Backtest completed in ${BACKTEST_DURATION}s${NC}"

# Step 4: Extract and validate metrics
echo -e "\n${YELLOW}Step 4/4: Extracting Wave D metrics...${NC}"

# Extract key metrics from backtest output
SHARPE=$(grep -oP "Sharpe Ratio[:\s]+\K[0-9.]+" backtest_225.log | tail -1 || echo "N/A")
WIN_RATE=$(grep -oP "Win Rate[:\s]+\K[0-9.]+" backtest_225.log | tail -1 || echo "N/A")
DRAWDOWN=$(grep -oP "Max Drawdown[:\s]+\K[0-9.]+" backtest_225.log | tail -1 || echo "N/A")
TOTAL_PNL=$(grep -oP "Total PnL[:\s]+\$?\K[0-9.]+" backtest_225.log | tail -1 || echo "N/A")
TOTAL_TRADES=$(grep -oP "Total Trades[:\s]+\K[0-9]+" backtest_225.log | tail -1 || echo "N/A")

echo ""
echo -e "${BLUE}========================================${NC}"
echo -e "${BLUE}Wave D Backtest Results${NC}"
echo -e "${BLUE}========================================${NC}"
echo ""

# Sharpe Ratio validation
echo -e "${BLUE}Sharpe Ratio:${NC} ${SHARPE}"
if [ "$SHARPE" != "N/A" ] && [ "$(echo "${SHARPE} >= 2.0" | bc -l 2>/dev/null || echo 0)" -eq 1 ]; then
    echo -e "  ${GREEN}✅ Target: ≥2.0 (PASSED)${NC}"
else
    echo -e "  ${RED}❌ Target: ≥2.0 (FAILED)${NC}"
fi
echo ""

# Win Rate validation
echo -e "${BLUE}Win Rate:${NC} ${WIN_RATE}%"
if [ "$WIN_RATE" != "N/A" ] && [ "$(echo "${WIN_RATE} >= 60.0" | bc -l 2>/dev/null || echo 0)" -eq 1 ]; then
    echo -e "  ${GREEN}✅ Target: ≥60% (PASSED)${NC}"
else
    echo -e "  ${RED}❌ Target: ≥60% (FAILED)${NC}"
fi
echo ""

# Drawdown validation
echo -e "${BLUE}Max Drawdown:${NC} ${DRAWDOWN}%"
if [ "$DRAWDOWN" != "N/A" ] && [ "$(echo "${DRAWDOWN} <= 15.0" | bc -l 2>/dev/null || echo 0)" -eq 1 ]; then
    echo -e "  ${GREEN}✅ Target: ≤15% (PASSED)${NC}"
else
    echo -e "  ${RED}❌ Target: ≤15% (FAILED)${NC}"
fi
echo ""

# Additional metrics
echo -e "${BLUE}Additional Metrics:${NC}"
echo "  • Total PnL: \$${TOTAL_PNL}"
echo "  • Total Trades: ${TOTAL_TRADES}"
echo "  • Backtest Duration: ${BACKTEST_DURATION}s"
echo ""

# Overall assessment
echo -e "${BLUE}========================================${NC}"
PASSED_COUNT=0

if [ "$SHARPE" != "N/A" ] && [ "$(echo "${SHARPE} >= 2.0" | bc -l 2>/dev/null || echo 0)" -eq 1 ]; then
    PASSED_COUNT=$((PASSED_COUNT + 1))
fi

if [ "$WIN_RATE" != "N/A" ] && [ "$(echo "${WIN_RATE} >= 60.0" | bc -l 2>/dev/null || echo 0)" -eq 1 ]; then
    PASSED_COUNT=$((PASSED_COUNT + 1))
fi

if [ "$DRAWDOWN" != "N/A" ] && [ "$(echo "${DRAWDOWN} <= 15.0" | bc -l 2>/dev/null || echo 0)" -eq 1 ]; then
    PASSED_COUNT=$((PASSED_COUNT + 1))
fi

if [ $PASSED_COUNT -eq 3 ]; then
    echo -e "${GREEN}✅ All Wave D Targets PASSED (3/3)${NC}"
    echo ""
    echo "Model is ready for production deployment!"
elif [ $PASSED_COUNT -ge 2 ]; then
    echo -e "${YELLOW}⚠️  Partial Success: ${PASSED_COUNT}/3 Targets Passed${NC}"
    echo ""
    echo "Model shows promise but may need fine-tuning."
else
    echo -e "${RED}❌ Wave D Targets NOT MET (${PASSED_COUNT}/3 Passed)${NC}"
    echo ""
    echo "Model requires additional training or hyperparameter tuning."
fi
echo -e "${BLUE}========================================${NC}"
echo ""

echo "Next Steps:"
echo "  1. Review detailed backtest log: backtest_225.log"
echo "  2. Generate results report: see RUNPOD_225_FEATURE_TRAINING_RESULTS.md"
echo "  3. Train additional models: DQN, PPO, MAMBA-2"
echo "  4. Multi-asset validation: NQ.FUT, 6E.FUT, ZN.FUT"
echo ""