foxhunt/run_cross_validation.sh

#!/bin/bash
# Cross-Validation: Test top 3 models on held-out May 2024 data
# Models: DQN-30, DQN-310, PPO-130
# Objective: Validate generalization (Sharpe drop <20%, win rate >55%, max drawdown <15%)

set -e

RESULTS_DIR="/home/jgrusewski/Work/foxhunt/results/cross_validation"
mkdir -p "$RESULTS_DIR"

echo "=========================================="
echo "CROSS-VALIDATION ON HELD-OUT DATA (May 2024)"
echo "=========================================="
echo ""
echo "Models Under Test:"
echo "  - DQN Epoch 30  (Early exploration, high Q-value)"
echo "  - DQN Epoch 310 (Late convergence, conservative)"
echo "  - PPO Epoch 130 (Mid-training, balanced)"
echo ""
echo "Held-Out Dataset: May 2024 (4 days × 4 symbols = 16 files)"
echo "Training Dataset: Jan-April 2024 (361 files)"
echo ""
echo "Success Criteria:"
echo "  ✅ Sharpe ratio >8.0 on held-out (vs 10+ on training)"
echo "  ✅ Win rate >55%"
echo "  ✅ Max drawdown <15%"
echo "  ✅ Generalization gap <20% (held-out Sharpe / training Sharpe)"
echo ""

# Test each model on each symbol's May data
MODELS=(
  "dqn_epoch_30:DQN"
  "dqn_epoch_310:DQN"
  "ppo_actor_epoch_130:PPO"
)

SYMBOLS=("ES.FUT" "NQ.FUT" "ZN.FUT" "6E.FUT")

for model_info in "${MODELS[@]}"; do
  IFS=':' read -r model_file model_type <<< "$model_info"

  echo "=========================================="
  echo "Testing: $model_file ($model_type)"
  echo "=========================================="

  for symbol in "${SYMBOLS[@]}"; do
    echo ""
    echo "📊 Symbol: $symbol (May 2024 held-out data)"

    # Find May 2024 files for this symbol
    DATA_FILES=$(find /home/jgrusewski/Work/foxhunt/test_data/real/databento/ml_training \
      -name "${symbol}_ohlcv-1m_2024-05-*.dbn" | sort)

    if [ -z "$DATA_FILES" ]; then
      echo "  ⚠️  No held-out data found for $symbol"
      continue
    fi

    NUM_FILES=$(echo "$DATA_FILES" | wc -l)
    echo "  Found $NUM_FILES May 2024 data files"

    # Create temporary directory for this symbol's May data
    TEMP_DATA_DIR="$RESULTS_DIR/temp_${symbol}_may2024"
    mkdir -p "$TEMP_DATA_DIR"

    # Copy May files to temp directory
    echo "$DATA_FILES" | while read -r file; do
      cp "$file" "$TEMP_DATA_DIR/"
    done

    # Determine model path based on type
    if [ "$model_type" = "DQN" ]; then
      MODEL_PATH="/home/jgrusewski/Work/foxhunt/ml/trained_models/production/dqn_real_data/${model_file}.safetensors"
    else
      MODEL_PATH="/home/jgrusewski/Work/foxhunt/ml/trained_models/production/ppo_real_data/${model_file}.safetensors"
    fi

    # Run backtest
    OUTPUT_FILE="$RESULTS_DIR/${model_file}_${symbol}_may2024.json"

    echo "  🔄 Running backtest..."
    echo "  Model: $MODEL_PATH"
    echo "  Data: $TEMP_DATA_DIR"
    echo "  Output: $OUTPUT_FILE"

    # Run comprehensive backtest (Note: This is a placeholder - actual implementation needed)
    # The comprehensive_model_backtest.rs needs to be updated to accept CLI args
    echo "  ⏳ Backtest execution placeholder (requires CLI args implementation)"

    # Cleanup temp directory
    rm -rf "$TEMP_DATA_DIR"

    echo "  ✅ Backtest complete"
  done

  echo ""
done

echo ""
echo "=========================================="
echo "CROSS-VALIDATION COMPLETE"
echo "=========================================="
echo ""
echo "Results saved to: $RESULTS_DIR"
echo ""
echo "Next Steps:"
echo "  1. Analyze results: Compare training metrics vs held-out"
echo "  2. Calculate generalization gap: (training_sharpe - held_out_sharpe) / training_sharpe"
echo "  3. Identify overfitting: Gap >20% indicates poor generalization"
echo "  4. Generate CROSS_VALIDATION_REPORT.md"
echo ""