foxhunt/scripts/validate_epsilon_fix3.sh

#!/bin/bash
# Validation Test Script for Fix #3: Epsilon Decay Range Change
# Expected: Break 100% HOLD bias, enable action diversity
# Change: epsilon_decay [0.990, 0.999] → [0.95, 0.99]

set -e

# Configuration
OUTPUT_DIR="/tmp/ml_training/fix3_validation"
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
LOG_FILE="${OUTPUT_DIR}/test_${TIMESTAMP}.log"
SUMMARY_FILE="${OUTPUT_DIR}/summary_${TIMESTAMP}.txt"
TRIALS=5
EPOCHS=10
PARQUET_FILE="test_data/ES_FUT_180d.parquet"

# Colors for output
GREEN='\033[0;32m'
RED='\033[0;31m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

echo "=================================================="
echo "Fix #3 Epsilon Decay Validation Test"
echo "=================================================="
echo "Output Directory: ${OUTPUT_DIR}"
echo "Log File: ${LOG_FILE}"
echo "Summary File: ${SUMMARY_FILE}"
echo "Trials: ${TRIALS}"
echo "Epochs: ${EPOCHS}"
echo "Parquet File: ${PARQUET_FILE}"
echo ""

# Create output directory
mkdir -p "${OUTPUT_DIR}"

# Check if parquet file exists
if [ ! -f "${PARQUET_FILE}" ]; then
    echo -e "${RED}ERROR: Parquet file not found: ${PARQUET_FILE}${NC}"
    exit 1
fi

# Run hyperopt
echo "Starting hyperopt test run..."
echo "Command: cargo run --release -p ml --example hyperopt_dqn_demo --features cuda -- --parquet-file ${PARQUET_FILE} --trials ${TRIALS} --epochs ${EPOCHS}"
echo ""

cargo run --release -p ml --example hyperopt_dqn_demo --features cuda -- \
    --parquet-file "${PARQUET_FILE}" --trials ${TRIALS} --epochs ${EPOCHS} \
    2>&1 | tee "${LOG_FILE}"

# Parse results
echo ""
echo "=================================================="
echo "Analyzing Results..."
echo "=================================================="

# Initialize counters
total_trials=0
trials_with_diversity=0
min_hold_pct=100
max_hold_pct=0
epsilon_values=()

# Extract epsilon_decay values and action distributions
echo "Extracting epsilon_decay values and action distributions from log..."
while IFS= read -r line; do
    # Extract epsilon_decay values (example: "epsilon_decay: 0.976")
    if echo "$line" | grep -q "epsilon_decay:"; then
        epsilon=$(echo "$line" | grep -oP 'epsilon_decay:\s*\K[\d.]+' || echo "")
        if [ -n "$epsilon" ]; then
            epsilon_values+=("$epsilon")
        fi
    fi

    # Extract action distributions (example: "Action distribution: BUY=5.2%, SELL=3.1%, HOLD=91.7%")
    if echo "$line" | grep -q "Action distribution:"; then
        hold_pct=$(echo "$line" | grep -oP 'HOLD=\K[\d.]+' || echo "")
        if [ -n "$hold_pct" ]; then
            total_trials=$((total_trials + 1))

            # Convert to integer for comparison
            hold_int=$(printf "%.0f" "$hold_pct")

            if [ "$hold_int" -lt 90 ]; then
                trials_with_diversity=$((trials_with_diversity + 1))
            fi

            if [ "$hold_int" -lt "$min_hold_pct" ]; then
                min_hold_pct=$hold_int
            fi

            if [ "$hold_int" -gt "$max_hold_pct" ]; then
                max_hold_pct=$hold_int
            fi
        fi
    fi
done < "${LOG_FILE}"

# Generate summary report
{
    echo "=================================================="
    echo "Fix #3 Epsilon Decay Validation Summary"
    echo "=================================================="
    echo "Timestamp: $(date)"
    echo "Log File: ${LOG_FILE}"
    echo ""
    echo "Configuration:"
    echo "  Trials: ${TRIALS}"
    echo "  Epochs: ${EPOCHS}"
    echo "  Parquet File: ${PARQUET_FILE}"
    echo ""
    echo "Expected Epsilon Decay Range: [0.95, 0.99]"
    echo "Actual Epsilon Values Observed:"

    if [ ${#epsilon_values[@]} -gt 0 ]; then
        for i in "${!epsilon_values[@]}"; do
            epsilon="${epsilon_values[$i]}"
            echo "  Trial $((i+1)): ${epsilon}"

            # Validate range
            if (( $(echo "$epsilon >= 0.95" | bc -l) )) && (( $(echo "$epsilon <= 0.99" | bc -l) )); then
                echo "    ✅ Within expected range [0.95, 0.99]"
            else
                echo "    ❌ Outside expected range [0.95, 0.99]"
            fi
        done
    else
        echo "  ⚠️  No epsilon values extracted from log"
    fi

    echo ""
    echo "Action Distribution Analysis:"
    echo "  Trials Analyzed: ${total_trials}"
    echo "  Trials with <90% HOLD: ${trials_with_diversity}"
    echo "  Min HOLD %: ${min_hold_pct}%"
    echo "  Max HOLD %: ${max_hold_pct}%"
    echo ""

    echo "=================================================="
    echo "Success Criteria Validation"
    echo "=================================================="

    # Criteria 1: At least 1 trial with <90% HOLD
    if [ "$trials_with_diversity" -ge 1 ]; then
        echo "✅ Criterion 1 PASS: At least 1 trial with <90% HOLD (${trials_with_diversity} trials)"
    else
        echo "❌ Criterion 1 FAIL: No trials with <90% HOLD (all trials ≥90% HOLD)"
    fi

    # Criteria 2: Action diversity >0% (BUY or SELL observed)
    if [ "$min_hold_pct" -lt 100 ]; then
        echo "✅ Criterion 2 PASS: Action diversity detected (min HOLD=${min_hold_pct}%)"
    else
        echo "❌ Criterion 2 FAIL: 100% HOLD bias persists"
    fi

    # Criteria 3: Epsilon values varying across trials
    if [ ${#epsilon_values[@]} -gt 1 ]; then
        unique_epsilons=$(printf '%s\n' "${epsilon_values[@]}" | sort -u | wc -l)
        if [ "$unique_epsilons" -gt 1 ]; then
            echo "✅ Criterion 3 PASS: Epsilon values varying across trials (${unique_epsilons} unique values)"
        else
            echo "⚠️  Criterion 3 WARNING: All epsilon values identical (no variation)"
        fi
    else
        echo "⚠️  Criterion 3 WARNING: Insufficient epsilon values to assess variation"
    fi

    echo ""
    echo "=================================================="
    echo "Overall Assessment"
    echo "=================================================="

    # Overall pass/fail
    if [ "$trials_with_diversity" -ge 1 ] && [ "$min_hold_pct" -lt 100 ]; then
        echo "✅ VALIDATION PASSED: Fix #3 successfully breaks 100% HOLD bias"
        echo ""
        echo "Key Improvements:"
        echo "  - Action diversity enabled (BUY/SELL actions observed)"
        echo "  - HOLD percentage reduced to ${min_hold_pct}% (min)"
        echo "  - ${trials_with_diversity}/${total_trials} trials show <90% HOLD"
    else
        echo "❌ VALIDATION FAILED: 100% HOLD bias persists"
        echo ""
        echo "Possible Issues:"
        echo "  - Epsilon decay range change not effective"
        echo "  - Other hyperparameters overriding epsilon effect"
        echo "  - Training epochs insufficient for exploration"
    fi

    echo ""
    echo "Full logs available at: ${LOG_FILE}"

} > "${SUMMARY_FILE}"

# Display summary
cat "${SUMMARY_FILE}"

# Exit with appropriate code
if [ "$trials_with_diversity" -ge 1 ] && [ "$min_hold_pct" -lt 100 ]; then
    echo ""
    echo -e "${GREEN}✅ VALIDATION PASSED${NC}"
    exit 0
else
    echo ""
    echo -e "${RED}❌ VALIDATION FAILED${NC}"
    exit 1
fi