foxhunt/entrypoint_debug.sh

#!/bin/bash
# DEBUG VERSION - Enhanced error capture for binary crash diagnosis
set -uo pipefail

# =============================================================================
# RUNPOD DEBUG ENTRYPOINT - ENHANCED ERROR CAPTURE
# =============================================================================
# Purpose: Capture detailed crash diagnostics when binary fails
# Changes from production entrypoint.sh:
#   1. Capture stderr to /tmp/training_error.log
#   2. Run binary with strace for system call tracing
#   3. Print detailed error diagnostics on crash
#   4. Test binary with --help first to isolate loading vs execution errors
# =============================================================================

echo "=========================================="
echo "Foxhunt HFT - DEBUG MODE (Enhanced Error Capture)"
echo "=========================================="

# Default binary name if not specified
BINARY_NAME=${BINARY_NAME:-train_tft_parquet}

echo "Configuration:"
echo "  Binary: ${BINARY_NAME}"
echo "  Volume Path: /runpod-volume/"
echo "  Debug Mode: ENABLED (stderr capture + strace)"

# =============================================================================
# VERIFY VOLUME IS MOUNTED
# =============================================================================

echo ""
echo "=========================================="
echo "Verifying Runpod Network Volume Mount"
echo "=========================================="

if [ ! -d "/runpod-volume" ]; then
    echo "ERROR: /runpod-volume directory does not exist"
    echo "Runpod Network Volume is NOT mounted!"
    exit 1
fi

if [ ! -d "/runpod-volume/binaries" ]; then
    echo "ERROR: /runpod-volume/binaries directory does not exist"
    echo "Volume is mounted but binaries directory is missing!"
    exit 1
fi

if [ ! -d "/runpod-volume/test_data" ]; then
    echo "ERROR: /runpod-volume/test_data directory does not exist"
    echo "Volume is mounted but test_data directory is missing!"
    exit 1
fi

echo "✓ Volume mounted successfully at /runpod-volume/"

# =============================================================================
# LOAD CREDENTIALS FROM .env FILE (IF EXISTS)
# =============================================================================

echo ""
echo "=========================================="
echo "Loading Credentials"
echo "=========================================="

if [ -f /runpod-volume/.env ]; then
  chmod 600 /runpod-volume/.env 2>/dev/null || true
  set -a
  source /runpod-volume/.env || {
    echo "ERROR: Failed to source /runpod-volume/.env"
    exit 1
  }
  set +a
  echo "✓ Loaded credentials from /runpod-volume/.env"
else
  echo "⚠ WARNING: .env file not found at /runpod-volume/.env"
  echo "Continuing with environment defaults..."
fi

# =============================================================================
# VERIFY TRAINING BINARY EXISTS
# =============================================================================

echo ""
echo "=========================================="
echo "Verifying Training Binary"
echo "=========================================="

BINARY_PATH="/runpod-volume/binaries/${BINARY_NAME}"

if [ ! -f "${BINARY_PATH}" ]; then
    echo "ERROR: Training binary not found: ${BINARY_PATH}"
    echo ""
    echo "Available binaries in /runpod-volume/binaries/:"
    ls -lh /runpod-volume/binaries/ 2>/dev/null || echo "  (none)"
    exit 1
fi

# Check if binary is executable
if [ ! -x "${BINARY_PATH}" ]; then
    echo "WARNING: Binary is not executable, attempting to fix..."
    chmod +x "${BINARY_PATH}" || {
        echo "ERROR: Failed to make binary executable"
        exit 1
    }
    echo "✓ Binary is now executable"
fi

BINARY_SIZE=$(stat -c%s "${BINARY_PATH}" 2>/dev/null || stat -f%z "${BINARY_PATH}")
echo "✓ Binary found: ${BINARY_PATH}"
echo "  Size: $(numfmt --to=iec-i --suffix=B ${BINARY_SIZE} 2>/dev/null || echo ${BINARY_SIZE} bytes)"

# =============================================================================
# DEBUG: ANALYZE BINARY DEPENDENCIES (ldd)
# =============================================================================

echo ""
echo "=========================================="
echo "Binary Dependency Analysis (ldd)"
echo "=========================================="

echo "Checking dynamic library dependencies..."
ldd "${BINARY_PATH}" 2>&1 | tee /tmp/ldd_output.log || {
    echo "ERROR: ldd command failed (binary might be corrupted)"
    exit 1
}

echo ""
echo "Checking for missing dependencies..."
MISSING_LIBS=$(ldd "${BINARY_PATH}" 2>&1 | grep "not found" || echo "")
if [ -n "${MISSING_LIBS}" ]; then
    echo "❌ MISSING LIBRARIES DETECTED:"
    echo "${MISSING_LIBS}"
    echo ""
    echo "This is likely the root cause of the crash!"
    echo "See /tmp/ldd_output.log for full details"
else
    echo "✓ All dynamic libraries resolved successfully"
fi

# =============================================================================
# VERIFY GPU HARDWARE
# =============================================================================

echo ""
echo "=========================================="
echo "GPU Hardware Status"
echo "=========================================="

if command -v nvidia-smi &> /dev/null; then
    echo "GPU Information:"
    nvidia-smi --query-gpu=name,memory.total,memory.free,driver_version,compute_cap --format=csv,noheader 2>&1 || {
        echo "  WARNING: nvidia-smi query failed (GPU might be initializing)"
    }
else
    echo "WARNING: nvidia-smi not found - GPU may not be accessible"
fi

# =============================================================================
# TEST 1: BINARY HELP (Isolate Loading vs Execution Errors)
# =============================================================================

echo ""
echo "=========================================="
echo "TEST 1: Binary Help (Loading Test)"
echo "=========================================="
echo "Testing if binary can load successfully with --help..."
echo ""

# Capture stderr and stdout separately
"${BINARY_PATH}" --help 1>/tmp/help_stdout.log 2>/tmp/help_stderr.log
HELP_EXIT_CODE=$?

echo "Exit code: ${HELP_EXIT_CODE}"
echo ""
echo "--- STDOUT ---"
cat /tmp/help_stdout.log || echo "(no stdout)"
echo ""
echo "--- STDERR ---"
cat /tmp/help_stderr.log || echo "(no stderr)"
echo ""

if [ ${HELP_EXIT_CODE} -ne 0 ]; then
    echo "❌ CRASH DETECTED DURING LOADING (--help test failed)"
    echo ""
    echo "This indicates the binary cannot even load dependencies."
    echo "Root cause is likely:"
    echo "  1. Missing dynamic library (check ldd output above)"
    echo "  2. GLIBC version mismatch (check 'ldd --version')"
    echo "  3. Corrupted binary (re-upload from local machine)"
    echo ""
    echo "Full error log saved to /tmp/help_stderr.log"
    exit 1
else
    echo "✓ Binary loaded successfully (--help test passed)"
fi

# =============================================================================
# TEST 2: STRACE EXECUTION (System Call Tracing)
# =============================================================================

echo ""
echo "=========================================="
echo "TEST 2: Strace Execution (System Call Tracing)"
echo "=========================================="
echo "Running binary with strace to capture system calls..."
echo "Binary: ${BINARY_PATH}"
echo "Arguments: $*"
echo ""

# Install strace if not available (common in minimal containers)
if ! command -v strace &> /dev/null; then
    echo "Installing strace..."
    apt-get update && apt-get install -y strace || {
        echo "WARNING: Could not install strace, proceeding without it"
        HAS_STRACE=false
    }
    HAS_STRACE=true
else
    HAS_STRACE=true
fi

# Run with strace if available, otherwise run directly
if [ "${HAS_STRACE}" = "true" ]; then
    echo "Executing with strace (last 500 syscalls will be saved)..."
    echo ""

    # Run binary with strace, capture last 500 lines of syscalls + full stderr
    strace -f -o /tmp/strace.log "${BINARY_PATH}" "$@" 2>&1 | tee /tmp/training_error.log
    BINARY_EXIT_CODE=$?

    echo ""
    echo "=========================================="
    echo "Execution Complete - Exit Code: ${BINARY_EXIT_CODE}"
    echo "=========================================="

    if [ ${BINARY_EXIT_CODE} -ne 0 ]; then
        echo "❌ CRASH DETECTED"
        echo ""
        echo "Last 50 system calls before crash:"
        echo "-----------------------------------"
        tail -50 /tmp/strace.log || echo "(strace log not available)"
        echo ""
        echo "Error messages from stderr:"
        echo "----------------------------"
        cat /tmp/training_error.log || echo "(no error log available)"
        echo ""
        echo "Full logs saved to:"
        echo "  - System calls: /tmp/strace.log"
        echo "  - Error output: /tmp/training_error.log"
        exit 1
    fi
else
    echo "Running without strace (not available)..."
    echo ""

    # Run binary directly with stderr capture
    "${BINARY_PATH}" "$@" 2>&1 | tee /tmp/training_error.log
    BINARY_EXIT_CODE=$?

    if [ ${BINARY_EXIT_CODE} -ne 0 ]; then
        echo ""
        echo "=========================================="
        echo "❌ CRASH DETECTED - Exit Code: ${BINARY_EXIT_CODE}"
        echo "=========================================="
        echo ""
        echo "Error messages from stderr:"
        echo "----------------------------"
        cat /tmp/training_error.log || echo "(no error log available)"
        echo ""
        echo "Full error log saved to: /tmp/training_error.log"
        exit 1
    fi
fi

echo ""
echo "✓ Training completed successfully"