chore: Second cleanup wave - organize root directory
- Archive: 85 agent .txt files → docs/archive/agents/legacy_txt/ - Scripts: Move 110 shell scripts → scripts/ (keep deploy.sh in root) - Models: Move 18 .safetensors → ml/models/checkpoints/training_artifacts/ - Delete: 34 directories (~33GB freed) - target/, coverage_*, test artifacts - Build: Clean 14 build artifacts (.rlib, .o, .pid, binaries) - Tests: Move 14 .rs files → tests/standalone/ - SQL: Move 5 files → sql/ (keep init-db*.sql for Docker) - Wave 153: Archive to docs/archive/historical/wave153/ - Docs: Archive 9 markdown files to wave_d/reports/ and historical/ Total impact: ~34GB freed (both waves), root directory cleaned from 583 to ~40 essential files Directory count reduced from 65 to 31 (52% reduction) All historical data preserved in organized archive structure
This commit is contained in:
jgrusewski
277
scripts/entrypoint_debug.sh
Executable file
277
scripts/entrypoint_debug.sh
Executable file
@@ -0,0 +1,277 @@
|
||||
#!/bin/bash
|
||||
# DEBUG VERSION - Enhanced error capture for binary crash diagnosis
|
||||
set -uo pipefail
|
||||
|
||||
# =============================================================================
|
||||
# RUNPOD DEBUG ENTRYPOINT - ENHANCED ERROR CAPTURE
|
||||
# =============================================================================
|
||||
# Purpose: Capture detailed crash diagnostics when binary fails
|
||||
# Changes from production entrypoint.sh:
|
||||
# 1. Capture stderr to /tmp/training_error.log
|
||||
# 2. Run binary with strace for system call tracing
|
||||
# 3. Print detailed error diagnostics on crash
|
||||
# 4. Test binary with --help first to isolate loading vs execution errors
|
||||
# =============================================================================
|
||||
|
||||
echo "=========================================="
|
||||
echo "Foxhunt HFT - DEBUG MODE (Enhanced Error Capture)"
|
||||
echo "=========================================="
|
||||
|
||||
# Default binary name if not specified
|
||||
BINARY_NAME=${BINARY_NAME:-train_tft_parquet}
|
||||
|
||||
echo "Configuration:"
|
||||
echo " Binary: ${BINARY_NAME}"
|
||||
echo " Volume Path: /runpod-volume/"
|
||||
echo " Debug Mode: ENABLED (stderr capture + strace)"
|
||||
|
||||
# =============================================================================
|
||||
# VERIFY VOLUME IS MOUNTED
|
||||
# =============================================================================
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Verifying Runpod Network Volume Mount"
|
||||
echo "=========================================="
|
||||
|
||||
if [ ! -d "/runpod-volume" ]; then
|
||||
echo "ERROR: /runpod-volume directory does not exist"
|
||||
echo "Runpod Network Volume is NOT mounted!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d "/runpod-volume/binaries" ]; then
|
||||
echo "ERROR: /runpod-volume/binaries directory does not exist"
|
||||
echo "Volume is mounted but binaries directory is missing!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d "/runpod-volume/test_data" ]; then
|
||||
echo "ERROR: /runpod-volume/test_data directory does not exist"
|
||||
echo "Volume is mounted but test_data directory is missing!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ Volume mounted successfully at /runpod-volume/"
|
||||
|
||||
# =============================================================================
|
||||
# LOAD CREDENTIALS FROM .env FILE (IF EXISTS)
|
||||
# =============================================================================
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Loading Credentials"
|
||||
echo "=========================================="
|
||||
|
||||
if [ -f /runpod-volume/.env ]; then
|
||||
chmod 600 /runpod-volume/.env 2>/dev/null || true
|
||||
set -a
|
||||
source /runpod-volume/.env || {
|
||||
echo "ERROR: Failed to source /runpod-volume/.env"
|
||||
exit 1
|
||||
}
|
||||
set +a
|
||||
echo "✓ Loaded credentials from /runpod-volume/.env"
|
||||
else
|
||||
echo "⚠ WARNING: .env file not found at /runpod-volume/.env"
|
||||
echo "Continuing with environment defaults..."
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# VERIFY TRAINING BINARY EXISTS
|
||||
# =============================================================================
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Verifying Training Binary"
|
||||
echo "=========================================="
|
||||
|
||||
BINARY_PATH="/runpod-volume/binaries/${BINARY_NAME}"
|
||||
|
||||
if [ ! -f "${BINARY_PATH}" ]; then
|
||||
echo "ERROR: Training binary not found: ${BINARY_PATH}"
|
||||
echo ""
|
||||
echo "Available binaries in /runpod-volume/binaries/:"
|
||||
ls -lh /runpod-volume/binaries/ 2>/dev/null || echo " (none)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if binary is executable
|
||||
if [ ! -x "${BINARY_PATH}" ]; then
|
||||
echo "WARNING: Binary is not executable, attempting to fix..."
|
||||
chmod +x "${BINARY_PATH}" || {
|
||||
echo "ERROR: Failed to make binary executable"
|
||||
exit 1
|
||||
}
|
||||
echo "✓ Binary is now executable"
|
||||
fi
|
||||
|
||||
BINARY_SIZE=$(stat -c%s "${BINARY_PATH}" 2>/dev/null || stat -f%z "${BINARY_PATH}")
|
||||
echo "✓ Binary found: ${BINARY_PATH}"
|
||||
echo " Size: $(numfmt --to=iec-i --suffix=B ${BINARY_SIZE} 2>/dev/null || echo ${BINARY_SIZE} bytes)"
|
||||
|
||||
# =============================================================================
|
||||
# DEBUG: ANALYZE BINARY DEPENDENCIES (ldd)
|
||||
# =============================================================================
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Binary Dependency Analysis (ldd)"
|
||||
echo "=========================================="
|
||||
|
||||
echo "Checking dynamic library dependencies..."
|
||||
ldd "${BINARY_PATH}" 2>&1 | tee /tmp/ldd_output.log || {
|
||||
echo "ERROR: ldd command failed (binary might be corrupted)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo ""
|
||||
echo "Checking for missing dependencies..."
|
||||
MISSING_LIBS=$(ldd "${BINARY_PATH}" 2>&1 | grep "not found" || echo "")
|
||||
if [ -n "${MISSING_LIBS}" ]; then
|
||||
echo "❌ MISSING LIBRARIES DETECTED:"
|
||||
echo "${MISSING_LIBS}"
|
||||
echo ""
|
||||
echo "This is likely the root cause of the crash!"
|
||||
echo "See /tmp/ldd_output.log for full details"
|
||||
else
|
||||
echo "✓ All dynamic libraries resolved successfully"
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# VERIFY GPU HARDWARE
|
||||
# =============================================================================
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "GPU Hardware Status"
|
||||
echo "=========================================="
|
||||
|
||||
if command -v nvidia-smi &> /dev/null; then
|
||||
echo "GPU Information:"
|
||||
nvidia-smi --query-gpu=name,memory.total,memory.free,driver_version,compute_cap --format=csv,noheader 2>&1 || {
|
||||
echo " WARNING: nvidia-smi query failed (GPU might be initializing)"
|
||||
}
|
||||
else
|
||||
echo "WARNING: nvidia-smi not found - GPU may not be accessible"
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# TEST 1: BINARY HELP (Isolate Loading vs Execution Errors)
|
||||
# =============================================================================
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "TEST 1: Binary Help (Loading Test)"
|
||||
echo "=========================================="
|
||||
echo "Testing if binary can load successfully with --help..."
|
||||
echo ""
|
||||
|
||||
# Capture stderr and stdout separately
|
||||
"${BINARY_PATH}" --help 1>/tmp/help_stdout.log 2>/tmp/help_stderr.log
|
||||
HELP_EXIT_CODE=$?
|
||||
|
||||
echo "Exit code: ${HELP_EXIT_CODE}"
|
||||
echo ""
|
||||
echo "--- STDOUT ---"
|
||||
cat /tmp/help_stdout.log || echo "(no stdout)"
|
||||
echo ""
|
||||
echo "--- STDERR ---"
|
||||
cat /tmp/help_stderr.log || echo "(no stderr)"
|
||||
echo ""
|
||||
|
||||
if [ ${HELP_EXIT_CODE} -ne 0 ]; then
|
||||
echo "❌ CRASH DETECTED DURING LOADING (--help test failed)"
|
||||
echo ""
|
||||
echo "This indicates the binary cannot even load dependencies."
|
||||
echo "Root cause is likely:"
|
||||
echo " 1. Missing dynamic library (check ldd output above)"
|
||||
echo " 2. GLIBC version mismatch (check 'ldd --version')"
|
||||
echo " 3. Corrupted binary (re-upload from local machine)"
|
||||
echo ""
|
||||
echo "Full error log saved to /tmp/help_stderr.log"
|
||||
exit 1
|
||||
else
|
||||
echo "✓ Binary loaded successfully (--help test passed)"
|
||||
fi
|
||||
|
||||
# =============================================================================
|
||||
# TEST 2: STRACE EXECUTION (System Call Tracing)
|
||||
# =============================================================================
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "TEST 2: Strace Execution (System Call Tracing)"
|
||||
echo "=========================================="
|
||||
echo "Running binary with strace to capture system calls..."
|
||||
echo "Binary: ${BINARY_PATH}"
|
||||
echo "Arguments: $*"
|
||||
echo ""
|
||||
|
||||
# Install strace if not available (common in minimal containers)
|
||||
if ! command -v strace &> /dev/null; then
|
||||
echo "Installing strace..."
|
||||
apt-get update && apt-get install -y strace || {
|
||||
echo "WARNING: Could not install strace, proceeding without it"
|
||||
HAS_STRACE=false
|
||||
}
|
||||
HAS_STRACE=true
|
||||
else
|
||||
HAS_STRACE=true
|
||||
fi
|
||||
|
||||
# Run with strace if available, otherwise run directly
|
||||
if [ "${HAS_STRACE}" = "true" ]; then
|
||||
echo "Executing with strace (last 500 syscalls will be saved)..."
|
||||
echo ""
|
||||
|
||||
# Run binary with strace, capture last 500 lines of syscalls + full stderr
|
||||
strace -f -o /tmp/strace.log "${BINARY_PATH}" "$@" 2>&1 | tee /tmp/training_error.log
|
||||
BINARY_EXIT_CODE=$?
|
||||
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "Execution Complete - Exit Code: ${BINARY_EXIT_CODE}"
|
||||
echo "=========================================="
|
||||
|
||||
if [ ${BINARY_EXIT_CODE} -ne 0 ]; then
|
||||
echo "❌ CRASH DETECTED"
|
||||
echo ""
|
||||
echo "Last 50 system calls before crash:"
|
||||
echo "-----------------------------------"
|
||||
tail -50 /tmp/strace.log || echo "(strace log not available)"
|
||||
echo ""
|
||||
echo "Error messages from stderr:"
|
||||
echo "----------------------------"
|
||||
cat /tmp/training_error.log || echo "(no error log available)"
|
||||
echo ""
|
||||
echo "Full logs saved to:"
|
||||
echo " - System calls: /tmp/strace.log"
|
||||
echo " - Error output: /tmp/training_error.log"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "Running without strace (not available)..."
|
||||
echo ""
|
||||
|
||||
# Run binary directly with stderr capture
|
||||
"${BINARY_PATH}" "$@" 2>&1 | tee /tmp/training_error.log
|
||||
BINARY_EXIT_CODE=$?
|
||||
|
||||
if [ ${BINARY_EXIT_CODE} -ne 0 ]; then
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo "❌ CRASH DETECTED - Exit Code: ${BINARY_EXIT_CODE}"
|
||||
echo "=========================================="
|
||||
echo ""
|
||||
echo "Error messages from stderr:"
|
||||
echo "----------------------------"
|
||||
cat /tmp/training_error.log || echo "(no error log available)"
|
||||
echo ""
|
||||
echo "Full error log saved to: /tmp/training_error.log"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "✓ Training completed successfully"
|
||||
Reference in New Issue
Block a user