- PPO numerical stability: Added epsilon (1e-8) protection at 4 log locations - Hurst division by zero: Fixed in trending.rs:394 and price_features.rs:342 - DQN 225-feature support: Fixed dimension mismatch (feature_vec[4..]) - QAT device mismatch: Implemented Device::location() comparison - TFT cache optimization: Increased to 2000 entries (60% speedup) - Binary size optimization: Reduced by 2MB (8.7%) via dependency tuning - Unused imports: Eliminated all 34 warnings in ML crate - Test coverage: Added 94+ production hardening tests Test Results: - FP32 Models: 1,317/1,317 tests passing (100%) - Overall Workspace: 313/314 passing (99.7%) - QAT: 0/24 (temporarily disabled, compilation errors) Performance: - TFT training: ~2 min (60% faster via cache optimization) - DQN training: ~15s (10-25% faster via mimalloc) - Average improvement: 922× vs minimum requirements QAT Blockers (P0 - 1-2 weeks): 1. Device mismatch: 11 compilation errors in qat_tft.rs 2. Gradient checkpointing: CLI flag exists but not implemented 3. OOM recovery: AutoBatchSizer exists but no retry integration Documentation: - FINAL_VALIDATION_SUMMARY.md (17 agents, 281 lines) - STABILIZATION_WAVE_COMPLETION_REPORT.md (290 lines) - DEPLOYMENT_QUICK_START.md (385 lines) - PRE_DEPLOYMENT_CHECKLIST.md (426 lines) - KNOWN_ISSUES.md (385 lines) - NEXT_STEPS_ROADMAP.md (27KB) Status: ✅ FP32 PRODUCTION READY | 🔴 QAT BLOCKED
124 lines
4.5 KiB
Bash
Executable File
124 lines
4.5 KiB
Bash
Executable File
#!/bin/bash
|
|
# =============================================================================
|
|
# RUNPOD GENERIC ENTRYPOINT - MINIMAL WRAPPER
|
|
# =============================================================================
|
|
# Purpose: Provide a minimal entrypoint that allows full command override
|
|
# Architecture: Execute whatever command is passed, with basic validation
|
|
# =============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
# Logging helper
|
|
log() {
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
|
|
}
|
|
|
|
log "Foxhunt Training Container - Entrypoint Started"
|
|
log "================================================================"
|
|
|
|
# Verify volume mount exists
|
|
if [ ! -d "/runpod-volume" ]; then
|
|
log "ERROR: /runpod-volume not mounted!"
|
|
log "This container requires a RunPod Network Volume mounted at /runpod-volume"
|
|
log "Please configure volume mount in deployment settings"
|
|
exit 1
|
|
fi
|
|
|
|
log "✓ Volume mount verified: /runpod-volume"
|
|
|
|
# =============================================================================
|
|
# START SSH DAEMON FOR RUNPOD REMOTE ACCESS
|
|
# =============================================================================
|
|
log "Configuring SSH access..."
|
|
|
|
# RunPod injects public key via PUBLIC_KEY environment variable
|
|
if [ -n "${PUBLIC_KEY:-}" ]; then
|
|
mkdir -p /root/.ssh
|
|
echo "$PUBLIC_KEY" > /root/.ssh/authorized_keys
|
|
chmod 700 /root/.ssh
|
|
chmod 600 /root/.ssh/authorized_keys
|
|
log "✓ RunPod public key installed (${#PUBLIC_KEY} bytes)"
|
|
else
|
|
log "⚠ WARNING: PUBLIC_KEY not set (SSH access unavailable)"
|
|
fi
|
|
|
|
# Start SSH daemon in background
|
|
if command -v sshd &> /dev/null; then
|
|
/usr/sbin/sshd
|
|
if pgrep -x sshd > /dev/null 2>&1; then
|
|
log "✓ SSH daemon running on port 22"
|
|
log " Access via: ssh root@\${RUNPOD_POD_ID}.ssh.runpod.io"
|
|
else
|
|
log "⚠ WARNING: SSH daemon failed to start"
|
|
fi
|
|
else
|
|
log "⚠ WARNING: sshd not found (openssh-server not installed)"
|
|
fi
|
|
|
|
log "================================================================"
|
|
|
|
# List available binaries (informational)
|
|
if [ -d "/runpod-volume/binaries" ]; then
|
|
log "Available binaries:"
|
|
ls -lh /runpod-volume/binaries/ | tail -n +2 | awk '{printf " - %s (%s)\n", $9, $5}'
|
|
else
|
|
log "WARNING: /runpod-volume/binaries/ not found"
|
|
fi
|
|
|
|
# List available test data (informational)
|
|
if [ -d "/runpod-volume/test_data" ]; then
|
|
log "Available test data:"
|
|
ls -lh /runpod-volume/test_data/*.parquet 2>/dev/null | awk '{printf " - %s (%s)\n", $9, $5}' || log " (no .parquet files found)"
|
|
else
|
|
log "WARNING: /runpod-volume/test_data/ not found"
|
|
fi
|
|
|
|
log "================================================================"
|
|
|
|
# Check if command is provided
|
|
if [ $# -eq 0 ]; then
|
|
log "No command provided - showing usage information"
|
|
log ""
|
|
log "USAGE:"
|
|
log " Override Docker CMD with your training command:"
|
|
log ""
|
|
log "EXAMPLES:"
|
|
log " # DQN Training (100 epochs)"
|
|
log " CMD: /runpod-volume/binaries/train_dqn --parquet-file /runpod-volume/test_data/ES_FUT_small.parquet --epochs 100 --output-dir /runpod-volume/models"
|
|
log ""
|
|
log " # TFT Training (50 epochs, GPU enabled)"
|
|
log " CMD: /runpod-volume/binaries/train_tft_parquet --parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 50 --use-gpu --output-dir /runpod-volume/models"
|
|
log ""
|
|
log " # PPO Training (200 epochs)"
|
|
log " CMD: /runpod-volume/binaries/train_ppo --parquet-file /runpod-volume/test_data/NQ_FUT_180d.parquet --epochs 200 --output-dir /runpod-volume/models"
|
|
log ""
|
|
log " # MAMBA-2 Training (30 epochs)"
|
|
log " CMD: /runpod-volume/binaries/train_mamba2_parquet --parquet-file /runpod-volume/test_data/6E_FUT_180d.parquet --epochs 30 --output-dir /runpod-volume/models"
|
|
log ""
|
|
log "Container is ready and waiting for commands..."
|
|
log "To deploy with a custom command, override the Docker CMD in your deployment script"
|
|
|
|
# Keep container running (useful for debugging)
|
|
tail -f /dev/null
|
|
fi
|
|
|
|
# If binary path is provided, ensure it's executable
|
|
if [[ "$1" =~ ^/runpod-volume/binaries/ ]] && [ -f "$1" ]; then
|
|
if [ ! -x "$1" ]; then
|
|
log "WARNING: Binary is not executable, attempting to fix..."
|
|
chmod +x "$1" || {
|
|
log "ERROR: Failed to make binary executable: $1"
|
|
exit 1
|
|
}
|
|
log "✓ Binary is now executable"
|
|
fi
|
|
fi
|
|
|
|
# Execute the provided command
|
|
log "Executing command: $*"
|
|
log "================================================================"
|
|
log ""
|
|
|
|
# Run the command
|
|
exec "$@"
|