86ed7af58f
Added is_final parameter to checkpoint callback to distinguish final
checkpoints from regular epoch checkpoints. Early stopping now saves
as dqn_final_epoch{N}.safetensors instead of dqn_epoch_{N}.safetensors.
Changes:
- Updated callback signature: Fn(usize, Vec<u8>, bool)
- Early stopping passes is_final=true
- Regular checkpoints pass is_final=false
- Callback uses final naming when is_final=true
Fixes checkpoint overwrite bug where final model was indistinguishable
from regular epoch checkpoints.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
93 lines
3.5 KiB
Bash
Executable File
93 lines
3.5 KiB
Bash
Executable File
#!/bin/bash
|
|
# =============================================================================
|
|
# RUNPOD GENERIC ENTRYPOINT - MINIMAL WRAPPER
|
|
# =============================================================================
|
|
# Purpose: Provide a minimal entrypoint that allows full command override
|
|
# Architecture: Execute whatever command is passed, with basic validation
|
|
# =============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
# Logging helper
|
|
log() {
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
|
|
}
|
|
|
|
log "Foxhunt Training Container - Entrypoint Started"
|
|
log "================================================================"
|
|
|
|
# Verify volume mount exists
|
|
if [ ! -d "/runpod-volume" ]; then
|
|
log "ERROR: /runpod-volume not mounted!"
|
|
log "This container requires a RunPod Network Volume mounted at /runpod-volume"
|
|
log "Please configure volume mount in deployment settings"
|
|
exit 1
|
|
fi
|
|
|
|
log "✓ Volume mount verified: /runpod-volume"
|
|
|
|
# List available binaries (informational)
|
|
if [ -d "/runpod-volume/binaries" ]; then
|
|
log "Available binaries:"
|
|
ls -lh /runpod-volume/binaries/ | tail -n +2 | awk '{printf " - %s (%s)\n", $9, $5}'
|
|
else
|
|
log "WARNING: /runpod-volume/binaries/ not found"
|
|
fi
|
|
|
|
# List available test data (informational)
|
|
if [ -d "/runpod-volume/test_data" ]; then
|
|
log "Available test data:"
|
|
ls -lh /runpod-volume/test_data/*.parquet 2>/dev/null | awk '{printf " - %s (%s)\n", $9, $5}' || log " (no .parquet files found)"
|
|
else
|
|
log "WARNING: /runpod-volume/test_data/ not found"
|
|
fi
|
|
|
|
log "================================================================"
|
|
|
|
# Check if command is provided
|
|
if [ $# -eq 0 ]; then
|
|
log "No command provided - showing usage information"
|
|
log ""
|
|
log "USAGE:"
|
|
log " Override Docker CMD with your training command:"
|
|
log ""
|
|
log "EXAMPLES:"
|
|
log " # DQN Training (100 epochs)"
|
|
log " CMD: /runpod-volume/binaries/train_dqn --parquet-file /runpod-volume/test_data/ES_FUT_small.parquet --epochs 100 --output-dir /runpod-volume/models"
|
|
log ""
|
|
log " # TFT Training (50 epochs, GPU enabled)"
|
|
log " CMD: /runpod-volume/binaries/train_tft_parquet --parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet --epochs 50 --use-gpu --output-dir /runpod-volume/models"
|
|
log ""
|
|
log " # PPO Training (200 epochs)"
|
|
log " CMD: /runpod-volume/binaries/train_ppo --parquet-file /runpod-volume/test_data/NQ_FUT_180d.parquet --epochs 200 --output-dir /runpod-volume/models"
|
|
log ""
|
|
log " # MAMBA-2 Training (30 epochs)"
|
|
log " CMD: /runpod-volume/binaries/train_mamba2_parquet --parquet-file /runpod-volume/test_data/6E_FUT_180d.parquet --epochs 30 --output-dir /runpod-volume/models"
|
|
log ""
|
|
log "Container is ready and waiting for commands..."
|
|
log "To deploy with a custom command, override the Docker CMD in your deployment script"
|
|
|
|
# Keep container running (useful for debugging)
|
|
tail -f /dev/null
|
|
fi
|
|
|
|
# If binary path is provided, ensure it's executable
|
|
if [[ "$1" =~ ^/runpod-volume/binaries/ ]] && [ -f "$1" ]; then
|
|
if [ ! -x "$1" ]; then
|
|
log "WARNING: Binary is not executable, attempting to fix..."
|
|
chmod +x "$1" || {
|
|
log "ERROR: Failed to make binary executable: $1"
|
|
exit 1
|
|
}
|
|
log "✓ Binary is now executable"
|
|
fi
|
|
fi
|
|
|
|
# Execute the provided command
|
|
log "Executing command: $*"
|
|
log "================================================================"
|
|
log ""
|
|
|
|
# Run the command
|
|
exec "$@"
|