feat(ml): Complete hyperopt infrastructure + documentation
Changes: - CLAUDE.md: Update OOM fix validation status - Add comprehensive documentation (30+ markdown reports) - LSTM encoder varmap bug fix (tft/lstm_encoder.rs:290) - Quantized LSTM layer matching fix (tft/quantized_lstm.rs) - Hyperopt paths module (ml/src/hyperopt/paths.rs) - Training path tests for all adapters (DQN, MAMBA-2, PPO, TFT) - Checkpoint integrity tests - Script cleanup: Remove 29 obsolete deployment scripts - Archive old scripts to scripts/archive/ - New deployment utilities: check_gpu_availability.py, monitor_hyperopt.sh Validation: - OOM fixes validated: 5/5 trials successful (pod b6kc3mc5lbjiro) - Batch-size-max 256 tested successfully - All hyperopt adapters working correctly 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
121
scripts/validate_binary.sh
Executable file
121
scripts/validate_binary.sh
Executable file
@@ -0,0 +1,121 @@
|
||||
#!/bin/bash
|
||||
# Binary Validation Script - Prevents deploying wrong binaries
|
||||
# Usage: ./validate_binary.sh <binary_name>
|
||||
|
||||
set -e
|
||||
|
||||
BINARY_NAME="$1"
|
||||
if [ -z "$BINARY_NAME" ]; then
|
||||
echo "❌ Usage: $0 <binary_name>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
LOCAL_PATH="target/release/examples/${BINARY_NAME}"
|
||||
S3_KEY="binaries/current/${BINARY_NAME}"
|
||||
S3_ENDPOINT="https://s3api-eur-is-1.runpod.io"
|
||||
S3_BUCKET="se3zdnb5o4"
|
||||
|
||||
echo "========================================"
|
||||
echo "Binary Validation: ${BINARY_NAME}"
|
||||
echo "========================================"
|
||||
|
||||
# 1. Check local binary exists
|
||||
if [ ! -f "$LOCAL_PATH" ]; then
|
||||
echo "❌ FAIL: Local binary not found at $LOCAL_PATH"
|
||||
exit 1
|
||||
fi
|
||||
echo "✅ Local binary exists"
|
||||
|
||||
# 2. Test local binary works (basic smoke test)
|
||||
echo "🧪 Testing local binary..."
|
||||
if [[ "$BINARY_NAME" == hyperopt* ]]; then
|
||||
# Test hyperopt binaries with --help
|
||||
if ! $LOCAL_PATH --help &>/dev/null; then
|
||||
echo "❌ FAIL: Local binary --help failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify critical arguments present
|
||||
if ! $LOCAL_PATH --help 2>&1 | grep -q -- "--base-dir"; then
|
||||
echo "❌ FAIL: Local binary missing --base-dir argument"
|
||||
echo "This binary was built BEFORE VarMap fix!"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
echo "✅ Local binary works and has correct arguments"
|
||||
|
||||
# 3. Calculate local SHA256
|
||||
echo "🔍 Calculating local SHA256..."
|
||||
LOCAL_SHA256=$(sha256sum "$LOCAL_PATH" | awk '{print $1}')
|
||||
echo "Local SHA256: $LOCAL_SHA256"
|
||||
|
||||
# 4. Check if S3 binary exists
|
||||
echo "🔍 Checking S3 binary..."
|
||||
if ! aws s3api head-object \
|
||||
--bucket "$S3_BUCKET" \
|
||||
--key "$S3_KEY" \
|
||||
--endpoint-url "$S3_ENDPOINT" \
|
||||
--profile runpod &>/dev/null; then
|
||||
echo "⚠️ S3 binary does not exist - will be uploaded"
|
||||
echo "VALIDATION: LOCAL_ONLY"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# 5. Get S3 metadata
|
||||
S3_SIZE=$(aws s3api head-object \
|
||||
--bucket "$S3_BUCKET" \
|
||||
--key "$S3_KEY" \
|
||||
--endpoint-url "$S3_ENDPOINT" \
|
||||
--profile runpod \
|
||||
| jq -r '.ContentLength')
|
||||
|
||||
S3_MODIFIED=$(aws s3api head-object \
|
||||
--bucket "$S3_BUCKET" \
|
||||
--key "$S3_KEY" \
|
||||
--endpoint-url "$S3_ENDPOINT" \
|
||||
--profile runpod \
|
||||
| jq -r '.LastModified')
|
||||
|
||||
echo "S3 Size: $S3_SIZE bytes"
|
||||
echo "S3 Modified: $S3_MODIFIED"
|
||||
|
||||
# 6. Download S3 binary to temp location for checksum
|
||||
echo "⬇️ Downloading S3 binary for checksum..."
|
||||
TEMP_S3="/tmp/${BINARY_NAME}_s3_$$"
|
||||
aws s3 cp \
|
||||
"s3://${S3_BUCKET}/${S3_KEY}" \
|
||||
"$TEMP_S3" \
|
||||
--endpoint-url "$S3_ENDPOINT" \
|
||||
--profile runpod \
|
||||
--quiet
|
||||
|
||||
# 7. Calculate S3 SHA256
|
||||
echo "🔍 Calculating S3 SHA256..."
|
||||
S3_SHA256=$(sha256sum "$TEMP_S3" | awk '{print $1}')
|
||||
echo "S3 SHA256: $S3_SHA256"
|
||||
|
||||
# 8. Cleanup temp file
|
||||
rm -f "$TEMP_S3"
|
||||
|
||||
# 9. Compare checksums
|
||||
echo ""
|
||||
echo "========================================"
|
||||
if [ "$LOCAL_SHA256" = "$S3_SHA256" ]; then
|
||||
echo "✅ VALIDATION PASSED"
|
||||
echo "Local and S3 binaries match perfectly"
|
||||
echo "========================================"
|
||||
exit 0
|
||||
else
|
||||
echo "❌ VALIDATION FAILED"
|
||||
echo "Local and S3 binaries DO NOT MATCH"
|
||||
echo ""
|
||||
echo "Local: $LOCAL_SHA256"
|
||||
echo "S3: $S3_SHA256"
|
||||
echo ""
|
||||
echo "REQUIRED ACTION:"
|
||||
echo "1. Delete S3 binary: aws s3 rm s3://${S3_BUCKET}/${S3_KEY} --endpoint-url $S3_ENDPOINT --profile runpod"
|
||||
echo "2. Upload local binary: aws s3 cp $LOCAL_PATH s3://${S3_BUCKET}/${S3_KEY} --endpoint-url $S3_ENDPOINT --profile runpod"
|
||||
echo "3. Re-run validation"
|
||||
echo "========================================"
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user