feat(ml): Complete hyperopt infrastructure + documentation

Changes:
- CLAUDE.md: Update OOM fix validation status
- Add comprehensive documentation (30+ markdown reports)
- LSTM encoder varmap bug fix (tft/lstm_encoder.rs:290)
- Quantized LSTM layer matching fix (tft/quantized_lstm.rs)
- Hyperopt paths module (ml/src/hyperopt/paths.rs)
- Training path tests for all adapters (DQN, MAMBA-2, PPO, TFT)
- Checkpoint integrity tests
- Script cleanup: Remove 29 obsolete deployment scripts
- Archive old scripts to scripts/archive/
- New deployment utilities: check_gpu_availability.py, monitor_hyperopt.sh

Validation:
- OOM fixes validated: 5/5 trials successful (pod b6kc3mc5lbjiro)
- Batch-size-max 256 tested successfully
- All hyperopt adapters working correctly

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2025-10-29 19:52:21 +01:00
parent 59cce96d9d
commit e61e8f54da
111 changed files with 15450 additions and 838 deletions

121
scripts/validate_binary.sh Executable file
View File

@@ -0,0 +1,121 @@
#!/bin/bash
# Binary Validation Script - Prevents deploying wrong binaries
# Usage: ./validate_binary.sh <binary_name>
set -e
BINARY_NAME="$1"
if [ -z "$BINARY_NAME" ]; then
echo "❌ Usage: $0 <binary_name>"
exit 1
fi
LOCAL_PATH="target/release/examples/${BINARY_NAME}"
S3_KEY="binaries/current/${BINARY_NAME}"
S3_ENDPOINT="https://s3api-eur-is-1.runpod.io"
S3_BUCKET="se3zdnb5o4"
echo "========================================"
echo "Binary Validation: ${BINARY_NAME}"
echo "========================================"
# 1. Check local binary exists
if [ ! -f "$LOCAL_PATH" ]; then
echo "❌ FAIL: Local binary not found at $LOCAL_PATH"
exit 1
fi
echo "✅ Local binary exists"
# 2. Test local binary works (basic smoke test)
echo "🧪 Testing local binary..."
if [[ "$BINARY_NAME" == hyperopt* ]]; then
# Test hyperopt binaries with --help
if ! $LOCAL_PATH --help &>/dev/null; then
echo "❌ FAIL: Local binary --help failed"
exit 1
fi
# Verify critical arguments present
if ! $LOCAL_PATH --help 2>&1 | grep -q -- "--base-dir"; then
echo "❌ FAIL: Local binary missing --base-dir argument"
echo "This binary was built BEFORE VarMap fix!"
exit 1
fi
fi
echo "✅ Local binary works and has correct arguments"
# 3. Calculate local SHA256
echo "🔍 Calculating local SHA256..."
LOCAL_SHA256=$(sha256sum "$LOCAL_PATH" | awk '{print $1}')
echo "Local SHA256: $LOCAL_SHA256"
# 4. Check if S3 binary exists
echo "🔍 Checking S3 binary..."
if ! aws s3api head-object \
--bucket "$S3_BUCKET" \
--key "$S3_KEY" \
--endpoint-url "$S3_ENDPOINT" \
--profile runpod &>/dev/null; then
echo "⚠️ S3 binary does not exist - will be uploaded"
echo "VALIDATION: LOCAL_ONLY"
exit 0
fi
# 5. Get S3 metadata
S3_SIZE=$(aws s3api head-object \
--bucket "$S3_BUCKET" \
--key "$S3_KEY" \
--endpoint-url "$S3_ENDPOINT" \
--profile runpod \
| jq -r '.ContentLength')
S3_MODIFIED=$(aws s3api head-object \
--bucket "$S3_BUCKET" \
--key "$S3_KEY" \
--endpoint-url "$S3_ENDPOINT" \
--profile runpod \
| jq -r '.LastModified')
echo "S3 Size: $S3_SIZE bytes"
echo "S3 Modified: $S3_MODIFIED"
# 6. Download S3 binary to temp location for checksum
echo "⬇️ Downloading S3 binary for checksum..."
TEMP_S3="/tmp/${BINARY_NAME}_s3_$$"
aws s3 cp \
"s3://${S3_BUCKET}/${S3_KEY}" \
"$TEMP_S3" \
--endpoint-url "$S3_ENDPOINT" \
--profile runpod \
--quiet
# 7. Calculate S3 SHA256
echo "🔍 Calculating S3 SHA256..."
S3_SHA256=$(sha256sum "$TEMP_S3" | awk '{print $1}')
echo "S3 SHA256: $S3_SHA256"
# 8. Cleanup temp file
rm -f "$TEMP_S3"
# 9. Compare checksums
echo ""
echo "========================================"
if [ "$LOCAL_SHA256" = "$S3_SHA256" ]; then
echo "✅ VALIDATION PASSED"
echo "Local and S3 binaries match perfectly"
echo "========================================"
exit 0
else
echo "❌ VALIDATION FAILED"
echo "Local and S3 binaries DO NOT MATCH"
echo ""
echo "Local: $LOCAL_SHA256"
echo "S3: $S3_SHA256"
echo ""
echo "REQUIRED ACTION:"
echo "1. Delete S3 binary: aws s3 rm s3://${S3_BUCKET}/${S3_KEY} --endpoint-url $S3_ENDPOINT --profile runpod"
echo "2. Upload local binary: aws s3 cp $LOCAL_PATH s3://${S3_BUCKET}/${S3_KEY} --endpoint-url $S3_ENDPOINT --profile runpod"
echo "3. Re-run validation"
echo "========================================"
exit 1
fi