feat(ml): Complete hyperopt infrastructure + documentation

Changes:
- CLAUDE.md: Update OOM fix validation status
- Add comprehensive documentation (30+ markdown reports)
- LSTM encoder varmap bug fix (tft/lstm_encoder.rs:290)
- Quantized LSTM layer matching fix (tft/quantized_lstm.rs)
- Hyperopt paths module (ml/src/hyperopt/paths.rs)
- Training path tests for all adapters (DQN, MAMBA-2, PPO, TFT)
- Checkpoint integrity tests
- Script cleanup: Remove 29 obsolete deployment scripts
- Archive old scripts to scripts/archive/
- New deployment utilities: check_gpu_availability.py, monitor_hyperopt.sh

Validation:
- OOM fixes validated: 5/5 trials successful (pod b6kc3mc5lbjiro)
- Batch-size-max 256 tested successfully
- All hyperopt adapters working correctly

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

scripts/archive/test_binary_sync.sh

@@ -0,0 +1,241 @@
+#!/bin/bash
+# =============================================================================
+# Binary Sync Validation Test Script
+# =============================================================================
+# Purpose: Validate the binary volume sync fix works correctly
+# Tests:
+#   1. Local binary has correct timestamp metadata
+#   2. S3 upload includes timestamp in metadata
+#   3. S3 binary can be downloaded and verified
+#   4. Timestamps are preserved and comparable
+#
+# Usage: ./test_binary_sync.sh <binary_name>
+# Example: ./test_binary_sync.sh hyperopt_mamba2_demo
+# =============================================================================
+
+set -e
+
+BINARY_NAME="$1"
+if [ -z "$BINARY_NAME" ]; then
+    echo "Usage: $0 <binary_name>"
+    echo "Example: $0 hyperopt_mamba2_demo"
+    exit 1
+fi
+
+# Configuration
+LOCAL_PATH="target/release/examples/${BINARY_NAME}"
+S3_KEY_CURRENT="binaries/current/${BINARY_NAME}"
+S3_ENDPOINT="https://s3api-eur-is-1.runpod.io"
+S3_BUCKET="se3zdnb5o4"
+S3_PROFILE="runpod"
+
+# Note: Upload creates two S3 objects:
+#   1. binaries/timestamped/${BINARY_NAME}_YYYYMMDD_HHMMSS (timestamped version)
+#   2. binaries/current/${BINARY_NAME} (symlink/copy to latest)
+# This test validates the 'current' version which is what deployments use
+
+echo "========================================================================"
+echo "BINARY SYNC VALIDATION TEST"
+echo "========================================================================"
+echo "Binary: $BINARY_NAME"
+echo "Local:  $LOCAL_PATH"
+echo "S3:     s3://$S3_BUCKET/$S3_KEY_CURRENT"
+echo "========================================================================"
+
+# =============================================================================
+# TEST 1: Verify local binary exists and has timestamp
+# =============================================================================
+echo ""
+echo "TEST 1: Local Binary Validation"
+echo "------------------------------------------------------------------------"
+
+if [ ! -f "$LOCAL_PATH" ]; then
+    echo "❌ FAIL: Local binary not found at $LOCAL_PATH"
+    echo "   Build it with: cargo build --release --example $BINARY_NAME"
+    exit 1
+fi
+echo "✅ Local binary exists"
+
+# Get local file modification time
+LOCAL_MTIME=$(stat -c %Y "$LOCAL_PATH" 2>/dev/null || stat -f %m "$LOCAL_PATH" 2>/dev/null)
+LOCAL_SIZE=$(stat -c %s "$LOCAL_PATH" 2>/dev/null || stat -f %z "$LOCAL_PATH" 2>/dev/null)
+LOCAL_TIMESTAMP=$(date -Iseconds -d @"$LOCAL_MTIME" 2>/dev/null || date -Iseconds -r "$LOCAL_MTIME" 2>/dev/null)
+
+echo "✅ Local metadata extracted:"
+echo "   Size:      $LOCAL_SIZE bytes ($(echo "scale=1; $LOCAL_SIZE/1024/1024" | bc) MB)"
+echo "   Modified:  $LOCAL_TIMESTAMP"
+
+# Calculate local SHA256
+LOCAL_SHA256=$(sha256sum "$LOCAL_PATH" | awk '{print $1}')
+echo "   SHA256:    $LOCAL_SHA256"
+
+# =============================================================================
+# TEST 2: Check S3 binary metadata (if exists)
+# =============================================================================
+echo ""
+echo "TEST 2: S3 Binary Metadata Validation"
+echo "------------------------------------------------------------------------"
+
+# Check if S3 binary exists
+if ! aws s3api head-object \
+    --bucket "$S3_BUCKET" \
+    --key "$S3_KEY_CURRENT" \
+    --endpoint-url "$S3_ENDPOINT" \
+    --profile "$S3_PROFILE" &>/dev/null; then
+    echo "⚠️  S3 binary does not exist yet"
+    echo "   Run: python3 scripts/runpod_deploy.py --skip-upload=false --dry-run"
+    echo "   This will upload the binary with timestamp metadata"
+    exit 0
+fi
+
+echo "✅ S3 binary exists"
+
+# Get S3 metadata
+S3_METADATA=$(aws s3api head-object \
+    --bucket "$S3_BUCKET" \
+    --key "$S3_KEY_CURRENT" \
+    --endpoint-url "$S3_ENDPOINT" \
+    --profile "$S3_PROFILE")
+
+S3_SIZE=$(echo "$S3_METADATA" | jq -r '.ContentLength')
+S3_MODIFIED=$(echo "$S3_METADATA" | jq -r '.LastModified')
+S3_SHA256=$(echo "$S3_METADATA" | jq -r '.Metadata.sha256 // "N/A"')
+S3_BUILD_TIMESTAMP=$(echo "$S3_METADATA" | jq -r '.Metadata.build_timestamp // "N/A"')
+S3_UPLOADED=$(echo "$S3_METADATA" | jq -r '.Metadata.uploaded // "N/A"')
+
+echo "✅ S3 metadata extracted:"
+echo "   Size:            $S3_SIZE bytes ($(echo "scale=1; $S3_SIZE/1024/1024" | bc) MB)"
+echo "   Last Modified:   $S3_MODIFIED"
+echo "   SHA256:          $S3_SHA256"
+echo "   Build Timestamp: $S3_BUILD_TIMESTAMP"
+echo "   Uploaded:        $S3_UPLOADED"
+
+# =============================================================================
+# TEST 3: Validate timestamp metadata
+# =============================================================================
+echo ""
+echo "TEST 3: Timestamp Validation"
+echo "------------------------------------------------------------------------"
+
+if [ "$S3_BUILD_TIMESTAMP" = "N/A" ]; then
+    echo "⚠️  WARNING: S3 binary missing build_timestamp metadata"
+    echo "   This binary was uploaded with old version of runpod_deploy.py"
+    echo "   Re-upload to add timestamp: python3 scripts/runpod_deploy.py --force-upload --dry-run"
+    exit 0
+fi
+
+echo "✅ S3 binary has timestamp metadata"
+
+# Compare timestamps (allowing for timezone differences)
+if [ "$LOCAL_TIMESTAMP" = "$S3_BUILD_TIMESTAMP" ]; then
+    echo "✅ PASS: Timestamps match exactly"
+    echo "   Local: $LOCAL_TIMESTAMP"
+    echo "   S3:    $S3_BUILD_TIMESTAMP"
+else
+    echo "⚠️  Timestamps differ (expected if binary was rebuilt):"
+    echo "   Local: $LOCAL_TIMESTAMP"
+    echo "   S3:    $S3_BUILD_TIMESTAMP"
+
+    # Check if local is newer (requires re-upload)
+    LOCAL_EPOCH=$(date -d "$LOCAL_TIMESTAMP" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%S" "$LOCAL_TIMESTAMP" +%s 2>/dev/null)
+    S3_EPOCH=$(date -d "$S3_BUILD_TIMESTAMP" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%S" "$S3_BUILD_TIMESTAMP" +%s 2>/dev/null)
+
+    if [ "$LOCAL_EPOCH" -gt "$S3_EPOCH" ]; then
+        echo "   ⚠️  Local binary is NEWER - re-upload required!"
+        echo "      Run: python3 scripts/runpod_deploy.py --force-upload --dry-run"
+    else
+        echo "   ℹ️  S3 binary is same or newer - deployment will use S3 version"
+    fi
+fi
+
+# =============================================================================
+# TEST 4: Validate SHA256 checksum
+# =============================================================================
+echo ""
+echo "TEST 4: Checksum Validation"
+echo "------------------------------------------------------------------------"
+
+if [ "$S3_SHA256" = "N/A" ]; then
+    echo "⚠️  WARNING: S3 binary missing SHA256 metadata"
+    echo "   Re-upload to add checksum: python3 scripts/runpod_deploy.py --force-upload --dry-run"
+    exit 0
+fi
+
+echo "✅ S3 binary has SHA256 metadata"
+
+# Compare checksums
+if [ "$LOCAL_SHA256" = "$S3_SHA256" ]; then
+    echo "✅ PASS: Checksums match - binaries are identical"
+    echo "   SHA256: $LOCAL_SHA256"
+else
+    echo "❌ FAIL: Checksums DO NOT MATCH"
+    echo "   Local:  $LOCAL_SHA256"
+    echo "   S3:     $S3_SHA256"
+    echo ""
+    echo "   REQUIRED ACTION:"
+    echo "   1. Force re-upload: python3 scripts/runpod_deploy.py --force-upload --dry-run"
+    echo "   2. Or delete and re-upload:"
+    echo "      aws s3 rm s3://$S3_BUCKET/$S3_KEY_CURRENT --endpoint-url $S3_ENDPOINT --profile $S3_PROFILE"
+    echo "      python3 scripts/runpod_deploy.py --dry-run"
+    exit 1
+fi
+
+# =============================================================================
+# TEST 5: Validate binary arguments (for hyperopt binaries)
+# =============================================================================
+echo ""
+echo "TEST 5: Binary Arguments Validation"
+echo "------------------------------------------------------------------------"
+
+if [[ "$BINARY_NAME" == hyperopt* ]]; then
+    if ! "$LOCAL_PATH" --help 2>&1 | grep -q -- "--base-dir"; then
+        echo "❌ FAIL: Local binary missing --base-dir argument"
+        echo "   This binary was built BEFORE VarMap fix!"
+        echo "   Rebuild: cargo clean && cargo build --release --example $BINARY_NAME"
+        exit 1
+    fi
+    echo "✅ PASS: Local binary has --base-dir argument (VarMap fix applied)"
+
+    # Test S3 binary (download temporarily)
+    TEMP_S3="/tmp/${BINARY_NAME}_s3_test_$$"
+    echo "   Downloading S3 binary for argument test..."
+    aws s3 cp \
+        "s3://${S3_BUCKET}/${S3_KEY_CURRENT}" \
+        "$TEMP_S3" \
+        --endpoint-url "$S3_ENDPOINT" \
+        --profile "$S3_PROFILE" \
+        --quiet
+
+    chmod +x "$TEMP_S3"
+
+    if ! "$TEMP_S3" --help 2>&1 | grep -q -- "--base-dir"; then
+        echo "❌ FAIL: S3 binary missing --base-dir argument"
+        echo "   This binary was uploaded BEFORE VarMap fix!"
+        echo "   Re-upload: python3 scripts/runpod_deploy.py --force-upload --dry-run"
+        rm -f "$TEMP_S3"
+        exit 1
+    fi
+    echo "✅ PASS: S3 binary has --base-dir argument (VarMap fix applied)"
+
+    rm -f "$TEMP_S3"
+else
+    echo "ℹ️  SKIP: Not a hyperopt binary, no argument validation needed"
+fi
+
+# =============================================================================
+# SUMMARY
+# =============================================================================
+echo ""
+echo "========================================================================"
+echo "✅ ALL TESTS PASSED"
+echo "========================================================================"
+echo "Binary:           $BINARY_NAME"
+echo "Local Size:       $(echo "scale=1; $LOCAL_SIZE/1024/1024" | bc) MB"
+echo "Local Timestamp:  $LOCAL_TIMESTAMP"
+echo "S3 Timestamp:     $S3_BUILD_TIMESTAMP"
+echo "Checksum:         $LOCAL_SHA256"
+echo ""
+echo "DEPLOYMENT STATUS: ✅ READY"
+echo "This binary is safe to deploy to Runpod."
+echo "The entrypoint will sync it from S3 to volume on pod startup."
+echo "========================================================================"

scripts/archive/test_binary_validation.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+# Test binary validation system
+
+set -e
+
+echo "========================================"
+echo "Binary Validation System Test Suite"
+echo "========================================"
+
+PROJECT_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+cd "$PROJECT_ROOT"
+
+# Test 1: Validate existing binary
+echo ""
+echo "Test 1: Validate hyperopt_mamba2_demo binary"
+echo "----------------------------------------"
+
+if [ ! -f "target/release/examples/hyperopt_mamba2_demo" ]; then
+    echo "⚠️  Binary not found, building first..."
+    cargo build -p ml --example hyperopt_mamba2_demo --release --features cuda
+fi
+
+./scripts/validate_binary.sh hyperopt_mamba2_demo
+echo "✅ Test 1 passed: Validation script works"
+
+# Test 2: Local binary smoke test
+echo ""
+echo "Test 2: Smoke test binary functionality"
+echo "----------------------------------------"
+
+if target/release/examples/hyperopt_mamba2_demo --help | grep -q "base-dir"; then
+    echo "✅ Test 2 passed: Binary has correct CLI arguments"
+else
+    echo "❌ Test 2 failed: Binary missing --base-dir argument"
+    exit 1
+fi
+
+# Test 3: Checksum comparison
+echo ""
+echo "Test 3: Checksum calculation"
+echo "----------------------------------------"
+
+LOCAL_SHA=$(sha256sum target/release/examples/hyperopt_mamba2_demo | awk '{print $1}')
+echo "Local SHA256: $LOCAL_SHA"
+
+if [ -n "$LOCAL_SHA" ] && [ ${#LOCAL_SHA} -eq 64 ]; then
+    echo "✅ Test 3 passed: Checksum calculated correctly (64 chars)"
+else
+    echo "❌ Test 3 failed: Invalid checksum format"
+    exit 1
+fi
+
+# Test 4: Python validation integration
+echo ""
+echo "Test 4: Python script integration"
+echo "----------------------------------------"
+
+if python3 -c "
+import subprocess
+import os
+
+project_root = os.getcwd()
+result = subprocess.run(
+    [os.path.join(project_root, 'scripts/validate_binary.sh'), 'hyperopt_mamba2_demo'],
+    capture_output=True,
+    text=True,
+    cwd=project_root
+)
+
+if result.returncode == 0:
+    print('✅ Test 4 passed: Python can call validation script')
+    exit(0)
+elif 'VALIDATION: LOCAL_ONLY' in result.stdout:
+    print('✅ Test 4 passed: Validation works (local-only mode)')
+    exit(0)
+else:
+    print('❌ Test 4 failed: Python integration broken')
+    print(result.stdout)
+    print(result.stderr)
+    exit(1)
+"; then
+    :
+else
+    exit 1
+fi
+
+echo ""
+echo "========================================"
+echo "✅ All validation tests passed"
+echo "========================================"
+echo ""
+echo "Validation system is ready for production use."
+echo ""
+echo "Next steps:"
+echo "1. Test deployment with: python3 scripts/runpod_deploy.py --dry-run"
+echo "2. Review SAFE_DEPLOYMENT_CHECKLIST.md for full workflow"

scripts/check_all_available_gpus.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+import os
+from pathlib import Path
+from dotenv import load_dotenv
+import runpod
+
+# Load environment
+env_path = Path.cwd().parent / '.env.runpod'
+load_dotenv(env_path)
+
+runpod.api_key = os.getenv('RUNPOD_API_KEY')
+
+# Get all GPU types with any availability
+gpu_types = runpod.get_gpus()
+available_count = 0
+
+print("ALL GPUs with ≥16GB VRAM and ANY availability:\n")
+for gpu in gpu_types:
+    memory_gb = gpu.get('memoryInGb', 0)
+    secure = gpu.get('secureCloud', 0)
+    community = gpu.get('communityCloud', 0)
+    total = secure + community
+    name = gpu.get('displayName', 'Unknown')
+    
+    if memory_gb >= 16 and total > 0:
+        price_info = gpu.get('lowestPrice', {})
+        price = price_info.get('uninterruptablePrice', 'N/A') if price_info else 'N/A'
+        
+        print(f"{name}:")
+        print(f"  VRAM: {memory_gb}GB")
+        print(f"  Secure cloud: {secure}")
+        print(f"  Community cloud: {community}")
+        print(f"  Total: {total}")
+        print(f"  Price: ${price}/hr" if price != 'N/A' else f"  Price: {price}")
+        print()
+        available_count += 1
+
+if available_count == 0:
+    print("NO GPUs with ≥16GB VRAM are available right now.")
+    print("\nMost common GPUs with ≥16GB VRAM (showing current status):")
+    
+    common_gpus = ['RTX 4090', 'RTX 4000 Ada', 'RTX 5090', 'RTX A4000', 'A100', 'H100']
+    for target in common_gpus:
+        for gpu in gpu_types:
+            if target in gpu.get('displayName', ''):
+                memory_gb = gpu.get('memoryInGb', 0)
+                if memory_gb >= 16:
+                    secure = gpu.get('secureCloud', 0)
+                    community = gpu.get('communityCloud', 0)
+                    print(f"  {gpu['displayName']}: {secure + community} available")
+                break

scripts/check_gpu_availability.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+import os
+import sys
+from pathlib import Path
+from dotenv import load_dotenv
+import runpod
+
+# Load environment
+env_path = Path.cwd().parent / '.env.runpod'
+load_dotenv(env_path)
+
+runpod.api_key = os.getenv('RUNPOD_API_KEY')
+
+# Get GPU types and check specific GPUs
+gpu_types = runpod.get_gpus()
+target_gpus = ['RTX 4090', 'RTX 4000 Ada', 'RTX 5090']
+
+for target in target_gpus:
+    for gpu in gpu_types:
+        if target in gpu.get('displayName', ''):
+            print(f"\n{target} details:")
+            print(f"  secureCloud: {gpu.get('secureCloud', 0)}")
+            print(f"  communityCloud: {gpu.get('communityCloud', 0)}")
+            print(f"  memoryInGb: {gpu.get('memoryInGb', 0)}")
+            price_info = gpu.get('lowestPrice', {})
+            if price_info:
+                print(f"  uninterruptablePrice: {price_info.get('uninterruptablePrice', 'N/A')}")
+            else:
+                print(f"  price: N/A")
+            break

scripts/deployment_log.txt

@@ -0,0 +1,55 @@
+2025-10-29 10:57:04 - INFO - Restarting script with venv Python...
+2025-10-29 10:57:06 - INFO - ✓ Loaded environment from /home/jgrusewski/Work/foxhunt/.env.runpod
+2025-10-29 10:57:06 - INFO - ✓ Environment variables validated
+2025-10-29 10:57:06 - INFO - 
+ℹ Skipping binary upload check (--skip-upload)
+2025-10-29 10:57:06 - INFO - 
+2025-10-29 10:57:06 - INFO - ======================================================================
+2025-10-29 10:57:06 - INFO - STEP 2: GPU AVAILABILITY SCAN
+2025-10-29 10:57:06 - INFO - ======================================================================
+2025-10-29 10:57:06 - INFO - 🔍 Querying available GPU types...
+2025-10-29 10:57:06 - INFO -    ⏭ MI300X: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ A100 PCIe: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ A100 SXM: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ A40: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ B200: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX 3070: Skipped (VRAM 8GB < 16GB)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX 3080: Skipped (VRAM 10GB < 16GB)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX 3080 Ti: Skipped (VRAM 12GB < 16GB)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX 3090: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX 3090 Ti: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX 4070 Ti: Skipped (VRAM 12GB < 16GB)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX 4080: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX 4080 SUPER: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX 4090: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX 5080: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX 5090: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ H100 SXM: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ H100 NVL: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ H100 PCIe: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ H200 SXM: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ NVIDIA H200 NVL: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ L4: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ L40: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ L40S: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX 2000 Ada: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX 4000 Ada: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX 4000 Ada SFF: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX 5000 Ada: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX 6000 Ada: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX A2000: Skipped (VRAM 6GB < 16GB)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX A4000: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX A4500: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX A5000: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX A6000: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX PRO 6000 MaxQ: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX PRO 6000: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ RTX PRO 6000 WK: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ V100 FHHL: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ Tesla V100: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ V100 SXM2: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ V100 SXM2 32GB: Skipped (0 instances available)
+2025-10-29 10:57:06 - INFO -    ⏭ unknown: Skipped (VRAM 0GB < 16GB)
+2025-10-29 10:57:06 - WARNING - ⚠ No GPU types found with ≥16GB VRAM
+2025-10-29 10:57:06 - ERROR - 
+✗ No GPUs available with ≥16GB VRAM in SECURE or COMMUNITY cloud

scripts/monitor_hyperopt.sh

@@ -0,0 +1,131 @@
+#!/bin/bash
+# MAMBA-2 Hyperopt Monitoring Script
+# Pod ID: w4srx0tgm5hfgu
+# Created: 2025-10-28
+
+set -e
+
+POD_ID="w4srx0tgm5hfgu"
+S3_BUCKET="s3://se3zdnb5o4"
+S3_ENDPOINT="https://s3api-eur-is-1.runpod.io"
+AWS_PROFILE="runpod"
+
+echo "========================================"
+echo "MAMBA-2 Hyperopt Monitor"
+echo "========================================"
+echo "Pod ID: $POD_ID"
+echo "GPU: RTX A4000 (16GB VRAM)"
+echo "Cost: \$0.25/hr"
+echo "Expected Duration: ~60 minutes"
+echo "========================================"
+echo ""
+
+# Function to check S3 results
+check_s3_results() {
+    echo "📦 Checking S3 for results..."
+    aws s3 ls "$S3_BUCKET/models/" \
+        --profile "$AWS_PROFILE" \
+        --endpoint-url "$S3_ENDPOINT" \
+        --recursive \
+        --human-readable \
+        | grep "mamba2_hyperopt" || echo "  (No results yet)"
+    echo ""
+}
+
+# Function to show monitoring commands
+show_commands() {
+    echo "📝 Monitoring Commands:"
+    echo ""
+    echo "1. SSH into pod:"
+    echo "   ssh root@$POD_ID.ssh.runpod.io"
+    echo ""
+    echo "2. View logs (inside pod):"
+    echo "   docker logs -f \$(docker ps -q)"
+    echo ""
+    echo "3. Monitor GPU (inside pod):"
+    echo "   watch -n 1 nvidia-smi"
+    echo ""
+    echo "4. Check process (inside pod):"
+    echo "   ps aux | grep hyperopt_mamba2_demo"
+    echo ""
+    echo "5. Check outputs (inside pod):"
+    echo "   ls -lh /runpod-volume/models/"
+    echo ""
+    echo "6. Runpod Console:"
+    echo "   https://www.runpod.io/console/pods"
+    echo ""
+    echo "7. Jupyter Notebook:"
+    echo "   https://$POD_ID-8888.proxy.runpod.net"
+    echo ""
+}
+
+# Function to download results
+download_results() {
+    echo "📥 Downloading results from S3..."
+    RESULTS_DIR="./runpod_hyperopt_results_$(date +%Y%m%d_%H%M%S)"
+    mkdir -p "$RESULTS_DIR"
+
+    aws s3 sync "$S3_BUCKET/models/" "$RESULTS_DIR/" \
+        --profile "$AWS_PROFILE" \
+        --endpoint-url "$S3_ENDPOINT" \
+        --exclude "*" \
+        --include "mamba2_hyperopt*" \
+        --no-progress
+
+    echo "  ✅ Results downloaded to: $RESULTS_DIR"
+    ls -lh "$RESULTS_DIR/"
+    echo ""
+}
+
+# Function to estimate completion time
+estimate_completion() {
+    echo "⏱️  Estimated Timeline:"
+    echo "  - Initialization: ~2-3 minutes"
+    echo "  - Trial 1: ~5 minutes"
+    echo "  - Trial 10: ~20 minutes"
+    echo "  - Trial 20: ~40 minutes"
+    echo "  - Trial 30 (Complete): ~60 minutes"
+    echo "  - Auto-Termination: ~61 minutes"
+    echo ""
+    echo "💰 Cost Estimate: \$0.19-\$0.38 (45-90 minutes)"
+    echo ""
+}
+
+# Main menu
+case "${1:-help}" in
+    status|s)
+        check_s3_results
+        estimate_completion
+        ;;
+    commands|c)
+        show_commands
+        ;;
+    download|d)
+        download_results
+        ;;
+    watch|w)
+        echo "🔄 Watching for results (checks every 30 seconds)..."
+        echo "   Press Ctrl+C to stop"
+        echo ""
+        while true; do
+            check_s3_results
+            sleep 30
+        done
+        ;;
+    help|h|*)
+        echo "Usage: $0 [command]"
+        echo ""
+        echo "Commands:"
+        echo "  status (s)     - Check S3 for results and show timeline"
+        echo "  commands (c)   - Show monitoring commands"
+        echo "  download (d)   - Download results from S3"
+        echo "  watch (w)      - Watch for results (checks every 30s)"
+        echo "  help (h)       - Show this help"
+        echo ""
+        echo "Examples:"
+        echo "  $0 status      # Check current status"
+        echo "  $0 watch       # Monitor in real-time"
+        echo "  $0 download    # Download completed results"
+        echo ""
+        ;;
+esac

scripts/requirements.txt

@@ -0,0 +1,88 @@
+aiodns==3.5.0
+aiohappyeyeballs==2.6.1
+aiohttp==3.13.2
+aiohttp-retry==2.9.1
+aiosignal==1.4.0
+annotated-doc==0.0.3
+annotated-types==0.7.0
+anyio==4.11.0
+attrs==25.4.0
+backoff==2.2.1
+backports.zstd==1.0.0
+bcrypt==5.0.0
+boto3==1.40.61
+botocore==1.40.61
+Brotli==1.1.0
+certifi==2025.10.5
+cffi==2.0.0
+charset-normalizer==3.4.4
+click==8.3.0
+colorama==0.4.6
+cryptography==45.0.7
+dnspython==2.8.0
+email-validator==2.3.0
+fastapi==0.120.1
+fastapi-cli==0.0.14
+fastapi-cloud-cli==0.3.1
+filelock==3.20.0
+frozenlist==1.8.0
+h11==0.16.0
+httpcore==1.0.9
+httptools==0.7.1
+httpx==0.28.1
+idna==3.11
+inquirerpy==0.3.4
+invoke==2.2.1
+itsdangerous==2.2.0
+Jinja2==3.1.6
+jmespath==1.0.1
+markdown-it-py==4.0.0
+MarkupSafe==3.0.3
+mdurl==0.1.2
+multidict==6.7.0
+orjson==3.11.4
+paramiko==4.0.0
+pfzy==0.3.4
+prettytable==3.16.0
+prompt_toolkit==3.0.52
+propcache==0.4.1
+py-cpuinfo==9.0.0
+pycares==4.11.0
+pycparser==2.23
+pydantic==2.12.3
+pydantic-extra-types==2.10.6
+pydantic-settings==2.11.0
+pydantic_core==2.41.4
+Pygments==2.19.2
+PyNaCl==1.6.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.2.1
+python-multipart==0.0.20
+PyYAML==6.0.3
+requests==2.32.5
+rich==14.2.0
+rich-toolkit==0.15.1
+rignore==0.7.1
+runpod==1.7.13
+s3transfer==0.14.0
+sentry-sdk==2.42.1
+shellingham==1.5.4
+six==1.17.0
+sniffio==1.3.1
+starlette==0.49.1
+tomli==2.3.0
+tomlkit==0.13.3
+tqdm==4.67.1
+tqdm-loggable==0.2
+typer==0.20.0
+typing-inspection==0.4.2
+typing_extensions==4.15.0
+ujson==5.11.0
+urllib3==2.5.0
+uvicorn==0.38.0
+uvloop==0.22.1
+watchdog==6.0.0
+watchfiles==1.1.1
+wcwidth==0.2.14
+websockets==15.0.1
+yarl==1.22.0

scripts/validate_binary.sh

@@ -0,0 +1,121 @@
+#!/bin/bash
+# Binary Validation Script - Prevents deploying wrong binaries
+# Usage: ./validate_binary.sh <binary_name>
+
+set -e
+
+BINARY_NAME="$1"
+if [ -z "$BINARY_NAME" ]; then
+    echo "❌ Usage: $0 <binary_name>"
+    exit 1
+fi
+
+LOCAL_PATH="target/release/examples/${BINARY_NAME}"
+S3_KEY="binaries/current/${BINARY_NAME}"
+S3_ENDPOINT="https://s3api-eur-is-1.runpod.io"
+S3_BUCKET="se3zdnb5o4"
+
+echo "========================================"
+echo "Binary Validation: ${BINARY_NAME}"
+echo "========================================"
+
+# 1. Check local binary exists
+if [ ! -f "$LOCAL_PATH" ]; then
+    echo "❌ FAIL: Local binary not found at $LOCAL_PATH"
+    exit 1
+fi
+echo "✅ Local binary exists"
+
+# 2. Test local binary works (basic smoke test)
+echo "🧪 Testing local binary..."
+if [[ "$BINARY_NAME" == hyperopt* ]]; then
+    # Test hyperopt binaries with --help
+    if ! $LOCAL_PATH --help &>/dev/null; then
+        echo "❌ FAIL: Local binary --help failed"
+        exit 1
+    fi
+
+    # Verify critical arguments present
+    if ! $LOCAL_PATH --help 2>&1 | grep -q -- "--base-dir"; then
+        echo "❌ FAIL: Local binary missing --base-dir argument"
+        echo "This binary was built BEFORE VarMap fix!"
+        exit 1
+    fi
+fi
+echo "✅ Local binary works and has correct arguments"
+
+# 3. Calculate local SHA256
+echo "🔍 Calculating local SHA256..."
+LOCAL_SHA256=$(sha256sum "$LOCAL_PATH" | awk '{print $1}')
+echo "Local SHA256:  $LOCAL_SHA256"
+
+# 4. Check if S3 binary exists
+echo "🔍 Checking S3 binary..."
+if ! aws s3api head-object \
+    --bucket "$S3_BUCKET" \
+    --key "$S3_KEY" \
+    --endpoint-url "$S3_ENDPOINT" \
+    --profile runpod &>/dev/null; then
+    echo "⚠️  S3 binary does not exist - will be uploaded"
+    echo "VALIDATION: LOCAL_ONLY"
+    exit 0
+fi
+
+# 5. Get S3 metadata
+S3_SIZE=$(aws s3api head-object \
+    --bucket "$S3_BUCKET" \
+    --key "$S3_KEY" \
+    --endpoint-url "$S3_ENDPOINT" \
+    --profile runpod \
+    | jq -r '.ContentLength')
+
+S3_MODIFIED=$(aws s3api head-object \
+    --bucket "$S3_BUCKET" \
+    --key "$S3_KEY" \
+    --endpoint-url "$S3_ENDPOINT" \
+    --profile runpod \
+    | jq -r '.LastModified')
+
+echo "S3 Size:       $S3_SIZE bytes"
+echo "S3 Modified:   $S3_MODIFIED"
+
+# 6. Download S3 binary to temp location for checksum
+echo "⬇️  Downloading S3 binary for checksum..."
+TEMP_S3="/tmp/${BINARY_NAME}_s3_$$"
+aws s3 cp \
+    "s3://${S3_BUCKET}/${S3_KEY}" \
+    "$TEMP_S3" \
+    --endpoint-url "$S3_ENDPOINT" \
+    --profile runpod \
+    --quiet
+
+# 7. Calculate S3 SHA256
+echo "🔍 Calculating S3 SHA256..."
+S3_SHA256=$(sha256sum "$TEMP_S3" | awk '{print $1}')
+echo "S3 SHA256:     $S3_SHA256"
+
+# 8. Cleanup temp file
+rm -f "$TEMP_S3"
+
+# 9. Compare checksums
+echo ""
+echo "========================================"
+if [ "$LOCAL_SHA256" = "$S3_SHA256" ]; then
+    echo "✅ VALIDATION PASSED"
+    echo "Local and S3 binaries match perfectly"
+    echo "========================================"
+    exit 0
+else
+    echo "❌ VALIDATION FAILED"
+    echo "Local and S3 binaries DO NOT MATCH"
+    echo ""
+    echo "Local:  $LOCAL_SHA256"
+    echo "S3:     $S3_SHA256"
+    echo ""
+    echo "REQUIRED ACTION:"
+    echo "1. Delete S3 binary: aws s3 rm s3://${S3_BUCKET}/${S3_KEY} --endpoint-url $S3_ENDPOINT --profile runpod"
+    echo "2. Upload local binary: aws s3 cp $LOCAL_PATH s3://${S3_BUCKET}/${S3_KEY} --endpoint-url $S3_ENDPOINT --profile runpod"
+    echo "3. Re-run validation"
+    echo "========================================"
+    exit 1
+fi