e61e8f54da
Changes: - CLAUDE.md: Update OOM fix validation status - Add comprehensive documentation (30+ markdown reports) - LSTM encoder varmap bug fix (tft/lstm_encoder.rs:290) - Quantized LSTM layer matching fix (tft/quantized_lstm.rs) - Hyperopt paths module (ml/src/hyperopt/paths.rs) - Training path tests for all adapters (DQN, MAMBA-2, PPO, TFT) - Checkpoint integrity tests - Script cleanup: Remove 29 obsolete deployment scripts - Archive old scripts to scripts/archive/ - New deployment utilities: check_gpu_availability.py, monitor_hyperopt.sh Validation: - OOM fixes validated: 5/5 trials successful (pod b6kc3mc5lbjiro) - Batch-size-max 256 tested successfully - All hyperopt adapters working correctly 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
51 lines
1.6 KiB
Bash
51 lines
1.6 KiB
Bash
#!/bin/bash
|
|
# VarMap Fix Validation - Runpod Deployment
|
|
# Deploys MAMBA-2 with fixed binary for checkpoint integrity validation
|
|
|
|
set -e
|
|
|
|
echo "=========================================="
|
|
echo "MAMBA-2 VarMap Fix Validation"
|
|
echo "=========================================="
|
|
echo ""
|
|
echo "Configuration:"
|
|
echo " GPU: RTX A4000 (16GB, \$0.25/hr)"
|
|
echo " Binary: hyperopt_mamba2_demo (FIXED - uploaded 2025-10-29 08:43 UTC)"
|
|
echo " Dataset: ES_FUT_180d.parquet (225 features)"
|
|
echo " Trials: 2 (quick validation)"
|
|
echo " Epochs: 1 (fast cycle)"
|
|
echo " Batch size: 256 (optimal)"
|
|
echo " Expected runtime: ~10 minutes"
|
|
echo " Expected cost: ~\$0.04"
|
|
echo ""
|
|
|
|
# Correct command for the pod
|
|
COMMAND="/runpod-volume/binaries/hyperopt_mamba2_demo \
|
|
--parquet-file /runpod-volume/test_data/ES_FUT_180d.parquet \
|
|
--base-dir /runpod-volume/ml_training \
|
|
--run-type hyperopt \
|
|
--trials 2 \
|
|
--epochs 1 \
|
|
--batch-size-max 256 \
|
|
--seed 42"
|
|
|
|
echo "Command to run in pod:"
|
|
echo "$COMMAND"
|
|
echo ""
|
|
|
|
# Deploy using runpod_deploy.py
|
|
echo "Deploying pod..."
|
|
python3 scripts/runpod_deploy.py \
|
|
--gpu-type "RTX A4000" \
|
|
--binary hyperopt_mamba2_demo \
|
|
--dataset ES_FUT_180d.parquet \
|
|
--extra-args "--trials 2 --epochs 1 --batch-size-max 256 --base-dir /runpod-volume/ml_training"
|
|
|
|
echo ""
|
|
echo "=========================================="
|
|
echo "Pod deployed! Monitor with:"
|
|
echo " aws s3 ls s3://se3zdnb5o4/training_runs/mamba2/ --endpoint-url https://s3api-eur-is-1.runpod.io --recursive --human-readable"
|
|
echo ""
|
|
echo "Expected checkpoint size: 2-8MB (NOT 842KB)"
|
|
echo "=========================================="
|