6da9d262db
CRITICAL P0 FIXES (Validated - Loss 0.87 → 0.07): - Add sigmoid activation to inference and training (ml/src/mamba/mod.rs:798, 1538) - Fix config.total_decay_steps (was hardcoded 10000) (ml/src/mamba/mod.rs:2271) - Update d_state: 16→64, 32→64 (Mamba-2 spec) (ml/src/mamba/mod.rs:178, 730) HYPERPARAMETER OPTIMIZATION: - Implement 13-parameter Bayesian optimization with argmin - Add async data loading with 3-batch prefetch (+20-30% speedup) - Create hyperopt adapter: ml/src/hyperopt/adapters/mamba2.rs - Add example: ml/examples/hyperopt_mamba2_demo.rs VALIDATION: - Local test: Loss 0.07 vs 0.87 (12× improvement) - Val loss: 0.04-0.14 vs 1.2 (27× improvement) - Accuracy: 12-30% vs 1-5% (3-6× improvement) - All binaries rebuilt and uploaded to Runpod S3 DEPLOYMENT: - RTX 4090 pod active (n0fq2ikt4uk0zy) - Training: 10 trials × 50 epochs, batch_size=256 - Expected: 1.3 days, $10.41 cost Fixes #P0-sigmoid #P0-decay-steps #hyperopt-mamba2
48 lines
1.6 KiB
Bash
Executable File
48 lines
1.6 KiB
Bash
Executable File
#!/bin/bash
|
|
# Monitor MAMBA2 Hyperopt Results on Runpod S3
|
|
|
|
ENDPOINT="https://s3api-eur-is-1.runpod.io"
|
|
PROFILE="runpod"
|
|
BUCKET="s3://se3zdnb5o4"
|
|
|
|
echo "╔════════════════════════════════════════════════════════════════════╗"
|
|
echo "║ MAMBA2 Hyperopt Results Monitor ║"
|
|
echo "╚════════════════════════════════════════════════════════════════════╝"
|
|
echo ""
|
|
|
|
# Check if results directory exists
|
|
echo "Checking S3 for hyperopt results..."
|
|
echo ""
|
|
|
|
aws s3 ls ${BUCKET}/results/ \
|
|
--profile ${PROFILE} \
|
|
--endpoint-url ${ENDPOINT} \
|
|
--human-readable \
|
|
--recursive 2>/dev/null
|
|
|
|
if [ $? -eq 0 ]; then
|
|
echo ""
|
|
echo "To download best parameters:"
|
|
echo ""
|
|
echo " aws s3 cp ${BUCKET}/results/mamba2_13param_best_params_*.json \\"
|
|
echo " /tmp/mamba2_best_params.json \\"
|
|
echo " --profile ${PROFILE} \\"
|
|
echo " --endpoint-url ${ENDPOINT}"
|
|
echo ""
|
|
echo "To download full log:"
|
|
echo ""
|
|
echo " aws s3 cp ${BUCKET}/results/mamba2_13param_hyperopt_*.log \\"
|
|
echo " /tmp/mamba2_hyperopt.log \\"
|
|
echo " --profile ${PROFILE} \\"
|
|
echo " --endpoint-url ${ENDPOINT}"
|
|
echo ""
|
|
else
|
|
echo ""
|
|
echo "No results found yet. Hyperopt is likely still running."
|
|
echo ""
|
|
echo "Expected completion time: 60-90 minutes from deployment"
|
|
echo ""
|
|
echo "Run this script again in 10-15 minutes to check progress."
|
|
echo ""
|
|
fi
|