8d89fe80ff
- Archive: 85 agent .txt files → docs/archive/agents/legacy_txt/ - Scripts: Move 110 shell scripts → scripts/ (keep deploy.sh in root) - Models: Move 18 .safetensors → ml/models/checkpoints/training_artifacts/ - Delete: 34 directories (~33GB freed) - target/, coverage_*, test artifacts - Build: Clean 14 build artifacts (.rlib, .o, .pid, binaries) - Tests: Move 14 .rs files → tests/standalone/ - SQL: Move 5 files → sql/ (keep init-db*.sql for Docker) - Wave 153: Archive to docs/archive/historical/wave153/ - Docs: Archive 9 markdown files to wave_d/reports/ and historical/ Total impact: ~34GB freed (both waves), root directory cleaned from 583 to ~40 essential files Directory count reduced from 65 to 31 (52% reduction) All historical data preserved in organized archive structure
118 lines
3.9 KiB
Bash
Executable File
118 lines
3.9 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Agent 142: TFT CUDA Fix Verification Script
|
|
#
|
|
# This script verifies that the tensor contiguity fix allows TFT training
|
|
# to proceed on CUDA GPU without errors.
|
|
|
|
set -e
|
|
|
|
echo "=================================================="
|
|
echo "Agent 142: TFT CUDA Fix Verification"
|
|
echo "=================================================="
|
|
echo ""
|
|
|
|
# 1. Check CUDA availability
|
|
echo "Step 1: Checking CUDA availability..."
|
|
if command -v nvidia-smi &> /dev/null; then
|
|
echo "✅ CUDA available"
|
|
nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader
|
|
else
|
|
echo "❌ CUDA not available - cannot verify GPU fix"
|
|
exit 1
|
|
fi
|
|
echo ""
|
|
|
|
# 2. Check if modified file exists and has the fix
|
|
echo "Step 2: Verifying fix is applied..."
|
|
FIX_FILE="ml/src/tft/quantile_outputs.rs"
|
|
if grep -q "last_step_contiguous" "$FIX_FILE"; then
|
|
echo "✅ Tensor contiguity fix found in $FIX_FILE"
|
|
else
|
|
echo "❌ Fix not found in $FIX_FILE"
|
|
echo "Expected to find: last_step_contiguous = last_step.contiguous()?"
|
|
exit 1
|
|
fi
|
|
echo ""
|
|
|
|
# 3. Build with CUDA support
|
|
echo "Step 3: Building ML crate with CUDA support..."
|
|
echo "Command: cargo build --release -p ml --features cuda"
|
|
if cargo build --release -p ml --features cuda 2>&1 | tee /tmp/tft_cuda_build.log | tail -20; then
|
|
echo "✅ Build successful"
|
|
else
|
|
echo "❌ Build failed"
|
|
echo "See /tmp/tft_cuda_build.log for details"
|
|
exit 1
|
|
fi
|
|
echo ""
|
|
|
|
# 4. Check for TFT training example
|
|
echo "Step 4: Checking for TFT training example..."
|
|
if [ -f "ml/examples/train_tft.rs" ]; then
|
|
echo "✅ TFT training example found"
|
|
TRAIN_CMD="cargo run --release -p ml --example train_tft --features cuda"
|
|
else
|
|
echo "⚠️ TFT training example not found (train_tft.rs)"
|
|
echo "Looking for alternative training examples..."
|
|
|
|
# Check for other training examples
|
|
TRAIN_EXAMPLES=$(find ml/examples -name "*.rs" -type f | grep -i "train\|tft" || true)
|
|
if [ -n "$TRAIN_EXAMPLES" ]; then
|
|
echo "Found alternative examples:"
|
|
echo "$TRAIN_EXAMPLES"
|
|
echo ""
|
|
echo "Manual command to test (adjust example name):"
|
|
echo " cargo run --release -p ml --example <example_name> --features cuda"
|
|
else
|
|
echo "⚠️ No training examples found"
|
|
echo "To test the fix, you can:"
|
|
echo " 1. Run unit tests: cargo test -p ml --features cuda -- tft"
|
|
echo " 2. Create a test script that instantiates TFT and runs forward pass"
|
|
fi
|
|
TRAIN_CMD=""
|
|
fi
|
|
echo ""
|
|
|
|
# 5. Run unit tests
|
|
echo "Step 5: Running TFT unit tests..."
|
|
echo "Command: cargo test -p ml --features cuda -- tft"
|
|
if cargo test -p ml --features cuda -- tft 2>&1 | tee /tmp/tft_cuda_tests.log | tail -30; then
|
|
echo "✅ Unit tests passed"
|
|
else
|
|
echo "⚠️ Some tests failed (see /tmp/tft_cuda_tests.log)"
|
|
echo "Note: Tests may fail if they don't account for CUDA-specific behavior"
|
|
fi
|
|
echo ""
|
|
|
|
# 6. Summary and next steps
|
|
echo "=================================================="
|
|
echo "Verification Summary"
|
|
echo "=================================================="
|
|
echo ""
|
|
echo "✅ CUDA available and detected"
|
|
echo "✅ Tensor contiguity fix applied in quantile_outputs.rs"
|
|
echo "✅ ML crate builds successfully with CUDA support"
|
|
echo ""
|
|
echo "Next Steps:"
|
|
echo "----------"
|
|
if [ -n "$TRAIN_CMD" ]; then
|
|
echo "1. Start TFT training:"
|
|
echo " $TRAIN_CMD"
|
|
echo ""
|
|
fi
|
|
echo "2. Monitor GPU utilization (in another terminal):"
|
|
echo " nvidia-smi -l 1"
|
|
echo ""
|
|
echo "3. Expected results:"
|
|
echo " - No 'matmul is only supported for contiguous tensors' error"
|
|
echo " - GPU utilization: 80-95%"
|
|
echo " - Epoch time: <10 seconds (vs 43-55s on CPU)"
|
|
echo ""
|
|
echo "4. If training runs successfully for 10+ epochs:"
|
|
echo " ✅ Fix is validated and production-ready"
|
|
echo ""
|
|
echo "=================================================="
|
|
echo "Agent 142: Verification Complete"
|
|
echo "=================================================="
|