foxhunt/scripts/LEVEL_3_ROLLBACK_TEST.sh

#!/bin/bash
# ================================================================================================
# Level 3 Rollback Test: Full Rollback to Wave C (Target: ~15 minutes)
# Agent R1 - Rollback & Disaster Recovery Specialist
# ================================================================================================
#
# SCENARIO: Complete redeployment to Wave C baseline (before Wave D)
# EXPECTED: System rolled back to last known good Wave C state
#
# ================================================================================================

set -e  # Exit on error

echo "===================================================================================================="
echo "LEVEL 3 ROLLBACK TEST: Full Rollback to Wave C"
echo "===================================================================================================="
echo ""

# Start timer
START_TIME=$(date +%s)

# Configuration
WAVE_C_TAG="wave-c-baseline"  # Git tag for Wave C baseline
WAVE_D_TAG="wave-d-v1.0"      # Git tag for current Wave D deployment
BACKUP_DIR="/tmp/foxhunt_rollback_$(date +%s)"

# Step 1: Pre-rollback preparation
echo "Step 1: Pre-rollback preparation"
echo "------------------------------------------------------------------------------------"

# Create backup directory
mkdir -p "$BACKUP_DIR"
echo "  ✓ Created backup directory: $BACKUP_DIR"

# Tag current Wave D state (if not already tagged)
cd /home/jgrusewski/Work/foxhunt
CURRENT_COMMIT=$(git rev-parse HEAD)
echo "  ✓ Current commit: $CURRENT_COMMIT"

if ! git tag -l | grep -q "^$WAVE_D_TAG$"; then
    git tag "$WAVE_D_TAG" "$CURRENT_COMMIT"
    echo "  ✓ Tagged Wave D deployment: $WAVE_D_TAG"
else
    echo "  ⚠ Tag $WAVE_D_TAG already exists"
fi

# Backup database
echo "  ✓ Backing up database..."
BACKUP_FILE="$BACKUP_DIR/foxhunt_wave_d_backup.sql"
PGPASSWORD=foxhunt_dev_password pg_dump -h localhost -U foxhunt -d foxhunt -f "$BACKUP_FILE" 2>/dev/null || {
    echo "  ⚠ WARNING: Database backup failed"
    BACKUP_FILE=""
}
if [ -n "$BACKUP_FILE" ] && [ -f "$BACKUP_FILE" ]; then
    BACKUP_SIZE=$(du -h "$BACKUP_FILE" | cut -f1)
    echo "  ✅ Database backup created: $BACKUP_FILE ($BACKUP_SIZE)"
fi

# Backup .env files
echo "  ✓ Backing up environment files..."
cp /home/jgrusewski/Work/foxhunt/.env "$BACKUP_DIR/.env.wave_d" 2>/dev/null || true
cp /home/jgrusewski/Work/foxhunt/.env.production "$BACKUP_DIR/.env.production.wave_d" 2>/dev/null || true
echo "  ✓ Environment files backed up"

echo ""

# Step 2: Stop all services
echo "Step 2: Stopping all services"
echo "------------------------------------------------------------------------------------"
SHUTDOWN_START=$(date +%s)

# Stop Foxhunt services
PIDS=$(pgrep -f "api_gateway|trading_service|backtesting_service|ml_training_service|trading_agent_service" || true)
if [ -n "$PIDS" ]; then
    echo "  ✓ Stopping Foxhunt services (PIDs: $PIDS)..."
    kill -TERM $PIDS 2>/dev/null || true

    # Wait for graceful shutdown
    for i in {1..30}; do
        REMAINING=$(pgrep -f "api_gateway|trading_service|backtesting_service|ml_training_service|trading_agent_service" | wc -l)
        if [ "$REMAINING" -eq 0 ]; then
            echo "  ✅ Services stopped in ${i}s"
            break
        fi
        sleep 1
    done

    # Force kill if needed
    REMAINING=$(pgrep -f "api_gateway|trading_service|backtesting_service|ml_training_service|trading_agent_service" | wc -l)
    if [ "$REMAINING" -gt 0 ]; then
        echo "  ⚠ Forcing shutdown of $REMAINING processes..."
        kill -9 $(pgrep -f "api_gateway|trading_service|backtesting_service|ml_training_service|trading_agent_service") 2>/dev/null || true
    fi
else
    echo "  ⚠ No services running"
fi

SHUTDOWN_END=$(date +%s)
SHUTDOWN_TIME=$((SHUTDOWN_END - SHUTDOWN_START))
echo "  ✓ Shutdown completed in ${SHUTDOWN_TIME}s"

echo ""

# Step 3: Rollback database (Level 2 rollback)
echo "Step 3: Rolling back database to Wave C state"
echo "------------------------------------------------------------------------------------"
MIGRATION_START=$(date +%s)

# Run Level 2 rollback (database migration revert)
echo "  ✓ Executing database rollback..."
if [ -f "/home/jgrusewski/Work/foxhunt/migrations/045_wave_d_regime_tracking.down.sql" ]; then
    PGPASSWORD=foxhunt_dev_password psql -h localhost -U foxhunt -d foxhunt \
        -f /home/jgrusewski/Work/foxhunt/migrations/045_wave_d_regime_tracking.down.sql \
        2>&1 | grep -E "DROP|REVOKE|NOTICE|ERROR|Wave D" || echo "  ✓ Database rollback completed"
else
    echo "  ⚠ WARNING: Down migration not found, attempting alternative method..."
    # Use migration 046 if available
    if [ -f "/home/jgrusewski/Work/foxhunt/migrations/046_rollback_regime_detection.sql" ]; then
        PGPASSWORD=foxhunt_dev_password psql -h localhost -U foxhunt -d foxhunt \
            -f /home/jgrusewski/Work/foxhunt/migrations/046_rollback_regime_detection.sql \
            2>&1 | grep -E "DROP|REVOKE|NOTICE|ERROR|Wave D" || echo "  ✓ Alternative rollback completed"
    fi
fi

# Verify database state
REGIME_TABLES=$(PGPASSWORD=foxhunt_dev_password psql -h localhost -U foxhunt -d foxhunt -tAc "
SELECT COUNT(*) FROM information_schema.tables
WHERE table_name IN ('regime_states', 'regime_transitions', 'adaptive_strategy_metrics');
" 2>/dev/null || echo "unknown")

if [ "$REGIME_TABLES" = "0" ]; then
    echo "  ✅ Database rollback successful (Wave D tables removed)"
else
    echo "  ❌ Database rollback failed ($REGIME_TABLES Wave D tables remain)"
fi

MIGRATION_END=$(date +%s)
MIGRATION_TIME=$((MIGRATION_END - MIGRATION_START))
echo "  ✓ Database rollback completed in ${MIGRATION_TIME}s"

echo ""

# Step 4: Checkout Wave C baseline code
echo "Step 4: Checking out Wave C baseline code"
echo "------------------------------------------------------------------------------------"
CHECKOUT_START=$(date +%s)

# Find Wave C baseline commit
if git tag -l | grep -q "^$WAVE_C_TAG$"; then
    echo "  ✓ Found Wave C tag: $WAVE_C_TAG"
    WAVE_C_COMMIT=$(git rev-list -n 1 "$WAVE_C_TAG")
else
    # Fallback: Find commit before Wave D Phase 3 (feature implementation)
    echo "  ⚠ No Wave C tag found, searching for baseline commit..."
    WAVE_C_COMMIT=$(git log --all --oneline | grep -E "Wave C.*COMPLETE|WAVE_C.*IMPLEMENTATION.*COMPLETE" | head -1 | awk '{print $1}')
    if [ -z "$WAVE_C_COMMIT" ]; then
        # Ultimate fallback: commit before "Wave D Phase 3"
        WAVE_C_COMMIT=$(git log --all --oneline --before="2025-10-17" | head -1 | awk '{print $1}')
    fi
fi

echo "  ✓ Wave C baseline commit: $WAVE_C_COMMIT"

# Stash current changes (if any)
if ! git diff-index --quiet HEAD --; then
    echo "  ✓ Stashing uncommitted changes..."
    git stash push -m "Rollback: Stashing Wave D changes before rollback to Wave C"
fi

# Checkout Wave C baseline
echo "  ✓ Checking out Wave C baseline..."
git checkout "$WAVE_C_COMMIT" 2>&1 | grep -E "HEAD|Previous|error" || echo "  ✓ Checkout completed"

# Verify checkout
CURRENT_COMMIT=$(git rev-parse HEAD)
if [ "$CURRENT_COMMIT" = "$WAVE_C_COMMIT" ]; then
    echo "  ✅ Successfully checked out Wave C baseline"
else
    echo "  ❌ Checkout failed (HEAD: $CURRENT_COMMIT, Expected: $WAVE_C_COMMIT)"
fi

CHECKOUT_END=$(date +%s)
CHECKOUT_TIME=$((CHECKOUT_END - CHECKOUT_START))
echo "  ✓ Git checkout completed in ${CHECKOUT_TIME}s"

echo ""

# Step 5: Rebuild services with Wave C codebase
echo "Step 5: Rebuilding services from Wave C codebase"
echo "------------------------------------------------------------------------------------"
REBUILD_START=$(date +%s)

# Clean build to ensure no Wave D artifacts
echo "  ✓ Cleaning previous build artifacts..."
cd /home/jgrusewski/Work/foxhunt
cargo clean 2>&1 | grep -E "Removing|error" || echo "  ✓ Clean completed"

# Rebuild entire workspace
echo "  ✓ Rebuilding workspace (release mode)..."
cargo build --workspace --release 2>&1 | grep -E "Compiling|Finished|error" || echo "  ✓ Build completed"

REBUILD_END=$(date +%s)
REBUILD_TIME=$((REBUILD_END - REBUILD_START))
echo "  ✓ Rebuild completed in ${REBUILD_TIME}s"

echo ""

# Step 6: Smoke test Wave C deployment
echo "Step 6: Smoke testing Wave C deployment"
echo "------------------------------------------------------------------------------------"

# Check feature count
echo "  ✓ Checking feature count..."
FEATURE_COUNT=$(cargo run --release -p ml --example check_feature_count 2>&1 | grep "Wave C: feature_count:" | grep -oP '\d+' || echo "unknown")
if [ "$FEATURE_COUNT" = "201" ]; then
    echo "  ✅ VERIFIED: Wave C feature count = 201"
elif [ "$FEATURE_COUNT" = "225" ]; then
    echo "  ❌ FAILED: Feature count = 225 (Wave D still active)"
else
    echo "  ⚠ WARNING: Could not verify feature count (got: $FEATURE_COUNT)"
fi

# Check for Wave D code
WAVE_D_CODE=$(grep -r "enable_wave_d_regime" /home/jgrusewski/Work/foxhunt/ml/src/features/ 2>/dev/null | wc -l)
if [ "$WAVE_D_CODE" -eq 0 ]; then
    echo "  ✅ VERIFIED: No Wave D code present"
else
    echo "  ⚠ WARNING: Wave D code references still exist ($WAVE_D_CODE occurrences)"
    echo "  ⚠ This may be expected if Wave C already had placeholders"
fi

# Test basic compilation
echo "  ✓ Running basic compilation test..."
cargo check --workspace 2>&1 | grep -E "Checking|Finished|error" || echo "  ✓ Check completed"

echo "  ⚠ NOTE: Services NOT started automatically (manual start required)"

echo ""

# Step 7: Calculate rollback time
echo "Step 7: Rollback Performance Metrics"
echo "------------------------------------------------------------------------------------"
END_TIME=$(date +%s)
TOTAL_TIME=$((END_TIME - START_TIME))

echo "  • Total rollback time: ${TOTAL_TIME}s"
echo "  • Shutdown time: ${SHUTDOWN_TIME}s"
echo "  • Database rollback time: ${MIGRATION_TIME}s"
echo "  • Git checkout time: ${CHECKOUT_TIME}s"
echo "  • Rebuild time: ${REBUILD_TIME}s"
echo "  • Target time: <900s (15 minutes)"

if [ $TOTAL_TIME -lt 900 ]; then
    echo "  ✅ PASSED: Rollback completed within 15-minute target"
else
    echo "  ⚠ MISSED TARGET: Rollback took ${TOTAL_TIME}s (>900s)"
fi

echo ""

# Summary
echo "===================================================================================================="
echo "LEVEL 3 ROLLBACK TEST: SUMMARY"
echo "===================================================================================================="
echo ""
echo "Result:"
echo "  • Git state: Wave C baseline ($WAVE_C_COMMIT)"
echo "  • Database: Wave D tables removed ($REGIME_TABLES tables remain)"
echo "  • Feature count: $FEATURE_COUNT (expected: 201)"
echo "  • Services: STOPPED (manual restart required)"
echo "  • Rollback time: ${TOTAL_TIME}s (~$((TOTAL_TIME / 60)) minutes)"
echo ""
echo "Data Loss:"
echo "  • All Wave D regime detection data DELETED"
echo "  • Wave D code changes REVERTED"
if [ -n "$BACKUP_FILE" ] && [ -f "$BACKUP_FILE" ]; then
    echo "  • Full backup: $BACKUP_FILE ($BACKUP_SIZE)"
fi
echo "  • Backup directory: $BACKUP_DIR"
echo ""
echo "Recovery Path:"
echo "  To re-deploy Wave D:"
echo "  1. Checkout Wave D tag: git checkout $WAVE_D_TAG"
echo "  2. Restore database (optional): psql < $BACKUP_FILE"
echo "  3. Re-apply migrations: sqlx migrate run"
echo "  4. Rebuild services: cargo build --workspace --release"
echo "  5. Restart all services"
echo ""
echo "Manual Steps Required:"
echo "  1. Start services: cargo run -p api_gateway &"
echo "  2. Verify health checks: curl http://localhost:8080/health"
echo "  3. Run integration tests: cargo test --workspace"
echo "  4. Monitor Grafana dashboards"
echo ""
echo "===================================================================================================="

exit 0