foxhunt/tests/paper_trading_smoke_test.sh

#!/bin/bash
# ================================================================================================
# Paper Trading Smoke Test
# ================================================================================================
# Purpose: Verify ensemble infrastructure works BEFORE deploying to paper trading
# Usage: bash tests/paper_trading_smoke_test.sh
# Duration: ~2-3 minutes
# ================================================================================================

set -e

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Helper functions
print_test() {
    echo -e "${BLUE}[TEST]${NC} $1"
}

print_pass() {
    echo -e "${GREEN}[PASS]${NC} $1"
}

print_fail() {
    echo -e "${RED}[FAIL]${NC} $1"
    exit 1
}

print_info() {
    echo -e "${BLUE}[INFO]${NC} $1"
}

# Test counter
TESTS_RUN=0
TESTS_PASSED=0

run_test() {
    ((TESTS_RUN++))
    print_test "$1"
}

pass_test() {
    ((TESTS_PASSED++))
    print_pass "$1"
}

# ================================================================================================
# TEST 1: Configuration File Validation
# ================================================================================================
run_test "Configuration file exists and is valid YAML"

CONFIG_FILE="/home/jgrusewski/Work/foxhunt/config/paper_trading_config.yaml"
if [ ! -f "$CONFIG_FILE" ]; then
    print_fail "Config file not found: $CONFIG_FILE"
fi

# Validate YAML syntax (basic check)
if grep -q "paper_trading:" "$CONFIG_FILE" && grep -q "ensemble:" "$CONFIG_FILE"; then
    pass_test "Config file valid"
else
    print_fail "Config file malformed (missing required sections)"
fi

# ================================================================================================
# TEST 2: Model Checkpoints Exist
# ================================================================================================
run_test "All 3 model checkpoints exist"

CHECKPOINT_DIR="/home/jgrusewski/Work/foxhunt/ml/trained_models/production"

# DQN
if [ ! -f "$CHECKPOINT_DIR/dqn/dqn_epoch_30.safetensors" ]; then
    print_fail "DQN checkpoint missing: dqn_epoch_30.safetensors"
fi

# PPO 130
if [ ! -f "$CHECKPOINT_DIR/ppo/ppo_actor_epoch_130.safetensors" ] || \
   [ ! -f "$CHECKPOINT_DIR/ppo/ppo_critic_epoch_130.safetensors" ]; then
    print_fail "PPO epoch 130 checkpoints missing"
fi

# PPO 420
if [ ! -f "$CHECKPOINT_DIR/ppo/ppo_actor_epoch_420.safetensors" ] || \
   [ ! -f "$CHECKPOINT_DIR/ppo/ppo_critic_epoch_420.safetensors" ]; then
    print_fail "PPO epoch 420 checkpoints missing"
fi

pass_test "All model checkpoints present (5 files)"

# ================================================================================================
# TEST 3: Services are Running
# ================================================================================================
run_test "All required services are running"

# Check Trading Service
if docker ps | grep -q "foxhunt-trading-service.*Up"; then
    print_info "Trading Service: Running"
else
    print_fail "Trading Service is not running"
fi

# Check PostgreSQL
if docker ps | grep -q "foxhunt-postgres.*Up"; then
    print_info "PostgreSQL: Running"
else
    print_fail "PostgreSQL is not running"
fi

# Check Prometheus
if docker ps | grep -q "foxhunt-prometheus.*Up"; then
    print_info "Prometheus: Running"
else
    print_fail "Prometheus is not running"
fi

pass_test "All services running"

# ================================================================================================
# TEST 4: Database Tables Exist
# ================================================================================================
run_test "Ensemble database tables exist"

TABLE_CHECK=$(psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -t -c \
    "SELECT COUNT(*) FROM pg_tables WHERE schemaname = 'public' AND tablename IN ('ensemble_predictions', 'model_performance_attribution', 'ab_test_experiments')" 2>/dev/null | tr -d ' ')

if [ "$TABLE_CHECK" = "3" ]; then
    pass_test "All 3 ensemble tables exist"
else
    print_info "Only $TABLE_CHECK/3 tables exist (may need migration 022)"
    print_info "This is OK for smoke test, but migration should be run before deployment"
fi

# ================================================================================================
# TEST 5: Prometheus Metrics Endpoint
# ================================================================================================
run_test "Trading Service metrics endpoint accessible"

if curl -s http://localhost:9092/metrics > /dev/null 2>&1; then
    METRIC_COUNT=$(curl -s http://localhost:9092/metrics | grep -c "^# HELP" || echo "0")
    print_info "Prometheus metrics: $METRIC_COUNT metrics available"
    pass_test "Metrics endpoint accessible"
else
    print_fail "Cannot access Trading Service metrics (port 9092)"
fi

# ================================================================================================
# TEST 6: Grafana Dashboard Import (Optional)
# ================================================================================================
run_test "Grafana dashboard file exists"

DASHBOARD_FILE="/home/jgrusewski/Work/foxhunt/monitoring/grafana/ensemble_ml_production.json"
if [ -f "$DASHBOARD_FILE" ]; then
    # Validate JSON syntax
    if jq empty "$DASHBOARD_FILE" 2>/dev/null; then
        pass_test "Grafana dashboard valid JSON"
    else
        print_fail "Grafana dashboard is not valid JSON"
    fi
else
    print_fail "Grafana dashboard not found: $DASHBOARD_FILE"
fi

# ================================================================================================
# TEST 7: Ensemble Coordinator Code Exists
# ================================================================================================
run_test "Ensemble coordinator code exists"

COORDINATOR_FILE="/home/jgrusewski/Work/foxhunt/services/trading_service/src/ensemble_coordinator.rs"
if [ -f "$COORDINATOR_FILE" ]; then
    # Check for key structs
    if grep -q "pub struct EnsembleCoordinator" "$COORDINATOR_FILE"; then
        pass_test "Ensemble coordinator code present"
    else
        print_fail "Ensemble coordinator code malformed (missing struct)"
    fi
else
    print_fail "Ensemble coordinator not found: $COORDINATOR_FILE"
fi

# ================================================================================================
# TEST 8: Ensemble Metrics Code Exists
# ================================================================================================
run_test "Ensemble metrics code exists"

METRICS_FILE="/home/jgrusewski/Work/foxhunt/services/trading_service/src/ensemble_metrics.rs"
if [ -f "$METRICS_FILE" ]; then
    # Check for Prometheus metrics
    if grep -q "ENSEMBLE_CONFIDENCE" "$METRICS_FILE" && grep -q "ENSEMBLE_DISAGREEMENT_RATE" "$METRICS_FILE"; then
        pass_test "Ensemble metrics defined"
    else
        print_fail "Ensemble metrics incomplete"
    fi
else
    print_fail "Ensemble metrics not found: $METRICS_FILE"
fi

# ================================================================================================
# TEST 9: Candle Device Check (GPU/CPU)
# ================================================================================================
run_test "Candle device availability (GPU or CPU)"

# Check if CUDA is available (optional, not required for smoke test)
if nvidia-smi > /dev/null 2>&1; then
    GPU_INFO=$(nvidia-smi --query-gpu=name,memory.total --format=csv,noheader,nounits | head -1)
    print_info "GPU available: $GPU_INFO"
    print_info "Ensemble will use GPU acceleration"
else
    print_info "No GPU detected, ensemble will use CPU"
    print_info "This is OK, but GPU recommended for <50μs latency"
fi

pass_test "Candle device check complete"

# ================================================================================================
# TEST 10: Database Write Access
# ================================================================================================
run_test "Database write access to ensemble_predictions table"

# Attempt a test insert (if table exists)
if [ "$TABLE_CHECK" = "3" ]; then
    TEST_INSERT=$(psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -t -c \
        "INSERT INTO ensemble_predictions (symbol, ensemble_action, ensemble_signal, ensemble_confidence, disagreement_rate) VALUES ('TEST', 'HOLD', 0.0, 0.5, 0.2) RETURNING id;" 2>&1)

    if echo "$TEST_INSERT" | grep -qE "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"; then
        print_info "Test insert successful, cleaning up..."
        psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -c "DELETE FROM ensemble_predictions WHERE symbol = 'TEST';" > /dev/null 2>&1
        pass_test "Database write access verified"
    else
        print_fail "Cannot write to ensemble_predictions table"
    fi
else
    print_info "Skipping write test (tables don't exist yet)"
fi

# ================================================================================================
# SUMMARY
# ================================================================================================
echo ""
echo "========================================"
echo "SMOKE TEST SUMMARY"
echo "========================================"
echo -e "${GREEN}Tests Passed: $TESTS_PASSED/$TESTS_RUN${NC}"

if [ "$TESTS_PASSED" -eq "$TESTS_RUN" ]; then
    echo -e "${GREEN}✅ ALL TESTS PASSED${NC}"
    echo ""
    echo "Next steps:"
    echo "  1. Run deployment script: bash scripts/deploy_paper_trading.sh"
    echo "  2. Monitor Grafana: http://localhost:3000/d/ensemble-ml-prod"
    echo "  3. Check first predictions: psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -c 'SELECT * FROM ensemble_predictions ORDER BY timestamp DESC LIMIT 5'"
    echo ""
    exit 0
else
    echo -e "${RED}❌ SOME TESTS FAILED${NC}"
    echo ""
    echo "Please fix issues before deploying to paper trading"
    echo ""
    exit 1
fi