#!/bin/bash # ================================================================================================ # Paper Trading Smoke Test # ================================================================================================ # Purpose: Verify ensemble infrastructure works BEFORE deploying to paper trading # Usage: bash tests/paper_trading_smoke_test.sh # Duration: ~2-3 minutes # ================================================================================================ set -e # Colors RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # Helper functions print_test() { echo -e "${BLUE}[TEST]${NC} $1" } print_pass() { echo -e "${GREEN}[PASS]${NC} $1" } print_fail() { echo -e "${RED}[FAIL]${NC} $1" exit 1 } print_info() { echo -e "${BLUE}[INFO]${NC} $1" } # Test counter TESTS_RUN=0 TESTS_PASSED=0 run_test() { ((TESTS_RUN++)) print_test "$1" } pass_test() { ((TESTS_PASSED++)) print_pass "$1" } # ================================================================================================ # TEST 1: Configuration File Validation # ================================================================================================ run_test "Configuration file exists and is valid YAML" CONFIG_FILE="/home/jgrusewski/Work/foxhunt/config/paper_trading_config.yaml" if [ ! -f "$CONFIG_FILE" ]; then print_fail "Config file not found: $CONFIG_FILE" fi # Validate YAML syntax (basic check) if grep -q "paper_trading:" "$CONFIG_FILE" && grep -q "ensemble:" "$CONFIG_FILE"; then pass_test "Config file valid" else print_fail "Config file malformed (missing required sections)" fi # ================================================================================================ # TEST 2: Model Checkpoints Exist # ================================================================================================ run_test "All 3 model checkpoints exist" CHECKPOINT_DIR="/home/jgrusewski/Work/foxhunt/ml/trained_models/production" # DQN if [ ! -f "$CHECKPOINT_DIR/dqn/dqn_epoch_30.safetensors" ]; then print_fail "DQN checkpoint missing: dqn_epoch_30.safetensors" fi # PPO 130 if [ ! -f "$CHECKPOINT_DIR/ppo/ppo_actor_epoch_130.safetensors" ] || \ [ ! -f "$CHECKPOINT_DIR/ppo/ppo_critic_epoch_130.safetensors" ]; then print_fail "PPO epoch 130 checkpoints missing" fi # PPO 420 if [ ! -f "$CHECKPOINT_DIR/ppo/ppo_actor_epoch_420.safetensors" ] || \ [ ! -f "$CHECKPOINT_DIR/ppo/ppo_critic_epoch_420.safetensors" ]; then print_fail "PPO epoch 420 checkpoints missing" fi pass_test "All model checkpoints present (5 files)" # ================================================================================================ # TEST 3: Services are Running # ================================================================================================ run_test "All required services are running" # Check Trading Service if docker ps | grep -q "foxhunt-trading-service.*Up"; then print_info "Trading Service: Running" else print_fail "Trading Service is not running" fi # Check PostgreSQL if docker ps | grep -q "foxhunt-postgres.*Up"; then print_info "PostgreSQL: Running" else print_fail "PostgreSQL is not running" fi # Check Prometheus if docker ps | grep -q "foxhunt-prometheus.*Up"; then print_info "Prometheus: Running" else print_fail "Prometheus is not running" fi pass_test "All services running" # ================================================================================================ # TEST 4: Database Tables Exist # ================================================================================================ run_test "Ensemble database tables exist" TABLE_CHECK=$(psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -t -c \ "SELECT COUNT(*) FROM pg_tables WHERE schemaname = 'public' AND tablename IN ('ensemble_predictions', 'model_performance_attribution', 'ab_test_experiments')" 2>/dev/null | tr -d ' ') if [ "$TABLE_CHECK" = "3" ]; then pass_test "All 3 ensemble tables exist" else print_info "Only $TABLE_CHECK/3 tables exist (may need migration 022)" print_info "This is OK for smoke test, but migration should be run before deployment" fi # ================================================================================================ # TEST 5: Prometheus Metrics Endpoint # ================================================================================================ run_test "Trading Service metrics endpoint accessible" if curl -s http://localhost:9092/metrics > /dev/null 2>&1; then METRIC_COUNT=$(curl -s http://localhost:9092/metrics | grep -c "^# HELP" || echo "0") print_info "Prometheus metrics: $METRIC_COUNT metrics available" pass_test "Metrics endpoint accessible" else print_fail "Cannot access Trading Service metrics (port 9092)" fi # ================================================================================================ # TEST 6: Grafana Dashboard Import (Optional) # ================================================================================================ run_test "Grafana dashboard file exists" DASHBOARD_FILE="/home/jgrusewski/Work/foxhunt/monitoring/grafana/ensemble_ml_production.json" if [ -f "$DASHBOARD_FILE" ]; then # Validate JSON syntax if jq empty "$DASHBOARD_FILE" 2>/dev/null; then pass_test "Grafana dashboard valid JSON" else print_fail "Grafana dashboard is not valid JSON" fi else print_fail "Grafana dashboard not found: $DASHBOARD_FILE" fi # ================================================================================================ # TEST 7: Ensemble Coordinator Code Exists # ================================================================================================ run_test "Ensemble coordinator code exists" COORDINATOR_FILE="/home/jgrusewski/Work/foxhunt/services/trading_service/src/ensemble_coordinator.rs" if [ -f "$COORDINATOR_FILE" ]; then # Check for key structs if grep -q "pub struct EnsembleCoordinator" "$COORDINATOR_FILE"; then pass_test "Ensemble coordinator code present" else print_fail "Ensemble coordinator code malformed (missing struct)" fi else print_fail "Ensemble coordinator not found: $COORDINATOR_FILE" fi # ================================================================================================ # TEST 8: Ensemble Metrics Code Exists # ================================================================================================ run_test "Ensemble metrics code exists" METRICS_FILE="/home/jgrusewski/Work/foxhunt/services/trading_service/src/ensemble_metrics.rs" if [ -f "$METRICS_FILE" ]; then # Check for Prometheus metrics if grep -q "ENSEMBLE_CONFIDENCE" "$METRICS_FILE" && grep -q "ENSEMBLE_DISAGREEMENT_RATE" "$METRICS_FILE"; then pass_test "Ensemble metrics defined" else print_fail "Ensemble metrics incomplete" fi else print_fail "Ensemble metrics not found: $METRICS_FILE" fi # ================================================================================================ # TEST 9: Candle Device Check (GPU/CPU) # ================================================================================================ run_test "Candle device availability (GPU or CPU)" # Check if CUDA is available (optional, not required for smoke test) if nvidia-smi > /dev/null 2>&1; then GPU_INFO=$(nvidia-smi --query-gpu=name,memory.total --format=csv,noheader,nounits | head -1) print_info "GPU available: $GPU_INFO" print_info "Ensemble will use GPU acceleration" else print_info "No GPU detected, ensemble will use CPU" print_info "This is OK, but GPU recommended for <50μs latency" fi pass_test "Candle device check complete" # ================================================================================================ # TEST 10: Database Write Access # ================================================================================================ run_test "Database write access to ensemble_predictions table" # Attempt a test insert (if table exists) if [ "$TABLE_CHECK" = "3" ]; then TEST_INSERT=$(psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -t -c \ "INSERT INTO ensemble_predictions (symbol, ensemble_action, ensemble_signal, ensemble_confidence, disagreement_rate) VALUES ('TEST', 'HOLD', 0.0, 0.5, 0.2) RETURNING id;" 2>&1) if echo "$TEST_INSERT" | grep -qE "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"; then print_info "Test insert successful, cleaning up..." psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -c "DELETE FROM ensemble_predictions WHERE symbol = 'TEST';" > /dev/null 2>&1 pass_test "Database write access verified" else print_fail "Cannot write to ensemble_predictions table" fi else print_info "Skipping write test (tables don't exist yet)" fi # ================================================================================================ # SUMMARY # ================================================================================================ echo "" echo "========================================" echo "SMOKE TEST SUMMARY" echo "========================================" echo -e "${GREEN}Tests Passed: $TESTS_PASSED/$TESTS_RUN${NC}" if [ "$TESTS_PASSED" -eq "$TESTS_RUN" ]; then echo -e "${GREEN}✅ ALL TESTS PASSED${NC}" echo "" echo "Next steps:" echo " 1. Run deployment script: bash scripts/deploy_paper_trading.sh" echo " 2. Monitor Grafana: http://localhost:3000/d/ensemble-ml-prod" echo " 3. Check first predictions: psql postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt -c 'SELECT * FROM ensemble_predictions ORDER BY timestamp DESC LIMIT 5'" echo "" exit 0 else echo -e "${RED}❌ SOME TESTS FAILED${NC}" echo "" echo "Please fix issues before deploying to paper trading" echo "" exit 1 fi