#!/bin/bash
# =============================================================================
# DQN PERFORMANCE VALIDATION SCRIPT
# =============================================================================
# Validates DQN model inference performance in staging environment
#
# Author: Agent F5
# Date: 2025-10-18
# Target: Inference latency < 100μs (current: 36.6μs)
# =============================================================================

set -euo pipefail

# Color codes
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Configuration
DB_URL="postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt_staging"
TARGET_LATENCY_US=100
EXPECTED_LATENCY_US=36.6

print_header() {
    echo ""
    echo "================================================================================"
    echo "$1"
    echo "================================================================================"
    echo ""
}

print_step() {
    echo -e "${BLUE}[STEP]${NC} $1"
}

print_success() {
    echo -e "${GREEN}[SUCCESS]${NC} $1"
}

print_error() {
    echo -e "${RED}[ERROR]${NC} $1"
}

print_warning() {
    echo -e "${YELLOW}[WARNING]${NC} $1"
}

# Validate database connectivity
validate_database() {
    print_step "Validating database connectivity..."

    if psql "$DB_URL" -c "SELECT 1" > /dev/null 2>&1; then
        print_success "Database connection established"
    else
        print_error "Cannot connect to staging database"
        exit 1
    fi
}

# Check model registration
check_model_registration() {
    print_step "Checking DQN model registration..."

    MODEL_STATUS=$(psql "$DB_URL" -t -c "SELECT status FROM ml_models WHERE model_id = 'DQN_v1';" | xargs)

    if [ "$MODEL_STATUS" == "active" ]; then
        print_success "DQN model is registered and active"
    else
        print_error "DQN model is not active (status: $MODEL_STATUS)"
        exit 1
    fi

    # Display model details
    echo ""
    psql "$DB_URL" -c "SELECT model_id, model_type, version, checkpoint_path, deployment_date FROM ml_models WHERE model_id = 'DQN_v1';"
}

# Check recent predictions
check_recent_predictions() {
    print_step "Checking recent ensemble predictions..."

    PREDICTION_COUNT=$(psql "$DB_URL" -t -c "SELECT COUNT(*) FROM ensemble_predictions WHERE prediction_timestamp > NOW() - INTERVAL '1 hour';" | xargs)

    if [ "$PREDICTION_COUNT" -gt "0" ]; then
        print_success "Found $PREDICTION_COUNT predictions in the last hour"

        echo ""
        echo "Recent predictions:"
        psql "$DB_URL" -c "
            SELECT
                prediction_timestamp,
                symbol,
                ensemble_action,
                ROUND(ensemble_confidence::numeric, 4) as confidence,
                ROUND(ensemble_signal::numeric, 4) as signal,
                inference_latency_us
            FROM ensemble_predictions
            WHERE prediction_timestamp > NOW() - INTERVAL '1 hour'
            ORDER BY prediction_timestamp DESC
            LIMIT 10;
        "
    else
        print_warning "No predictions found in the last hour. Model may not be running."
    fi
}

# Measure inference latency
measure_inference_latency() {
    print_step "Measuring inference latency from database records..."

    # Get latency statistics from recent predictions
    LATENCY_STATS=$(psql "$DB_URL" -t -c "
        SELECT
            COUNT(*) as sample_count,
            ROUND(AVG(inference_latency_us)::numeric, 2) as avg_latency_us,
            ROUND(MIN(inference_latency_us)::numeric, 2) as min_latency_us,
            ROUND(MAX(inference_latency_us)::numeric, 2) as max_latency_us,
            ROUND(PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY inference_latency_us)::numeric, 2) as p50_latency_us,
            ROUND(PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY inference_latency_us)::numeric, 2) as p95_latency_us,
            ROUND(PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY inference_latency_us)::numeric, 2) as p99_latency_us
        FROM ensemble_predictions
        WHERE prediction_timestamp > NOW() - INTERVAL '1 hour'
        AND inference_latency_us IS NOT NULL;
    ")

    if [ -z "$LATENCY_STATS" ] || [ "$LATENCY_STATS" == "0" ]; then
        print_warning "No latency data available. Inference may not be running."
        return
    fi

    echo ""
    echo "Inference Latency Statistics (Last 1 Hour):"
    echo "--------------------------------------------"
    psql "$DB_URL" -c "
        SELECT
            COUNT(*) as samples,
            ROUND(AVG(inference_latency_us)::numeric, 2) as avg_us,
            ROUND(MIN(inference_latency_us)::numeric, 2) as min_us,
            ROUND(MAX(inference_latency_us)::numeric, 2) as max_us,
            ROUND(PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY inference_latency_us)::numeric, 2) as p50_us,
            ROUND(PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY inference_latency_us)::numeric, 2) as p95_us,
            ROUND(PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY inference_latency_us)::numeric, 2) as p99_us
        FROM ensemble_predictions
        WHERE prediction_timestamp > NOW() - INTERVAL '1 hour'
        AND inference_latency_us IS NOT NULL;
    "

    # Extract P99 latency for validation
    P99_LATENCY=$(psql "$DB_URL" -t -c "
        SELECT ROUND(PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY inference_latency_us)::numeric, 2)
        FROM ensemble_predictions
        WHERE prediction_timestamp > NOW() - INTERVAL '1 hour'
        AND inference_latency_us IS NOT NULL;
    " | xargs)

    if [ ! -z "$P99_LATENCY" ]; then
        echo ""
        if (( $(echo "$P99_LATENCY < $TARGET_LATENCY_US" | bc -l) )); then
            print_success "✓ P99 latency ($P99_LATENCY μs) is below target ($TARGET_LATENCY_US μs)"
        else
            print_error "✗ P99 latency ($P99_LATENCY μs) exceeds target ($TARGET_LATENCY_US μs)"
        fi
    fi
}

# Check paper trading performance
check_paper_trading() {
    print_step "Checking paper trading performance..."

    ORDER_COUNT=$(psql "$DB_URL" -t -c "SELECT COUNT(*) FROM paper_trading_orders WHERE order_timestamp > NOW() - INTERVAL '24 hours';" | xargs)

    if [ "$ORDER_COUNT" -gt "0" ]; then
        print_success "Found $ORDER_COUNT paper trading orders in the last 24 hours"

        echo ""
        echo "Paper Trading Summary (Last 24 Hours):"
        psql "$DB_URL" -c "
            SELECT
                symbol,
                action,
                COUNT(*) as order_count,
                ROUND(AVG(executed_price / 100.0)::numeric, 2) as avg_price,
                SUM(pnl / 100.0) as total_pnl_usd
            FROM paper_trading_orders
            WHERE order_timestamp > NOW() - INTERVAL '24 hours'
            GROUP BY symbol, action
            ORDER BY symbol, action;
        "
    else
        print_warning "No paper trading orders found in the last 24 hours"
    fi
}

# Check model inference metrics
check_inference_metrics() {
    print_step "Checking model inference metrics..."

    INFERENCE_COUNT=$(psql "$DB_URL" -t -c "SELECT COUNT(*) FROM model_inference_metrics WHERE model_id = 'DQN_v1' AND inference_timestamp > NOW() - INTERVAL '1 hour';" | xargs)

    if [ "$INFERENCE_COUNT" -gt "0" ]; then
        print_success "Found $INFERENCE_COUNT inference records in the last hour"

        echo ""
        echo "Inference Metrics Summary:"
        psql "$DB_URL" -c "
            SELECT
                COUNT(*) as total_inferences,
                ROUND(AVG(inference_latency_us)::numeric, 2) as avg_latency_us,
                ROUND(AVG(prediction_confidence)::numeric, 4) as avg_confidence,
                SUM(CASE WHEN success THEN 1 ELSE 0 END) as successful,
                SUM(CASE WHEN NOT success THEN 1 ELSE 0 END) as failed,
                ROUND(AVG(gpu_memory_mb)::numeric, 2) as avg_gpu_mb
            FROM model_inference_metrics
            WHERE model_id = 'DQN_v1'
            AND inference_timestamp > NOW() - INTERVAL '1 hour';
        "
    else
        print_warning "No inference metrics found in the last hour"
    fi
}

# GPU status check
check_gpu_status() {
    print_step "Checking GPU status..."

    if command -v nvidia-smi &> /dev/null; then
        echo ""
        nvidia-smi --query-gpu=name,memory.total,memory.used,memory.free,utilization.gpu --format=csv
        print_success "GPU is available"
    else
        print_warning "nvidia-smi not found. GPU may not be available."
    fi
}

# Prometheus metrics check
check_prometheus_metrics() {
    print_step "Checking Prometheus metrics..."

    if curl -s http://localhost:9090/-/healthy > /dev/null 2>&1; then
        print_success "Prometheus is healthy"

        # Check if DQN metrics are being collected
        DQN_METRICS=$(curl -s "http://localhost:9090/api/v1/query?query=ml_model_predictions_total{model_id=\"DQN\"}" | jq -r '.data.result | length')

        if [ "$DQN_METRICS" -gt "0" ]; then
            print_success "DQN metrics are being collected in Prometheus"
        else
            print_warning "No DQN metrics found in Prometheus"
        fi
    else
        print_warning "Prometheus is not accessible"
    fi
}

# Generate validation report
generate_report() {
    print_step "Generating validation report..."

    REPORT_FILE="logs/staging/dqn_performance_validation_$(date +%Y%m%d_%H%M%S).txt"

    cat > "$REPORT_FILE" <<EOF
================================================================================
DQN MODEL PERFORMANCE VALIDATION REPORT
================================================================================

Validation Date: $(date -u +%Y-%m-%d\ %H:%M:%S\ UTC)
Model ID: DQN_v1
Environment: Staging
Target Latency: < $TARGET_LATENCY_US μs
Expected Latency: $EXPECTED_LATENCY_US μs

MODEL STATUS:
-------------
$(psql "$DB_URL" -t -c "SELECT model_id || ' - ' || status || ' (deployed: ' || deployment_date || ')' FROM ml_models WHERE model_id = 'DQN_v1';")

INFERENCE PERFORMANCE:
----------------------
$(psql "$DB_URL" -t -c "
SELECT
    'Samples: ' || COUNT(*) || E'\n' ||
    'Average Latency: ' || ROUND(AVG(inference_latency_us)::numeric, 2) || ' μs' || E'\n' ||
    'P50 Latency: ' || ROUND(PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY inference_latency_us)::numeric, 2) || ' μs' || E'\n' ||
    'P95 Latency: ' || ROUND(PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY inference_latency_us)::numeric, 2) || ' μs' || E'\n' ||
    'P99 Latency: ' || ROUND(PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY inference_latency_us)::numeric, 2) || ' μs' || E'\n' ||
    'Max Latency: ' || ROUND(MAX(inference_latency_us)::numeric, 2) || ' μs'
FROM ensemble_predictions
WHERE prediction_timestamp > NOW() - INTERVAL '1 hour'
AND inference_latency_us IS NOT NULL;
")

PAPER TRADING SUMMARY:
----------------------
$(psql "$DB_URL" -t -c "
SELECT
    'Total Orders (24h): ' || COUNT(*) || E'\n' ||
    'Total PnL: $' || ROUND((SUM(COALESCE(pnl, 0)) / 100.0)::numeric, 2)
FROM paper_trading_orders
WHERE order_timestamp > NOW() - INTERVAL '24 hours';
")

GPU STATUS:
-----------
$(nvidia-smi --query-gpu=name,memory.used,memory.total --format=csv,noheader 2>/dev/null || echo "GPU info not available")

VALIDATION RESULT:
------------------
$(psql "$DB_URL" -t -c "
SELECT CASE
    WHEN PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY inference_latency_us) < $TARGET_LATENCY_US
    THEN '✓ PASSED: P99 latency is below target ($TARGET_LATENCY_US μs)'
    ELSE '✗ FAILED: P99 latency exceeds target ($TARGET_LATENCY_US μs)'
END
FROM ensemble_predictions
WHERE prediction_timestamp > NOW() - INTERVAL '1 hour'
AND inference_latency_us IS NOT NULL;
" 2>/dev/null || echo "Insufficient data for validation")

================================================================================
EOF

    print_success "Validation report saved to: $REPORT_FILE"
    echo ""
    cat "$REPORT_FILE"
}

# Main validation flow
main() {
    print_header "DQN MODEL PERFORMANCE VALIDATION"

    validate_database
    check_model_registration
    check_recent_predictions
    measure_inference_latency
    check_paper_trading
    check_inference_metrics
    check_gpu_status
    check_prometheus_metrics
    generate_report

    echo ""
    print_success "✓ Performance validation completed"
    echo ""
}

main