#!/bin/bash
# Production Deployment Validation Script - Foxhunt HFT System
# Based on expert security and performance analysis
# 
# This script validates the system is ready for production deployment
# and checks for critical issues identified in expert review.

set -euo pipefail

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Counters
CRITICAL_ISSUES=0
HIGH_ISSUES=0
MEDIUM_ISSUES=0
WARNINGS=0

# Log function
log() {
    echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"
}

error() {
    echo -e "${RED}[ERROR]${NC} $1"
    ((CRITICAL_ISSUES++))
}

warning() {
    echo -e "${YELLOW}[WARNING]${NC} $1"
    ((WARNINGS++))
}

success() {
    echo -e "${GREEN}[SUCCESS]${NC} $1"
}

check_critical() {
    echo -e "${RED}[CRITICAL]${NC} $1"
    ((CRITICAL_ISSUES++))
}

check_high() {
    echo -e "${YELLOW}[HIGH]${NC} $1"
    ((HIGH_ISSUES++))
}

check_medium() {
    echo -e "${YELLOW}[MEDIUM]${NC} $1"
    ((MEDIUM_ISSUES++))
}

echo "=========================================="
echo "  Foxhunt HFT Production Validation"
echo "=========================================="
echo

# Check if we're in the right directory
if [[ ! -f "Cargo.toml" ]] || [[ ! -d "ml" ]] || [[ ! -d "risk" ]]; then
    error "Not in Foxhunt project root directory"
    exit 1
fi

log "Starting comprehensive production readiness validation..."

# =============================================================================
# CRITICAL ISSUE CHECKS - Based on Expert Analysis
# =============================================================================

echo
echo "🔴 CHECKING CRITICAL ISSUES (DEPLOYMENT BLOCKERS)"
echo "================================================="

# Check 1: Silent Health Monitoring
log "Checking ML metrics implementation..."
if grep -q "fn calculate_total_predictions(&self) -> u64 { 0 }" ml/src/observability/metrics.rs 2>/dev/null; then
    check_critical "ML metrics return hardcoded values - monitoring will always show 'healthy'"
    echo "   Fix: Implement real metric aggregation in ml/src/observability/metrics.rs:416-434"
elif grep -q "fn calculate_total_predictions(&self) -> u64 {" ml/src/observability/metrics.rs 2>/dev/null; then
    if grep -A 5 "fn calculate_total_predictions" ml/src/observability/metrics.rs | grep -q "self.predictions_total"; then
        success "ML metrics properly implemented"
    else
        check_critical "ML metrics may still be using dummy values"
    fi
else
    warning "Could not verify ML metrics implementation"
fi

# Check 2: High-frequency logging
log "Checking for performance-killing logs in hot paths..."
if grep -n "info!" risk/src/position_tracker.rs | grep -q "update.*position"; then
    check_critical "INFO logging in position update hot path - will spam millions of logs/second"
    echo "   Fix: Change to debug! in risk/src/position_tracker.rs position update functions"
else
    success "No high-frequency INFO logging detected in position tracker"
fi

# Check 3: O(n) position scanning
log "Checking position tracker performance..."
if grep -A 10 "update_market_data" risk/src/position_tracker.rs | grep -q "iter_mut()"; then
    check_critical "O(n) position scanning on every market tick - CPU bound under load"
    echo "   Fix: Implement instrument->position index in risk/src/position_tracker.rs:575-621"
else
    success "Position tracker appears to use efficient lookups"
fi

# =============================================================================
# HIGH PRIORITY CHECKS
# =============================================================================

echo
echo "🟡 CHECKING HIGH PRIORITY ISSUES"
echo "================================"

# Check 4: Metrics initialization race
log "Checking metrics initialization safety..."
if grep -A 10 "initialize_metrics" ml/src/observability/metrics.rs | grep -q "OnceCell\|is_some()"; then
    success "Metrics initialization is race-condition safe"
else
    check_high "Metrics initialization lacks race condition protection"
    echo "   Fix: Add initialization guard in ml/src/observability/metrics.rs:478-488"
fi

# Check 5: Secret generation security
log "Checking secret generation security..."
if [[ -f "scripts/generate-production-secrets.sh" ]]; then
    if grep -q "/tmp" scripts/generate-production-secrets.sh && ! grep -q "secure.*path\|custom.*path" scripts/generate-production-secrets.sh; then
        check_high "Secret generation uses insecure /tmp directory"
        echo "   Fix: Require explicit secure path in scripts/generate-production-secrets.sh"
    else
        success "Secret generation appears secure"
    fi
else
    warning "Secret generation script not found"
fi

# =============================================================================
# COMPILATION AND BUILD CHECKS
# =============================================================================

echo
echo "🔧 CHECKING SYSTEM COMPILATION"
echo "=============================="

log "Testing workspace compilation..."
if cargo check --workspace --all-targets >/dev/null 2>&1; then
    success "All modules compile successfully"
else
    error "Compilation failures detected - deployment blocked"
    echo "Run: cargo check --workspace --all-targets"
fi

log "Testing individual critical modules..."
for module in "ml" "risk" "foxhunt-core"; do
    if cargo check -p "$module" >/dev/null 2>&1; then
        success "Module '$module' compiles successfully"
    else
        error "Module '$module' compilation failed"
    fi
done

# =============================================================================
# SERVICE READINESS CHECKS
# =============================================================================

echo
echo "🔍 CHECKING SERVICE READINESS"
echo "============================="

# Check ML Training Service
log "Validating ML Training Service..."
if [[ -d "services/ml_training_service" ]]; then
    if [[ -f "services/ml_training_service/src/main.rs" ]] && \
       [[ -f "services/ml_training_service/src/service.rs" ]] && \
       [[ -f "services/ml_training_service/src/config.rs" ]]; then
        success "ML Training Service structure complete"
    else
        warning "ML Training Service missing core files"
    fi
else
    error "ML Training Service not found"
fi

# Check deployment scripts
log "Validating deployment infrastructure..."
if [[ -d "scripts" ]] || [[ -d "deployment/scripts" ]]; then
    success "Deployment scripts directory found"
else
    warning "Deployment scripts directory missing"
fi

# Check security documentation
log "Validating security documentation..."
if [[ -f "docs/SECURITY_INCIDENT_RESPONSE.md" ]]; then
    success "Security incident response documentation present"
else
    warning "Security incident response documentation missing"
fi

# =============================================================================
# PERFORMANCE AND MONITORING CHECKS
# =============================================================================

echo
echo "📊 CHECKING MONITORING AND PERFORMANCE"
echo "======================================"

# Check Prometheus integration
log "Validating Prometheus metrics integration..."
if grep -r "prometheus" ml/src/observability/metrics.rs >/dev/null 2>&1; then
    success "ML Prometheus metrics integration found"
else
    warning "ML Prometheus metrics not detected"
fi

if grep -r "prometheus" risk/src/position_tracker.rs >/dev/null 2>&1; then
    success "Risk Prometheus metrics integration found"
else
    warning "Risk Prometheus metrics not detected"
fi

# Check for performance tests
log "Checking performance validation..."
if [[ -f "tests/performance/critical_path_tests.rs" ]]; then
    success "Performance tests found"
else
    warning "Performance validation tests missing"
fi

# =============================================================================
# MEDIUM PRIORITY CHECKS
# =============================================================================

echo
echo "📝 CHECKING MEDIUM PRIORITY ISSUES"
echo "=================================="

# Check stress test math
log "Checking stress test percentile calculations..."
if grep -n "len.*99.*100" ml/src/stress_testing/mod.rs >/dev/null 2>&1; then
    check_medium "Potential percentile calculation off-by-one error"
    echo "   Fix: Use proper percentile calculation with bounds checking"
else
    success "Stress test calculations appear correct"
fi

# Check missing latency records
log "Checking inference latency recording..."
if grep -A 10 "record_inference_timing" ml/src/observability/metrics.rs | grep -q "Err.*record_failed_prediction" && \
   ! grep -A 10 "record_inference_timing" ml/src/observability/metrics.rs | grep -q "record_inference_latency.*Err"; then
    check_medium "Missing latency recording for failed inferences"
    echo "   Fix: Record latency for both success and failure cases"
else
    success "Inference latency recording appears complete"
fi

# =============================================================================
# FINAL SUMMARY
# =============================================================================

echo
echo "=========================================="
echo "         VALIDATION SUMMARY"
echo "=========================================="
echo

if [[ $CRITICAL_ISSUES -gt 0 ]]; then
    echo -e "${RED}❌ DEPLOYMENT BLOCKED${NC}"
    echo -e "Critical Issues: ${RED}$CRITICAL_ISSUES${NC}"
    echo -e "High Priority: ${YELLOW}$HIGH_ISSUES${NC}"
    echo -e "Medium Priority: ${YELLOW}$MEDIUM_ISSUES${NC}"
    echo -e "Warnings: ${YELLOW}$WARNINGS${NC}"
    echo
    echo "🚨 CRITICAL ISSUES MUST BE FIXED BEFORE DEPLOYMENT 🚨"
    echo
    echo "Next steps:"
    echo "1. Fix all critical issues above"
    echo "2. Re-run this validation script"
    echo "3. Review deployment checklist: deployment/production-deployment-checklist.md"
    exit 1
elif [[ $HIGH_ISSUES -gt 0 ]]; then
    echo -e "${YELLOW}⚠️ DEPLOYMENT NOT RECOMMENDED${NC}"
    echo -e "Critical Issues: ${GREEN}$CRITICAL_ISSUES${NC}"
    echo -e "High Priority: ${YELLOW}$HIGH_ISSUES${NC}"
    echo -e "Medium Priority: ${YELLOW}$MEDIUM_ISSUES${NC}"
    echo -e "Warnings: ${YELLOW}$WARNINGS${NC}"
    echo
    echo "🟡 HIGH PRIORITY ISSUES SHOULD BE FIXED BEFORE DEPLOYMENT"
    echo
    echo "Consider fixing high priority issues, then re-run validation"
    exit 2
else
    echo -e "${GREEN}✅ READY FOR DEPLOYMENT${NC}"
    echo -e "Critical Issues: ${GREEN}$CRITICAL_ISSUES${NC}"
    echo -e "High Priority: ${GREEN}$HIGH_ISSUES${NC}"
    echo -e "Medium Priority: ${YELLOW}$MEDIUM_ISSUES${NC}"
    echo -e "Warnings: ${YELLOW}$WARNINGS${NC}"
    echo
    echo "🎉 System is ready for production deployment!"
    echo
    echo "Next steps:"
    echo "1. Run full test suite: cargo test --workspace"
    echo "2. Build for production: cargo build --release"
    echo "3. Execute deployment: ./deployment/scripts/deploy-production.sh"
    exit 0
fi