foxhunt/deployment/scripts/log-pipeline.sh

#!/bin/bash
# Real-time log processing pipeline for Foxhunt HFT Trading System
# Aggregates logs, extracts metrics, and forwards to monitoring systems

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
LOG_PIPELINE_CONFIG="/etc/foxhunt/log-pipeline.conf"
FIFO_DIR="/tmp/foxhunt-logs"
METRICS_ENDPOINT="http://localhost:8080/metrics/logs"
LOKI_ENDPOINT="http://localhost:3100/loki/api/v1/push"

# Performance settings
BUFFER_SIZE=10000
BATCH_SIZE=100
FLUSH_INTERVAL=1

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] [LOG-PIPELINE] $1"
}

error() {
    echo -e "${RED}[ERROR]${NC} $1" >&2
}

success() {
    echo -e "${GREEN}[SUCCESS]${NC} $1"
}

warning() {
    echo -e "${YELLOW}[WARNING]${NC} $1"
}

cleanup() {
    log "Shutting down log pipeline..."
    
    # Clean up named pipes
    find "$FIFO_DIR" -name "*.fifo" -delete 2>/dev/null || true
    rmdir "$FIFO_DIR" 2>/dev/null || true
    
    # Kill background processes
    jobs -p | xargs -r kill 2>/dev/null || true
    
    exit "${1:-0}"
}

trap cleanup EXIT INT TERM

setup_fifos() {
    log "Setting up named pipes for log streaming..."
    
    mkdir -p "$FIFO_DIR"
    
    # Create FIFOs for each service
    local services=("core" "tli" "ml" "risk" "data")
    for service in "${services[@]}"; do
        local fifo_path="$FIFO_DIR/foxhunt-$service.fifo"
        mkfifo "$fifo_path" 2>/dev/null || true
        log "Created FIFO: $fifo_path"
    done
}

extract_latency_metrics() {
    local line="$1"
    
    # Extract latency from log lines (various formats)
    if echo "$line" | grep -q "order_latency"; then
        local latency
        latency=$(echo "$line" | grep -o '[0-9]\+μs\|[0-9]\+us\|latency.*[0-9]\+' | grep -o '[0-9]\+' | head -1)
        
        if [ -n "$latency" ]; then
            # Send to metrics endpoint
            curl -X POST "$METRICS_ENDPOINT" \
                 -H "Content-Type: application/json" \
                 -d "{\"metric\":\"latency\",\"value\":$latency,\"timestamp\":$(date +%s)}" \
                 --max-time 1 --silent &
        fi
    fi
}

extract_trading_metrics() {
    local line="$1"
    
    # Extract order information
    if echo "$line" | grep -q "order_filled\|order_executed"; then
        local order_id symbol quantity price
        
        order_id=$(echo "$line" | grep -o 'order_id[=:][[:space:]]*[^[:space:]]*' | cut -d'=' -f2 | cut -d':' -f2 | tr -d ' ')
        symbol=$(echo "$line" | grep -o 'symbol[=:][[:space:]]*[^[:space:]]*' | cut -d'=' -f2 | cut -d':' -f2 | tr -d ' ')
        quantity=$(echo "$line" | grep -o 'quantity[=:][[:space:]]*[0-9]*' | grep -o '[0-9]*')
        price=$(echo "$line" | grep -o 'price[=:][[:space:]]*[0-9.]*' | grep -o '[0-9.]*')
        
        if [ -n "$order_id" ]; then
            # Send trading metrics
            curl -X POST "$METRICS_ENDPOINT" \
                 -H "Content-Type: application/json" \
                 -d "{\"metric\":\"order_executed\",\"order_id\":\"$order_id\",\"symbol\":\"$symbol\",\"quantity\":$quantity,\"price\":$price,\"timestamp\":$(date +%s)}" \
                 --max-time 1 --silent &
        fi
    fi
}

extract_error_metrics() {
    local line="$1"
    
    # Extract error information
    if echo "$line" | grep -qE "ERROR|CRITICAL|FATAL"; then
        local error_type service_name
        
        error_type=$(echo "$line" | grep -oE "ERROR|CRITICAL|FATAL")
        service_name=$(echo "$line" | grep -o 'foxhunt-[a-z]*' | head -1)
        
        # Send error metrics
        curl -X POST "$METRICS_ENDPOINT" \
             -H "Content-Type: application/json" \
             -d "{\"metric\":\"error\",\"type\":\"$error_type\",\"service\":\"$service_name\",\"timestamp\":$(date +%s)}" \
             --max-time 1 --silent &
    fi
}

format_for_loki() {
    local line="$1"
    local service="$2"
    local timestamp="$3"
    
    # Format log line for Loki
    local json_line
    json_line=$(echo "$line" | jq -Rs . 2>/dev/null || echo "\"$line\"")
    
    cat << EOF
{
  "streams": [
    {
      "stream": {
        "service": "$service",
        "environment": "production",
        "job": "foxhunt-hft"
      },
      "values": [
        ["$timestamp", $json_line]
      ]
    }
  ]
}
EOF
}

send_to_loki() {
    local formatted_log="$1"
    
    # Send to Loki with retry logic
    local attempts=0
    local max_attempts=3
    
    while [ $attempts -lt $max_attempts ]; do
        if curl -X POST "$LOKI_ENDPOINT" \
                -H "Content-Type: application/json" \
                -d "$formatted_log" \
                --max-time 2 --silent; then
            return 0
        fi
        
        attempts=$((attempts + 1))
        sleep 0.1
    done
    
    return 1
}

process_log_line() {
    local line="$1"
    local service="$2"
    local timestamp_ns="$3"
    
    # Extract metrics from log line
    extract_latency_metrics "$line"
    extract_trading_metrics "$line"
    extract_error_metrics "$line"
    
    # Format and send to Loki
    local formatted_log
    formatted_log=$(format_for_loki "$line" "$service" "$timestamp_ns")
    send_to_loki "$formatted_log" &
}

stream_logs() {
    local service="$1"
    local log_file="$2"
    local fifo_path="$FIFO_DIR/foxhunt-$service.fifo"
    
    log "Starting log streaming for $service from $log_file"
    
    # Stream logs with high performance
    tail -F "$log_file" 2>/dev/null | \
    while IFS= read -r line; do
        local timestamp_ns=$(($(date +%s) * 1000000000))
        
        # Process line in background for high throughput
        process_log_line "$line" "$service" "$timestamp_ns" &
        
        # Rate limiting to prevent overwhelming
        local job_count
        job_count=$(jobs -r | wc -l)
        if [ "$job_count" -gt 50 ]; then
            wait
        fi
    done &
}

monitor_system_logs() {
    log "Setting up system log monitoring..."
    
    # Monitor syslog for system events
    tail -F /var/log/syslog 2>/dev/null | \
    grep "foxhunt" | \
    while IFS= read -r line; do
        local timestamp_ns=$(($(date +%s) * 1000000000))
        process_log_line "$line" "system" "$timestamp_ns" &
    done &
}

performance_monitor() {
    log "Starting performance monitoring loop..."
    
    while true; do
        # Collect performance metrics every second
        local timestamp=$(date +%s)
        
        # Memory usage
        local memory_usage
        memory_usage=$(ps -o %mem --no-headers -C foxhunt-core,foxhunt-tli,foxhunt-ml,foxhunt-risk,foxhunt-data | awk '{sum += $1} END {print sum}')
        
        # CPU usage
        local cpu_usage
        cpu_usage=$(ps -o %cpu --no-headers -C foxhunt-core,foxhunt-tli,foxhunt-ml,foxhunt-risk,foxhunt-data | awk '{sum += $1} END {print sum}')
        
        # Send performance metrics
        curl -X POST "$METRICS_ENDPOINT" \
             -H "Content-Type: application/json" \
             -d "{\"metric\":\"system_performance\",\"memory_pct\":$memory_usage,\"cpu_pct\":$cpu_usage,\"timestamp\":$timestamp}" \
             --max-time 1 --silent &
        
        sleep 1
    done &
}

main() {
    log "Starting Foxhunt log aggregation pipeline..."
    
    # Setup infrastructure
    setup_fifos
    
    # Start log streaming for each service
    local services=("core" "tli" "ml" "risk" "data")
    for service in "${services[@]}"; do
        local log_file="/home/jgrusewski/Work/foxhunt/logs/$service.log"
        
        if [ -f "$log_file" ]; then
            stream_logs "$service" "$log_file"
            success "Started log streaming for $service"
        else
            warning "Log file not found: $log_file"
        fi
    done
    
    # Start system monitoring
    monitor_system_logs
    performance_monitor
    
    success "Log pipeline fully operational"
    
    # Keep pipeline running
    while true; do
        sleep 10
        
        # Health check - ensure processes are still running
        local active_jobs
        active_jobs=$(jobs -r | wc -l)
        
        if [ "$active_jobs" -lt 3 ]; then
            warning "Some log streaming processes have stopped, restarting..."
            # Restart logic could go here
        fi
    done
}

# Execute main function
main "$@"