foxhunt/PERSISTENCE_PRODUCTION_DEPLOYMENT.md

Foxhunt Persistence Layer - Production Deployment Guide

🚀 Production-Ready Database Infrastructure

This guide provides comprehensive instructions for deploying the Foxhunt HFT persistence layer in production with sub-millisecond performance requirements.

📋 Prerequisites

Hardware Requirements

Minimum HFT Production Setup

• CPU: Intel Xeon Gold 6000+ series or AMD EPYC 7000+ series
• RAM: 32GB DDR4-3200+ (64GB recommended for full production)
• Storage: NVMe SSD (Samsung 980 PRO or Intel Optane recommended)
• Network: 10Gbps+ with <1ms latency to exchanges

Database Server Specifications

• PostgreSQL Server: 16+ cores, 64GB RAM, 2TB NVMe SSD
• InfluxDB Server: 8+ cores, 32GB RAM, 1TB NVMe SSD
• Redis Server: 4+ cores, 16GB RAM, 500GB NVMe SSD
• ClickHouse Server: 16+ cores, 128GB RAM, 4TB NVMe SSD (optional)

Software Requirements

• OS: Ubuntu 22.04 LTS or RHEL 9+
• PostgreSQL: 15+ with TimescaleDB extension
• InfluxDB: 2.7+
• Redis: 7.0+
• ClickHouse: 23.3+ (optional for analytics)

🗄️ Database Setup

1. PostgreSQL + TimescaleDB Setup

# Install PostgreSQL 15
sudo apt update
sudo apt install -y postgresql-15 postgresql-contrib-15

# Install TimescaleDB
echo "deb https://packagecloud.io/timescale/timescaledb/ubuntu/ jammy main" | sudo tee /etc/apt/sources.list.d/timescaledb.list
wget --quiet -O - https://packagecloud.io/timescale/timescaledb/gpgkey | sudo apt-key add -
sudo apt update
sudo apt install -y timescaledb-2-postgresql-15

# Tune PostgreSQL for HFT performance
sudo timescaledb-tune --quiet --yes

# Configure PostgreSQL for HFT
sudo tee -a /etc/postgresql/15/main/postgresql.conf << 'EOF'
# HFT Performance Optimizations
shared_buffers = 16GB                    # 25% of RAM
effective_cache_size = 48GB              # 75% of RAM
checkpoint_timeout = 15min
checkpoint_completion_target = 0.9
wal_buffers = 16MB
default_statistics_target = 100
random_page_cost = 1.1                   # SSD optimization
effective_io_concurrency = 200           # SSD optimization
work_mem = 256MB
maintenance_work_mem = 2GB
max_wal_size = 4GB
min_wal_size = 1GB
max_connections = 200

# HFT-specific settings
synchronous_commit = off                 # Async for speed
wal_writer_delay = 10ms                  # Fast WAL writes
commit_delay = 0                         # No artificial delays
commit_siblings = 5
tcp_keepalives_idle = 60
tcp_keepalives_interval = 10
tcp_keepalives_count = 3

# Enable TimescaleDB
shared_preload_libraries = 'timescaledb'
EOF

# Restart PostgreSQL
sudo systemctl restart postgresql
sudo systemctl enable postgresql

# Create trading database and user
sudo -u postgres psql << 'EOF'
CREATE DATABASE foxhunt_production;
CREATE USER foxhunt_prod WITH ENCRYPTED PASSWORD 'CHANGE_THIS_PASSWORD';
GRANT ALL PRIVILEGES ON DATABASE foxhunt_production TO foxhunt_prod;
ALTER USER foxhunt_prod CREATEDB;
\c foxhunt_production
CREATE EXTENSION IF NOT EXISTS timescaledb;
CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
EOF

2. InfluxDB Setup

# Install InfluxDB 2.7+
wget -q https://repos.influxdata.com/influxdata-archive_compat.key
echo '393e8779c89ac8d958f81f942f9ad7fb82a25e133faddaf92e15b16e6ac9ce4c influxdata-archive_compat.key' | sha256sum -c && cat influxdata-archive_compat.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg > /dev/null
echo 'deb [signed-by=/etc/apt/trusted.gpg.d/influxdata-archive_compat.gpg] https://repos.influxdata.com/debian stable main' | sudo tee /etc/apt/sources.list.d/influxdata.list

sudo apt update
sudo apt install -y influxdb2

# Configure InfluxDB for HFT performance
sudo tee /etc/influxdb/config.toml << 'EOF'
[meta]
  dir = "/var/lib/influxdb/meta"

[data]
  dir = "/var/lib/influxdb/data"
  wal-dir = "/var/lib/influxdb/wal"
  
  # HFT Performance optimizations
  cache-max-memory-size = "8g"
  cache-snapshot-memory-size = "256m"
  cache-snapshot-write-cold-duration = "10m"
  compact-full-write-cold-duration = "4h"
  max-concurrent-compactions = 8
  max-index-log-file-size = "1m"
  
[coordinator]
  write-timeout = "1s"
  max-concurrent-queries = 100
  query-timeout = "30s"
  log-queries-after = "5s"

[retention]
  enabled = true
  check-interval = "30m"

[http]
  enabled = true
  bind-address = ":8086"
  max-body-size = "25MB"
  max-concurrent-write-limit = 1000
  max-enqueued-write-limit = 10000
  enqueued-write-timeout = "30s"
EOF

# Start InfluxDB
sudo systemctl start influxdb
sudo systemctl enable influxdb

# Setup InfluxDB (interactive setup)
influx setup

3. Redis Setup

# Install Redis 7.0+
sudo apt install -y redis-server

# Configure Redis for HFT performance
sudo tee /etc/redis/redis.conf << 'EOF'
# Network and connections
bind 127.0.0.1
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 300
maxclients 10000

# Memory and persistence
maxmemory 8gb
maxmemory-policy allkeys-lru
save 900 1
save 300 10
save 60 10000

# Performance optimizations
# Disable slow operations in production
rename-command FLUSHDB ""
rename-command FLUSHALL ""
rename-command DEBUG ""

# Enable AOF for durability
appendonly yes
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb

# HFT-specific optimizations
hz 100
dynamic-hz yes
rdbcompression yes
rdbchecksum yes
stop-writes-on-bgsave-error yes

# Logging
loglevel notice
logfile /var/log/redis/redis-server.log
syslog-enabled yes
EOF

# Start Redis
sudo systemctl start redis-server
sudo systemctl enable redis-server

4. ClickHouse Setup (Optional - Analytics)

# Install ClickHouse
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754
echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee /etc/apt/sources.list.d/clickhouse.list
sudo apt update
sudo apt install -y clickhouse-server clickhouse-client

# Configure ClickHouse for analytics workload
sudo tee /etc/clickhouse-server/config.xml << 'EOF'
<?xml version="1.0"?>
<clickhouse>
    <logger>
        <level>warning</level>
        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
        <size>1000M</size>
        <count>10</count>
    </logger>
    
    <http_port>8123</http_port>
    <tcp_port>9000</tcp_port>
    
    <max_connections>4096</max_connections>
    <keep_alive_timeout>3</keep_alive_timeout>
    <max_concurrent_queries>100</max_concurrent_queries>
    <max_server_memory_usage>0</max_server_memory_usage>
    <max_thread_pool_size>10000</max_thread_pool_size>
    
    <users>
        <default>
            <password></password>
            <networks incl="networks" replace="replace">
                <ip>::/0</ip>
            </networks>
            <profile>default</profile>
            <quota>default</quota>
        </default>
    </users>
    
    <profiles>
        <default>
            <max_memory_usage>10000000000</max_memory_usage>
            <use_uncompressed_cache>0</use_uncompressed_cache>
            <load_balancing>in_order</load_balancing>
        </default>
    </profiles>
    
    <quotas>
        <default>
            <interval>
                <duration>3600</duration>
                <queries>0</queries>
                <errors>0</errors>
                <result_rows>0</result_rows>
                <read_rows>0</read_rows>
                <execution_time>0</execution_time>
            </interval>
        </default>
    </quotas>
</clickhouse>
EOF

# Start ClickHouse
sudo systemctl start clickhouse-server
sudo systemctl enable clickhouse-server

⚙️ Application Configuration

1. Environment Variables

Create production environment file:

# Create secure environment file
sudo tee /etc/foxhunt/production.env << 'EOF'
# Environment
ENVIRONMENT=production
RUST_LOG=info,foxhunt=debug

# PostgreSQL Configuration (HFT Optimized)
POSTGRES_URL=postgresql://foxhunt_prod:SECURE_PASSWORD@localhost:5432/foxhunt_production
POSTGRES_POOL_MAX=100
POSTGRES_POOL_MIN=20
POSTGRES_QUERY_TIMEOUT_MICROS=800    # <1ms for HFT
POSTGRES_CONNECT_TIMEOUT_MS=100
POSTGRES_ACQUIRE_TIMEOUT_MS=50

# Redis Configuration (HFT Optimized)
REDIS_URL=redis://localhost:6379
REDIS_POOL_SIZE=50
REDIS_MIN_CONNECTIONS=10
REDIS_COMMAND_TIMEOUT_MICROS=500     # <1ms for HFT
REDIS_CONNECT_TIMEOUT_MS=100

# InfluxDB Configuration
INFLUXDB_URL=http://localhost:8086
INFLUXDB_ORG=foxhunt
INFLUXDB_BUCKET=market_data_prod
INFLUXDB_TOKEN=YOUR_INFLUX_TOKEN_HERE

# ClickHouse Configuration (Optional)
CLICKHOUSE_URL=http://localhost:8123
CLICKHOUSE_DATABASE=foxhunt_analytics
CLICKHOUSE_USERNAME=default
CLICKHOUSE_PASSWORD=

# Backup Configuration
BACKUP_DIRECTORY=/var/backups/foxhunt
MIGRATIONS_PATH=/opt/foxhunt/migrations

# Performance Settings
MAX_QUERY_LATENCY_MICROS=800
ENABLE_QUERY_LOGGING=true
ENABLE_POOL_MONITORING=true
ENABLE_HEALTH_CHECKS=true
HEALTH_CHECK_INTERVAL_SECONDS=30
EOF

# Secure the environment file
sudo chmod 600 /etc/foxhunt/production.env
sudo chown foxhunt:foxhunt /etc/foxhunt/production.env

2. Database Configuration

Copy the HFT-optimized configuration:

sudo mkdir -p /etc/foxhunt/config
sudo cp /home/jgrusewski/Work/foxhunt/config/database/database-hft-optimized.toml /etc/foxhunt/config/

3. Migration Setup

# Copy migration files
sudo mkdir -p /opt/foxhunt/migrations
sudo cp -r /home/jgrusewski/Work/foxhunt/migrations/* /opt/foxhunt/migrations/
sudo chown -R foxhunt:foxhunt /opt/foxhunt/migrations

🚀 Deployment Steps

1. Create System User

# Create foxhunt user
sudo useradd -r -m -s /bin/bash foxhunt
sudo usermod -a -G postgres foxhunt

# Create necessary directories
sudo mkdir -p /opt/foxhunt/{bin,config,logs,backups}
sudo mkdir -p /var/log/foxhunt
sudo mkdir -p /var/lib/foxhunt
sudo chown -R foxhunt:foxhunt /opt/foxhunt /var/log/foxhunt /var/lib/foxhunt

2. Build and Install Application

# Build optimized release
cd /home/jgrusewski/Work/foxhunt
cargo build --release --features="persistence,influxdb-support,clickhouse-support"

# Install binary
sudo cp target/release/foxhunt-tli /opt/foxhunt/bin/
sudo chown foxhunt:foxhunt /opt/foxhunt/bin/foxhunt-tli
sudo chmod +x /opt/foxhunt/bin/foxhunt-tli

3. Run Database Migrations

# Switch to foxhunt user and run migrations
sudo -u foxhunt bash << 'EOF'
source /etc/foxhunt/production.env
/opt/foxhunt/bin/foxhunt-tli migrate
EOF

4. Create Systemd Service

sudo tee /etc/systemd/system/foxhunt-persistence.service << 'EOF'
[Unit]
Description=Foxhunt HFT Trading System - Persistence Layer
After=network.target postgresql.service redis-server.service influxdb.service
Wants=postgresql.service redis-server.service influxdb.service

[Service]
Type=notify
User=foxhunt
Group=foxhunt
WorkingDirectory=/opt/foxhunt
ExecStart=/opt/foxhunt/bin/foxhunt-tli server
EnvironmentFile=/etc/foxhunt/production.env
Restart=always
RestartSec=10
LimitNOFILE=65536
LimitMEMLOCK=infinity

# Security settings
NoNewPrivileges=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths=/opt/foxhunt /var/log/foxhunt /var/lib/foxhunt /tmp

# Performance settings
Nice=-10
IOSchedulingClass=1
IOSchedulingPriority=4

[Install]
WantedBy=multi-user.target
EOF

# Enable and start service
sudo systemctl daemon-reload
sudo systemctl enable foxhunt-persistence
sudo systemctl start foxhunt-persistence

📊 Performance Validation

1. Database Performance Tests

# PostgreSQL performance test
sudo -u foxhunt psql -d foxhunt_production << 'EOF'
-- Test query performance
EXPLAIN (ANALYZE, BUFFERS) 
SELECT * FROM market_data 
WHERE symbol = 'AAPL' AND timestamp > NOW() - INTERVAL '1 hour'
LIMIT 1000;

-- Check timing
\timing on
SELECT COUNT(*) FROM market_data;
\timing off
EOF

# Redis performance test
redis-cli --latency-history -i 1

# InfluxDB performance test
influx query --org foxhunt --token $INFLUXDB_TOKEN '
from(bucket: "market_data_prod")
  |> range(start: -1h)
  |> filter(fn: (r) => r._measurement == "tick_data")
  |> count()
'

2. Connection Pool Monitoring

# Monitor PostgreSQL connections
sudo -u postgres psql -c "
SELECT 
    state,
    COUNT(*) as connections
FROM pg_stat_activity 
WHERE datname = 'foxhunt_production'
GROUP BY state;
"

# Monitor Redis connections
redis-cli info clients

# Monitor system resources
sudo apt install -y htop iotop nethogs
htop  # CPU and memory usage
iotop # Disk I/O
nethogs # Network usage

3. Latency Validation

# Test application latency
sudo -u foxhunt /opt/foxhunt/bin/foxhunt-tli benchmark --test-type=persistence

# Monitor application logs
sudo journalctl -u foxhunt-persistence -f

# Check health status
curl -s http://localhost:8080/health | jq '.'

🔒 Security Hardening

1. Database Security

# PostgreSQL security
sudo -u postgres psql << 'EOF'
-- Remove default postgres user network access
ALTER USER postgres PASSWORD 'SECURE_POSTGRES_PASSWORD';

-- Create read-only monitoring user
CREATE USER foxhunt_monitor WITH PASSWORD 'SECURE_MONITOR_PASSWORD';
GRANT CONNECT ON DATABASE foxhunt_production TO foxhunt_monitor;
GRANT USAGE ON SCHEMA public TO foxhunt_monitor;
GRANT SELECT ON ALL TABLES IN SCHEMA public TO foxhunt_monitor;
EOF

# Redis security
echo "requirepass SECURE_REDIS_PASSWORD" | sudo tee -a /etc/redis/redis.conf
sudo systemctl restart redis-server

# InfluxDB security - create tokens with specific permissions
influx auth create --org foxhunt --description "Trading Service" --read-buckets --write-buckets

2. Network Security

# Configure firewall
sudo ufw allow from 10.0.0.0/24 to any port 5432  # PostgreSQL
sudo ufw allow from 10.0.0.0/24 to any port 6379  # Redis
sudo ufw allow from 10.0.0.0/24 to any port 8086  # InfluxDB
sudo ufw allow from 10.0.0.0/24 to any port 8123  # ClickHouse
sudo ufw --force enable

3. SSL/TLS Configuration

# Generate certificates for PostgreSQL
sudo -u postgres openssl req -new -x509 -days 365 -nodes -text \
    -out /etc/ssl/certs/postgresql.crt \
    -keyout /etc/ssl/private/postgresql.key \
    -subj "/CN=foxhunt-db"

sudo chown postgres:postgres /etc/ssl/private/postgresql.key
sudo chmod 600 /etc/ssl/private/postgresql.key

# Enable SSL in PostgreSQL
echo "ssl = on" | sudo tee -a /etc/postgresql/15/main/postgresql.conf
sudo systemctl restart postgresql

📈 Monitoring and Alerting

1. Setup Prometheus Monitoring

# Install Prometheus
sudo useradd --no-create-home --shell /bin/false prometheus
sudo mkdir /etc/prometheus /var/lib/prometheus
sudo chown prometheus:prometheus /etc/prometheus /var/lib/prometheus

# Download and install
cd /tmp
wget https://github.com/prometheus/prometheus/releases/download/v2.40.0/prometheus-2.40.0.linux-amd64.tar.gz
tar xvf prometheus-2.40.0.linux-amd64.tar.gz
sudo cp prometheus-2.40.0.linux-amd64/prometheus /usr/local/bin/
sudo cp prometheus-2.40.0.linux-amd64/promtool /usr/local/bin/
sudo chown prometheus:prometheus /usr/local/bin/prometheus /usr/local/bin/promtool

# Configure Prometheus
sudo tee /etc/prometheus/prometheus.yml << 'EOF'
global:
  scrape_interval: 1s
  evaluation_interval: 1s

scrape_configs:
  - job_name: 'foxhunt-persistence'
    static_configs:
      - targets: ['localhost:8080']
    scrape_interval: 1s
    metrics_path: /metrics

  - job_name: 'postgres'
    static_configs:
      - targets: ['localhost:9187']

  - job_name: 'redis'
    static_configs:
      - targets: ['localhost:9121']
EOF

2. Setup Grafana Dashboards

# Install Grafana
sudo apt-get install -y software-properties-common
sudo add-apt-repository "deb https://packages.grafana.com/oss/deb stable main"
wget -q -O - https://packages.grafana.com/gpg.key | sudo apt-key add -
sudo apt-get update
sudo apt-get install -y grafana

# Start Grafana
sudo systemctl start grafana-server
sudo systemctl enable grafana-server

# Grafana will be available at http://localhost:3000
# Default login: admin/admin

🔄 Backup and Recovery

1. Automated Backup Script

sudo tee /opt/foxhunt/bin/backup.sh << 'EOF'
#!/bin/bash
set -euo pipefail

# Load environment
source /etc/foxhunt/production.env

# Create timestamp
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
BACKUP_DIR="/var/backups/foxhunt/backup_${TIMESTAMP}"
mkdir -p "$BACKUP_DIR"

# PostgreSQL backup
pg_dump "$POSTGRES_URL" --format=custom --file="$BACKUP_DIR/postgresql_dump.sql"

# Redis backup  
redis-cli --rdb "$BACKUP_DIR/redis_dump.rdb"

# InfluxDB backup
influx backup --org foxhunt --token "$INFLUXDB_TOKEN" "$BACKUP_DIR/influxdb_backup"

# Configuration backup
tar -czf "$BACKUP_DIR/configuration.tar.gz" /etc/foxhunt /opt/foxhunt/migrations

# Create backup metadata
cat > "$BACKUP_DIR/backup_metadata.json" << JSON
{
  "timestamp": "$(date -Iseconds)",
  "backup_id": "backup_${TIMESTAMP}",
  "components": ["postgresql", "redis", "influxdb", "configuration"],
  "environment": "production"
}
JSON

echo "Backup completed: $BACKUP_DIR"
EOF

sudo chmod +x /opt/foxhunt/bin/backup.sh
sudo chown foxhunt:foxhunt /opt/foxhunt/bin/backup.sh

2. Setup Cron for Automated Backups

# Add to foxhunt user crontab
sudo -u foxhunt crontab << 'EOF'
# Daily backup at 2 AM
0 2 * * * /opt/foxhunt/bin/backup.sh >> /var/log/foxhunt/backup.log 2>&1

# Health check every minute
* * * * * curl -s http://localhost:8080/health > /dev/null || echo "Health check failed at $(date)" >> /var/log/foxhunt/health.log
EOF

✅ Production Checklist

Pre-Deployment

• Hardware meets HFT requirements
• All databases installed and configured
• Network latency tested (<1ms to exchanges)
• Security hardening completed
• SSL certificates configured
• Monitoring setup completed

Deployment

• Application built with release optimizations
• Database migrations executed successfully
• Environment variables configured
• Systemd service created and enabled
• Firewall rules configured
• Backup procedures tested

Post-Deployment Validation

• Database query latency <1ms verified
• Connection pools functioning correctly
• Health checks passing
• Monitoring dashboards operational
• Backup procedures validated
• Security audit completed
• Performance benchmarks met

🆘 Troubleshooting

Common Issues

1. High Query Latency

   # Check PostgreSQL slow queries
   SELECT query, mean_exec_time, calls 
   FROM pg_stat_statements 
   ORDER BY mean_exec_time DESC LIMIT 10;
   
   # Check connection pool status
   curl http://localhost:8080/metrics | grep pool
   

2. Connection Pool Exhaustion

   # Monitor pool usage
   SELECT state, COUNT(*) FROM pg_stat_activity GROUP BY state;
   
   # Check application logs
   journalctl -u foxhunt-persistence --since "10 minutes ago"
   

3. Memory Issues

   # Check memory usage
   free -h
   
   # Check PostgreSQL memory
   SELECT name, setting, unit FROM pg_settings WHERE name LIKE '%memory%';
   

Emergency Procedures

1. Database Recovery

   # Stop application
   sudo systemctl stop foxhunt-persistence
   
   # Restore from backup
   pg_restore -d foxhunt_production /var/backups/foxhunt/latest/postgresql_dump.sql
   
   # Restart application
   sudo systemctl start foxhunt-persistence
   

2. Performance Degradation

   # Enable detailed logging
   sudo sed -i 's/RUST_LOG=info/RUST_LOG=debug/' /etc/foxhunt/production.env
   sudo systemctl restart foxhunt-persistence
   
   # Monitor real-time performance
   watch -n 1 'curl -s http://localhost:8080/metrics | grep latency'
   

📞 Support

For production support:

• Logs: /var/log/foxhunt/ and journalctl -u foxhunt-persistence
• Metrics: http://localhost:8080/metrics
• Health: http://localhost:8080/health
• Configuration: /etc/foxhunt/

---

⚠️ CRITICAL: Always test deployment procedures in staging environment before applying to production. HFT systems require zero downtime and sub-millisecond performance.