# ====================================================================== # DATABASE OPTIMIZATION CONFIGURATION FOR HFT SYSTEMS # ====================================================================== # Optimized connection pools and performance settings for ultra-low latency [postgresql] # Primary transactional database optimized for HFT # Connection Pool Optimization pool_size = 50 # Optimal for high-throughput trading min_pool_size = 20 # Always keep warm connections max_pool_size = 100 # Scale under heavy load connection_timeout_seconds = 2 # Fast fail for HFT requirements idle_timeout_seconds = 300 # 5 minutes idle timeout max_lifetime_seconds = 3600 # 1 hour connection lifetime # Query Performance Optimization query_timeout_ms = 1000 # 1ms timeout for HFT queries statement_timeout_ms = 5000 # 5ms for complex queries prepared_statement_cache_size = 2000 # Cache prepared statements enable_query_plan_cache = true # Enable plan caching max_prepared_statements = 1000 # Limit prepared statements # HFT-Specific Optimizations enable_synchronous_commit = false # Async commit for speed (risk vs performance) wal_buffers_mb = 64 # Large WAL buffers shared_buffers_mb = 2048 # 2GB shared buffers work_mem_mb = 256 # 256MB work memory maintenance_work_mem_mb = 512 # 512MB maintenance memory # Connection Pool Behavior pool_pre_ping = true # Validate connections before use pool_recycle_seconds = 3600 # Recycle connections hourly enable_pool_overflow = true # Allow temporary overflow overflow_size = 20 # Additional overflow connections [redis] # High-speed cache optimized for sub-millisecond access # Connection Pool pool_size = 30 # Sufficient for high-frequency access min_pool_size = 10 # Minimum warm connections max_pool_size = 50 # Scale for burst traffic connection_timeout_ms = 500 # 0.5ms connection timeout socket_timeout_ms = 100 # 0.1ms socket timeout # Performance Settings enable_pipelining = true # Batch Redis commands pipeline_buffer_size = 1000 # Large pipeline buffer max_connections_per_pool = 10 # Connections per pool instance enable_connection_multiplexing = true # Share connections efficiently # Memory Optimization enable_compression = false # Disable compression for speed memory_policy = "allkeys-lru" # LRU eviction policy max_memory_mb = 8192 # 8GB memory limit # Clustering (if enabled) enable_cluster_mode = false # Single instance for low latency cluster_retry_attempts = 3 # Retry attempts for cluster cluster_retry_delay_ms = 10 # Fast retry delay [influxdb] # Time-series database for market data and analytics # Connection Settings pool_size = 20 # Moderate pool for time-series writes connection_timeout_seconds = 3 # 3 second timeout request_timeout_seconds = 10 # 10 second request timeout # Write Optimization batch_size = 10000 # Large batch sizes for efficiency flush_interval_ms = 100 # 100ms flush interval for real-time max_retries = 3 # Retry failed writes retry_interval_ms = 100 # 100ms retry interval # Query Performance enable_chunked_responses = true # Handle large result sets chunk_size = 10000 # Chunk size for large queries max_series_per_request = 1000 # Limit series per request # Retention and Compression default_retention_policy = "30d" # 30 days retention enable_compression = true # Enable compression for storage compression_level = 6 # Moderate compression [clickhouse] # Analytics database for complex queries and reporting # Connection Pool pool_size = 15 # Moderate pool for analytics queries max_pool_size = 30 # Allow scaling for complex queries connection_timeout_seconds = 5 # 5 second connection timeout query_timeout_seconds = 60 # 60 second query timeout # Query Optimization max_memory_usage_mb = 4096 # 4GB memory per query max_threads = 8 # 8 threads per query max_execution_time_seconds = 300 # 5 minute max execution enable_distributed_queries = true # Enable distributed processing # Insert Performance max_insert_block_size = 1048576 # 1MB insert blocks min_insert_block_size_rows = 1000 # Minimum rows per block enable_async_insert = true # Asynchronous inserts async_insert_timeout_ms = 1000 # 1 second async timeout # Compression and Storage enable_compression = true # Enable compression compression_method = "lz4" # Fast LZ4 compression enable_ttl = true # Enable TTL for data lifecycle # HFT-Specific Database Configurations [hft_optimizations] # Ultra-low latency optimizations enable_connection_warming = true # Pre-warm connections on startup connection_validation_query = "SELECT 1" # Fast validation query enable_connection_health_checks = true # Monitor connection health health_check_interval_seconds = 30 # Health check frequency # Memory Management enable_huge_pages = true # Use huge pages for performance buffer_pool_size_ratio = 0.8 # 80% of RAM for buffer pools enable_numa_awareness = true # NUMA-aware memory allocation # Network Optimization tcp_keepalive_time = 600 # 10 minutes keepalive tcp_keepalive_interval = 60 # 1 minute keepalive interval tcp_keepalive_probes = 3 # 3 keepalive probes enable_tcp_nodelay = true # Disable Nagle's algorithm # Monitoring and Observability [monitoring] enable_connection_pool_metrics = true # Monitor pool statistics enable_query_performance_tracking = true # Track query performance enable_slow_query_logging = true # Log slow queries slow_query_threshold_ms = 100 # 100ms slow query threshold # Pool Statistics Collection pool_stats_collection_interval_seconds = 10 # Collect stats every 10 seconds enable_connection_lifecycle_tracking = true # Track connection lifecycle enable_deadlock_detection = true # Monitor for deadlocks # Environment-Specific Overrides [environments.production] # Production-specific overrides postgresql.pool_size = 100 # Larger pool for production redis.pool_size = 50 # More Redis connections influxdb.batch_size = 20000 # Larger batches in production clickhouse.pool_size = 25 # More analytics connections [environments.development] # Development-specific settings (smaller pools) postgresql.pool_size = 10 redis.pool_size = 5 influxdb.batch_size = 1000 clickhouse.pool_size = 5 [environments.testing] # Testing environment settings postgresql.pool_size = 5 redis.pool_size = 3 influxdb.batch_size = 100 clickhouse.pool_size = 2