Files
foxhunt/crates/database/tests/connection_pool_tests.rs
jgrusewski 9c3d741a08 refactor: restructure repo — crates/, bin/, testing/ layout
Move 17 library crates into crates/, CLI binary into bin/fxt,
consolidate 10 test crates into testing/, split config crate
from deployment config files.

Root directory reduced from 38+ to ~17 directories.
All Cargo.toml paths and build.rs proto refs updated.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-25 11:56:00 +01:00

838 lines
25 KiB
Rust

//! Comprehensive connection pool edge case tests
//!
//! These tests cover connection pooling behavior including:
//! - Pool exhaustion scenarios
//! - Connection timeouts
//! - Connection leaks and recovery
//! - Concurrent access patterns
//! - Connection lifecycle (max lifetime, idle timeout)
//! - Failure scenarios and recovery
//! - Transaction rollback and cleanup
//!
//! Requires a running PostgreSQL database:
//! ```bash
//! docker-compose up -d postgres
//! ```
use config::database::PoolConfig;
use database::{DatabaseError, DatabasePool};
use futures::stream::StreamExt;
use sqlx::Acquire;
use std::sync::Arc;
use std::time::Duration;
use tokio::time::{sleep, timeout};
/// Helper to create a test pool configuration
fn create_test_pool_config(max_connections: u32, min_connections: u32) -> PoolConfig {
PoolConfig {
max_connections,
min_connections,
acquire_timeout_secs: 5,
idle_timeout_secs: 30,
max_lifetime_secs: 60,
test_before_acquire: true,
database_url: "postgresql://foxhunt:foxhunt_dev_password@localhost:5432/foxhunt"
.to_string(),
health_check_enabled: false, // Disable for deterministic testing
health_check_interval_secs: 60,
}
}
/// Helper to create a pool for testing
async fn create_test_pool(
max_connections: u32,
min_connections: u32,
) -> Result<DatabasePool, DatabaseError> {
let config = create_test_pool_config(max_connections, min_connections);
DatabasePool::new(config).await
}
#[cfg(test)]
mod pool_exhaustion_tests {
use super::*;
#[tokio::test]
async fn test_pool_exhaustion_with_small_pool() {
let pool = create_test_pool(2, 1).await.expect("Pool creation failed");
// Acquire all available connections
let conn1 = pool.acquire().await.expect("First connection failed");
let conn2 = pool.acquire().await.expect("Second connection failed");
// Next request should timeout
let result = timeout(Duration::from_millis(100), pool.acquire()).await;
assert!(result.is_err(), "Should timeout when pool is exhausted");
// Return one connection to pool
drop(conn1);
// Should now succeed
let conn3 = pool.acquire().await;
assert!(conn3.is_ok(), "Should succeed after connection returned");
drop(conn2);
drop(conn3);
pool.close().await;
}
#[tokio::test]
async fn test_pool_exhaustion_recovery() {
let pool = create_test_pool(3, 1).await.expect("Pool creation failed");
// Take all connections
let tasks = futures::stream::FuturesUnordered::new();
for _ in 0..3 {
tasks.push(pool.acquire());
}
let conns: Vec<_> = tasks.collect::<Vec<_>>().await;
assert_eq!(conns.len(), 3);
assert!(conns.iter().all(|c| c.is_ok()));
// Pool should be exhausted
let result = timeout(Duration::from_millis(100), pool.acquire()).await;
assert!(result.is_err());
// Release all connections
drop(conns);
// Pool should recover immediately
for _ in 0..3 {
let conn = pool.acquire().await;
assert!(
conn.is_ok(),
"Pool should recover after all connections returned"
);
}
pool.close().await;
}
#[tokio::test]
async fn test_minimum_pool_size_single_connection() {
// Test with minimum possible pool size
let pool = create_test_pool(1, 1).await.expect("Pool creation failed");
let conn1 = pool.acquire().await.expect("Should get connection");
// Second request should timeout
let result = timeout(Duration::from_millis(100), pool.acquire()).await;
assert!(
result.is_err(),
"Should timeout with single connection pool"
);
drop(conn1);
// Should succeed after release
let conn2 = pool.acquire().await;
assert!(conn2.is_ok());
drop(conn2);
pool.close().await;
}
#[tokio::test]
async fn test_large_pool_no_exhaustion() {
// Test with large pool size
let pool = create_test_pool(100, 10)
.await
.expect("Pool creation failed");
// Take many connections without exhaustion
let mut conns = Vec::new();
for _ in 0..50 {
let conn = pool.acquire().await.expect("Should not exhaust large pool");
conns.push(conn);
}
// Should still have capacity
let extra = pool.acquire().await;
assert!(extra.is_ok(), "Large pool should still have capacity");
drop(conns);
drop(extra);
pool.close().await;
}
}
#[cfg(test)]
mod concurrent_access_tests {
use super::*;
#[tokio::test]
async fn test_concurrent_connection_requests() {
let pool = Arc::new(create_test_pool(10, 2).await.expect("Pool creation failed"));
let mut handles = vec![];
// Spawn 50 concurrent tasks requesting connections
for i in 0..50 {
let pool_clone = pool.clone();
let handle = tokio::spawn(async move {
let _conn = pool_clone
.acquire()
.await
.expect(&format!("Task {} failed to acquire connection", i));
// Hold connection briefly
sleep(Duration::from_millis(10)).await;
});
handles.push(handle);
}
// All tasks should complete successfully
for handle in handles {
handle.await.expect("Task should complete");
}
pool.close().await;
}
#[tokio::test]
async fn test_high_concurrency_stress() {
let pool = Arc::new(create_test_pool(20, 5).await.expect("Pool creation failed"));
let mut handles = vec![];
// Spawn 200 tasks with high contention
for i in 0..200 {
let pool_clone = pool.clone();
let handle = tokio::spawn(async move {
let _conn = pool_clone
.acquire()
.await
.expect(&format!("Task {} failed to acquire", i));
// Very brief hold time to create high turnover
sleep(Duration::from_millis(1)).await;
});
handles.push(handle);
}
for handle in handles {
handle.await.expect("High concurrency task should complete");
}
pool.close().await;
}
#[tokio::test]
async fn test_concurrent_with_timeouts() {
let pool = Arc::new(create_test_pool(5, 1).await.expect("Pool creation failed"));
let mut handles = vec![];
// Spawn more tasks than pool capacity
for i in 0..15 {
let pool_clone = pool.clone();
let handle = tokio::spawn(async move {
// Some tasks will timeout, others succeed
let result = timeout(Duration::from_millis(500), pool_clone.acquire()).await;
if result.is_ok() && result.unwrap().is_ok() {
sleep(Duration::from_millis(100)).await;
}
// Don't panic on timeout - it's expected
i
});
handles.push(handle);
}
// Count successful vs timed out
let mut succeeded = 0;
for handle in handles {
let _ = handle.await.expect("Task should complete");
succeeded += 1;
}
assert_eq!(succeeded, 15, "All tasks should complete");
pool.close().await;
}
#[tokio::test]
async fn test_sequential_vs_concurrent_performance() {
let pool = create_test_pool(10, 2).await.expect("Pool creation failed");
// Sequential acquisitions
let sequential_start = std::time::Instant::now();
for _ in 0..20 {
let _conn = pool.acquire().await.expect("Sequential acquire failed");
sleep(Duration::from_millis(5)).await;
}
let sequential_duration = sequential_start.elapsed();
// Reset pool stats
pool.reset_stats().await;
// Concurrent acquisitions
let pool = Arc::new(pool);
let concurrent_start = std::time::Instant::now();
let handles: Vec<_> = (0..20)
.map(|_| {
let pool_clone = pool.clone();
tokio::spawn(async move {
let _conn = pool_clone
.acquire()
.await
.expect("Concurrent acquire failed");
sleep(Duration::from_millis(5)).await;
})
})
.collect();
for handle in handles {
handle.await.expect("Concurrent task failed");
}
let concurrent_duration = concurrent_start.elapsed();
// Concurrent should be faster (or at least not much slower)
assert!(
concurrent_duration < sequential_duration * 2,
"Concurrent should be reasonably efficient"
);
pool.close().await;
}
}
#[cfg(test)]
mod connection_timeout_tests {
use super::*;
#[tokio::test]
async fn test_acquire_timeout_honored() {
// Create pool with very short timeout
let mut config = create_test_pool_config(2, 1);
config.acquire_timeout_secs = 1; // 1 second timeout
let pool = DatabasePool::new(config)
.await
.expect("Pool creation failed");
// Take all connections
let _conn1 = pool.acquire().await.expect("First acquire failed");
let _conn2 = pool.acquire().await.expect("Second acquire failed");
// Next acquire should timeout within ~1 second
let start = std::time::Instant::now();
let result = pool.acquire().await;
let elapsed = start.elapsed();
assert!(result.is_err(), "Should timeout");
assert!(
elapsed < Duration::from_secs(2),
"Timeout should be honored"
);
pool.close().await;
}
#[tokio::test]
async fn test_zero_timeout_immediate_failure() {
// Very short timeout for immediate failure
let mut config = create_test_pool_config(1, 1);
config.acquire_timeout_secs = 1;
let pool = DatabasePool::new(config)
.await
.expect("Pool creation failed");
let _conn = pool.acquire().await.expect("First acquire failed");
// Should fail almost immediately
let start = std::time::Instant::now();
let result = timeout(Duration::from_millis(100), pool.acquire()).await;
let elapsed = start.elapsed();
assert!(result.is_err() || result.unwrap().is_err());
assert!(elapsed < Duration::from_millis(200), "Should fail quickly");
pool.close().await;
}
#[tokio::test]
async fn test_long_timeout_eventually_succeeds() {
let mut config = create_test_pool_config(2, 1);
config.acquire_timeout_secs = 10; // Long timeout
let pool = Arc::new(
DatabasePool::new(config)
.await
.expect("Pool creation failed"),
);
// Take all connections
let conn1 = pool.acquire().await.expect("First acquire failed");
let conn2 = pool.acquire().await.expect("Second acquire failed");
// Spawn task that will release connection after delay
tokio::spawn(async move {
let _c1 = conn1;
let _c2 = conn2;
sleep(Duration::from_millis(500)).await;
// Connections dropped here
});
// Should succeed once connection is released
let result = pool.acquire().await;
assert!(result.is_ok(), "Should succeed with long timeout");
pool.close().await;
}
}
#[cfg(test)]
mod connection_lifecycle_tests {
use super::*;
#[tokio::test]
async fn test_connection_max_lifetime() {
let mut config = create_test_pool_config(5, 2);
config.max_lifetime_secs = 2; // Very short lifetime
config.idle_timeout_secs = 10; // Longer idle timeout
let pool = DatabasePool::new(config)
.await
.expect("Pool creation failed");
// Acquire and use a connection
let conn = pool.acquire().await.expect("Acquire failed");
drop(conn);
// Wait for connection to exceed max lifetime
sleep(Duration::from_secs(3)).await;
// Acquire again - should get a fresh connection
let conn2 = pool.acquire().await.expect("Should get fresh connection");
drop(conn2);
pool.close().await;
}
#[tokio::test]
async fn test_connection_idle_timeout() {
let mut config = create_test_pool_config(10, 2);
config.idle_timeout_secs = 2; // Short idle timeout
config.max_lifetime_secs = 60; // Long max lifetime
let pool = DatabasePool::new(config)
.await
.expect("Pool creation failed");
// Create connections
let tasks = futures::stream::FuturesUnordered::new();
for _ in 0..5 {
tasks.push(pool.acquire());
}
let conns: Vec<_> = tasks.collect::<Vec<_>>().await;
// Release all connections
drop(conns);
// Wait for idle timeout
sleep(Duration::from_secs(3)).await;
// Pool should have cleaned up idle connections
let idle_count = pool.num_idle();
assert!(
idle_count >= 2,
"Should maintain minimum connections: {}",
idle_count
);
pool.close().await;
}
#[tokio::test]
async fn test_pool_maintains_minimum_connections() {
let config = create_test_pool_config(10, 5);
let pool = DatabasePool::new(config)
.await
.expect("Pool creation failed");
// Wait for pool to initialize
sleep(Duration::from_millis(500)).await;
// Check pool size is at least minimum
let size = pool.size();
assert!(
size >= 5,
"Pool should maintain minimum connections, got {}",
size
);
pool.close().await;
}
#[tokio::test]
async fn test_connection_recreation_after_expiry() {
let mut config = create_test_pool_config(3, 1);
config.max_lifetime_secs = 1;
let pool = DatabasePool::new(config)
.await
.expect("Pool creation failed");
let initial_size = pool.size();
// Use connections
for _ in 0..5 {
let _conn = pool.acquire().await.expect("Acquire failed");
sleep(Duration::from_millis(100)).await;
}
// Wait for connections to expire
sleep(Duration::from_secs(2)).await;
// Acquire new connections - should recreate
let _conn = pool.acquire().await.expect("Should recreate connection");
let final_size = pool.size();
assert!(
final_size >= 1,
"Pool should recreate connections: initial={}, final={}",
initial_size,
final_size
);
pool.close().await;
}
}
#[cfg(test)]
mod failure_recovery_tests {
use super::*;
#[tokio::test]
async fn test_invalid_connection_string() {
let mut config = create_test_pool_config(5, 2);
config.database_url = "postgresql://invalid:invalid@nonexistent:9999/invalid".to_string();
let result = DatabasePool::new(config).await;
assert!(
result.is_err(),
"Should fail with invalid connection string"
);
}
#[tokio::test]
async fn test_connection_validation_before_acquire() {
let mut config = create_test_pool_config(5, 2);
config.test_before_acquire = true;
let pool = DatabasePool::new(config)
.await
.expect("Pool creation failed");
// Acquire connection with validation
let mut conn = pool
.acquire()
.await
.expect("Acquire with validation failed");
// Connection should be valid
let result = sqlx::query("SELECT 1").fetch_one(&mut *conn).await;
assert!(result.is_ok(), "Connection should be valid");
pool.close().await;
}
#[tokio::test]
async fn test_pool_recovery_after_close() {
let pool = create_test_pool(5, 2).await.expect("Pool creation failed");
// Close the pool
pool.close().await;
assert!(pool.is_closed());
// Attempts to acquire should fail
let result = timeout(Duration::from_millis(100), pool.acquire()).await;
assert!(
result.is_err() || result.unwrap().is_err(),
"Acquire should fail on closed pool"
);
}
#[tokio::test]
async fn test_graceful_pool_shutdown() {
let pool = create_test_pool(5, 2).await.expect("Pool creation failed");
// Acquire some connections
let tasks = futures::stream::FuturesUnordered::new();
for _ in 0..3 {
tasks.push(pool.acquire());
}
let conns: Vec<_> = tasks.collect::<Vec<_>>().await;
// Close pool
pool.close().await;
// Existing connections should still work
for conn in conns {
if let Ok(mut c) = conn {
let result = sqlx::query("SELECT 1").fetch_one(&mut *c).await;
assert!(result.is_ok(), "Existing connection should still work");
}
}
}
#[tokio::test]
async fn test_pool_health_check() {
let pool = create_test_pool(5, 2).await.expect("Pool creation failed");
let health = pool.health_check().await.expect("Health check failed");
assert!(health, "Pool should be healthy");
pool.close().await;
// Health check on closed pool
let health_after_close = pool.health_check().await;
assert!(
!health_after_close.unwrap_or(false),
"Closed pool should be unhealthy"
);
}
}
#[cfg(test)]
mod transaction_tests {
use super::*;
#[tokio::test]
async fn test_transaction_connection_returned_on_commit() {
let pool = create_test_pool(3, 1).await.expect("Pool creation failed");
let initial_idle = pool.num_idle();
{
let mut conn = pool.acquire().await.expect("Acquire failed");
let mut tx = conn.begin().await.expect("Begin transaction failed");
sqlx::query("SELECT 1")
.execute(&mut *tx)
.await
.expect("Query failed");
tx.commit().await.expect("Commit failed");
} // Connection dropped here
// Wait for connection to return to pool
sleep(Duration::from_millis(100)).await;
let final_idle = pool.num_idle();
assert!(
final_idle >= initial_idle,
"Connection should return to pool after commit"
);
pool.close().await;
}
#[tokio::test]
async fn test_transaction_connection_returned_on_rollback() {
let pool = create_test_pool(3, 1).await.expect("Pool creation failed");
let initial_idle = pool.num_idle();
{
let mut conn = pool.acquire().await.expect("Acquire failed");
let mut tx = conn.begin().await.expect("Begin transaction failed");
sqlx::query("SELECT 1")
.execute(&mut *tx)
.await
.expect("Query failed");
tx.rollback().await.expect("Rollback failed");
} // Connection dropped here
// Wait for connection to return to pool
sleep(Duration::from_millis(100)).await;
let final_idle = pool.num_idle();
assert!(
final_idle >= initial_idle,
"Connection should return to pool after rollback"
);
pool.close().await;
}
#[tokio::test]
async fn test_failed_transaction_cleanup() {
let pool = create_test_pool(3, 1).await.expect("Pool creation failed");
let initial_idle = pool.num_idle();
{
let mut conn = pool.acquire().await.expect("Acquire failed");
let mut tx = conn.begin().await.expect("Begin transaction failed");
// Execute invalid query to cause failure
let _ = sqlx::query("SELECT * FROM nonexistent_table_xyz")
.execute(&mut *tx)
.await;
// Transaction will be rolled back on drop
} // Connection dropped here
// Wait for cleanup
sleep(Duration::from_millis(100)).await;
// Connection should still return to pool despite failed transaction
let final_idle = pool.num_idle();
assert!(
final_idle >= initial_idle,
"Connection should return even after failed transaction"
);
pool.close().await;
}
}
#[cfg(test)]
mod pool_statistics_tests {
use super::*;
#[tokio::test]
async fn test_pool_stats_acquisition_tracking() {
let pool = create_test_pool(5, 2).await.expect("Pool creation failed");
pool.reset_stats().await;
// Perform acquisitions
for _ in 0..10 {
let _conn = pool.acquire().await.expect("Acquire failed");
}
let stats = pool.stats().await;
assert_eq!(
stats.total_acquisitions, 10,
"Should track total acquisitions"
);
pool.close().await;
}
#[tokio::test]
async fn test_pool_stats_failed_acquisitions() {
let pool = create_test_pool(2, 1).await.expect("Pool creation failed");
pool.reset_stats().await;
// Take all connections
let _conn1 = pool.acquire().await.expect("First acquire failed");
let _conn2 = pool.acquire().await.expect("Second acquire failed");
// Try to acquire more (will timeout and fail)
for _ in 0..3 {
let _ = timeout(Duration::from_millis(50), pool.acquire()).await;
}
let stats = pool.stats().await;
assert!(
stats.total_acquisitions >= 5,
"Should track all acquisition attempts"
);
pool.close().await;
}
#[tokio::test]
async fn test_pool_stats_reset() {
let pool = create_test_pool(5, 2).await.expect("Pool creation failed");
// Perform some operations
for _ in 0..5 {
let _conn = pool.acquire().await.expect("Acquire failed");
}
let stats_before = pool.stats().await;
assert!(stats_before.total_acquisitions > 0);
// Reset stats
pool.reset_stats().await;
let stats_after = pool.stats().await;
assert_eq!(stats_after.total_acquisitions, 0, "Stats should be reset");
pool.close().await;
}
#[tokio::test]
async fn test_pool_size_metrics() {
let pool = create_test_pool(10, 3).await.expect("Pool creation failed");
// Wait for minimum connections to be established
sleep(Duration::from_millis(500)).await;
let size = pool.size();
let idle = pool.num_idle();
assert!(size >= 3, "Pool size should be at least minimum: {}", size);
assert!(idle <= size as usize, "Idle cannot exceed total size");
// Acquire connections
let tasks = futures::stream::FuturesUnordered::new();
for _ in 0..5 {
tasks.push(pool.acquire());
}
let conns: Vec<_> = tasks.collect::<Vec<_>>().await;
let idle_after = pool.num_idle();
assert!(
idle_after < idle,
"Idle connections should decrease after acquisitions"
);
drop(conns);
pool.close().await;
}
}
#[cfg(test)]
mod configuration_validation_tests {
use super::*;
#[tokio::test]
async fn test_invalid_min_max_configuration() {
let mut config = create_test_pool_config(5, 10);
config.min_connections = 10; // Min > Max
config.max_connections = 5;
let result = DatabasePool::new(config).await;
assert!(result.is_err(), "Should reject min > max configuration");
}
#[tokio::test]
async fn test_zero_max_connections() {
let mut config = create_test_pool_config(0, 0);
config.max_connections = 0;
let result = DatabasePool::new(config).await;
assert!(result.is_err(), "Should reject zero max connections");
}
#[tokio::test]
async fn test_invalid_database_url() {
let mut config = create_test_pool_config(5, 2);
config.database_url = "not-a-valid-url".to_string();
let result = DatabasePool::new(config).await;
assert!(result.is_err(), "Should reject invalid database URL");
}
#[tokio::test]
async fn test_min_equals_max_configuration() {
let pool = create_test_pool(5, 5).await.expect("Pool creation failed");
sleep(Duration::from_millis(500)).await;
let size = pool.size();
assert_eq!(size, 5, "Pool should maintain fixed size when min=max");
pool.close().await;
}
}