Files
foxhunt/testing/integration/config_hot_reload.rs
jgrusewski 9c3d741a08 refactor: restructure repo — crates/, bin/, testing/ layout
Move 17 library crates into crates/, CLI binary into bin/fxt,
consolidate 10 test crates into testing/, split config crate
from deployment config files.

Root directory reduced from 38+ to ~17 directories.
All Cargo.toml paths and build.rs proto refs updated.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-25 11:56:00 +01:00

760 lines
26 KiB
Rust

//! Configuration Hot-Reload Integration Tests
//!
//! Comprehensive test suite for PostgreSQL NOTIFY/LISTEN configuration hot-reload system.
//!
//! ## Test Categories
//!
//! 1. **Environment-Aware Defaults** - Verify dev/staging/prod default values
//! 2. **Environment Variable Parsing** - Test 60+ parameter overrides
//! 3. **PostgreSQL NOTIFY/LISTEN** - Hot-reload notification testing
//! 4. **Configuration Validation** - Boundary conditions and error handling
//! 5. **Service Integration** - Multi-service coordination
//!
//! ## Prerequisites
//!
//! - PostgreSQL database running
//! - Migrations applied (especially 007_configuration_schema.sql)
//! - DATABASE_URL environment variable set
//!
//! ## Running Tests
//!
//! ```bash
//! # Run all hot-reload tests
//! cargo test --test config_hot_reload --features postgres
//!
//! # Run specific category
//! cargo test --test config_hot_reload test_environment_ --features postgres
//!
//! # Run with output
//! cargo test --test config_hot_reload -- --nocapture
//! ```
use config::{DatabaseRuntimeConfig, Environment, LimitsConfig, RuntimeConfig};
use serde_json::json;
use sqlx::{Executor, PgPool};
use std::env;
use std::time::Duration;
use tokio::time::timeout;
// ============================================================================
// TEST HELPERS
// ============================================================================
/// Helper to clear and set environment variables for isolated tests
fn set_env_vars(vars: &[(&str, &str)]) {
for (key, _) in vars {
std::env::remove_var(key);
}
for (key, value) in vars {
std::env::set_var(key, value);
}
}
fn clear_env_vars(keys: &[&str]) {
for key in keys {
std::env::remove_var(key);
}
}
/// Get database URL from environment or use default test database
fn get_database_url() -> String {
env::var("DATABASE_URL").unwrap_or_else(|_| {
"postgresql://postgres:postgres@localhost:5432/foxhunt_test".to_string()
})
}
/// Create a separate connection pool for test operations
async fn create_test_pool() -> PgPool {
let url = get_database_url();
PgPool::connect(&url)
.await
.expect("Failed to create test pool")
}
/// Helper to cleanup test config settings
async fn cleanup_config_setting(pool: &PgPool, config_key: &str, environment: &str) {
let _ = sqlx::query("DELETE FROM config_settings WHERE config_key = $1 AND environment = $2")
.bind(config_key)
.bind(environment)
.execute(pool)
.await;
}
/// Helper to cleanup config categories
async fn cleanup_config_category(pool: &PgPool, category_name: &str) {
let _ = sqlx::query("DELETE FROM config_categories WHERE category_name = $1")
.bind(category_name)
.execute(pool)
.await;
}
/// Helper to insert a test category and return its ID
async fn insert_test_category(pool: &PgPool, name: &str, path: &str) -> i32 {
sqlx::query_scalar(
"INSERT INTO config_categories (category_name, category_path)
VALUES ($1, $2)
ON CONFLICT (category_path) DO UPDATE SET category_name = EXCLUDED.category_name
RETURNING id",
)
.bind(name)
.bind(path)
.fetch_one(pool)
.await
.unwrap()
}
/// Helper to insert a test config setting
async fn insert_test_config_setting(
pool: &PgPool,
config_key: &str,
category_id: i32,
category_path: &str,
value: serde_json::Value,
environment: &str,
) -> i32 {
sqlx::query_scalar(
"INSERT INTO config_settings (config_key, category_id, category_path, config_value, environment)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (config_key, environment) DO UPDATE SET config_value = EXCLUDED.config_value
RETURNING id"
)
.bind(config_key)
.bind(category_id)
.bind(category_path)
.bind(value)
.bind(environment)
.fetch_one(pool)
.await
.unwrap()
}
// ============================================================================
// CATEGORY 1: ENVIRONMENT-AWARE DEFAULTS TESTS
// ============================================================================
/// Test: Environment detection works for explicit and default cases.
///
/// Covers "Environment-Aware Defaults" and "Environment detection from ENVIRONMENT variable".
#[test]
fn test_environment_detection_explicit() {
// Context: Verify that setting the "ENVIRONMENT" variable correctly maps to the Environment enum.
// Also, ensure that invalid or missing variables default to Development.
set_env_vars(&[("ENVIRONMENT", "production")]);
assert_eq!(
Environment::detect(),
Environment::Production,
"Should detect Production environment"
);
clear_env_vars(&["ENVIRONMENT"]);
set_env_vars(&[("ENVIRONMENT", "stage")]);
assert_eq!(
Environment::detect(),
Environment::Staging,
"Should detect Staging environment"
);
clear_env_vars(&["ENVIRONMENT"]);
set_env_vars(&[("ENVIRONMENT", "DEV")]);
assert_eq!(
Environment::detect(),
Environment::Development,
"Should detect Development environment (case-insensitive)"
);
clear_env_vars(&["ENVIRONMENT"]);
set_env_vars(&[("ENVIRONMENT", "unknown_env")]);
assert_eq!(
Environment::detect(),
Environment::Development,
"Should fallback to Development for unknown environment"
);
clear_env_vars(&["ENVIRONMENT"]);
clear_env_vars(&["ENVIRONMENT"]); // Ensure it's not set
assert_eq!(
Environment::detect(),
Environment::Development,
"Should fallback to Development when ENVIRONMENT is not set"
);
}
/// Test: All sub-configurations follow graduated defaults (dev > staging > prod tightness).
///
/// Covers "Graduated defaults (dev > staging > prod tightness)" and "Database, cache, timeout, limits configurations".
#[test]
fn test_all_subconfigs_graduated_defaults() {
// Context: Verify that default values for different environments (Development, Staging, Production)
// adhere to the expected "graduated defaults" principle, where Production settings are generally
// tighter/more performant than Staging, which are tighter than Development.
let dev = RuntimeConfig::with_defaults(Environment::Development);
let staging = RuntimeConfig::with_defaults(Environment::Staging);
let prod = RuntimeConfig::with_defaults(Environment::Production);
// Database: Prod should be tighter/higher pool than Dev
assert!(
prod.database.query_timeout < dev.database.query_timeout,
"Prod query timeout should be tighter than Dev"
);
assert!(
prod.database.connection_timeout < dev.database.connection_timeout,
"Prod connection timeout should be tighter than Dev"
);
assert!(
prod.database.acquire_timeout < dev.database.acquire_timeout,
"Prod acquire timeout should be tighter than Dev"
);
assert!(
prod.database.pool_size > dev.database.pool_size,
"Prod pool size should be higher than Dev"
);
assert!(
prod.database.max_pool_size > dev.database.max_pool_size,
"Prod max pool size should be higher than Dev"
);
assert!(
staging.database.query_timeout < dev.database.query_timeout,
"Staging query timeout should be tighter than Dev"
);
assert!(
staging.database.query_timeout > prod.database.query_timeout,
"Staging query timeout should be looser than Prod"
);
// Cache: Prod should have shorter TTLs
assert!(
prod.cache.position_ttl < dev.cache.position_ttl,
"Prod position TTL should be shorter than Dev"
);
assert!(
prod.cache.var_ttl < dev.cache.var_ttl,
"Prod VaR TTL should be shorter than Dev"
);
assert!(
staging.cache.position_ttl < dev.cache.position_ttl,
"Staging position TTL should be shorter than Dev"
);
assert!(
staging.cache.position_ttl > prod.cache.position_ttl,
"Staging position TTL should be longer than Prod"
);
// Timeouts: Prod should have tighter network timeouts
assert!(
prod.timeouts.grpc_request_timeout < dev.timeouts.grpc_request_timeout,
"Prod gRPC request timeout should be tighter than Dev"
);
assert!(
prod.timeouts.keep_alive_interval < dev.timeouts.keep_alive_interval,
"Prod keep-alive interval should be tighter than Dev"
);
assert!(
staging.timeouts.grpc_request_timeout < dev.timeouts.grpc_request_timeout,
"Staging gRPC request timeout should be tighter than Dev"
);
assert!(
staging.timeouts.grpc_request_timeout > prod.timeouts.grpc_request_timeout,
"Staging gRPC request timeout should be looser than Prod"
);
// Limits: Prod should have tighter safety, faster ML, lower retry attempts, tighter risk
assert!(
prod.limits.safety_check_timeout < dev.limits.safety_check_timeout,
"Prod safety check timeout should be tighter than Dev"
);
assert!(
prod.limits.ml_inference_timeout < dev.limits.ml_inference_timeout,
"Prod ML inference timeout should be tighter than Dev"
);
assert!(
prod.limits.retry_max_attempts < dev.limits.retry_max_attempts,
"Prod retry max attempts should be lower than Dev"
);
assert!(
prod.limits.risk_max_drawdown_warning_pct < dev.limits.risk_max_drawdown_warning_pct,
"Prod max drawdown warning should be tighter than Dev"
);
assert!(
staging.limits.safety_check_timeout < dev.limits.safety_check_timeout,
"Staging safety check timeout should be tighter than Dev"
);
assert!(
staging.limits.safety_check_timeout > prod.limits.safety_check_timeout,
"Staging safety check timeout should be looser than Prod"
);
}
// ============================================================================
// CATEGORY 2: ENVIRONMENT VARIABLE PARSING TESTS
// ============================================================================
/// Test: Invalid environment variable values for DatabaseRuntimeConfig lead to errors.
///
/// Covers "Invalid value error handling" and "Type validation (u32, u64, f32, f64, Duration)".
#[test]
fn test_database_config_from_env_invalid_values() {
// Context: Ensure that parsing environment variables for database configuration
// correctly handles invalid input types (e.g., non-numeric strings for u32).
let keys = ["DATABASE_POOL_SIZE", "DATABASE_QUERY_TIMEOUT_MS"];
set_env_vars(&[
("DATABASE_POOL_SIZE", "invalid"),
("DATABASE_QUERY_TIMEOUT_MS", "-100"), // Negative for u64
]);
let result_pool_size = DatabaseRuntimeConfig::from_env(Environment::Production);
assert!(
result_pool_size.is_err(),
"Parsing invalid string for pool size should fail"
);
let err_msg = result_pool_size.unwrap_err().to_string();
assert!(
err_msg.contains("Invalid u32 for DATABASE_POOL_SIZE"),
"Error message should indicate invalid u32, got: {}",
err_msg
);
clear_env_vars(&keys);
set_env_vars(&[("DATABASE_QUERY_TIMEOUT_MS", "-100")]);
let result_query_timeout = DatabaseRuntimeConfig::from_env(Environment::Production);
assert!(
result_query_timeout.is_err(),
"Parsing negative duration should fail"
);
let err_msg = result_query_timeout.unwrap_err().to_string();
assert!(
err_msg.contains("Invalid duration for DATABASE_QUERY_TIMEOUT_MS"),
"Error message should indicate invalid duration, got: {}",
err_msg
);
clear_env_vars(&keys);
}
// ============================================================================
// CATEGORY 3: CONFIGURATION VALIDATION TESTS
// ============================================================================
/// Test: LimitsConfig validation handles boundary conditions and invalid ranges.
///
/// Covers "Boundary condition validation" and "Invalid configuration rejection".
#[test]
fn test_limits_config_validation_boundary_conditions() {
// Context: Validate that the LimitsConfig's internal validation logic correctly
// identifies and rejects configurations with invalid or out-of-bounds values.
let mut config = LimitsConfig::with_defaults(Environment::Production);
assert!(config.validate().is_ok(), "Default config should be valid");
// retry_max_attempts = 0
config.retry_max_attempts = 0;
assert!(
config.validate().is_err(),
"Retry max attempts cannot be zero"
);
assert_eq!(
config.validate().unwrap_err().to_string(),
"Invalid: Retry max attempts must be positive",
"Correct error message for zero retry attempts"
);
// retry_backoff_multiplier <= 1.0
config = LimitsConfig::with_defaults(Environment::Production);
config.retry_backoff_multiplier = 1.0;
assert!(
config.validate().is_err(),
"Backoff multiplier must be > 1.0"
);
assert_eq!(
config.validate().unwrap_err().to_string(),
"Invalid: Backoff multiplier must be > 1.0",
"Correct error message for backoff multiplier <= 1.0"
);
// ml_max_batch_size = 0
config = LimitsConfig::with_defaults(Environment::Production);
config.ml_max_batch_size = 0;
assert!(
config.validate().is_err(),
"ML max batch size cannot be zero"
);
assert_eq!(
config.validate().unwrap_err().to_string(),
"Invalid: ML max batch size must be positive",
"Correct error message for zero ML batch size"
);
// risk_var_confidence out of range (negative)
config = LimitsConfig::with_defaults(Environment::Production);
config.risk_var_confidence = -0.1;
assert!(
config.validate().is_err(),
"VaR confidence cannot be negative"
);
assert_eq!(
config.validate().unwrap_err().to_string(),
"Invalid: VaR confidence must be between 0.0 and 1.0",
"Correct error message for negative VaR confidence"
);
// risk_var_confidence out of range (greater than 1.0)
config = LimitsConfig::with_defaults(Environment::Production);
config.risk_var_confidence = 1.1;
assert!(
config.validate().is_err(),
"VaR confidence cannot be greater than 1.0"
);
assert_eq!(
config.validate().unwrap_err().to_string(),
"Invalid: VaR confidence must be between 0.0 and 1.0",
"Correct error message for VaR confidence > 1.0"
);
// risk_var_lookback_days = 0
config = LimitsConfig::with_defaults(Environment::Production);
config.risk_var_lookback_days = 0;
assert!(
config.validate().is_err(),
"VaR lookback days cannot be zero"
);
assert_eq!(
config.validate().unwrap_err().to_string(),
"Invalid: VaR lookback days must be positive",
"Correct error message for zero VaR lookback days"
);
}
// ============================================================================
// CATEGORY 4: POSTGRESQL NOTIFY/LISTEN HOT-RELOAD TESTS
// ============================================================================
/// Test: Verify basic NOTIFY/LISTEN for config_settings table updates.
///
/// Covers "Configuration change notification trigger" and "Notification payload format validation".
#[tokio::test]
async fn test_general_config_hot_reload_notification_on_update() {
// Context: Ensure that updating a configuration setting in the `config_settings` table
// triggers a PostgreSQL NOTIFY event on the `foxhunt_config_changes` channel, and that
// the payload contains the expected information about the change.
let pool = create_test_pool().await;
let mut listener = sqlx::postgres::PgListener::connect_with(&pool)
.await
.unwrap();
listener.listen("foxhunt_config_changes").await.unwrap();
let category_id =
insert_test_category(&pool, "test_category_notify", "test_category_notify").await;
let config_key = "test_setting_notify";
let environment = "development";
insert_test_config_setting(
&pool,
config_key,
category_id,
"test_category_notify",
json!("initial"),
environment,
)
.await;
// Update the config setting
sqlx::query(
"UPDATE config_settings SET config_value = $1, updated_by = $2 WHERE config_key = $3 AND environment = $4"
)
.bind(json!("updated_value"))
.bind("test_user")
.bind(config_key)
.bind(environment)
.execute(&pool)
.await
.unwrap();
// Wait for notification (with timeout to prevent hanging)
let notification = timeout(Duration::from_secs(5), listener.recv())
.await
.unwrap()
.unwrap();
let payload: serde_json::Value = serde_json::from_str(notification.payload()).unwrap();
assert_eq!(
payload["table"], "config_settings",
"Payload should indicate 'config_settings' table"
);
assert_eq!(
payload["operation"], "UPDATE",
"Payload should indicate 'UPDATE' operation"
);
assert_eq!(
payload["config_key"], config_key,
"Payload should contain the correct config_key"
);
assert_eq!(
payload["environment"], environment,
"Payload should contain the correct environment"
);
assert_eq!(
payload["old_value"], "initial",
"Payload should contain the old value"
);
assert_eq!(
payload["new_value"], "updated_value",
"Payload should contain the new value"
);
assert_eq!(
payload["changed_by"], "test_user",
"Payload should contain the user who made the change"
);
cleanup_config_setting(&pool, config_key, environment).await;
cleanup_config_category(&pool, "test_category_notify").await;
}
// ============================================================================
// CATEGORY 5: CONCURRENT UPDATE TESTS
// ============================================================================
/// Test: Concurrent updates to config_settings using optimistic locking (version column).
///
/// Covers "Concurrent config updates maintain consistency" and "Configuration history audit trail is maintained".
#[tokio::test]
async fn test_concurrent_config_settings_updates_optimistic_locking() {
// Context: Simulate two concurrent attempts to update the same configuration setting.
// This test uses optimistic locking based on the `version` column to ensure that
// only one update succeeds if both transactions read the same initial version.
let pool = create_test_pool().await;
let category_id = insert_test_category(&pool, "concurrent_cat", "concurrent_cat").await;
let config_key = "concurrent_key";
let environment = "development";
let setting_id = insert_test_config_setting(
&pool,
config_key,
category_id,
"concurrent_cat",
json!("initial"),
environment,
)
.await;
// Get initial version
let initial_version: i32 =
sqlx::query_scalar("SELECT version FROM config_settings WHERE id = $1")
.bind(setting_id)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(initial_version, 1, "Initial version should be 1");
let pool_clone1 = pool.clone();
let pool_clone2 = pool.clone();
let task1_key = config_key.to_string();
let task1_env = environment.to_string();
let task2_key = config_key.to_string();
let task2_env = environment.to_string();
// Task 1: Attempts to update the config setting
let task1 = tokio::spawn(async move {
// Read current version
let current_version: i32 = sqlx::query_scalar(
"SELECT version FROM config_settings WHERE config_key = $1 AND environment = $2",
)
.bind(&task1_key)
.bind(&task1_env)
.fetch_one(&pool_clone1)
.await
.unwrap();
// Attempt update with optimistic locking (WHERE version = current_version)
sqlx::query(
"UPDATE config_settings
SET config_value = $1, version = version + 1, updated_by = $2
WHERE config_key = $3 AND environment = $4 AND version = $5",
)
.bind(json!("value_from_task1"))
.bind("task1_user")
.bind(&task1_key)
.bind(&task1_env)
.bind(current_version)
.execute(&pool_clone1)
.await
});
// Task 2: Attempts to update the same config setting with a small delay
let task2 = tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(10)).await; // Ensure task1 likely reads first
// Read current version
let current_version: i32 = sqlx::query_scalar(
"SELECT version FROM config_settings WHERE config_key = $1 AND environment = $2",
)
.bind(&task2_key)
.bind(&task2_env)
.fetch_one(&pool_clone2)
.await
.unwrap();
// Attempt update with optimistic locking (WHERE version = current_version)
sqlx::query(
"UPDATE config_settings
SET config_value = $1, version = version + 1, updated_by = $2
WHERE config_key = $3 AND environment = $4 AND version = $5",
)
.bind(json!("value_from_task2"))
.bind("task2_user")
.bind(&task2_key)
.bind(&task2_env)
.bind(current_version)
.execute(&pool_clone2)
.await
});
let result1 = task1.await.unwrap().unwrap();
let result2 = task2.await.unwrap().unwrap();
// One update should succeed (rows_affected = 1), the other should fail (rows_affected = 0)
assert_eq!(
result1.rows_affected() + result2.rows_affected(),
1,
"Only one concurrent update should succeed with optimistic locking"
);
// Verify final state
let final_value: serde_json::Value =
sqlx::query_scalar("SELECT config_value FROM config_settings WHERE id = $1")
.bind(setting_id)
.fetch_one(&pool)
.await
.unwrap();
let final_version: i32 =
sqlx::query_scalar("SELECT version FROM config_settings WHERE id = $1")
.bind(setting_id)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(
final_version, 2,
"Version should be incremented exactly once"
);
assert!(
final_value == json!("value_from_task1") || final_value == json!("value_from_task2"),
"Final value should be from the successful task"
);
cleanup_config_setting(&pool, config_key, environment).await;
cleanup_config_category(&pool, "concurrent_cat").await;
}
// ============================================================================
// CATEGORY 6: SERVICE INTEGRATION TESTS
// ============================================================================
/// Test: RuntimeConfig::from_env() loads all configuration categories correctly.
///
/// Covers "Service Integration" and "RuntimeConfig::from_env() loads all categories".
#[test]
fn test_runtime_config_from_env_loads_all_categories() {
// Context: Verify that RuntimeConfig::from_env() successfully loads and validates
// all configuration categories (database, cache, timeouts, limits) using the
// detected environment and environment variables.
// Set up environment variables for testing
set_env_vars(&[
("ENVIRONMENT", "production"),
("DATABASE_QUERY_TIMEOUT_MS", "500"),
("CACHE_POSITION_TTL_SECS", "30"),
("NETWORK_GRPC_REQUEST_TIMEOUT_SECS", "5"),
("ML_MAX_BATCH_SIZE", "4096"),
]);
let result = RuntimeConfig::from_env();
assert!(result.is_ok(), "RuntimeConfig::from_env() should succeed");
let config = result.unwrap();
assert_eq!(
config.environment,
Environment::Production,
"Should detect Production environment"
);
// Verify database config was loaded
assert_eq!(
config.database.query_timeout,
Duration::from_millis(500),
"Database query timeout should be overridden by env var"
);
// Verify cache config was loaded
assert_eq!(
config.cache.position_ttl,
Duration::from_secs(30),
"Cache position TTL should be overridden by env var"
);
// Verify timeout config was loaded
assert_eq!(
config.timeouts.grpc_request_timeout,
Duration::from_secs(5),
"gRPC request timeout should be overridden by env var"
);
// Verify limits config was loaded
assert_eq!(
config.limits.ml_max_batch_size, 4096,
"ML max batch size should be overridden by env var"
);
// Verify validation passed
assert!(
config.validate().is_ok(),
"Config validation should pass for valid environment variables"
);
clear_env_vars(&[
"ENVIRONMENT",
"DATABASE_QUERY_TIMEOUT_MS",
"CACHE_POSITION_TTL_SECS",
"NETWORK_GRPC_REQUEST_TIMEOUT_SECS",
"ML_MAX_BATCH_SIZE",
]);
}
/// Test: RuntimeConfig::validate() catches validation errors across all categories.
///
/// Covers "Configuration Validation" and "RuntimeConfig::validate() catches all errors".
#[test]
fn test_runtime_config_validate_catches_all_errors() {
// Context: Verify that RuntimeConfig::validate() properly validates all sub-configurations
// and catches errors in any category.
let mut config = RuntimeConfig::with_defaults(Environment::Production);
assert!(config.validate().is_ok(), "Default config should be valid");
// Test database validation error
config.database.query_timeout = Duration::from_millis(0);
assert!(
config.validate().is_err(),
"Should catch database validation error"
);
// Reset and test cache validation error
config = RuntimeConfig::with_defaults(Environment::Production);
config.cache.position_ttl = Duration::from_secs(0);
assert!(
config.validate().is_err(),
"Should catch cache validation error"
);
// Reset and test timeout validation error
config = RuntimeConfig::with_defaults(Environment::Production);
config.timeouts.grpc_connect_timeout = Duration::from_secs(0);
assert!(
config.validate().is_err(),
"Should catch timeout validation error"
);
// Reset and test limits validation error
config = RuntimeConfig::with_defaults(Environment::Production);
config.limits.retry_max_attempts = 0;
assert!(
config.validate().is_err(),
"Should catch limits validation error"
);
}