Files
foxhunt/testing/vault-integration/certificate_lifecycle_tests.rs
jgrusewski 9c3d741a08 refactor: restructure repo — crates/, bin/, testing/ layout
Move 17 library crates into crates/, CLI binary into bin/fxt,
consolidate 10 test crates into testing/, split config crate
from deployment config files.

Root directory reduced from 38+ to ~17 directories.
All Cargo.toml paths and build.rs proto refs updated.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-25 11:56:00 +01:00

569 lines
18 KiB
Rust

//! Certificate lifecycle management tests
//!
//! This module tests the complete certificate lifecycle:
//! - Certificate generation from Vault PKI
//! - Certificate caching and persistence
//! - Certificate validation and parsing
//! - Certificate rotation without service disruption
//! - Certificate expiry handling
//! - Performance measurement of certificate operations
use anyhow::{Context, Result};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use tokio::fs;
use tokio::sync::RwLock;
use tracing::{debug, info, warn, error};
use crate::{VaultTestConfig, VaultTestResults, PerformanceMetrics};
/// Certificate test data structure
#[derive(Debug, Clone)]
pub struct TestCertificate {
/// PEM-encoded certificate
pub certificate: String,
/// PEM-encoded private key
pub private_key: String,
/// PEM-encoded CA chain
pub ca_chain: String,
/// Certificate serial number
pub serial_number: String,
/// Certificate common name
pub common_name: String,
/// Certificate expiration timestamp
pub expires_at: SystemTime,
/// Time when certificate was generated
pub generated_at: Instant,
}
impl TestCertificate {
/// Parse certificate from Vault response
pub fn from_vault_response(response: &serde_json::Value, common_name: String) -> Result<Self> {
let data = response.get("data")
.context("No data in certificate response")?;
let certificate = data.get("certificate")
.and_then(|v| v.as_str())
.context("No certificate in response")?
.to_string();
let private_key = data.get("private_key")
.and_then(|v| v.as_str())
.context("No private key in response")?
.to_string();
let ca_chain = if let Some(chain_array) = data.get("ca_chain").and_then(|v| v.as_array()) {
chain_array.iter()
.filter_map(|v| v.as_str())
.collect::<Vec<_>>()
.join("\n")
} else if let Some(issuing_ca) = data.get("issuing_ca").and_then(|v| v.as_str()) {
issuing_ca.to_string()
} else {
return Err(anyhow::anyhow!("No CA chain in response"));
};
let serial_number = data.get("serial_number")
.and_then(|v| v.as_str())
.unwrap_or("unknown")
.to_string();
// Parse expiration from certificate (simplified for testing)
let expires_at = SystemTime::now() + Duration::from_secs(3600); // Default 1 hour
Ok(Self {
certificate,
private_key,
ca_chain,
serial_number,
common_name,
expires_at,
generated_at: Instant::now(),
})
}
/// Check if certificate is valid for TLS use
pub fn validate_for_tls(&self) -> Result<()> {
// Basic validation - check PEM format
if !self.certificate.contains("BEGIN CERTIFICATE") {
return Err(anyhow::anyhow!("Invalid certificate PEM format"));
}
if !self.private_key.contains("BEGIN PRIVATE KEY") && !self.private_key.contains("BEGIN RSA PRIVATE KEY") {
return Err(anyhow::anyhow!("Invalid private key PEM format"));
}
if !self.ca_chain.contains("BEGIN CERTIFICATE") {
return Err(anyhow::anyhow!("Invalid CA chain PEM format"));
}
// Check certificate contains expected common name
if !self.certificate.contains(&self.common_name) {
debug!("Certificate might not contain expected CN: {}", self.common_name);
}
Ok(())
}
/// Get certificate age
pub fn age(&self) -> Duration {
self.generated_at.elapsed()
}
/// Check if certificate needs renewal (within threshold of expiry)
pub fn needs_renewal(&self, threshold: Duration) -> bool {
match self.expires_at.duration_since(UNIX_EPOCH) {
Ok(expires) => {
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default();
expires.saturating_sub(now) < threshold
}
Err(_) => true, // If we can't parse expiry, assume renewal needed
}
}
}
/// Certificate cache for testing
pub struct CertificateCache {
certificates: Arc<RwLock<HashMap<String, TestCertificate>>>,
cache_dir: String,
}
impl CertificateCache {
pub fn new(cache_dir: String) -> Self {
Self {
certificates: Arc::new(RwLock::new(HashMap::new())),
cache_dir,
}
}
/// Store certificate in cache
pub async fn store(&self, key: String, cert: TestCertificate) -> Result<()> {
// Store in memory
{
let mut certs = self.certificates.write().await;
certs.insert(key.clone(), cert.clone());
}
// Persist to disk
self.persist_certificate(&key, &cert).await?;
Ok(())
}
/// Retrieve certificate from cache
pub async fn get(&self, key: &str) -> Option<TestCertificate> {
let certs = self.certificates.read().await;
certs.get(key).cloned()
}
/// Check cache hit rate
pub async fn get_hit_rate(&self) -> f64 {
let certs = self.certificates.read().await;
if certs.is_empty() {
return 0.0;
}
// Simplified hit rate calculation
100.0 // For testing purposes
}
/// Persist certificate to disk
async fn persist_certificate(&self, key: &str, cert: &TestCertificate) -> Result<()> {
// Create cache directory if it doesn't exist
fs::create_dir_all(&self.cache_dir).await?;
// Write certificate files
let cert_file = format!("{}/{}.crt", self.cache_dir, key);
let key_file = format!("{}/{}.key", self.cache_dir, key);
let ca_file = format!("{}/{}.ca", self.cache_dir, key);
fs::write(&cert_file, &cert.certificate).await?;
fs::write(&key_file, &cert.private_key).await?;
fs::write(&ca_file, &cert.ca_chain).await?;
// Set restrictive permissions on private key
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mut perms = fs::metadata(&key_file).await?.permissions();
perms.set_mode(0o600);
fs::set_permissions(&key_file, perms).await?;
}
debug!("Certificate persisted to disk: {}", key);
Ok(())
}
}
/// Test certificate generation from Vault
pub async fn test_certificate_generation(
config: &VaultTestConfig,
results: &Arc<RwLock<VaultTestResults>>,
) -> Result<TestCertificate> {
info!("Testing certificate generation from Vault");
let start = Instant::now();
let common_name = "test-service.foxhunt.internal";
// Generate certificate via Vault API
let cert_request = serde_json::json!({
"common_name": common_name,
"ttl": "1h",
"format": "pem"
});
let response = make_vault_request(
&format!("{}/v1/pki_int/issue/hft-trading", config.vault_addr),
"POST",
Some(&cert_request),
&config.vault_token,
).await?;
let generation_time = start.elapsed();
// Parse certificate
let cert = TestCertificate::from_vault_response(&response, common_name.to_string())?;
// Validate certificate
cert.validate_for_tls()
.context("Generated certificate failed TLS validation")?;
// Update performance metrics
{
let mut test_results = results.write().await;
test_results.performance.cert_generation_time = Some(generation_time);
test_results.performance.vault_api_calls += 1;
}
// Add success to results
{
let mut test_results = results.write().await;
test_results.add_success("certificate_generation", generation_time);
}
info!("Certificate generated successfully in {:?}", generation_time);
Ok(cert)
}
/// Test certificate caching functionality
pub async fn test_certificate_caching(
config: &VaultTestConfig,
results: &Arc<RwLock<VaultTestResults>>,
) -> Result<()> {
info!("Testing certificate caching");
let cache = CertificateCache::new(config.cert_cache_dir.clone());
let service_name = "caching-test-service";
// Generate initial certificate
let cert = test_certificate_generation(config, results).await?;
// Test cache store operation
let store_start = Instant::now();
cache.store(service_name.to_string(), cert.clone()).await?;
let store_duration = store_start.elapsed();
// Test cache retrieve operation
let retrieve_start = Instant::now();
let cached_cert = cache.get(service_name).await
.context("Certificate not found in cache")?;
let retrieve_duration = retrieve_start.elapsed();
// Verify cached certificate matches
if cached_cert.serial_number != cert.serial_number {
return Err(anyhow::anyhow!("Cached certificate serial number mismatch"));
}
// Update performance metrics
{
let mut test_results = results.write().await;
test_results.performance.cache_lookup_time = Some(retrieve_duration);
test_results.performance.cache_hit_rate = Some(cache.get_hit_rate().await);
}
// Add success to results
{
let mut test_results = results.write().await;
test_results.add_success("certificate_caching", store_duration + retrieve_duration);
}
info!("Certificate caching test completed successfully");
Ok(())
}
/// Test certificate rotation scenario
pub async fn test_certificate_rotation(
config: &VaultTestConfig,
results: &Arc<RwLock<VaultTestResults>>,
) -> Result<()> {
info!("Testing certificate rotation");
let cache = CertificateCache::new(format!("{}/rotation", config.cert_cache_dir));
let service_name = "rotation-test-service";
// Generate initial certificate
let cert1 = test_certificate_generation(config, results).await?;
cache.store(service_name.to_string(), cert1.clone()).await?;
// Wait a short time to ensure different timestamps
tokio::time::sleep(Duration::from_millis(100)).await;
// Generate new certificate (simulate rotation)
let rotation_start = Instant::now();
let cert2 = test_certificate_generation(config, results).await?;
// Update cache with new certificate
cache.store(service_name.to_string(), cert2.clone()).await?;
let rotation_duration = rotation_start.elapsed();
// Verify certificates are different
if cert1.serial_number == cert2.serial_number {
return Err(anyhow::anyhow!("Certificate rotation did not generate new certificate"));
}
// Verify both certificates are valid
cert1.validate_for_tls()?;
cert2.validate_for_tls()?;
// Retrieve updated certificate from cache
let retrieved_cert = cache.get(service_name).await
.context("Rotated certificate not found in cache")?;
if retrieved_cert.serial_number != cert2.serial_number {
return Err(anyhow::anyhow!("Cache did not update with rotated certificate"));
}
// Add success to results
{
let mut test_results = results.write().await;
test_results.add_success("certificate_rotation", rotation_duration);
}
info!("Certificate rotation test completed successfully");
Ok(())
}
/// Test certificate expiry handling
pub async fn test_certificate_expiry_handling(
config: &VaultTestConfig,
results: &Arc<RwLock<VaultTestResults>>,
) -> Result<()> {
info!("Testing certificate expiry handling");
// Generate certificate with very short TTL
let cert_request = serde_json::json!({
"common_name": "expiry-test.foxhunt.internal",
"ttl": "30s", // Very short for testing
"format": "pem"
});
let response = make_vault_request(
&format!("{}/v1/pki_int/issue/hft-trading", config.vault_addr),
"POST",
Some(&cert_request),
&config.vault_token,
).await?;
let cert = TestCertificate::from_vault_response(&response, "expiry-test.foxhunt.internal".to_string())?;
// Test renewal threshold logic
let needs_renewal_soon = cert.needs_renewal(Duration::from_secs(60)); // Should be true
let needs_renewal_now = cert.needs_renewal(Duration::from_secs(1)); // Should be false initially
if !needs_renewal_soon {
return Err(anyhow::anyhow!("Certificate should need renewal within 60 seconds"));
}
if needs_renewal_now {
warn!("Certificate needs immediate renewal (may be expected for short TTL)");
}
// Add success to results
{
let mut test_results = results.write().await;
test_results.add_success("certificate_expiry_handling", Duration::from_millis(1));
}
info!("Certificate expiry handling test completed successfully");
Ok(())
}
/// Test multiple certificate generation for different services
pub async fn test_multi_service_certificates(
config: &VaultTestConfig,
results: &Arc<RwLock<VaultTestResults>>,
) -> Result<()> {
info!("Testing certificate generation for multiple services");
let services = vec![
("trading-service", "trading.foxhunt.internal"),
("backtesting-service", "backtesting.foxhunt.internal"),
("tli-service", "tli.foxhunt.internal"),
];
let mut certificates = Vec::new();
let start = Instant::now();
for (service_name, common_name) in &services {
let cert_request = serde_json::json!({
"common_name": common_name,
"ttl": "2h",
"format": "pem"
});
let response = make_vault_request(
&format!("{}/v1/pki_int/issue/hft-trading", config.vault_addr),
"POST",
Some(&cert_request),
&config.vault_token,
).await?;
let cert = TestCertificate::from_vault_response(&response, common_name.to_string())?;
cert.validate_for_tls()?;
certificates.push((service_name.to_string(), cert));
debug!("Certificate generated for service: {}", service_name);
}
let total_duration = start.elapsed();
// Verify all certificates are unique
let mut serial_numbers = std::collections::HashSet::new();
for (_, cert) in &certificates {
if !serial_numbers.insert(&cert.serial_number) {
return Err(anyhow::anyhow!("Duplicate certificate serial number found"));
}
}
// Update performance metrics
{
let mut test_results = results.write().await;
test_results.performance.vault_api_calls += services.len() as u64;
}
// Add success to results
{
let mut test_results = results.write().await;
test_results.add_success("multi_service_certificates", total_duration);
}
info!("Multi-service certificate generation completed successfully");
Ok(())
}
/// Helper function to make Vault HTTP requests
async fn make_vault_request(
url: &str,
method: &str,
body: Option<&serde_json::Value>,
token: &str,
) -> Result<serde_json::Value> {
let mut cmd = tokio::process::Command::new("curl");
cmd.args(["-s", "-f", "-X", method]);
cmd.args(["-H", &format!("X-Vault-Token: {}", token)]);
cmd.args(["-H", "Content-Type: application/json"]);
if let Some(body_data) = body {
cmd.args(["-d", &body_data.to_string()]);
}
cmd.arg(url);
let output = cmd.output().await
.context("Failed to execute Vault request")?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(anyhow::anyhow!("Vault request failed: {}", stderr));
}
let response_text = String::from_utf8_lossy(&output.stdout);
serde_json::from_str(&response_text)
.context("Failed to parse Vault response JSON")
}
/// Run all certificate lifecycle tests
pub async fn run_certificate_tests(
config: &VaultTestConfig,
results: &Arc<RwLock<VaultTestResults>>,
) -> Result<()> {
info!("Running certificate lifecycle tests");
// Test 1: Basic certificate generation
test_certificate_generation(config, results).await?;
// Test 2: Certificate caching
test_certificate_caching(config, results).await?;
// Test 3: Certificate rotation
test_certificate_rotation(config, results).await?;
// Test 4: Certificate expiry handling
test_certificate_expiry_handling(config, results).await?;
// Test 5: Multiple service certificates
test_multi_service_certificates(config, results).await?;
info!("All certificate lifecycle tests completed successfully");
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_certificate_validation() {
let cert = TestCertificate {
certificate: "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----".to_string(),
private_key: "-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----".to_string(),
ca_chain: "-----BEGIN CERTIFICATE-----\nca\n-----END CERTIFICATE-----".to_string(),
serial_number: "123".to_string(),
common_name: "test.example.com".to_string(),
expires_at: SystemTime::now() + Duration::from_secs(3600),
generated_at: Instant::now(),
};
assert!(cert.validate_for_tls().is_ok());
}
#[test]
fn test_certificate_renewal_logic() {
let cert = TestCertificate {
certificate: "test".to_string(),
private_key: "test".to_string(),
ca_chain: "test".to_string(),
serial_number: "123".to_string(),
common_name: "test.example.com".to_string(),
expires_at: SystemTime::now() + Duration::from_secs(1800), // 30 minutes
generated_at: Instant::now(),
};
// Should need renewal if threshold is 1 hour
assert!(cert.needs_renewal(Duration::from_secs(3600)));
// Should not need renewal if threshold is 15 minutes
assert!(!cert.needs_renewal(Duration::from_secs(900)));
}
#[tokio::test]
async fn test_certificate_cache() {
let cache = CertificateCache::new("/tmp/test-certs".to_string());
let cert = TestCertificate {
certificate: "test".to_string(),
private_key: "test".to_string(),
ca_chain: "test".to_string(),
serial_number: "123".to_string(),
common_name: "test.example.com".to_string(),
expires_at: SystemTime::now() + Duration::from_secs(3600),
generated_at: Instant::now(),
};
// This test would require proper file system setup
// Just verify the cache structure
assert!(cache.certificates.read().await.is_empty());
}
}