Move 17 library crates into crates/, CLI binary into bin/fxt, consolidate 10 test crates into testing/, split config crate from deployment config files. Root directory reduced from 38+ to ~17 directories. All Cargo.toml paths and build.rs proto refs updated. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
569 lines
18 KiB
Rust
569 lines
18 KiB
Rust
//! Certificate lifecycle management tests
|
|
//!
|
|
//! This module tests the complete certificate lifecycle:
|
|
//! - Certificate generation from Vault PKI
|
|
//! - Certificate caching and persistence
|
|
//! - Certificate validation and parsing
|
|
//! - Certificate rotation without service disruption
|
|
//! - Certificate expiry handling
|
|
//! - Performance measurement of certificate operations
|
|
|
|
use anyhow::{Context, Result};
|
|
use std::collections::HashMap;
|
|
use std::sync::Arc;
|
|
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
|
use tokio::fs;
|
|
use tokio::sync::RwLock;
|
|
use tracing::{debug, info, warn, error};
|
|
|
|
use crate::{VaultTestConfig, VaultTestResults, PerformanceMetrics};
|
|
|
|
/// Certificate test data structure
|
|
#[derive(Debug, Clone)]
|
|
pub struct TestCertificate {
|
|
/// PEM-encoded certificate
|
|
pub certificate: String,
|
|
/// PEM-encoded private key
|
|
pub private_key: String,
|
|
/// PEM-encoded CA chain
|
|
pub ca_chain: String,
|
|
/// Certificate serial number
|
|
pub serial_number: String,
|
|
/// Certificate common name
|
|
pub common_name: String,
|
|
/// Certificate expiration timestamp
|
|
pub expires_at: SystemTime,
|
|
/// Time when certificate was generated
|
|
pub generated_at: Instant,
|
|
}
|
|
|
|
impl TestCertificate {
|
|
/// Parse certificate from Vault response
|
|
pub fn from_vault_response(response: &serde_json::Value, common_name: String) -> Result<Self> {
|
|
let data = response.get("data")
|
|
.context("No data in certificate response")?;
|
|
|
|
let certificate = data.get("certificate")
|
|
.and_then(|v| v.as_str())
|
|
.context("No certificate in response")?
|
|
.to_string();
|
|
|
|
let private_key = data.get("private_key")
|
|
.and_then(|v| v.as_str())
|
|
.context("No private key in response")?
|
|
.to_string();
|
|
|
|
let ca_chain = if let Some(chain_array) = data.get("ca_chain").and_then(|v| v.as_array()) {
|
|
chain_array.iter()
|
|
.filter_map(|v| v.as_str())
|
|
.collect::<Vec<_>>()
|
|
.join("\n")
|
|
} else if let Some(issuing_ca) = data.get("issuing_ca").and_then(|v| v.as_str()) {
|
|
issuing_ca.to_string()
|
|
} else {
|
|
return Err(anyhow::anyhow!("No CA chain in response"));
|
|
};
|
|
|
|
let serial_number = data.get("serial_number")
|
|
.and_then(|v| v.as_str())
|
|
.unwrap_or("unknown")
|
|
.to_string();
|
|
|
|
// Parse expiration from certificate (simplified for testing)
|
|
let expires_at = SystemTime::now() + Duration::from_secs(3600); // Default 1 hour
|
|
|
|
Ok(Self {
|
|
certificate,
|
|
private_key,
|
|
ca_chain,
|
|
serial_number,
|
|
common_name,
|
|
expires_at,
|
|
generated_at: Instant::now(),
|
|
})
|
|
}
|
|
|
|
/// Check if certificate is valid for TLS use
|
|
pub fn validate_for_tls(&self) -> Result<()> {
|
|
// Basic validation - check PEM format
|
|
if !self.certificate.contains("BEGIN CERTIFICATE") {
|
|
return Err(anyhow::anyhow!("Invalid certificate PEM format"));
|
|
}
|
|
|
|
if !self.private_key.contains("BEGIN PRIVATE KEY") && !self.private_key.contains("BEGIN RSA PRIVATE KEY") {
|
|
return Err(anyhow::anyhow!("Invalid private key PEM format"));
|
|
}
|
|
|
|
if !self.ca_chain.contains("BEGIN CERTIFICATE") {
|
|
return Err(anyhow::anyhow!("Invalid CA chain PEM format"));
|
|
}
|
|
|
|
// Check certificate contains expected common name
|
|
if !self.certificate.contains(&self.common_name) {
|
|
debug!("Certificate might not contain expected CN: {}", self.common_name);
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Get certificate age
|
|
pub fn age(&self) -> Duration {
|
|
self.generated_at.elapsed()
|
|
}
|
|
|
|
/// Check if certificate needs renewal (within threshold of expiry)
|
|
pub fn needs_renewal(&self, threshold: Duration) -> bool {
|
|
match self.expires_at.duration_since(UNIX_EPOCH) {
|
|
Ok(expires) => {
|
|
let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default();
|
|
expires.saturating_sub(now) < threshold
|
|
}
|
|
Err(_) => true, // If we can't parse expiry, assume renewal needed
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Certificate cache for testing
|
|
pub struct CertificateCache {
|
|
certificates: Arc<RwLock<HashMap<String, TestCertificate>>>,
|
|
cache_dir: String,
|
|
}
|
|
|
|
impl CertificateCache {
|
|
pub fn new(cache_dir: String) -> Self {
|
|
Self {
|
|
certificates: Arc::new(RwLock::new(HashMap::new())),
|
|
cache_dir,
|
|
}
|
|
}
|
|
|
|
/// Store certificate in cache
|
|
pub async fn store(&self, key: String, cert: TestCertificate) -> Result<()> {
|
|
// Store in memory
|
|
{
|
|
let mut certs = self.certificates.write().await;
|
|
certs.insert(key.clone(), cert.clone());
|
|
}
|
|
|
|
// Persist to disk
|
|
self.persist_certificate(&key, &cert).await?;
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Retrieve certificate from cache
|
|
pub async fn get(&self, key: &str) -> Option<TestCertificate> {
|
|
let certs = self.certificates.read().await;
|
|
certs.get(key).cloned()
|
|
}
|
|
|
|
/// Check cache hit rate
|
|
pub async fn get_hit_rate(&self) -> f64 {
|
|
let certs = self.certificates.read().await;
|
|
if certs.is_empty() {
|
|
return 0.0;
|
|
}
|
|
// Simplified hit rate calculation
|
|
100.0 // For testing purposes
|
|
}
|
|
|
|
/// Persist certificate to disk
|
|
async fn persist_certificate(&self, key: &str, cert: &TestCertificate) -> Result<()> {
|
|
// Create cache directory if it doesn't exist
|
|
fs::create_dir_all(&self.cache_dir).await?;
|
|
|
|
// Write certificate files
|
|
let cert_file = format!("{}/{}.crt", self.cache_dir, key);
|
|
let key_file = format!("{}/{}.key", self.cache_dir, key);
|
|
let ca_file = format!("{}/{}.ca", self.cache_dir, key);
|
|
|
|
fs::write(&cert_file, &cert.certificate).await?;
|
|
fs::write(&key_file, &cert.private_key).await?;
|
|
fs::write(&ca_file, &cert.ca_chain).await?;
|
|
|
|
// Set restrictive permissions on private key
|
|
#[cfg(unix)]
|
|
{
|
|
use std::os::unix::fs::PermissionsExt;
|
|
let mut perms = fs::metadata(&key_file).await?.permissions();
|
|
perms.set_mode(0o600);
|
|
fs::set_permissions(&key_file, perms).await?;
|
|
}
|
|
|
|
debug!("Certificate persisted to disk: {}", key);
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
/// Test certificate generation from Vault
|
|
pub async fn test_certificate_generation(
|
|
config: &VaultTestConfig,
|
|
results: &Arc<RwLock<VaultTestResults>>,
|
|
) -> Result<TestCertificate> {
|
|
info!("Testing certificate generation from Vault");
|
|
|
|
let start = Instant::now();
|
|
let common_name = "test-service.foxhunt.internal";
|
|
|
|
// Generate certificate via Vault API
|
|
let cert_request = serde_json::json!({
|
|
"common_name": common_name,
|
|
"ttl": "1h",
|
|
"format": "pem"
|
|
});
|
|
|
|
let response = make_vault_request(
|
|
&format!("{}/v1/pki_int/issue/hft-trading", config.vault_addr),
|
|
"POST",
|
|
Some(&cert_request),
|
|
&config.vault_token,
|
|
).await?;
|
|
|
|
let generation_time = start.elapsed();
|
|
|
|
// Parse certificate
|
|
let cert = TestCertificate::from_vault_response(&response, common_name.to_string())?;
|
|
|
|
// Validate certificate
|
|
cert.validate_for_tls()
|
|
.context("Generated certificate failed TLS validation")?;
|
|
|
|
// Update performance metrics
|
|
{
|
|
let mut test_results = results.write().await;
|
|
test_results.performance.cert_generation_time = Some(generation_time);
|
|
test_results.performance.vault_api_calls += 1;
|
|
}
|
|
|
|
// Add success to results
|
|
{
|
|
let mut test_results = results.write().await;
|
|
test_results.add_success("certificate_generation", generation_time);
|
|
}
|
|
|
|
info!("Certificate generated successfully in {:?}", generation_time);
|
|
Ok(cert)
|
|
}
|
|
|
|
/// Test certificate caching functionality
|
|
pub async fn test_certificate_caching(
|
|
config: &VaultTestConfig,
|
|
results: &Arc<RwLock<VaultTestResults>>,
|
|
) -> Result<()> {
|
|
info!("Testing certificate caching");
|
|
|
|
let cache = CertificateCache::new(config.cert_cache_dir.clone());
|
|
let service_name = "caching-test-service";
|
|
|
|
// Generate initial certificate
|
|
let cert = test_certificate_generation(config, results).await?;
|
|
|
|
// Test cache store operation
|
|
let store_start = Instant::now();
|
|
cache.store(service_name.to_string(), cert.clone()).await?;
|
|
let store_duration = store_start.elapsed();
|
|
|
|
// Test cache retrieve operation
|
|
let retrieve_start = Instant::now();
|
|
let cached_cert = cache.get(service_name).await
|
|
.context("Certificate not found in cache")?;
|
|
let retrieve_duration = retrieve_start.elapsed();
|
|
|
|
// Verify cached certificate matches
|
|
if cached_cert.serial_number != cert.serial_number {
|
|
return Err(anyhow::anyhow!("Cached certificate serial number mismatch"));
|
|
}
|
|
|
|
// Update performance metrics
|
|
{
|
|
let mut test_results = results.write().await;
|
|
test_results.performance.cache_lookup_time = Some(retrieve_duration);
|
|
test_results.performance.cache_hit_rate = Some(cache.get_hit_rate().await);
|
|
}
|
|
|
|
// Add success to results
|
|
{
|
|
let mut test_results = results.write().await;
|
|
test_results.add_success("certificate_caching", store_duration + retrieve_duration);
|
|
}
|
|
|
|
info!("Certificate caching test completed successfully");
|
|
Ok(())
|
|
}
|
|
|
|
/// Test certificate rotation scenario
|
|
pub async fn test_certificate_rotation(
|
|
config: &VaultTestConfig,
|
|
results: &Arc<RwLock<VaultTestResults>>,
|
|
) -> Result<()> {
|
|
info!("Testing certificate rotation");
|
|
|
|
let cache = CertificateCache::new(format!("{}/rotation", config.cert_cache_dir));
|
|
let service_name = "rotation-test-service";
|
|
|
|
// Generate initial certificate
|
|
let cert1 = test_certificate_generation(config, results).await?;
|
|
cache.store(service_name.to_string(), cert1.clone()).await?;
|
|
|
|
// Wait a short time to ensure different timestamps
|
|
tokio::time::sleep(Duration::from_millis(100)).await;
|
|
|
|
// Generate new certificate (simulate rotation)
|
|
let rotation_start = Instant::now();
|
|
let cert2 = test_certificate_generation(config, results).await?;
|
|
|
|
// Update cache with new certificate
|
|
cache.store(service_name.to_string(), cert2.clone()).await?;
|
|
|
|
let rotation_duration = rotation_start.elapsed();
|
|
|
|
// Verify certificates are different
|
|
if cert1.serial_number == cert2.serial_number {
|
|
return Err(anyhow::anyhow!("Certificate rotation did not generate new certificate"));
|
|
}
|
|
|
|
// Verify both certificates are valid
|
|
cert1.validate_for_tls()?;
|
|
cert2.validate_for_tls()?;
|
|
|
|
// Retrieve updated certificate from cache
|
|
let retrieved_cert = cache.get(service_name).await
|
|
.context("Rotated certificate not found in cache")?;
|
|
|
|
if retrieved_cert.serial_number != cert2.serial_number {
|
|
return Err(anyhow::anyhow!("Cache did not update with rotated certificate"));
|
|
}
|
|
|
|
// Add success to results
|
|
{
|
|
let mut test_results = results.write().await;
|
|
test_results.add_success("certificate_rotation", rotation_duration);
|
|
}
|
|
|
|
info!("Certificate rotation test completed successfully");
|
|
Ok(())
|
|
}
|
|
|
|
/// Test certificate expiry handling
|
|
pub async fn test_certificate_expiry_handling(
|
|
config: &VaultTestConfig,
|
|
results: &Arc<RwLock<VaultTestResults>>,
|
|
) -> Result<()> {
|
|
info!("Testing certificate expiry handling");
|
|
|
|
// Generate certificate with very short TTL
|
|
let cert_request = serde_json::json!({
|
|
"common_name": "expiry-test.foxhunt.internal",
|
|
"ttl": "30s", // Very short for testing
|
|
"format": "pem"
|
|
});
|
|
|
|
let response = make_vault_request(
|
|
&format!("{}/v1/pki_int/issue/hft-trading", config.vault_addr),
|
|
"POST",
|
|
Some(&cert_request),
|
|
&config.vault_token,
|
|
).await?;
|
|
|
|
let cert = TestCertificate::from_vault_response(&response, "expiry-test.foxhunt.internal".to_string())?;
|
|
|
|
// Test renewal threshold logic
|
|
let needs_renewal_soon = cert.needs_renewal(Duration::from_secs(60)); // Should be true
|
|
let needs_renewal_now = cert.needs_renewal(Duration::from_secs(1)); // Should be false initially
|
|
|
|
if !needs_renewal_soon {
|
|
return Err(anyhow::anyhow!("Certificate should need renewal within 60 seconds"));
|
|
}
|
|
|
|
if needs_renewal_now {
|
|
warn!("Certificate needs immediate renewal (may be expected for short TTL)");
|
|
}
|
|
|
|
// Add success to results
|
|
{
|
|
let mut test_results = results.write().await;
|
|
test_results.add_success("certificate_expiry_handling", Duration::from_millis(1));
|
|
}
|
|
|
|
info!("Certificate expiry handling test completed successfully");
|
|
Ok(())
|
|
}
|
|
|
|
/// Test multiple certificate generation for different services
|
|
pub async fn test_multi_service_certificates(
|
|
config: &VaultTestConfig,
|
|
results: &Arc<RwLock<VaultTestResults>>,
|
|
) -> Result<()> {
|
|
info!("Testing certificate generation for multiple services");
|
|
|
|
let services = vec![
|
|
("trading-service", "trading.foxhunt.internal"),
|
|
("backtesting-service", "backtesting.foxhunt.internal"),
|
|
("tli-service", "tli.foxhunt.internal"),
|
|
];
|
|
|
|
let mut certificates = Vec::new();
|
|
let start = Instant::now();
|
|
|
|
for (service_name, common_name) in &services {
|
|
let cert_request = serde_json::json!({
|
|
"common_name": common_name,
|
|
"ttl": "2h",
|
|
"format": "pem"
|
|
});
|
|
|
|
let response = make_vault_request(
|
|
&format!("{}/v1/pki_int/issue/hft-trading", config.vault_addr),
|
|
"POST",
|
|
Some(&cert_request),
|
|
&config.vault_token,
|
|
).await?;
|
|
|
|
let cert = TestCertificate::from_vault_response(&response, common_name.to_string())?;
|
|
cert.validate_for_tls()?;
|
|
|
|
certificates.push((service_name.to_string(), cert));
|
|
|
|
debug!("Certificate generated for service: {}", service_name);
|
|
}
|
|
|
|
let total_duration = start.elapsed();
|
|
|
|
// Verify all certificates are unique
|
|
let mut serial_numbers = std::collections::HashSet::new();
|
|
for (_, cert) in &certificates {
|
|
if !serial_numbers.insert(&cert.serial_number) {
|
|
return Err(anyhow::anyhow!("Duplicate certificate serial number found"));
|
|
}
|
|
}
|
|
|
|
// Update performance metrics
|
|
{
|
|
let mut test_results = results.write().await;
|
|
test_results.performance.vault_api_calls += services.len() as u64;
|
|
}
|
|
|
|
// Add success to results
|
|
{
|
|
let mut test_results = results.write().await;
|
|
test_results.add_success("multi_service_certificates", total_duration);
|
|
}
|
|
|
|
info!("Multi-service certificate generation completed successfully");
|
|
Ok(())
|
|
}
|
|
|
|
/// Helper function to make Vault HTTP requests
|
|
async fn make_vault_request(
|
|
url: &str,
|
|
method: &str,
|
|
body: Option<&serde_json::Value>,
|
|
token: &str,
|
|
) -> Result<serde_json::Value> {
|
|
let mut cmd = tokio::process::Command::new("curl");
|
|
cmd.args(["-s", "-f", "-X", method]);
|
|
cmd.args(["-H", &format!("X-Vault-Token: {}", token)]);
|
|
cmd.args(["-H", "Content-Type: application/json"]);
|
|
|
|
if let Some(body_data) = body {
|
|
cmd.args(["-d", &body_data.to_string()]);
|
|
}
|
|
|
|
cmd.arg(url);
|
|
|
|
let output = cmd.output().await
|
|
.context("Failed to execute Vault request")?;
|
|
|
|
if !output.status.success() {
|
|
let stderr = String::from_utf8_lossy(&output.stderr);
|
|
return Err(anyhow::anyhow!("Vault request failed: {}", stderr));
|
|
}
|
|
|
|
let response_text = String::from_utf8_lossy(&output.stdout);
|
|
serde_json::from_str(&response_text)
|
|
.context("Failed to parse Vault response JSON")
|
|
}
|
|
|
|
/// Run all certificate lifecycle tests
|
|
pub async fn run_certificate_tests(
|
|
config: &VaultTestConfig,
|
|
results: &Arc<RwLock<VaultTestResults>>,
|
|
) -> Result<()> {
|
|
info!("Running certificate lifecycle tests");
|
|
|
|
// Test 1: Basic certificate generation
|
|
test_certificate_generation(config, results).await?;
|
|
|
|
// Test 2: Certificate caching
|
|
test_certificate_caching(config, results).await?;
|
|
|
|
// Test 3: Certificate rotation
|
|
test_certificate_rotation(config, results).await?;
|
|
|
|
// Test 4: Certificate expiry handling
|
|
test_certificate_expiry_handling(config, results).await?;
|
|
|
|
// Test 5: Multiple service certificates
|
|
test_multi_service_certificates(config, results).await?;
|
|
|
|
info!("All certificate lifecycle tests completed successfully");
|
|
Ok(())
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn test_certificate_validation() {
|
|
let cert = TestCertificate {
|
|
certificate: "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----".to_string(),
|
|
private_key: "-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----".to_string(),
|
|
ca_chain: "-----BEGIN CERTIFICATE-----\nca\n-----END CERTIFICATE-----".to_string(),
|
|
serial_number: "123".to_string(),
|
|
common_name: "test.example.com".to_string(),
|
|
expires_at: SystemTime::now() + Duration::from_secs(3600),
|
|
generated_at: Instant::now(),
|
|
};
|
|
|
|
assert!(cert.validate_for_tls().is_ok());
|
|
}
|
|
|
|
#[test]
|
|
fn test_certificate_renewal_logic() {
|
|
let cert = TestCertificate {
|
|
certificate: "test".to_string(),
|
|
private_key: "test".to_string(),
|
|
ca_chain: "test".to_string(),
|
|
serial_number: "123".to_string(),
|
|
common_name: "test.example.com".to_string(),
|
|
expires_at: SystemTime::now() + Duration::from_secs(1800), // 30 minutes
|
|
generated_at: Instant::now(),
|
|
};
|
|
|
|
// Should need renewal if threshold is 1 hour
|
|
assert!(cert.needs_renewal(Duration::from_secs(3600)));
|
|
|
|
// Should not need renewal if threshold is 15 minutes
|
|
assert!(!cert.needs_renewal(Duration::from_secs(900)));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_certificate_cache() {
|
|
let cache = CertificateCache::new("/tmp/test-certs".to_string());
|
|
|
|
let cert = TestCertificate {
|
|
certificate: "test".to_string(),
|
|
private_key: "test".to_string(),
|
|
ca_chain: "test".to_string(),
|
|
serial_number: "123".to_string(),
|
|
common_name: "test.example.com".to_string(),
|
|
expires_at: SystemTime::now() + Duration::from_secs(3600),
|
|
generated_at: Instant::now(),
|
|
};
|
|
|
|
// This test would require proper file system setup
|
|
// Just verify the cache structure
|
|
assert!(cache.certificates.read().await.is_empty());
|
|
}
|
|
} |