8950831817
COMPLETED: ✅ Created 3 shared libraries: common, config (foxhunt-config), storage ✅ Config library: PostgreSQL hot-reload, Vault integration, unified ConfigManager ✅ Storage library: S3 with Vault credentials, model checkpoints, zero hardcoded keys ✅ Common library: Shared types, database connections, error handling ✅ Fixed TLI protobuf compilation issues (duplicate health_check, Aad types) ✅ Trading Service migrated to use centralized config SECURITY IMPROVEMENTS: 🔒 ALL AWS credentials now from Vault (no environment variables) 🔒 Circuit breaker patterns for external services 🔒 Secure error messages that don't leak credentials 🔒 Automatic credential refresh with 5-minute TTL ARCHITECTURE: - Single source of truth for configuration - Zero code duplication for common functionality - Hot-reload capability via PostgreSQL NOTIFY/LISTEN - Multi-tier storage with compression and lifecycle management - Type-safe configuration with comprehensive error handling Next: Complete service migrations to use shared libraries
591 lines
21 KiB
Rust
591 lines
21 KiB
Rust
//! Local filesystem storage implementation
|
|
|
|
use std::collections::HashMap;
|
|
use std::path::{Path, PathBuf};
|
|
use std::time::{SystemTime, UNIX_EPOCH};
|
|
|
|
use async_trait::async_trait;
|
|
use chrono::{DateTime, Utc};
|
|
use serde::{Deserialize, Serialize};
|
|
use tokio::fs;
|
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
use tracing::{debug, info, warn};
|
|
|
|
use crate::{Storage, StorageError, StorageMetadata, StorageResult};
|
|
|
|
/// Configuration for local filesystem storage
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct LocalStorageConfig {
|
|
/// Base directory for storage operations
|
|
pub base_path: PathBuf,
|
|
/// Enable atomic writes using temporary files
|
|
pub atomic_writes: bool,
|
|
/// Enable file locking for concurrent access
|
|
pub enable_locking: bool,
|
|
/// Create parent directories if they don't exist
|
|
pub create_directories: bool,
|
|
/// File permissions (Unix only)
|
|
pub file_permissions: Option<u32>,
|
|
/// Directory permissions (Unix only)
|
|
pub directory_permissions: Option<u32>,
|
|
/// Enable compression for stored files
|
|
pub enable_compression: bool,
|
|
/// Buffer size for file operations
|
|
pub buffer_size: usize,
|
|
}
|
|
|
|
impl Default for LocalStorageConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
base_path: PathBuf::from("./storage"),
|
|
atomic_writes: true,
|
|
enable_locking: true,
|
|
create_directories: true,
|
|
file_permissions: Some(0o644),
|
|
directory_permissions: Some(0o755),
|
|
enable_compression: false,
|
|
buffer_size: 8192,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl LocalStorageConfig {
|
|
/// Load configuration from environment variables
|
|
pub fn from_env() -> StorageResult<Self> {
|
|
let mut config = Self::default();
|
|
|
|
if let Ok(base_path) = std::env::var("STORAGE_LOCAL_BASE_PATH") {
|
|
config.base_path = PathBuf::from(base_path);
|
|
}
|
|
|
|
if let Ok(atomic_str) = std::env::var("STORAGE_LOCAL_ATOMIC_WRITES") {
|
|
config.atomic_writes = atomic_str.to_lowercase() == "true";
|
|
}
|
|
|
|
if let Ok(locking_str) = std::env::var("STORAGE_LOCAL_ENABLE_LOCKING") {
|
|
config.enable_locking = locking_str.to_lowercase() == "true";
|
|
}
|
|
|
|
if let Ok(create_str) = std::env::var("STORAGE_LOCAL_CREATE_DIRECTORIES") {
|
|
config.create_directories = create_str.to_lowercase() == "true";
|
|
}
|
|
|
|
if let Ok(compression_str) = std::env::var("STORAGE_LOCAL_ENABLE_COMPRESSION") {
|
|
config.enable_compression = compression_str.to_lowercase() == "true";
|
|
}
|
|
|
|
if let Ok(buffer_str) = std::env::var("STORAGE_LOCAL_BUFFER_SIZE") {
|
|
if let Ok(buffer_size) = buffer_str.parse::<usize>() {
|
|
config.buffer_size = buffer_size;
|
|
}
|
|
}
|
|
|
|
Ok(config)
|
|
}
|
|
|
|
/// Validate the configuration
|
|
pub fn validate(&self) -> StorageResult<()> {
|
|
if !self.base_path.is_absolute() {
|
|
return Err(StorageError::ConfigError {
|
|
message: "Base path must be absolute".to_string(),
|
|
});
|
|
}
|
|
|
|
if self.buffer_size == 0 {
|
|
return Err(StorageError::ConfigError {
|
|
message: "Buffer size must be greater than 0".to_string(),
|
|
});
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
/// File operation types for metrics and logging
|
|
#[derive(Debug, Clone, Copy)]
|
|
pub enum FileOperation {
|
|
Read,
|
|
Write,
|
|
Delete,
|
|
List,
|
|
Exists,
|
|
Metadata,
|
|
}
|
|
|
|
impl FileOperation {
|
|
pub fn as_str(&self) -> &'static str {
|
|
match self {
|
|
FileOperation::Read => "read",
|
|
FileOperation::Write => "write",
|
|
FileOperation::Delete => "delete",
|
|
FileOperation::List => "list",
|
|
FileOperation::Exists => "exists",
|
|
FileOperation::Metadata => "metadata",
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Local filesystem storage implementation
|
|
pub struct LocalStorage {
|
|
config: LocalStorageConfig,
|
|
base_path: PathBuf,
|
|
}
|
|
|
|
impl LocalStorage {
|
|
/// Create new local storage instance
|
|
pub async fn new(config: LocalStorageConfig) -> StorageResult<Self> {
|
|
config.validate()?;
|
|
|
|
let base_path = config.base_path.clone();
|
|
|
|
// Create base directory if it doesn't exist
|
|
if config.create_directories {
|
|
Self::ensure_directory_exists(&base_path, config.directory_permissions).await?;
|
|
}
|
|
|
|
info!("Local storage initialized at: {}", base_path.display());
|
|
|
|
Ok(Self {
|
|
config,
|
|
base_path,
|
|
})
|
|
}
|
|
|
|
/// Get the full filesystem path for a storage path
|
|
fn get_full_path(&self, path: &str) -> PathBuf {
|
|
// Sanitize path to prevent directory traversal
|
|
let sanitized = path.replace("..", "").replace("\\", "/");
|
|
let sanitized = sanitized.trim_start_matches('/');
|
|
|
|
self.base_path.join(sanitized)
|
|
}
|
|
|
|
/// Ensure a directory exists, creating it if necessary
|
|
async fn ensure_directory_exists(path: &Path, permissions: Option<u32>) -> StorageResult<()> {
|
|
if !path.exists() {
|
|
fs::create_dir_all(path).await.map_err(|e| StorageError::IoError {
|
|
message: format!("Failed to create directory {}: {}", path.display(), e),
|
|
})?;
|
|
|
|
// Set permissions on Unix systems
|
|
#[cfg(unix)]
|
|
if let Some(perms) = permissions {
|
|
use std::os::unix::fs::PermissionsExt;
|
|
let std_perms = std::fs::Permissions::from_mode(perms);
|
|
std::fs::set_permissions(path, std_perms).map_err(|e| StorageError::IoError {
|
|
message: format!("Failed to set directory permissions for {}: {}", path.display(), e),
|
|
})?;
|
|
}
|
|
|
|
debug!("Created directory: {}", path.display());
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Compress data if compression is enabled
|
|
fn compress_data(&self, data: &[u8]) -> StorageResult<Vec<u8>> {
|
|
if !self.config.enable_compression {
|
|
return Ok(data.to_vec());
|
|
}
|
|
|
|
use flate2::write::GzEncoder;
|
|
use flate2::Compression;
|
|
use std::io::Write;
|
|
|
|
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
|
|
encoder.write_all(data).map_err(|e| StorageError::CompressionError {
|
|
message: format!("Compression failed: {}", e),
|
|
})?;
|
|
|
|
let compressed = encoder.finish().map_err(|e| StorageError::CompressionError {
|
|
message: format!("Compression finalization failed: {}", e),
|
|
})?;
|
|
|
|
debug!("Compressed {} bytes to {} bytes", data.len(), compressed.len());
|
|
Ok(compressed)
|
|
}
|
|
|
|
/// Decompress data if it was compressed
|
|
fn decompress_data(&self, data: &[u8]) -> StorageResult<Vec<u8>> {
|
|
if !self.config.enable_compression {
|
|
return Ok(data.to_vec());
|
|
}
|
|
|
|
use flate2::read::GzDecoder;
|
|
use std::io::Read;
|
|
|
|
let mut decoder = GzDecoder::new(data);
|
|
let mut decompressed = Vec::new();
|
|
decoder.read_to_end(&mut decompressed).map_err(|e| StorageError::CompressionError {
|
|
message: format!("Decompression failed: {}", e),
|
|
})?;
|
|
|
|
debug!("Decompressed {} bytes to {} bytes", data.len(), decompressed.len());
|
|
Ok(decompressed)
|
|
}
|
|
|
|
/// Write data to file with optional atomic operation
|
|
async fn write_file_data(&self, full_path: &Path, data: &[u8]) -> StorageResult<()> {
|
|
// Ensure parent directory exists
|
|
if let Some(parent) = full_path.parent() {
|
|
if self.config.create_directories {
|
|
Self::ensure_directory_exists(parent, self.config.directory_permissions).await?;
|
|
}
|
|
}
|
|
|
|
let compressed_data = self.compress_data(data)?;
|
|
|
|
if self.config.atomic_writes {
|
|
// Write to temporary file first, then rename
|
|
let temp_path = full_path.with_extension(format!(
|
|
"{}.tmp.{}",
|
|
full_path.extension().and_then(|s| s.to_str()).unwrap_or(""),
|
|
std::process::id()
|
|
));
|
|
|
|
fs::write(&temp_path, &compressed_data).await.map_err(|e| StorageError::IoError {
|
|
message: format!("Failed to write temporary file {}: {}", temp_path.display(), e),
|
|
})?;
|
|
|
|
// Set file permissions
|
|
#[cfg(unix)]
|
|
if let Some(perms) = self.config.file_permissions {
|
|
use std::os::unix::fs::PermissionsExt;
|
|
let std_perms = std::fs::Permissions::from_mode(perms);
|
|
std::fs::set_permissions(&temp_path, std_perms).map_err(|e| StorageError::IoError {
|
|
message: format!("Failed to set file permissions for {}: {}", temp_path.display(), e),
|
|
})?;
|
|
}
|
|
|
|
// Atomic rename
|
|
fs::rename(&temp_path, full_path).await.map_err(|e| StorageError::IoError {
|
|
message: format!("Failed to rename {} to {}: {}", temp_path.display(), full_path.display(), e),
|
|
})?;
|
|
|
|
debug!("Atomically wrote {} bytes to {}", compressed_data.len(), full_path.display());
|
|
} else {
|
|
// Direct write
|
|
fs::write(full_path, &compressed_data).await.map_err(|e| StorageError::IoError {
|
|
message: format!("Failed to write file {}: {}", full_path.display(), e),
|
|
})?;
|
|
|
|
// Set file permissions
|
|
#[cfg(unix)]
|
|
if let Some(perms) = self.config.file_permissions {
|
|
use std::os::unix::fs::PermissionsExt;
|
|
let std_perms = std::fs::Permissions::from_mode(perms);
|
|
std::fs::set_permissions(full_path, std_perms).map_err(|e| StorageError::IoError {
|
|
message: format!("Failed to set file permissions for {}: {}", full_path.display(), e),
|
|
})?;
|
|
}
|
|
|
|
debug!("Wrote {} bytes to {}", compressed_data.len(), full_path.display());
|
|
}
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Record operation metrics
|
|
fn record_metrics(&self, operation: FileOperation, duration: std::time::Duration, success: bool) {
|
|
crate::metrics::get_metrics().record_operation(operation.as_str(), "local", duration, success);
|
|
}
|
|
}
|
|
|
|
#[async_trait]
|
|
impl Storage for LocalStorage {
|
|
async fn store(&self, path: &str, data: &[u8]) -> StorageResult<()> {
|
|
let start = std::time::Instant::now();
|
|
let full_path = self.get_full_path(path);
|
|
|
|
let result = self.write_file_data(&full_path, data).await;
|
|
|
|
let duration = start.elapsed();
|
|
self.record_metrics(FileOperation::Write, duration, result.is_ok());
|
|
|
|
if result.is_ok() {
|
|
crate::metrics::get_metrics().record_transfer("store", "local", data.len() as u64, duration);
|
|
}
|
|
|
|
result
|
|
}
|
|
|
|
async fn retrieve(&self, path: &str) -> StorageResult<Vec<u8>> {
|
|
let start = std::time::Instant::now();
|
|
let full_path = self.get_full_path(path);
|
|
|
|
let result = async {
|
|
let compressed_data = fs::read(&full_path).await.map_err(|e| match e.kind() {
|
|
std::io::ErrorKind::NotFound => StorageError::NotFound {
|
|
path: path.to_string(),
|
|
},
|
|
std::io::ErrorKind::PermissionDenied => StorageError::PermissionDenied {
|
|
path: path.to_string(),
|
|
},
|
|
_ => StorageError::IoError {
|
|
message: format!("Failed to read file {}: {}", full_path.display(), e),
|
|
},
|
|
})?;
|
|
|
|
let data = self.decompress_data(&compressed_data)?;
|
|
debug!("Retrieved {} bytes from {}", data.len(), full_path.display());
|
|
Ok(data)
|
|
}.await;
|
|
|
|
let duration = start.elapsed();
|
|
self.record_metrics(FileOperation::Read, duration, result.is_ok());
|
|
|
|
if let Ok(ref data) = result {
|
|
crate::metrics::get_metrics().record_transfer("retrieve", "local", data.len() as u64, duration);
|
|
}
|
|
|
|
result
|
|
}
|
|
|
|
async fn exists(&self, path: &str) -> StorageResult<bool> {
|
|
let start = std::time::Instant::now();
|
|
let full_path = self.get_full_path(path);
|
|
|
|
let exists = full_path.exists();
|
|
debug!("Path {} exists: {}", path, exists);
|
|
|
|
let duration = start.elapsed();
|
|
self.record_metrics(FileOperation::Exists, duration, true);
|
|
|
|
Ok(exists)
|
|
}
|
|
|
|
async fn delete(&self, path: &str) -> StorageResult<bool> {
|
|
let start = std::time::Instant::now();
|
|
let full_path = self.get_full_path(path);
|
|
|
|
let result = if full_path.exists() {
|
|
fs::remove_file(&full_path).await.map_err(|e| match e.kind() {
|
|
std::io::ErrorKind::NotFound => StorageError::NotFound {
|
|
path: path.to_string(),
|
|
},
|
|
std::io::ErrorKind::PermissionDenied => StorageError::PermissionDenied {
|
|
path: path.to_string(),
|
|
},
|
|
_ => StorageError::IoError {
|
|
message: format!("Failed to delete file {}: {}", full_path.display(), e),
|
|
},
|
|
})?;
|
|
|
|
debug!("Deleted file: {}", full_path.display());
|
|
Ok(true)
|
|
} else {
|
|
debug!("File not found for deletion: {}", full_path.display());
|
|
Ok(false)
|
|
};
|
|
|
|
let duration = start.elapsed();
|
|
self.record_metrics(FileOperation::Delete, duration, result.is_ok());
|
|
|
|
result
|
|
}
|
|
|
|
async fn list(&self, prefix: &str) -> StorageResult<Vec<String>> {
|
|
let start = std::time::Instant::now();
|
|
let prefix_path = self.get_full_path(prefix);
|
|
|
|
let result = async {
|
|
let mut paths = Vec::new();
|
|
|
|
// Handle both directory listing and prefix matching
|
|
let (search_dir, file_prefix) = if prefix_path.is_dir() {
|
|
(prefix_path, String::new())
|
|
} else {
|
|
let parent = prefix_path.parent().unwrap_or(&self.base_path);
|
|
let filename = prefix_path.file_name()
|
|
.and_then(|n| n.to_str())
|
|
.unwrap_or("")
|
|
.to_string();
|
|
(parent.to_path_buf(), filename)
|
|
};
|
|
|
|
if !search_dir.exists() {
|
|
return Ok(paths);
|
|
}
|
|
|
|
let mut entries = fs::read_dir(&search_dir).await.map_err(|e| StorageError::IoError {
|
|
message: format!("Failed to read directory {}: {}", search_dir.display(), e),
|
|
})?;
|
|
|
|
while let Some(entry) = entries.next_entry().await.map_err(|e| StorageError::IoError {
|
|
message: format!("Failed to read directory entry: {}", e),
|
|
})? {
|
|
let entry_path = entry.path();
|
|
|
|
if entry_path.is_file() {
|
|
if let Some(filename) = entry_path.file_name().and_then(|n| n.to_str()) {
|
|
if file_prefix.is_empty() || filename.starts_with(&file_prefix) {
|
|
// Convert back to relative path
|
|
if let Ok(relative) = entry_path.strip_prefix(&self.base_path) {
|
|
if let Some(path_str) = relative.to_str() {
|
|
paths.push(path_str.replace('\\', "/"));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
paths.sort();
|
|
debug!("Listed {} files with prefix '{}'", paths.len(), prefix);
|
|
Ok(paths)
|
|
}.await;
|
|
|
|
let duration = start.elapsed();
|
|
self.record_metrics(FileOperation::List, duration, result.is_ok());
|
|
|
|
result
|
|
}
|
|
|
|
async fn metadata(&self, path: &str) -> StorageResult<StorageMetadata> {
|
|
let start = std::time::Instant::now();
|
|
let full_path = self.get_full_path(path);
|
|
|
|
let result = async {
|
|
let metadata = fs::metadata(&full_path).await.map_err(|e| match e.kind() {
|
|
std::io::ErrorKind::NotFound => StorageError::NotFound {
|
|
path: path.to_string(),
|
|
},
|
|
std::io::ErrorKind::PermissionDenied => StorageError::PermissionDenied {
|
|
path: path.to_string(),
|
|
},
|
|
_ => StorageError::IoError {
|
|
message: format!("Failed to get metadata for {}: {}", full_path.display(), e),
|
|
},
|
|
})?;
|
|
|
|
let last_modified = metadata.modified()
|
|
.unwrap_or(SystemTime::UNIX_EPOCH)
|
|
.duration_since(UNIX_EPOCH)
|
|
.unwrap_or_default();
|
|
|
|
let last_modified_dt = DateTime::from_timestamp(last_modified.as_secs() as i64, 0)
|
|
.unwrap_or_else(|| Utc::now());
|
|
|
|
let storage_metadata = StorageMetadata {
|
|
path: path.to_string(),
|
|
size: metadata.len(),
|
|
content_type: Some("application/octet-stream".to_string()),
|
|
last_modified: last_modified_dt,
|
|
etag: None, // Could implement checksum-based ETag
|
|
tags: HashMap::new(),
|
|
};
|
|
|
|
debug!("Retrieved metadata for {}: {} bytes", path, metadata.len());
|
|
Ok(storage_metadata)
|
|
}.await;
|
|
|
|
let duration = start.elapsed();
|
|
self.record_metrics(FileOperation::Metadata, duration, result.is_ok());
|
|
|
|
result
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use tempfile::TempDir;
|
|
|
|
async fn create_test_storage() -> (LocalStorage, TempDir) {
|
|
let temp_dir = TempDir::new().unwrap();
|
|
let config = LocalStorageConfig {
|
|
base_path: temp_dir.path().to_path_buf(),
|
|
..Default::default()
|
|
};
|
|
let storage = LocalStorage::new(config).await.unwrap();
|
|
(storage, temp_dir)
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_store_and_retrieve() {
|
|
let (storage, _temp_dir) = create_test_storage().await;
|
|
let test_data = b"Hello, World!";
|
|
|
|
storage.store("test.txt", test_data).await.unwrap();
|
|
let retrieved = storage.retrieve("test.txt").await.unwrap();
|
|
|
|
assert_eq!(retrieved, test_data);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_exists() {
|
|
let (storage, _temp_dir) = create_test_storage().await;
|
|
|
|
assert!(!storage.exists("nonexistent.txt").await.unwrap());
|
|
|
|
storage.store("test.txt", b"data").await.unwrap();
|
|
assert!(storage.exists("test.txt").await.unwrap());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_delete() {
|
|
let (storage, _temp_dir) = create_test_storage().await;
|
|
|
|
storage.store("test.txt", b"data").await.unwrap();
|
|
assert!(storage.exists("test.txt").await.unwrap());
|
|
|
|
let deleted = storage.delete("test.txt").await.unwrap();
|
|
assert!(deleted);
|
|
assert!(!storage.exists("test.txt").await.unwrap());
|
|
|
|
// Deleting non-existent file should return false
|
|
let deleted = storage.delete("nonexistent.txt").await.unwrap();
|
|
assert!(!deleted);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_list() {
|
|
let (storage, _temp_dir) = create_test_storage().await;
|
|
|
|
storage.store("file1.txt", b"data1").await.unwrap();
|
|
storage.store("file2.txt", b"data2").await.unwrap();
|
|
storage.store("other.dat", b"data3").await.unwrap();
|
|
|
|
let all_files = storage.list("").await.unwrap();
|
|
assert_eq!(all_files.len(), 3);
|
|
assert!(all_files.contains(&"file1.txt".to_string()));
|
|
assert!(all_files.contains(&"file2.txt".to_string()));
|
|
assert!(all_files.contains(&"other.dat".to_string()));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_metadata() {
|
|
let (storage, _temp_dir) = create_test_storage().await;
|
|
let test_data = b"Hello, World!";
|
|
|
|
storage.store("test.txt", test_data).await.unwrap();
|
|
let metadata = storage.metadata("test.txt").await.unwrap();
|
|
|
|
assert_eq!(metadata.path, "test.txt");
|
|
assert!(metadata.size > 0); // May be larger due to compression
|
|
assert_eq!(metadata.content_type, Some("application/octet-stream".to_string()));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_nested_directories() {
|
|
let (storage, _temp_dir) = create_test_storage().await;
|
|
|
|
storage.store("nested/dir/file.txt", b"nested data").await.unwrap();
|
|
let retrieved = storage.retrieve("nested/dir/file.txt").await.unwrap();
|
|
|
|
assert_eq!(retrieved, b"nested data");
|
|
assert!(storage.exists("nested/dir/file.txt").await.unwrap());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_path_sanitization() {
|
|
let (storage, _temp_dir) = create_test_storage().await;
|
|
|
|
// Directory traversal attempts should be sanitized
|
|
storage.store("../../../etc/passwd", b"malicious").await.unwrap();
|
|
|
|
// Should be stored safely within base directory
|
|
assert!(storage.exists("etc/passwd").await.unwrap());
|
|
}
|
|
} |