diff --git a/Cargo.lock b/Cargo.lock index 901352c92..202b1a882 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -102,10 +102,10 @@ dependencies = [ "candle-core", "candle-nn", "chrono", - "config", "criterion", "cudarc 0.12.1", "data", + "foxhunt-config", "foxhunt-core", "futures", "linfa", @@ -1747,6 +1747,17 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b7e4c2464d97fe331d41de9d5db0def0a96f4d823b8b32a2efd503578988973" +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "getrandom 0.2.16", + "instant", + "rand 0.8.5", +] + [[package]] name = "backtesting" version = "1.0.0" @@ -1792,11 +1803,13 @@ dependencies = [ "anyhow", "async-stream", "chrono", + "common", "config", "crossbeam", "dashmap", "data", "dotenvy", + "foxhunt-config", "foxhunt-core", "influxdb2", "num_cpus", @@ -1808,6 +1821,7 @@ dependencies = [ "serde_json", "serial_test", "sqlx", + "storage", "tempfile", "thiserror 1.0.69", "tokio", @@ -2837,6 +2851,31 @@ dependencies = [ "unicode-width 0.2.1", ] +[[package]] +name = "common" +version = "1.0.0" +dependencies = [ + "anyhow", + "async-trait", + "chrono", + "config", + "futures", + "once_cell", + "redis 0.27.6", + "rust_decimal", + "serde", + "serde_json", + "sqlx", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tokio-test", + "toml", + "tracing", + "tracing-subscriber", + "uuid 1.16.0", +] + [[package]] name = "compact_str" version = "0.8.1" @@ -3532,13 +3571,13 @@ dependencies = [ "bincode", "bytes", "chrono", - "config", "crossbeam", "crossbeam-channel", "dashmap", "databento", "fastrand 2.3.0", "flate2", + "foxhunt-config", "foxhunt-core", "futures", "futures-util", @@ -4864,6 +4903,39 @@ dependencies = [ "uuid 1.16.0", ] +[[package]] +name = "foxhunt-config" +version = "1.0.0" +dependencies = [ + "anyhow", + "async-trait", + "base64 0.22.1", + "chrono", + "dashmap", + "futures", + "num_cpus", + "once_cell", + "parking_lot 0.12.4", + "reqwest 0.12.4", + "rustc-hash 1.1.0", + "serde", + "serde_json", + "serde_yaml", + "sha2", + "sqlx", + "tempfile", + "test-case", + "testcontainers", + "thiserror 1.0.69", + "tokio", + "tokio-test", + "toml", + "tracing", + "uuid 1.16.0", + "vaultrs", + "wiremock", +] + [[package]] name = "foxhunt-core" version = "1.0.0" @@ -8068,6 +8140,38 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom-tracable" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a39d3ec4e5bc9816ca540bd6b1e4885c0275536eb3293d317d984bb17f9a294" +dependencies = [ + "nom 7.1.3", + "nom-tracable-macros", + "nom_locate", +] + +[[package]] +name = "nom-tracable-macros" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9c68f5316254dae193b3ce083f6caf19ae1a58471e6585e89f0796b9e5bdf4a" +dependencies = [ + "quote 1.0.40", + "syn 1.0.109", +] + +[[package]] +name = "nom_locate" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e3c83c053b0713da60c5b8de47fe8e494fe3ece5267b2f23090a07a053ba8f3" +dependencies = [ + "bytecount", + "memchr 2.7.5", + "nom 7.1.3", +] + [[package]] name = "normalize-line-endings" version = "0.3.0" @@ -13706,6 +13810,41 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "storage" +version = "1.0.0" +dependencies = [ + "anyhow", + "async-trait", + "aws-config", + "aws-sdk-s3", + "aws-types", + "backoff", + "bincode", + "chrono", + "dashmap", + "flate2", + "fs2", + "futures", + "indexmap 2.11.4", + "lru", + "parking_lot 0.12.4", + "rustc-hash 1.1.0", + "serde", + "serde_json", + "serial_test", + "sha2", + "tempfile", + "thiserror 1.0.69", + "tokio", + "tokio-test", + "tokio-util", + "tracing", + "uuid 1.16.0", + "vault", + "wiremock", +] + [[package]] name = "streaming-iterator" version = "0.1.9" @@ -14909,8 +15048,10 @@ dependencies = [ "blake3", "chrono", "clap 4.5.48", + "common", "config", "data", + "foxhunt-config", "foxhunt-core", "futures", "hdrhistogram", @@ -14925,6 +15066,7 @@ dependencies = [ "serde_json", "sha2", "sqlx", + "storage", "tokio", "tokio-stream", "toml", @@ -15287,6 +15429,19 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "943ce29a8a743eb10d6082545d861b24f9d1b160b7d741e0f2cdf726bec909c5" +[[package]] +name = "vault" +version = "10.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "746d7814ee13975d8eb857ac881c34058aa802eae8065024bc92b7762e6d18b6" +dependencies = [ + "byteorder", + "nom 7.1.3", + "nom-tracable", + "nom_locate", + "uuid 1.16.0", +] + [[package]] name = "vaultrs" version = "0.7.4" diff --git a/Cargo.toml b/Cargo.toml index 6c9b94c2a..93ab1fbc7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -161,6 +161,9 @@ members = [ "data", "backtesting", "adaptive-strategy", + "common", + "storage", + "crates/config", "services/backtesting_service", "services/trading_service", "services/ml_training_service", @@ -357,6 +360,9 @@ risk = { path = "risk" } backtesting = { path = "backtesting" } ml = { path = "ml" } adaptive-strategy = { path = "adaptive-strategy" } +common = { path = "common" } +storage = { path = "storage" } +foxhunt-config = { path = "crates/config" } # Enable CUDA features by default for GPU acceleration [features] diff --git a/adaptive-strategy/Cargo.toml b/adaptive-strategy/Cargo.toml index 4f08643f6..63ed6f519 100644 --- a/adaptive-strategy/Cargo.toml +++ b/adaptive-strategy/Cargo.toml @@ -37,7 +37,7 @@ statrs = { workspace = true } ta = { workspace = true } # Configuration (using workspace dependencies) -config = { workspace = true } +foxhunt-config = { workspace = true } # GPU acceleration dependencies (coordinated with workspace) cudarc = { workspace = true, optional = true } diff --git a/adaptive-strategy/src/lib.rs b/adaptive-strategy/src/lib.rs index f68eff388..65ffb0f7f 100644 --- a/adaptive-strategy/src/lib.rs +++ b/adaptive-strategy/src/lib.rs @@ -52,7 +52,7 @@ pub mod risk; use foxhunt_core::types::prelude::*; use anyhow::Result; -use config::StrategyConfig; +use foxhunt_config::StrategyConfig; use ensemble::EnsembleCoordinator; use serde::{Deserialize, Serialize}; use std::sync::Arc; diff --git a/common/Cargo.toml b/common/Cargo.toml index 733cc82ad..f05ed0666 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -34,7 +34,7 @@ chrono = { workspace = true, features = ["serde"] } rust_decimal = { workspace = true, features = ["serde", "macros"] } # Database dependencies -sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres", "chrono", "uuid"] } +sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres", "chrono", "uuid"], optional = true } redis.workspace = true # Logging and tracing diff --git a/core/src/storage/mod.rs b/core/src/storage/mod.rs index c74eb4d91..f2e0c7bbd 100644 --- a/core/src/storage/mod.rs +++ b/core/src/storage/mod.rs @@ -1,11 +1,20 @@ //! Storage Management Module //! -//! This module provides various storage backends and archival solutions -//! for the HFT system, including local storage and AWS S3 integration. +//! DEPRECATED: Storage functionality has been moved to the dedicated `storage` crate. +//! This module is kept for backward compatibility during migration. +//! +//! Please use `storage` crate for all storage operations: +//! - S3 operations with Vault integration +//! - Local filesystem storage +//! - Model checkpoint management +//! - Archival and lifecycle management -pub mod s3_archival; - -pub use s3_archival::{ - S3Archival, S3ArchivalService, S3ArchivalConfig, - ArchivalDataType, ArchivalMetadata, ArchivalStats +// Re-export from storage crate for backward compatibility +pub use storage::{ + S3Storage as S3ArchivalService, + S3StorageConfig as S3ArchivalConfig, + ArchivalDataType, + ArchivalMetadata, + ArchivalStats, + Storage as S3Archival, }; \ No newline at end of file diff --git a/crates/config/Cargo.toml b/crates/config/Cargo.toml index 680bbac13..343e7dcf0 100644 --- a/crates/config/Cargo.toml +++ b/crates/config/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "config" +name = "foxhunt-config" version.workspace = true edition.workspace = true rust-version.workspace = true @@ -53,6 +53,9 @@ reqwest.workspace = true sha2.workspace = true base64.workspace = true +# System info +num_cpus = "1.16" + [dev-dependencies] tokio-test.workspace = true tempfile.workspace = true diff --git a/crates/config/src/database.rs b/crates/config/src/database.rs new file mode 100644 index 000000000..13d98a721 --- /dev/null +++ b/crates/config/src/database.rs @@ -0,0 +1,643 @@ +//! PostgreSQL-based Configuration Database Module +//! +//! This module provides PostgreSQL-backed configuration storage with: +//! - Hot-reload via PostgreSQL NOTIFY/LISTEN +//! - In-memory caching with TTL for performance +//! - Type-safe configuration getters +//! - Automatic table creation and migration +//! - Configuration change notifications + +use crate::{ConfigCategory, ConfigError, ConfigResult, ConfigValue, ConfigSource}; +use anyhow::Context; +// Utc removed - not used in current implementation +use serde::{Deserialize, Serialize}; +use sqlx::{PgPool, Row}; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::{mpsc, RwLock}; +use tokio::time::interval; +use tracing::{debug, error, info, warn}; + +/// Database configuration for PostgreSQL connection +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DatabaseConfig { + /// PostgreSQL connection URL + pub url: String, + /// Maximum number of connections in the pool + pub max_connections: u32, + /// Connection timeout in seconds + pub connect_timeout: u64, + /// Query timeout in seconds + pub query_timeout: u64, + /// Whether to run migrations on startup + pub auto_migrate: bool, +} + +impl DatabaseConfig { + /// Create database configuration from environment variables + pub fn from_env() -> ConfigResult { + let url = std::env::var("DATABASE_URL") + .or_else(|_| std::env::var("FOXHUNT_POSTGRES_URL")) + .unwrap_or_else(|_| "postgresql://postgres:password@localhost/foxhunt".to_string()); + + let max_connections = std::env::var("DATABASE_MAX_CONNECTIONS") + .unwrap_or_else(|_| "10".to_string()) + .parse() + .unwrap_or(10); + + let connect_timeout = std::env::var("DATABASE_CONNECT_TIMEOUT") + .unwrap_or_else(|_| "30".to_string()) + .parse() + .unwrap_or(30); + + let query_timeout = std::env::var("DATABASE_QUERY_TIMEOUT") + .unwrap_or_else(|_| "60".to_string()) + .parse() + .unwrap_or(60); + + let auto_migrate = std::env::var("DATABASE_AUTO_MIGRATE") + .unwrap_or_else(|_| "true".to_string()) + .parse() + .unwrap_or(true); + + Ok(Self { + url, + max_connections, + connect_timeout, + query_timeout, + auto_migrate, + }) + } +} + +impl Default for DatabaseConfig { + fn default() -> Self { + Self { + url: "postgresql://postgres:password@localhost/foxhunt".to_string(), + max_connections: 10, + connect_timeout: 30, + query_timeout: 60, + auto_migrate: true, + } + } +} + +/// Cached configuration entry with TTL +#[derive(Debug, Clone)] +struct CachedConfig { + /// The configuration value + value: ConfigValue, + /// When this entry was cached + cached_at: Instant, + /// TTL for this entry + ttl: Duration, +} + +impl CachedConfig { + /// Check if this cached entry has expired + fn is_expired(&self) -> bool { + self.cached_at.elapsed() > self.ttl + } +} + +/// PostgreSQL Configuration Loader with hot-reload support +pub struct PostgresConfigLoader { + /// PostgreSQL connection pool + pool: PgPool, + /// In-memory cache of configurations + cache: Arc>>, + /// Default TTL for cached entries + default_ttl: Duration, + /// Channel for hot-reload notifications + reload_tx: mpsc::UnboundedSender<(ConfigCategory, String)>, + /// Receiver for hot-reload notifications (for internal use) + reload_rx: Arc>>>, + /// Environment for configuration loading + environment: String, +} + +impl PostgresConfigLoader { + /// Create a new PostgreSQL configuration loader + pub async fn new(config: DatabaseConfig, default_ttl: Duration) -> ConfigResult { + let pool = PgPool::connect(&config.url) + .await + .context("Failed to connect to PostgreSQL")?; + + let environment = std::env::var("FOXHUNT_ENV").unwrap_or_else(|_| "development".to_string()); + + let (reload_tx, reload_rx) = mpsc::unbounded_channel(); + + let loader = Self { + pool, + cache: Arc::new(RwLock::new(HashMap::new())), + default_ttl, + reload_tx, + reload_rx: Arc::new(RwLock::new(Some(reload_rx))), + environment, + }; + + // Create configuration tables and triggers if needed + if config.auto_migrate { + loader.create_tables().await?; + } + + // Start the hot-reload listener + loader.start_notify_listener().await?; + + info!( + "PostgreSQL ConfigLoader initialized for environment '{}' with TTL {:?}", + loader.environment, default_ttl + ); + + Ok(loader) + } + + /// Create configuration tables and triggers if they don't exist + async fn create_tables(&self) -> ConfigResult<()> { + let categories = [ + ConfigCategory::Trading, + ConfigCategory::Risk, + ConfigCategory::MarketData, + ConfigCategory::MachineLearning, + ConfigCategory::Brokers, + ConfigCategory::Performance, + ConfigCategory::Security, + ConfigCategory::Environment, + ]; + + for category in &categories { + let table_name = category.table_name(); + let sql = format!( + r#" + CREATE TABLE IF NOT EXISTS {} ( + id SERIAL PRIMARY KEY, + key VARCHAR(255) NOT NULL, + value JSONB NOT NULL, + environment VARCHAR(50) NOT NULL DEFAULT 'development', + description TEXT, + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW(), + UNIQUE(key, environment) + ); + + CREATE INDEX IF NOT EXISTS idx_{}_key_env ON {} (key, environment); + CREATE INDEX IF NOT EXISTS idx_{}_updated_at ON {} (updated_at); + CREATE INDEX IF NOT EXISTS idx_{}_active ON {} (is_active) WHERE is_active = TRUE; + + CREATE OR REPLACE FUNCTION notify_{}_changes() + RETURNS trigger AS $$ + BEGIN + PERFORM pg_notify('{}', NEW.key || ':' || NEW.environment); + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + DROP TRIGGER IF EXISTS {}_notify_trigger ON {}; + CREATE TRIGGER {}_notify_trigger + AFTER INSERT OR UPDATE ON {} + FOR EACH ROW EXECUTE FUNCTION notify_{}_changes(); + "#, + table_name, + table_name, table_name, + table_name, table_name, + table_name, table_name, + table_name, + category.notify_channel(), + table_name, table_name, + table_name, + table_name, + table_name + ); + + sqlx::query(&sql) + .execute(&self.pool) + .await + .with_context(|| format!("Failed to create table {}", table_name))?; + } + + info!("Configuration tables and triggers created successfully"); + Ok(()) + } + + /// Start the PostgreSQL NOTIFY listener for hot-reload + async fn start_notify_listener(&self) -> ConfigResult<()> { + let pool = self.pool.clone(); + let reload_tx = self.reload_tx.clone(); + + tokio::spawn(async move { + let mut listener = match sqlx::postgres::PgListener::connect_with(&pool).await { + Ok(listener) => listener, + Err(e) => { + error!("Failed to create NOTIFY listener: {}", e); + return; + } + }; + + // Subscribe to all configuration change channels + let categories = [ + ConfigCategory::Trading, + ConfigCategory::Risk, + ConfigCategory::MarketData, + ConfigCategory::MachineLearning, + ConfigCategory::Brokers, + ConfigCategory::Performance, + ConfigCategory::Security, + ConfigCategory::Environment, + ]; + + for category in &categories { + if let Err(e) = listener.listen(category.notify_channel()).await { + error!( + "Failed to listen on channel {}: {}", + category.notify_channel(), + e + ); + return; + } + } + + info!("NOTIFY listener started for configuration hot-reload"); + + loop { + match listener.recv().await { + Ok(notification) => { + let channel = notification.channel(); + let payload = notification.payload(); + + debug!("Received NOTIFY on channel {}: {}", channel, payload); + + // Determine which category was updated + let category = match channel { + "config_trading_changes" => ConfigCategory::Trading, + "config_risk_changes" => ConfigCategory::Risk, + "config_market_data_changes" => ConfigCategory::MarketData, + "config_ml_changes" => ConfigCategory::MachineLearning, + "config_broker_changes" => ConfigCategory::Brokers, + "config_performance_changes" => ConfigCategory::Performance, + "config_security_changes" => ConfigCategory::Security, + "config_environment_changes" => ConfigCategory::Environment, + _ => { + warn!("Unknown notification channel: {}", channel); + continue; + } + }; + + // Extract key from payload (format: "key:environment") + let key = payload.split(':').next().unwrap_or(payload).to_string(); + + // Send reload notification + if let Err(e) = reload_tx.send((category, key)) { + error!("Failed to send reload notification: {}", e); + break; + } + } + Err(e) => { + error!("Error receiving NOTIFY: {}", e); + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + } + }); + + // Start cache cleanup task + self.start_cache_cleanup().await; + + Ok(()) + } + + /// Start background task to clean up expired cache entries + async fn start_cache_cleanup(&self) { + let cache = self.cache.clone(); + let cleanup_interval = self.default_ttl / 4; // Clean up 4x more frequently than TTL + + tokio::spawn(async move { + let mut interval = interval(cleanup_interval); + + loop { + interval.tick().await; + + let mut cache_guard = cache.write().await; + let initial_size = cache_guard.len(); + + cache_guard.retain(|_, cached| !cached.is_expired()); + + let final_size = cache_guard.len(); + if initial_size != final_size { + debug!( + "Cache cleanup: removed {} expired entries", + initial_size - final_size + ); + } + } + }); + } + + /// Get a configuration value with caching + pub async fn get_config(&self, category: ConfigCategory, key: &str) -> ConfigResult> + where + T: for<'de> Deserialize<'de>, + { + let cache_key = (category.clone(), key.to_string()); + + // Check cache first + { + let cache_guard = self.cache.read().await; + if let Some(cached) = cache_guard.get(&cache_key) { + if !cached.is_expired() { + debug!("Cache hit for {}.{}", category.table_name(), key); + return Ok(Some(serde_json::from_value(cached.value.value.clone())?)); + } + } + } + + // Cache miss or expired - fetch from database + debug!( + "Cache miss for {}.{}, fetching from database", + category.table_name(), + key + ); + + let table_name = category.table_name(); + let sql = format!( + "SELECT key, value, updated_at, description, is_active FROM {} + WHERE key = $1 AND environment = $2 AND is_active = TRUE + ORDER BY updated_at DESC LIMIT 1", + table_name + ); + + let row = sqlx::query(&sql) + .bind(key) + .bind(&self.environment) + .fetch_optional(&self.pool) + .await + .with_context(|| format!("Failed to fetch config {}.{}", table_name, key))?; + + if let Some(row) = row { + let config_value = ConfigValue { + key: row.try_get("key")?, + value: row.try_get("value")?, + category: category.clone(), + environment: self.environment.clone(), + updated_at: row.try_get("updated_at")?, + description: row.try_get("description")?, + is_active: row.try_get("is_active")?, + source: ConfigSource::Database, + }; + + // Cache the result + let cached = CachedConfig { + value: config_value.clone(), + cached_at: Instant::now(), + ttl: self.default_ttl, + }; + + { + let mut cache_guard = self.cache.write().await; + cache_guard.insert(cache_key, cached); + } + + Ok(Some(serde_json::from_value(config_value.value)?)) + } else { + Ok(None) + } + } + + /// Set a configuration value + pub async fn set_config( + &self, + category: ConfigCategory, + key: &str, + value: &T, + description: Option<&str>, + ) -> ConfigResult<()> + where + T: Serialize, + { + self.set_config_for_environment(category, key, value, &self.environment, description) + .await + } + + /// Set a configuration value for a specific environment + pub async fn set_config_for_environment( + &self, + category: ConfigCategory, + key: &str, + value: &T, + environment: &str, + description: Option<&str>, + ) -> ConfigResult<()> + where + T: Serialize, + { + let json_value = serde_json::to_value(value)?; + let table_name = category.table_name(); + + let sql = format!( + "INSERT INTO {} (key, value, environment, description, updated_at) + VALUES ($1, $2, $3, $4, NOW()) + ON CONFLICT (key, environment) + DO UPDATE SET value = $2, description = $4, updated_at = NOW()", + table_name + ); + + sqlx::query(&sql) + .bind(key) + .bind(&json_value) + .bind(environment) + .bind(description) + .execute(&self.pool) + .await + .with_context(|| format!("Failed to set config {}.{}", table_name, key))?; + + // Invalidate cache entry + let cache_key = (category, key.to_string()); + { + let mut cache_guard = self.cache.write().await; + cache_guard.remove(&cache_key); + } + + info!("Updated configuration {}.{} for {}", table_name, key, environment); + Ok(()) + } + + /// Get all configurations for a category + pub async fn get_category_configs(&self, category: ConfigCategory) -> ConfigResult> { + self.get_category_configs_for_environment(category, &self.environment).await + } + + /// Get all configurations for a category and environment + pub async fn get_category_configs_for_environment( + &self, + category: ConfigCategory, + environment: &str + ) -> ConfigResult> { + let table_name = category.table_name(); + let sql = format!( + "SELECT key, value, updated_at, description, is_active FROM {} + WHERE environment = $1 AND is_active = TRUE + ORDER BY key", + table_name + ); + + let rows = sqlx::query(&sql) + .bind(environment) + .fetch_all(&self.pool) + .await + .with_context(|| format!("Failed to fetch configs for category {}", table_name))?; + + let mut configs = Vec::new(); + for row in rows { + configs.push(ConfigValue { + key: row.try_get("key")?, + value: row.try_get("value")?, + category: category.clone(), + environment: environment.to_string(), + updated_at: row.try_get("updated_at")?, + description: row.try_get("description")?, + is_active: row.try_get("is_active")?, + source: ConfigSource::Database, + }); + } + + Ok(configs) + } + + /// Subscribe to configuration changes (returns receiver for hot-reload notifications) + pub async fn subscribe_to_changes( + &self, + ) -> ConfigResult> { + let mut reload_rx_guard = self.reload_rx.write().await; + reload_rx_guard + .take() + .ok_or_else(|| ConfigError::ValidationError { + message: "Configuration change subscription already taken".to_string(), + }) + } + + /// Get cache statistics + pub async fn cache_stats(&self) -> (usize, usize) { + let cache_guard = self.cache.read().await; + let total = cache_guard.len(); + let expired = cache_guard.values().filter(|c| c.is_expired()).count(); + (total, expired) + } + + /// Clear the entire cache + pub async fn clear_cache(&self) { + let mut cache_guard = self.cache.write().await; + let size = cache_guard.len(); + cache_guard.clear(); + info!("Cleared {} entries from configuration cache", size); + } + + /// Get current environment + pub fn environment(&self) -> &str { + &self.environment + } + + /// Test database connection + pub async fn test_connection(&self) -> ConfigResult<()> { + sqlx::query("SELECT 1") + .fetch_one(&self.pool) + .await + .context("Failed to test database connection")?; + Ok(()) + } +} + +/// Type-safe configuration getters for common configuration parameters +impl PostgresConfigLoader { + /// Get trading configuration + pub async fn get_trading_config(&self, key: &str) -> ConfigResult> { + self.get_config(ConfigCategory::Trading, key).await + } + + /// Get risk configuration + pub async fn get_risk_config(&self, key: &str) -> ConfigResult> { + self.get_config(ConfigCategory::Risk, key).await + } + + /// Get market data configuration + pub async fn get_market_data_config(&self, key: &str) -> ConfigResult> { + self.get_config(ConfigCategory::MarketData, key).await + } + + /// Get machine learning configuration + pub async fn get_ml_config(&self, key: &str) -> ConfigResult> { + self.get_config(ConfigCategory::MachineLearning, key).await + } + + /// Get broker configuration + pub async fn get_broker_config(&self, key: &str) -> ConfigResult> { + self.get_config(ConfigCategory::Brokers, key).await + } + + /// Get performance configuration + pub async fn get_performance_config(&self, key: &str) -> ConfigResult> { + self.get_config(ConfigCategory::Performance, key).await + } + + /// Get security configuration + pub async fn get_security_config(&self, key: &str) -> ConfigResult> { + self.get_config(ConfigCategory::Security, key).await + } + + /// Get environment configuration + pub async fn get_environment_config(&self, key: &str) -> ConfigResult> { + self.get_config(ConfigCategory::Environment, key).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + #[test] + fn test_database_config_from_env() { + std::env::set_var("DATABASE_URL", "postgresql://test:test@localhost/test"); + std::env::set_var("DATABASE_MAX_CONNECTIONS", "5"); + std::env::set_var("DATABASE_CONNECT_TIMEOUT", "10"); + + let config = DatabaseConfig::from_env().unwrap(); + assert_eq!(config.url, "postgresql://test:test@localhost/test"); + assert_eq!(config.max_connections, 5); + assert_eq!(config.connect_timeout, 10); + + std::env::remove_var("DATABASE_URL"); + std::env::remove_var("DATABASE_MAX_CONNECTIONS"); + std::env::remove_var("DATABASE_CONNECT_TIMEOUT"); + } + + #[test] + fn test_cached_config_expiry() { + let config_value = ConfigValue { + key: "test".to_string(), + value: serde_json::json!("test_value"), + category: ConfigCategory::Trading, + environment: "test".to_string(), + updated_at: Utc::now(), + description: None, + is_active: true, + source: ConfigSource::Database, + }; + + let cached = CachedConfig { + value: config_value.clone(), + cached_at: Instant::now() - Duration::from_secs(10), + ttl: Duration::from_secs(5), + }; + + assert!(cached.is_expired()); + + let fresh_cached = CachedConfig { + value: config_value, + cached_at: Instant::now(), + ttl: Duration::from_secs(60), + }; + + assert!(!fresh_cached.is_expired()); + } +} \ No newline at end of file diff --git a/crates/config/src/error.rs b/crates/config/src/error.rs index c73114d2c..a6c3b076c 100644 --- a/crates/config/src/error.rs +++ b/crates/config/src/error.rs @@ -42,6 +42,18 @@ pub enum ConfigError { #[error("Configuration validation failed: {message}")] ValidationError { message: String }, + + #[error("Connection error: {message}")] + ConnectionError { message: String }, + + #[error("Write error: {message}")] + WriteError { message: String }, + + #[error("Retrieval error: {message}")] + RetrievalError { message: String }, + + #[error("Service unavailable: {message}")] + ServiceUnavailable { message: String }, #[error("Configuration key '{key}' not found in category '{category}'")] KeyNotFound { category: String, key: String }, @@ -179,6 +191,34 @@ impl ConfigError { message: message.into(), } } + + /// Create a connection error + pub fn connection>(message: S) -> Self { + ConfigError::ConnectionError { + message: message.into(), + } + } + + /// Create a write error + pub fn write>(message: S) -> Self { + ConfigError::WriteError { + message: message.into(), + } + } + + /// Create a retrieval error + pub fn retrieval>(message: S) -> Self { + ConfigError::RetrievalError { + message: message.into(), + } + } + + /// Create a service unavailable error + pub fn service_unavailable>(message: S) -> Self { + ConfigError::ServiceUnavailable { + message: message.into(), + } + } /// Check if this error is retryable pub fn is_retryable(&self) -> bool { @@ -202,6 +242,10 @@ impl ConfigError { ConfigError::NotInitialized => false, ConfigError::CacheError { .. } => false, ConfigError::HotReloadError { .. } => true, + ConfigError::ConnectionError { .. } => true, + ConfigError::WriteError { .. } => true, + ConfigError::RetrievalError { .. } => true, + ConfigError::ServiceUnavailable { .. } => true, ConfigError::Unknown { .. } => false, } } @@ -228,6 +272,10 @@ impl ConfigError { ConfigError::NotInitialized => "initialization", ConfigError::CacheError { .. } => "cache", ConfigError::HotReloadError { .. } => "hot_reload", + ConfigError::ConnectionError { .. } => "connection", + ConfigError::WriteError { .. } => "write", + ConfigError::RetrievalError { .. } => "retrieval", + ConfigError::ServiceUnavailable { .. } => "service_unavailable", ConfigError::Unknown { .. } => "unknown", } } @@ -253,7 +301,7 @@ impl From for ConfigError { impl From for ConfigError { fn from(err: serde_yaml::Error) -> Self { ConfigError::ParseError { - source: serde_json::Error::custom(err.to_string()), + source: serde_json::Error::io(std::io::Error::new(std::io::ErrorKind::InvalidData, err.to_string())), } } } @@ -261,7 +309,7 @@ impl From for ConfigError { impl From for ConfigError { fn from(err: toml::de::Error) -> Self { ConfigError::ParseError { - source: serde_json::Error::custom(err.to_string()), + source: serde_json::Error::io(std::io::Error::new(std::io::ErrorKind::InvalidData, err.to_string())), } } } diff --git a/crates/config/src/manager.rs b/crates/config/src/manager.rs new file mode 100644 index 000000000..d769c68eb --- /dev/null +++ b/crates/config/src/manager.rs @@ -0,0 +1,659 @@ +//! Unified Configuration Manager +//! +//! This module provides the main ConfigManager that integrates: +//! - PostgreSQL configuration storage with hot-reload +//! - HashiCorp Vault for secure credential management +//! - Environment variable overrides +//! - File-based configuration fallbacks +//! - Unified caching and change notifications + +use crate::{ + ConfigCategory, ConfigChange, ConfigError, ConfigHealth, ConfigResult, ConfigSource, + ConfigValue, DatabaseConfig, PostgresConfigLoader, VaultConfig, VaultSecrets, +}; +use chrono::Utc; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::{mpsc, RwLock}; +use tracing::{debug, info, warn}; + +/// Configuration manager settings +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ConfigManagerSettings { + /// Default cache TTL + pub cache_ttl_seconds: u64, + /// Health check interval + pub health_check_interval_seconds: u64, + /// Environment name + pub environment: String, + /// Service name + pub service_name: String, + /// Whether to enable Vault integration + pub enable_vault: bool, + /// Whether to enable PostgreSQL integration + pub enable_postgres: bool, + /// Configuration sources priority (in order of preference) + pub source_priority: Vec, +} + +impl Default for ConfigManagerSettings { + fn default() -> Self { + Self { + cache_ttl_seconds: 300, // 5 minutes + health_check_interval_seconds: 60, // 1 minute + environment: std::env::var("FOXHUNT_ENV").unwrap_or_else(|_| "development".to_string()), + service_name: std::env::var("FOXHUNT_SERVICE_NAME").unwrap_or_else(|_| "foxhunt".to_string()), + enable_vault: true, + enable_postgres: true, + source_priority: vec![ + ConfigSource::Environment, + ConfigSource::Vault, + ConfigSource::Database, + ConfigSource::File, + ConfigSource::Default, + ], + } + } +} + +/// Unified Configuration Manager +pub struct ConfigManager { + /// PostgreSQL configuration loader + postgres_loader: Option>, + /// Vault secrets manager + vault_secrets: Option>, + /// Manager settings + settings: ConfigManagerSettings, + /// Configuration change notifications + change_tx: mpsc::UnboundedSender, + /// Health status for components + health_status: Arc>>, +} + +impl ConfigManager { + /// Create new configuration manager with PostgreSQL and Vault + pub async fn new( + db_config: Option, + vault_config: Option, + settings: Option, + ) -> ConfigResult { + let settings = settings.unwrap_or_default(); + let cache_ttl = Duration::from_secs(settings.cache_ttl_seconds); + + // Initialize PostgreSQL loader if enabled and configured + let postgres_loader = if settings.enable_postgres { + if let Some(db_config) = db_config { + match PostgresConfigLoader::new(db_config, cache_ttl).await { + Ok(loader) => { + info!("PostgreSQL configuration loader initialized"); + Some(Arc::new(loader)) + } + Err(e) => { + warn!("Failed to initialize PostgreSQL loader: {}", e); + None + } + } + } else { + warn!("PostgreSQL enabled but no database config provided"); + None + } + } else { + None + }; + + // Initialize Vault secrets manager if enabled and configured + let vault_secrets = if settings.enable_vault { + if let Some(vault_config) = vault_config { + match VaultSecrets::new(vault_config).await { + Ok(vault) => { + info!("Vault secrets manager initialized"); + Some(Arc::new(vault)) + } + Err(e) => { + warn!("Failed to initialize Vault secrets: {}", e); + None + } + } + } else { + warn!("Vault enabled but no vault config provided"); + None + } + } else { + None + }; + + let (change_tx, _change_rx) = mpsc::unbounded_channel(); + + let manager = Self { + postgres_loader, + vault_secrets, + settings, + change_tx, + health_status: Arc::new(RwLock::new(HashMap::new())), + }; + + // Start health monitoring + manager.start_health_monitoring().await; + + // Start configuration change monitoring + manager.start_change_monitoring().await?; + + info!( + "ConfigManager initialized for service '{}' in environment '{}'", + manager.settings.service_name, manager.settings.environment + ); + + Ok(manager) + } + + /// Create new configuration manager from environment + pub async fn from_env() -> ConfigResult { + let db_config = if std::env::var("DATABASE_URL").is_ok() || std::env::var("FOXHUNT_POSTGRES_URL").is_ok() { + Some(DatabaseConfig::from_env()?) + } else { + None + }; + + let vault_config = if std::env::var("VAULT_ADDR").is_ok() && std::env::var("VAULT_ROLE_ID").is_ok() { + Some(VaultConfig::from_env()?) + } else { + None + }; + + Self::new(db_config, vault_config, None).await + } + + /// Get configuration value with source priority + pub async fn get_config( + &self, + category: ConfigCategory, + key: &str, + ) -> ConfigResult> + where + T: for<'de> Deserialize<'de>, + { + for source in &self.settings.source_priority { + match self.get_config_from_source(source, category.clone(), key).await? { + Some(value) => { + debug!( + "Found config {}.{} from source {:?}", + category.table_name(), + key, + source + ); + return Ok(Some(serde_json::from_value(value.value)?)); + } + None => continue, + } + } + + debug!("Config {}.{} not found in any source", category.table_name(), key); + Ok(None) + } + + /// Get configuration value from specific source + async fn get_config_from_source( + &self, + source: &ConfigSource, + category: ConfigCategory, + key: &str, + ) -> ConfigResult> { + match source { + ConfigSource::Environment => self.get_config_from_env(category, key).await, + ConfigSource::Vault => { + if let Some(ref vault) = self.vault_secrets { + vault.get_config(category, key).await + } else { + Ok(None) + } + } + ConfigSource::Database => { + if let Some(ref postgres) = self.postgres_loader { + match postgres.get_config::(category.clone(), key).await? { + Some(value) => Ok(Some(ConfigValue { + key: key.to_string(), + value, + category, + environment: self.settings.environment.clone(), + updated_at: Utc::now(), + description: None, + is_active: true, + source: ConfigSource::Database, + })), + None => Ok(None), + } + } else { + Ok(None) + } + } + ConfigSource::File => { + // TODO: Implement file-based configuration loading + Ok(None) + } + ConfigSource::Default => { + // TODO: Implement default configuration values + Ok(None) + } + } + } + + /// Get configuration from environment variables + async fn get_config_from_env( + &self, + category: ConfigCategory, + key: &str, + ) -> ConfigResult> { + // Build environment variable name + let env_key = format!( + "FOXHUNT_{}_{}", + category.table_name().to_uppercase(), + key.to_uppercase() + ); + + if let Ok(env_value) = std::env::var(&env_key) { + let json_value = if env_value.starts_with('{') || env_value.starts_with('[') { + // Try to parse as JSON + serde_json::from_str(&env_value).unwrap_or_else(|_| serde_json::Value::String(env_value)) + } else { + // Try to parse as number, boolean, or keep as string + env_value.parse::().map(serde_json::Value::from) + .or_else(|_| env_value.parse::().map(serde_json::Value::from)) + .unwrap_or_else(|_| serde_json::Value::String(env_value)) + }; + + Ok(Some(ConfigValue { + key: key.to_string(), + value: json_value, + category, + environment: self.settings.environment.clone(), + updated_at: Utc::now(), + description: Some(format!("Loaded from environment variable {}", env_key)), + is_active: true, + source: ConfigSource::Environment, + })) + } else { + Ok(None) + } + } + + /// Set configuration value (writes to all available backends) + pub async fn set_config( + &self, + category: ConfigCategory, + key: &str, + value: &T, + description: Option<&str>, + ) -> ConfigResult<()> + where + T: Serialize, + { + let mut errors = Vec::new(); + + // Write to PostgreSQL if available + if let Some(ref postgres) = self.postgres_loader { + if let Err(e) = postgres.set_config(category.clone(), key, value, description).await { + errors.push(format!("PostgreSQL: {}", e)); + } + } + + // Write to Vault if available + if let Some(ref vault) = self.vault_secrets { + if let Err(e) = vault.set_config(category.clone(), key, value).await { + errors.push(format!("Vault: {}", e)); + } + } + + if !errors.is_empty() && errors.len() == 2 { + // Both backends failed + return Err(ConfigError::WriteError { + message: format!("Failed to write to all backends: {}", errors.join(", ")), + }); + } + + // Send change notification + let config_value = ConfigValue { + key: key.to_string(), + value: serde_json::to_value(value)?, + category: category.clone(), + environment: self.settings.environment.clone(), + updated_at: Utc::now(), + description: description.map(|s| s.to_string()), + is_active: true, + source: ConfigSource::Database, // Assuming database is primary + }; + + let change = ConfigChange { + category, + key: key.to_string(), + new_value: config_value, + old_value: None, + changed_at: Utc::now(), + changed_by: format!("ConfigManager[{}]", self.settings.service_name), + }; + + let _ = self.change_tx.send(change); + + if !errors.is_empty() { + warn!("Partial failure writing configuration: {}", errors.join(", ")); + } + + Ok(()) + } + + /// Get all configurations for a category + pub async fn get_category_configs( + &self, + category: ConfigCategory, + ) -> ConfigResult> { + let mut all_configs: HashMap = HashMap::new(); + + // Collect configurations from all sources in reverse priority order + for source in self.settings.source_priority.iter().rev() { + let configs = match source { + ConfigSource::Environment => { + // TODO: Scan environment variables for category + Vec::new() + } + ConfigSource::Vault => { + if let Some(ref vault) = self.vault_secrets { + vault.get_category_configs(category.clone()).await.unwrap_or_default() + } else { + Vec::new() + } + } + ConfigSource::Database => { + if let Some(ref postgres) = self.postgres_loader { + postgres.get_category_configs(category.clone()).await.unwrap_or_default() + } else { + Vec::new() + } + } + ConfigSource::File | ConfigSource::Default => Vec::new(), + }; + + // Add configs, but don't overwrite higher priority ones + for config in configs { + all_configs.entry(config.key.clone()).or_insert(config); + } + } + + Ok(all_configs.into_values().collect()) + } + + /// Subscribe to configuration changes + pub async fn subscribe_to_changes(&self) -> ConfigResult> { + let (_tx, rx) = mpsc::unbounded_channel(); + // TODO: Implement proper change subscription multiplexing + Ok(rx) + } + + /// Get health status for all components + pub async fn get_health_status(&self) -> HashMap { + let health = self.health_status.read().await; + health.clone() + } + + /// Start health monitoring for all components + async fn start_health_monitoring(&self) { + let health_status = self.health_status.clone(); + let postgres_loader = self.postgres_loader.clone(); + let vault_secrets = self.vault_secrets.clone(); + let interval_secs = self.settings.health_check_interval_seconds; + + tokio::spawn(async move { + let mut interval = tokio::time::interval(Duration::from_secs(interval_secs)); + + loop { + interval.tick().await; + + let mut health = health_status.write().await; + + // Check PostgreSQL health + if let Some(ref postgres) = postgres_loader { + let pg_health = match postgres.test_connection().await { + Ok(()) => ConfigHealth { + is_healthy: true, + message: "PostgreSQL connection healthy".to_string(), + last_success: Some(Utc::now()), + failure_count: 0, + metrics: HashMap::new(), + }, + Err(e) => { + let current_failure_count = health + .get("postgresql") + .map(|h| h.failure_count) + .unwrap_or(0) + 1; + + ConfigHealth { + is_healthy: false, + message: format!("PostgreSQL connection failed: {}", e), + last_success: health.get("postgresql").and_then(|h| h.last_success), + failure_count: current_failure_count, + metrics: HashMap::new(), + } + } + }; + health.insert("postgresql".to_string(), pg_health); + } + + // Check Vault health + if let Some(ref vault) = vault_secrets { + let vault_health = if vault.health_check().await { + ConfigHealth { + is_healthy: true, + message: "Vault connection healthy".to_string(), + last_success: Some(Utc::now()), + failure_count: 0, + metrics: HashMap::new(), + } + } else { + let current_failure_count = health + .get("vault") + .map(|h| h.failure_count) + .unwrap_or(0) + 1; + + ConfigHealth { + is_healthy: false, + message: "Vault connection failed".to_string(), + last_success: health.get("vault").and_then(|h| h.last_success), + failure_count: current_failure_count, + metrics: HashMap::new(), + } + }; + health.insert("vault".to_string(), vault_health); + } + + // Overall health check + let overall_healthy = health.values().all(|h| h.is_healthy); + let overall_health = ConfigHealth { + is_healthy: overall_healthy, + message: if overall_healthy { + "All configuration components healthy".to_string() + } else { + "Some configuration components unhealthy".to_string() + }, + last_success: if overall_healthy { Some(Utc::now()) } else { None }, + failure_count: if overall_healthy { 0 } else { 1 }, + metrics: HashMap::new(), + }; + health.insert("overall".to_string(), overall_health); + } + }); + } + + /// Start monitoring configuration changes from backends + async fn start_change_monitoring(&self) -> ConfigResult<()> { + // Monitor PostgreSQL changes + if let Some(ref postgres) = self.postgres_loader { + let change_tx = self.change_tx.clone(); + let postgres_clone = postgres.clone(); + + tokio::spawn(async move { + if let Ok(mut changes) = postgres_clone.subscribe_to_changes().await { + while let Some((category, key)) = changes.recv().await { + debug!("PostgreSQL configuration changed: {}.{}", category.table_name(), key); + + // Create change notification + let change = ConfigChange { + category, + key, + new_value: ConfigValue { + key: "unknown".to_string(), + value: serde_json::Value::Null, + category: ConfigCategory::Environment, + environment: "unknown".to_string(), + updated_at: Utc::now(), + description: None, + is_active: true, + source: ConfigSource::Database, + }, + old_value: None, + changed_at: Utc::now(), + changed_by: "PostgreSQL NOTIFY".to_string(), + }; + + let _ = change_tx.send(change); + } + } + }); + } + + // TODO: Monitor Vault changes if it supports notifications + + Ok(()) + } + + /// Test all connections and configurations + pub async fn test_all_connections(&self) -> ConfigResult> { + let mut results = HashMap::new(); + + // Test PostgreSQL + if let Some(ref postgres) = self.postgres_loader { + results.insert( + "postgresql".to_string(), + postgres.test_connection().await.is_ok(), + ); + } + + // Test Vault + if let Some(ref vault) = self.vault_secrets { + results.insert("vault".to_string(), vault.health_check().await); + } + + Ok(results) + } + + /// Get cache statistics for all components + pub async fn get_cache_stats(&self) -> HashMap { + let mut stats = HashMap::new(); + + if let Some(ref postgres) = self.postgres_loader { + stats.insert("postgresql".to_string(), postgres.cache_stats().await); + } + + if let Some(ref vault) = self.vault_secrets { + stats.insert("vault".to_string(), vault.cache_stats().await); + } + + stats + } + + /// Clear all caches + pub async fn clear_all_caches(&self) { + if let Some(ref postgres) = self.postgres_loader { + postgres.clear_cache().await; + } + + if let Some(ref vault) = self.vault_secrets { + vault.clear_cache().await; + } + + info!("Cleared all configuration caches"); + } + + /// Get current environment + pub fn environment(&self) -> &str { + &self.settings.environment + } + + /// Get service name + pub fn service_name(&self) -> &str { + &self.settings.service_name + } +} + +/// Convenience methods for common configuration types +impl ConfigManager { + /// Get string configuration value + pub async fn get_string( + &self, + category: ConfigCategory, + key: &str, + ) -> ConfigResult> { + self.get_config(category, key).await + } + + /// Get integer configuration value + pub async fn get_int( + &self, + category: ConfigCategory, + key: &str, + ) -> ConfigResult> { + self.get_config(category, key).await + } + + /// Get float configuration value + pub async fn get_float( + &self, + category: ConfigCategory, + key: &str, + ) -> ConfigResult> { + self.get_config(category, key).await + } + + /// Get boolean configuration value + pub async fn get_bool( + &self, + category: ConfigCategory, + key: &str, + ) -> ConfigResult> { + self.get_config(category, key).await + } + + /// Get configuration value with default + pub async fn get_with_default( + &self, + category: ConfigCategory, + key: &str, + default: T, + ) -> ConfigResult + where + T: for<'de> Deserialize<'de>, + { + Ok(self.get_config(category, key).await?.unwrap_or(default)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_config_manager_settings_default() { + let settings = ConfigManagerSettings::default(); + assert_eq!(settings.cache_ttl_seconds, 300); + assert_eq!(settings.health_check_interval_seconds, 60); + assert!(settings.enable_vault); + assert!(settings.enable_postgres); + assert!(!settings.source_priority.is_empty()); + } + + #[tokio::test] + async fn test_config_manager_from_env_no_backends() { + // This should succeed even without any backends configured + let result = ConfigManager::from_env().await; + assert!(result.is_ok()); + } +} \ No newline at end of file diff --git a/crates/config/src/structures.rs b/crates/config/src/structures.rs new file mode 100644 index 000000000..cdba34885 --- /dev/null +++ b/crates/config/src/structures.rs @@ -0,0 +1,995 @@ +//! Unified Configuration Structures +//! +//! This module defines the configuration structures used across all Foxhunt services. +//! These structures provide type-safe access to configuration values and ensure +//! consistency across the trading system. + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +// Duration is used in default values + +/// Trading engine configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TradingConfig { + /// Maximum order size (in base currency) + pub max_order_size: f64, + /// Maximum position size (in base currency) + pub max_position_size: f64, + /// Maximum daily trading volume + pub max_daily_volume: f64, + /// Order timeout in seconds + pub order_timeout_seconds: u64, + /// Minimum order size + pub min_order_size: f64, + /// Symbols to trade + pub symbols_to_trade: Vec, + /// Trading hours (UTC) + pub trading_hours: TradingHours, + /// Order execution settings + pub execution: OrderExecutionConfig, + /// Position management settings + pub position_management: PositionManagementConfig, +} + +impl Default for TradingConfig { + fn default() -> Self { + Self { + max_order_size: 10000.0, + max_position_size: 50000.0, + max_daily_volume: 1000000.0, + order_timeout_seconds: 30, + min_order_size: 1.0, + symbols_to_trade: vec![ + "EURUSD".to_string(), + "GBPUSD".to_string(), + "USDJPY".to_string(), + "BTCUSD".to_string(), + "ETHUSD".to_string(), + ], + trading_hours: TradingHours::default(), + execution: OrderExecutionConfig::default(), + position_management: PositionManagementConfig::default(), + } + } +} + +/// Trading hours configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TradingHours { + /// Trading session start (UTC hour, 0-23) + pub start_hour: u8, + /// Trading session end (UTC hour, 0-23) + pub end_hour: u8, + /// Trading days (0 = Sunday, 6 = Saturday) + pub trading_days: Vec, + /// Timezone for trading hours + pub timezone: String, +} + +impl Default for TradingHours { + fn default() -> Self { + Self { + start_hour: 0, + end_hour: 23, + trading_days: vec![1, 2, 3, 4, 5], // Monday to Friday + timezone: "UTC".to_string(), + } + } +} + +/// Order execution configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OrderExecutionConfig { + /// Maximum slippage tolerance (in basis points) + pub max_slippage_bps: f64, + /// Order retry attempts + pub retry_attempts: u32, + /// Retry delay in milliseconds + pub retry_delay_ms: u64, + /// Use IOC (Immediate or Cancel) orders + pub use_ioc_orders: bool, + /// Partial fill acceptance threshold + pub partial_fill_threshold: f64, +} + +impl Default for OrderExecutionConfig { + fn default() -> Self { + Self { + max_slippage_bps: 10.0, + retry_attempts: 3, + retry_delay_ms: 100, + use_ioc_orders: true, + partial_fill_threshold: 0.95, + } + } +} + +/// Position management configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PositionManagementConfig { + /// Enable position netting + pub enable_netting: bool, + /// Maximum positions per symbol + pub max_positions_per_symbol: u32, + /// Position size limits by symbol + pub symbol_position_limits: HashMap, + /// Auto-close positions on trading session end + pub auto_close_on_session_end: bool, +} + +impl Default for PositionManagementConfig { + fn default() -> Self { + Self { + enable_netting: true, + max_positions_per_symbol: 5, + symbol_position_limits: HashMap::new(), + auto_close_on_session_end: false, + } + } +} + +/// Risk management configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RiskConfig { + /// VaR confidence level (0.95, 0.99, etc.) + pub var_confidence: f64, + /// VaR horizon in days + pub var_horizon_days: u32, + /// Maximum drawdown limit (as percentage) + pub max_drawdown_percent: f64, + /// Daily loss limit (in base currency) + pub daily_loss_limit: f64, + /// Portfolio value for risk calculations + pub portfolio_value: f64, + /// Risk limits by symbol + pub symbol_risk_limits: HashMap, + /// Circuit breaker configuration + pub circuit_breaker: CircuitBreakerConfig, + /// Correlation monitoring + pub correlation_monitoring: CorrelationConfig, +} + +impl Default for RiskConfig { + fn default() -> Self { + Self { + var_confidence: 0.95, + var_horizon_days: 1, + max_drawdown_percent: 10.0, + daily_loss_limit: 50000.0, + portfolio_value: 1000000.0, + symbol_risk_limits: HashMap::new(), + circuit_breaker: CircuitBreakerConfig::default(), + correlation_monitoring: CorrelationConfig::default(), + } + } +} + +/// Risk limits for individual symbols +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SymbolRiskLimit { + /// Maximum position size + pub max_position: f64, + /// Maximum daily volume + pub max_daily_volume: f64, + /// VaR limit for this symbol + pub var_limit: f64, + /// Volatility threshold + pub volatility_threshold: f64, +} + +/// Circuit breaker configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CircuitBreakerConfig { + /// Enable circuit breaker + pub enabled: bool, + /// Loss threshold to trigger circuit breaker + pub loss_threshold: f64, + /// Time window for loss calculation (seconds) + pub time_window_seconds: u64, + /// Cool-down period after trigger (seconds) + pub cooldown_seconds: u64, + /// Auto-resume trading after cooldown + pub auto_resume: bool, +} + +impl Default for CircuitBreakerConfig { + fn default() -> Self { + Self { + enabled: true, + loss_threshold: 25000.0, + time_window_seconds: 3600, // 1 hour + cooldown_seconds: 1800, // 30 minutes + auto_resume: false, + } + } +} + +/// Correlation monitoring configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CorrelationConfig { + /// Enable correlation monitoring + pub enabled: bool, + /// Correlation threshold for warnings + pub warning_threshold: f64, + /// Correlation threshold for blocking trades + pub blocking_threshold: f64, + /// Lookback period for correlation calculation (days) + pub lookback_days: u32, + /// Update frequency (seconds) + pub update_frequency_seconds: u64, +} + +impl Default for CorrelationConfig { + fn default() -> Self { + Self { + enabled: true, + warning_threshold: 0.7, + blocking_threshold: 0.9, + lookback_days: 30, + update_frequency_seconds: 300, // 5 minutes + } + } +} + +/// Market data provider configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MarketDataConfig { + /// Primary data providers + pub providers: Vec, + /// Data refresh intervals + pub refresh_intervals: RefreshIntervalConfig, + /// Data quality settings + pub quality_settings: DataQualityConfig, + /// Subscription management + pub subscriptions: SubscriptionConfig, +} + +impl Default for MarketDataConfig { + fn default() -> Self { + Self { + providers: vec![ + DataProviderConfig { + name: "databento".to_string(), + enabled: true, + priority: 1, + api_key_env: "DATABENTO_API_KEY".to_string(), + base_url: "https://hist.databento.com".to_string(), + rate_limit_per_second: 10, + timeout_seconds: 30, + retry_attempts: 3, + }, + DataProviderConfig { + name: "benzinga".to_string(), + enabled: true, + priority: 2, + api_key_env: "BENZINGA_API_KEY".to_string(), + base_url: "https://api.benzinga.com".to_string(), + rate_limit_per_second: 5, + timeout_seconds: 30, + retry_attempts: 3, + }, + ], + refresh_intervals: RefreshIntervalConfig::default(), + quality_settings: DataQualityConfig::default(), + subscriptions: SubscriptionConfig::default(), + } + } +} + +/// Data provider configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DataProviderConfig { + /// Provider name + pub name: String, + /// Whether this provider is enabled + pub enabled: bool, + /// Provider priority (1 = highest) + pub priority: u8, + /// Environment variable containing API key + pub api_key_env: String, + /// Base URL for API + pub base_url: String, + /// Rate limit (requests per second) + pub rate_limit_per_second: u32, + /// Request timeout + pub timeout_seconds: u64, + /// Retry attempts on failure + pub retry_attempts: u32, +} + +/// Data refresh interval configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RefreshIntervalConfig { + /// Real-time tick data (milliseconds) + pub tick_data_ms: u64, + /// Level 1 quotes (milliseconds) + pub level1_quotes_ms: u64, + /// Level 2 order book (milliseconds) + pub level2_orderbook_ms: u64, + /// Trade data (milliseconds) + pub trade_data_ms: u64, + /// News updates (seconds) + pub news_updates_seconds: u64, +} + +impl Default for RefreshIntervalConfig { + fn default() -> Self { + Self { + tick_data_ms: 10, + level1_quotes_ms: 50, + level2_orderbook_ms: 100, + trade_data_ms: 10, + news_updates_seconds: 5, + } + } +} + +/// Data quality configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DataQualityConfig { + /// Maximum allowed age for data (seconds) + pub max_data_age_seconds: u64, + /// Minimum required data sources + pub min_data_sources: u8, + /// Enable data validation + pub enable_validation: bool, + /// Outlier detection settings + pub outlier_detection: OutlierDetectionConfig, +} + +impl Default for DataQualityConfig { + fn default() -> Self { + Self { + max_data_age_seconds: 30, + min_data_sources: 1, + enable_validation: true, + outlier_detection: OutlierDetectionConfig::default(), + } + } +} + +/// Outlier detection configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OutlierDetectionConfig { + /// Enable outlier detection + pub enabled: bool, + /// Z-score threshold for outlier detection + pub z_score_threshold: f64, + /// Lookback period for outlier calculation + pub lookback_periods: u32, +} + +impl Default for OutlierDetectionConfig { + fn default() -> Self { + Self { + enabled: true, + z_score_threshold: 3.0, + lookback_periods: 20, + } + } +} + +/// Subscription configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SubscriptionConfig { + /// Auto-subscribe to new symbols + pub auto_subscribe_new_symbols: bool, + /// Maximum concurrent subscriptions + pub max_subscriptions: u32, + /// Subscription buffer size + pub buffer_size: u32, +} + +impl Default for SubscriptionConfig { + fn default() -> Self { + Self { + auto_subscribe_new_symbols: true, + max_subscriptions: 1000, + buffer_size: 10000, + } + } +} + +/// Machine learning configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MLConfig { + /// Model configurations + pub models: HashMap, + /// Training settings + pub training: TrainingConfig, + /// Inference settings + pub inference: InferenceConfig, + /// Model management + pub model_management: ModelManagementConfig, +} + +impl Default for MLConfig { + fn default() -> Self { + let mut models = HashMap::new(); + models.insert( + "mamba_signals".to_string(), + MLModelConfig { + model_type: "mamba".to_string(), + enabled: true, + model_path: "/opt/foxhunt/models/mamba_signals.pt".to_string(), + input_features: vec![ + "price".to_string(), + "volume".to_string(), + "volatility".to_string(), + ], + output_features: vec!["signal".to_string()], + inference_timeout_ms: 100, + batch_size: 32, + max_sequence_length: 256, + device: "cuda".to_string(), + }, + ); + + Self { + models, + training: TrainingConfig::default(), + inference: InferenceConfig::default(), + model_management: ModelManagementConfig::default(), + } + } +} + +/// ML model configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MLModelConfig { + /// Model type (mamba, transformer, dqn, etc.) + pub model_type: String, + /// Whether this model is enabled + pub enabled: bool, + /// Path to model file + pub model_path: String, + /// Input features + pub input_features: Vec, + /// Output features + pub output_features: Vec, + /// Inference timeout in milliseconds + pub inference_timeout_ms: u64, + /// Batch size for inference + pub batch_size: u32, + /// Maximum sequence length + pub max_sequence_length: u32, + /// Compute device (cpu, cuda, mps) + pub device: String, +} + +/// Training configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TrainingConfig { + /// Enable automatic retraining + pub auto_retrain: bool, + /// Retraining interval in hours + pub retrain_interval_hours: u64, + /// Minimum training data points + pub min_training_data_points: u64, + /// Training batch size + pub batch_size: u32, + /// Learning rate + pub learning_rate: f64, + /// Number of training epochs + pub epochs: u32, + /// Early stopping patience + pub early_stopping_patience: u32, +} + +impl Default for TrainingConfig { + fn default() -> Self { + Self { + auto_retrain: true, + retrain_interval_hours: 24, + min_training_data_points: 10000, + batch_size: 64, + learning_rate: 0.001, + epochs: 100, + early_stopping_patience: 10, + } + } +} + +/// Inference configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InferenceConfig { + /// Maximum inference latency in milliseconds + pub max_inference_latency_ms: u64, + /// Enable inference caching + pub enable_caching: bool, + /// Cache TTL in seconds + pub cache_ttl_seconds: u64, + /// Inference threading + pub num_inference_threads: u32, +} + +impl Default for InferenceConfig { + fn default() -> Self { + Self { + max_inference_latency_ms: 50, + enable_caching: true, + cache_ttl_seconds: 300, + num_inference_threads: 4, + } + } +} + +/// Model management configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ModelManagementConfig { + /// Model storage path + pub model_storage_path: String, + /// Model versioning enabled + pub enable_versioning: bool, + /// Maximum model versions to keep + pub max_versions: u32, + /// Model backup settings + pub backup: ModelBackupConfig, +} + +impl Default for ModelManagementConfig { + fn default() -> Self { + Self { + model_storage_path: "/opt/foxhunt/models".to_string(), + enable_versioning: true, + max_versions: 10, + backup: ModelBackupConfig::default(), + } + } +} + +/// Model backup configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ModelBackupConfig { + /// Enable automatic backups + pub enabled: bool, + /// Backup interval in hours + pub backup_interval_hours: u64, + /// Backup storage path + pub backup_path: String, + /// Compress backups + pub compress: bool, +} + +impl Default for ModelBackupConfig { + fn default() -> Self { + Self { + enabled: true, + backup_interval_hours: 6, + backup_path: "/opt/foxhunt/backups/models".to_string(), + compress: true, + } + } +} + +/// Broker connection configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BrokerConfig { + /// Broker connections + pub brokers: HashMap, + /// Default broker + pub default_broker: String, + /// Failover settings + pub failover: BrokerFailoverConfig, +} + +impl Default for BrokerConfig { + fn default() -> Self { + let mut brokers = HashMap::new(); + brokers.insert( + "interactive_brokers".to_string(), + BrokerConnectionConfig { + broker_type: "interactive_brokers".to_string(), + enabled: true, + host: "127.0.0.1".to_string(), + port: 7497, + client_id: 1, + timeout_seconds: 30, + retry_attempts: 3, + paper_trading: true, + credentials: BrokerCredentials::default(), + }, + ); + + Self { + brokers, + default_broker: "interactive_brokers".to_string(), + failover: BrokerFailoverConfig::default(), + } + } +} + +/// Broker connection configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BrokerConnectionConfig { + /// Broker type + pub broker_type: String, + /// Whether this broker is enabled + pub enabled: bool, + /// Broker host + pub host: String, + /// Broker port + pub port: u16, + /// Client ID + pub client_id: u32, + /// Connection timeout + pub timeout_seconds: u64, + /// Retry attempts + pub retry_attempts: u32, + /// Paper trading mode + pub paper_trading: bool, + /// Broker credentials + pub credentials: BrokerCredentials, +} + +/// Broker credentials +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BrokerCredentials { + /// Username (if applicable) + pub username: Option, + /// Password (if applicable) + pub password: Option, + /// API key (if applicable) + pub api_key: Option, + /// API secret (if applicable) + pub api_secret: Option, +} + +impl Default for BrokerCredentials { + fn default() -> Self { + Self { + username: None, + password: None, + api_key: None, + api_secret: None, + } + } +} + +/// Broker failover configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BrokerFailoverConfig { + /// Enable automatic failover + pub enabled: bool, + /// Failover detection timeout + pub detection_timeout_seconds: u64, + /// Failover retry interval + pub retry_interval_seconds: u64, + /// Maximum failover attempts + pub max_attempts: u32, +} + +impl Default for BrokerFailoverConfig { + fn default() -> Self { + Self { + enabled: true, + detection_timeout_seconds: 30, + retry_interval_seconds: 60, + max_attempts: 3, + } + } +} + +/// Performance configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PerformanceConfig { + /// Latency targets + pub latency_targets: LatencyTargets, + /// Thread pool configuration + pub thread_pools: ThreadPoolConfig, + /// Memory management + pub memory: MemoryConfig, + /// Cache configuration + pub cache: CacheConfig, +} + +impl Default for PerformanceConfig { + fn default() -> Self { + Self { + latency_targets: LatencyTargets::default(), + thread_pools: ThreadPoolConfig::default(), + memory: MemoryConfig::default(), + cache: CacheConfig::default(), + } + } +} + +/// Latency target configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LatencyTargets { + /// Order execution target (microseconds) + pub order_execution_us: u64, + /// Market data processing target (microseconds) + pub market_data_processing_us: u64, + /// Risk calculation target (microseconds) + pub risk_calculation_us: u64, + /// ML inference target (microseconds) + pub ml_inference_us: u64, +} + +impl Default for LatencyTargets { + fn default() -> Self { + Self { + order_execution_us: 1000, // 1ms + market_data_processing_us: 100, // 100μs + risk_calculation_us: 500, // 500μs + ml_inference_us: 50000, // 50ms + } + } +} + +/// Thread pool configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ThreadPoolConfig { + /// Trading thread pool size + pub trading_threads: u32, + /// Market data thread pool size + pub market_data_threads: u32, + /// Risk calculation thread pool size + pub risk_threads: u32, + /// ML inference thread pool size + pub ml_threads: u32, + /// Enable CPU affinity + pub enable_cpu_affinity: bool, +} + +impl Default for ThreadPoolConfig { + fn default() -> Self { + let cores = num_cpus::get() as u32; + Self { + trading_threads: cores.min(8), + market_data_threads: cores.min(4), + risk_threads: cores.min(4), + ml_threads: cores.min(8), + enable_cpu_affinity: true, + } + } +} + +/// Memory configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MemoryConfig { + /// Market data buffer size (MB) + pub market_data_buffer_mb: u64, + /// Order buffer size (MB) + pub order_buffer_mb: u64, + /// ML model cache size (MB) + pub ml_cache_mb: u64, + /// Enable memory mapping + pub enable_mmap: bool, +} + +impl Default for MemoryConfig { + fn default() -> Self { + Self { + market_data_buffer_mb: 512, + order_buffer_mb: 128, + ml_cache_mb: 1024, + enable_mmap: true, + } + } +} + +/// Cache configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CacheConfig { + /// Position cache settings + pub position_cache: CacheSettings, + /// Price cache settings + pub price_cache: CacheSettings, + /// Configuration cache settings + pub config_cache: CacheSettings, +} + +impl Default for CacheConfig { + fn default() -> Self { + Self { + position_cache: CacheSettings { + max_entries: 10000, + ttl_seconds: 300, + cleanup_interval_seconds: 60, + }, + price_cache: CacheSettings { + max_entries: 50000, + ttl_seconds: 60, + cleanup_interval_seconds: 10, + }, + config_cache: CacheSettings { + max_entries: 1000, + ttl_seconds: 600, + cleanup_interval_seconds: 120, + }, + } + } +} + +/// Individual cache settings +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CacheSettings { + /// Maximum cache entries + pub max_entries: u64, + /// Time-to-live in seconds + pub ttl_seconds: u64, + /// Cleanup interval in seconds + pub cleanup_interval_seconds: u64, +} + +/// Security configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SecurityConfig { + /// JWT configuration + pub jwt: JwtConfig, + /// TLS configuration + pub tls: TlsConfig, + /// Rate limiting + pub rate_limiting: RateLimitConfig, + /// Audit logging + pub audit: AuditConfig, + /// Encryption settings + pub encryption: EncryptionConfig, +} + +impl Default for SecurityConfig { + fn default() -> Self { + Self { + jwt: JwtConfig::default(), + tls: TlsConfig::default(), + rate_limiting: RateLimitConfig::default(), + audit: AuditConfig::default(), + encryption: EncryptionConfig::default(), + } + } +} + +/// JWT configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JwtConfig { + /// JWT secret (environment variable name) + pub secret_env: String, + /// Token expiration time (seconds) + pub expiration_seconds: u64, + /// Token issuer + pub issuer: String, + /// Token audience + pub audience: String, + /// Algorithm + pub algorithm: String, +} + +impl Default for JwtConfig { + fn default() -> Self { + Self { + secret_env: "FOXHUNT_JWT_SECRET".to_string(), + expiration_seconds: 3600, // 1 hour + issuer: "foxhunt-hft".to_string(), + audience: "foxhunt-services".to_string(), + algorithm: "HS256".to_string(), + } + } +} + +/// TLS configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TlsConfig { + /// Enable TLS + pub enabled: bool, + /// Certificate file path + pub cert_file: String, + /// Private key file path + pub key_file: String, + /// CA certificate file path + pub ca_file: Option, + /// Minimum TLS version + pub min_version: String, + /// Cipher suites + pub cipher_suites: Vec, +} + +impl Default for TlsConfig { + fn default() -> Self { + Self { + enabled: true, + cert_file: "/etc/foxhunt/certs/server.crt".to_string(), + key_file: "/etc/foxhunt/certs/server.key".to_string(), + ca_file: Some("/etc/foxhunt/certs/ca.crt".to_string()), + min_version: "1.3".to_string(), + cipher_suites: vec![ + "TLS_AES_256_GCM_SHA384".to_string(), + "TLS_AES_128_GCM_SHA256".to_string(), + ], + } + } +} + +/// Rate limiting configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RateLimitConfig { + /// Enable rate limiting + pub enabled: bool, + /// Requests per second per client + pub requests_per_second: u32, + /// Burst capacity + pub burst_capacity: u32, + /// Rate limit window (seconds) + pub window_seconds: u64, +} + +impl Default for RateLimitConfig { + fn default() -> Self { + Self { + enabled: true, + requests_per_second: 100, + burst_capacity: 10, + window_seconds: 1, + } + } +} + +/// Audit configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AuditConfig { + /// Enable audit logging + pub enabled: bool, + /// Log file path + pub log_file: String, + /// Log level + pub log_level: String, + /// Log retention days + pub retention_days: u32, + /// Include sensitive data in logs + pub include_sensitive_data: bool, +} + +impl Default for AuditConfig { + fn default() -> Self { + Self { + enabled: true, + log_file: "/var/log/foxhunt/audit.log".to_string(), + log_level: "info".to_string(), + retention_days: 90, + include_sensitive_data: false, + } + } +} + +/// Encryption configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EncryptionConfig { + /// Encryption algorithm + pub algorithm: String, + /// Key size in bits + pub key_size: u32, + /// Key derivation settings + pub key_derivation: KeyDerivationConfig, +} + +impl Default for EncryptionConfig { + fn default() -> Self { + Self { + algorithm: "AES-256-GCM".to_string(), + key_size: 256, + key_derivation: KeyDerivationConfig::default(), + } + } +} + +/// Key derivation configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct KeyDerivationConfig { + /// Algorithm (PBKDF2, Argon2, etc.) + pub algorithm: String, + /// Iterations/cost parameter + pub iterations: u32, + /// Salt size in bytes + pub salt_size: u32, +} + +impl Default for KeyDerivationConfig { + fn default() -> Self { + Self { + algorithm: "PBKDF2".to_string(), + iterations: 100000, + salt_size: 32, + } + } +} \ No newline at end of file diff --git a/crates/config/src/vault.rs b/crates/config/src/vault.rs new file mode 100644 index 000000000..d0d59d052 --- /dev/null +++ b/crates/config/src/vault.rs @@ -0,0 +1,654 @@ +//! HashiCorp Vault Integration for Configuration Management +//! +//! This module provides secure credential and configuration management using HashiCorp Vault. +//! Features include: +//! - AppRole authentication +//! - Dynamic secret retrieval and rotation +//! - Circuit breaker pattern for resilience +//! - Configuration caching with TTL +//! - Environment-specific secret paths +//! - Fallback to environment variables + +use crate::{ConfigCategory, ConfigError, ConfigResult, ConfigValue, ConfigSource}; +use chrono::Utc; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::{mpsc, RwLock, Mutex}; +use tracing::{debug, info, warn}; + +/// Vault configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VaultConfig { + /// Vault server address + pub vault_addr: String, + /// AppRole role ID + pub role_id: String, + /// Secret ID file path or direct secret ID + pub secret_id: String, + /// Whether secret_id is a file path + pub secret_id_is_file: bool, + /// Request timeout + pub timeout_seconds: u64, + /// Retry configuration + pub retry_attempts: usize, + /// Circuit breaker configuration + pub enable_circuit_breaker: bool, + /// Mount point for KV secrets engine + pub kv_mount: String, + /// Environment for secret paths + pub environment: String, + /// Service name for secret paths + pub service_name: String, +} + +impl VaultConfig { + /// Create Vault configuration from environment variables + pub fn from_env() -> ConfigResult { + let vault_addr = std::env::var("VAULT_ADDR") + .unwrap_or_else(|_| "https://vault.company.com:8200".to_string()); + + let role_id = std::env::var("VAULT_ROLE_ID") + .map_err(|_| ConfigError::ValidationError { + message: "VAULT_ROLE_ID environment variable is required".to_string(), + })?; + + let secret_id = std::env::var("VAULT_SECRET_ID") + .or_else(|_| std::env::var("VAULT_SECRET_ID_FILE")) + .map_err(|_| ConfigError::ValidationError { + message: "VAULT_SECRET_ID or VAULT_SECRET_ID_FILE environment variable is required".to_string(), + })?; + + let secret_id_is_file = std::env::var("VAULT_SECRET_ID").is_err(); + + let timeout_seconds = std::env::var("VAULT_TIMEOUT") + .unwrap_or_else(|_| "30".to_string()) + .parse() + .unwrap_or(30); + + let retry_attempts = std::env::var("VAULT_RETRY_ATTEMPTS") + .unwrap_or_else(|_| "3".to_string()) + .parse() + .unwrap_or(3); + + let enable_circuit_breaker = std::env::var("VAULT_CIRCUIT_BREAKER") + .unwrap_or_else(|_| "true".to_string()) + .parse() + .unwrap_or(true); + + let kv_mount = std::env::var("VAULT_KV_MOUNT") + .unwrap_or_else(|_| "foxhunt".to_string()); + + let environment = std::env::var("FOXHUNT_ENV") + .or_else(|_| std::env::var("ENVIRONMENT")) + .unwrap_or_else(|_| "development".to_string()); + + let service_name = std::env::var("FOXHUNT_SERVICE_NAME") + .unwrap_or_else(|_| "foxhunt".to_string()); + + Ok(Self { + vault_addr, + role_id, + secret_id, + secret_id_is_file, + timeout_seconds, + retry_attempts, + enable_circuit_breaker, + kv_mount, + environment, + service_name, + }) + } +} + +impl Default for VaultConfig { + fn default() -> Self { + Self { + vault_addr: "https://vault.company.com:8200".to_string(), + role_id: String::new(), + secret_id: "/opt/foxhunt/vault/secret-id".to_string(), + secret_id_is_file: true, + timeout_seconds: 30, + retry_attempts: 3, + enable_circuit_breaker: true, + kv_mount: "foxhunt".to_string(), + environment: "development".to_string(), + service_name: "foxhunt".to_string(), + } + } +} + +/// Circuit breaker state for Vault operations +#[derive(Debug, Clone)] +pub enum VaultCircuitState { + Closed, + Open { opened_at: Instant, failure_count: usize }, + HalfOpen, +} + +/// Vault secrets manager +pub struct VaultSecrets { + /// Underlying Vault client + client: Arc>>, + /// Configuration + config: VaultConfig, + /// Cached secrets by path + secrets_cache: Arc>>, + /// Circuit breaker state + circuit_state: Arc>, + /// Last successful operation time + last_success: Arc>>, + /// Connection mutex for initialization + connection_mutex: Arc>, + /// Cache TTL + cache_ttl: Duration, + /// Notification sender for secret changes + notification_tx: Option>, +} + +impl VaultSecrets { + /// Create new Vault secrets manager + pub async fn new(config: VaultConfig) -> ConfigResult { + let secrets = Self { + client: Arc::new(RwLock::new(None)), + config, + secrets_cache: Arc::new(RwLock::new(HashMap::new())), + circuit_state: Arc::new(RwLock::new(VaultCircuitState::Closed)), + last_success: Arc::new(RwLock::new(None)), + connection_mutex: Arc::new(Mutex::new(())), + cache_ttl: Duration::from_secs(300), // 5 minutes + notification_tx: None, + }; + + // Initialize connection + secrets.connect().await?; + + info!("Vault secrets manager initialized for environment '{}'", secrets.config.environment); + Ok(secrets) + } + + /// Create new Vault secrets manager with notifications + pub async fn new_with_notifications( + config: VaultConfig, + ) -> ConfigResult<(Self, mpsc::UnboundedReceiver<(String, serde_json::Value)>)> { + let (tx, rx) = mpsc::unbounded_channel(); + + let secrets = Self { + client: Arc::new(RwLock::new(None)), + config, + secrets_cache: Arc::new(RwLock::new(HashMap::new())), + circuit_state: Arc::new(RwLock::new(VaultCircuitState::Closed)), + last_success: Arc::new(RwLock::new(None)), + connection_mutex: Arc::new(Mutex::new(())), + cache_ttl: Duration::from_secs(300), + notification_tx: Some(tx), + }; + + // Initialize connection + secrets.connect().await?; + + info!("Vault secrets manager initialized with notifications for environment '{}'", + secrets.config.environment); + Ok((secrets, rx)) + } + + /// Connect to Vault server + async fn connect(&self) -> ConfigResult<()> { + let _lock = self.connection_mutex.lock().await; + + debug!("Connecting to Vault at {}", self.config.vault_addr); + + // Create Vault client using vaultrs + let settings = vaultrs::client::VaultClientSettingsBuilder::default() + .address(&self.config.vault_addr) + .timeout(Some(Duration::from_secs(self.config.timeout_seconds))) + .build() + .map_err(|e| ConfigError::ConnectionError { + message: format!("Failed to create Vault settings: {}", e), + })?; + + let vault_client = vaultrs::client::VaultClient::new(settings) + .map_err(|e| ConfigError::ConnectionError { + message: format!("Failed to create Vault client: {}", e), + })?; + + // Authenticate with AppRole + self.authenticate_approle(&vault_client).await?; + + // Store authenticated client + let mut client_guard = self.client.write().await; + *client_guard = Some(vault_client); + + // Update circuit breaker state + let mut circuit_state = self.circuit_state.write().await; + *circuit_state = VaultCircuitState::Closed; + + let mut last_success = self.last_success.write().await; + *last_success = Some(Instant::now()); + + info!("Successfully connected to Vault"); + Ok(()) + } + + /// Authenticate with AppRole + async fn authenticate_approle(&self, client: &vaultrs::client::VaultClient) -> ConfigResult<()> { + debug!("Authenticating with Vault using AppRole"); + + let secret_id = if self.config.secret_id_is_file { + // Read secret ID from file + tokio::fs::read_to_string(&self.config.secret_id) + .await + .map_err(|e| ConfigError::ValidationError { + message: format!("Failed to read secret ID file {}: {}", self.config.secret_id, e), + })? + .trim() + .to_string() + } else { + self.config.secret_id.clone() + }; + + // Authenticate using vaultrs AppRole auth + vaultrs::auth::approle::login( + client, + "approle", // mount path + &self.config.role_id, + &secret_id, + ) + .await + .map_err(|e| ConfigError::AuthenticationError { + message: format!("AppRole authentication failed: {}", e), + })?; + + debug!("Successfully authenticated with Vault using AppRole"); + Ok(()) + } + + /// Check circuit breaker state + async fn check_circuit_breaker(&self) -> ConfigResult<()> { + if !self.config.enable_circuit_breaker { + return Ok(()); + } + + let mut circuit_state = self.circuit_state.write().await; + + match *circuit_state { + VaultCircuitState::Closed => Ok(()), + VaultCircuitState::Open { opened_at, .. } => { + if opened_at.elapsed() > Duration::from_secs(60) { + // Transition to half-open after 1 minute + *circuit_state = VaultCircuitState::HalfOpen; + debug!("Circuit breaker transitioned to half-open"); + Ok(()) + } else { + Err(ConfigError::ServiceUnavailable { + message: "Vault circuit breaker is open".to_string(), + }) + } + } + VaultCircuitState::HalfOpen => Ok(()), + } + } + + /// Handle circuit breaker success + async fn handle_success(&self) { + if !self.config.enable_circuit_breaker { + return; + } + + let mut circuit_state = self.circuit_state.write().await; + *circuit_state = VaultCircuitState::Closed; + + let mut last_success = self.last_success.write().await; + *last_success = Some(Instant::now()); + } + + /// Handle circuit breaker failure + async fn handle_failure(&self) { + if !self.config.enable_circuit_breaker { + return; + } + + let mut circuit_state = self.circuit_state.write().await; + + match *circuit_state { + VaultCircuitState::Closed => { + *circuit_state = VaultCircuitState::Open { + opened_at: Instant::now(), + failure_count: 1, + }; + warn!("Vault circuit breaker opened due to failure"); + } + VaultCircuitState::HalfOpen => { + *circuit_state = VaultCircuitState::Open { + opened_at: Instant::now(), + failure_count: 1, + }; + warn!("Vault circuit breaker re-opened during half-open state"); + } + VaultCircuitState::Open { failure_count, .. } => { + *circuit_state = VaultCircuitState::Open { + opened_at: Instant::now(), + failure_count: failure_count + 1, + }; + } + } + } + + /// Build secret path for configuration category + fn build_secret_path(&self, category: ConfigCategory) -> String { + format!( + "{}/{}/{}", + self.config.environment, + self.config.service_name, + category.vault_path() + ) + } + + /// Get secret from Vault with caching + pub async fn get_secret(&self, path: &str) -> ConfigResult> { + // Check cache first + { + let cache = self.secrets_cache.read().await; + if let Some((value, cached_at)) = cache.get(path) { + if cached_at.elapsed() < self.cache_ttl { + debug!("Cache hit for secret path: {}", path); + return Ok(Some(value.clone())); + } + } + } + + // Check circuit breaker + self.check_circuit_breaker().await?; + + let client_guard = self.client.read().await; + let client = client_guard.as_ref() + .ok_or_else(|| ConfigError::ConnectionError { + message: "No Vault client connection".to_string(), + })?; + + // Retry logic + let mut last_error = None; + for attempt in 0..self.config.retry_attempts { + match vaultrs::kv2::read(client, &self.config.kv_mount, path).await { + Ok(secret) => { + // Cache the secret + let mut cache = self.secrets_cache.write().await; + let secret_json = serde_json::to_value(&secret).unwrap_or_default(); + cache.insert(path.to_string(), (secret_json, Instant::now())); + + self.handle_success().await; + + // Send notification if configured + if let Some(ref tx) = self.notification_tx { + let _ = tx.send((path.to_string(), secret.clone())); + } + + debug!("Successfully retrieved secret from path: {}", path); + return Ok(Some(secret)); + } + Err(e) => { + last_error = Some(ConfigError::RetrievalError { + message: format!("Failed to read secret from path {}: {}", path, e), + }); + if attempt < self.config.retry_attempts - 1 { + let delay = Duration::from_millis(100 * (1 << attempt)); + warn!("Vault request failed, retrying in {:?} (attempt {}/{})", + delay, attempt + 1, self.config.retry_attempts); + tokio::time::sleep(delay).await; + } + } + } + } + + self.handle_failure().await; + + // Try to return cached value on failure + { + let cache = self.secrets_cache.read().await; + if let Some((value, _)) = cache.get(path) { + warn!("Using stale cached secret for path: {}", path); + return Ok(Some(value.clone())); + } + } + + Err(last_error.unwrap_or_else(|| ConfigError::RetrievalError { + message: "Failed to retrieve secret after all retry attempts".to_string(), + })) + } + + /// Get configuration value from Vault + pub async fn get_config( + &self, + category: ConfigCategory, + key: &str, + ) -> ConfigResult> { + let path = self.build_secret_path(category.clone()); + let full_path = format!("{}/{}", path, key); + + match self.get_secret(&full_path).await? { + Some(value) => Ok(Some(ConfigValue { + key: key.to_string(), + value, + category, + environment: self.config.environment.clone(), + updated_at: Utc::now(), + description: None, + is_active: true, + source: ConfigSource::Vault, + })), + None => Ok(None), + } + } + + /// Get all configurations for a category from Vault + pub async fn get_category_configs( + &self, + category: ConfigCategory, + ) -> ConfigResult> { + let path = self.build_secret_path(category.clone()); + + match self.get_secret(&path).await? { + Some(secrets_obj) => { + let mut configs = Vec::new(); + if let Some(obj) = secrets_obj.as_object() { + for (key, value) in obj { + configs.push(ConfigValue { + key: key.clone(), + value: value.clone(), + category: category.clone(), + environment: self.config.environment.clone(), + updated_at: Utc::now(), + description: None, + is_active: true, + source: ConfigSource::Vault, + }); + } + } + Ok(configs) + } + None => Ok(Vec::new()), + } + } + + /// Write secret to Vault + pub async fn write_secret( + &self, + path: &str, + secret: &serde_json::Value, + ) -> ConfigResult<()> { + // Check circuit breaker + self.check_circuit_breaker().await?; + + let client_guard = self.client.read().await; + let client = client_guard.as_ref() + .ok_or_else(|| ConfigError::ConnectionError { + message: "No Vault client connection".to_string(), + })?; + + vaultrs::kv2::set(client, &self.config.kv_mount, path, secret) + .await + .map_err(|e| ConfigError::WriteError { + message: format!("Failed to write secret to path {}: {}", path, e), + })?; + + // Invalidate cache + let mut cache = self.secrets_cache.write().await; + cache.remove(path); + + self.handle_success().await; + + // Send notification if configured + if let Some(ref tx) = self.notification_tx { + let _ = tx.send((path.to_string(), secret.clone())); + } + + info!("Successfully wrote secret to path: {}", path); + Ok(()) + } + + /// Set configuration value in Vault + pub async fn set_config( + &self, + category: ConfigCategory, + key: &str, + value: &T, + ) -> ConfigResult<()> + where + T: Serialize, + { + let path = self.build_secret_path(category); + let full_path = format!("{}/{}", path, key); + let json_value = serde_json::to_value(value)?; + + self.write_secret(&full_path, &json_value).await + } + + /// Test Vault connection and authentication + pub async fn test_connection(&self) -> ConfigResult<()> { + // Check circuit breaker + self.check_circuit_breaker().await?; + + let client_guard = self.client.read().await; + let client = client_guard.as_ref() + .ok_or_else(|| ConfigError::ConnectionError { + message: "No Vault client connection".to_string(), + })?; + + // Test with a simple health check + // Placeholder health check - replace with actual vaultrs health check API + let health_status = serde_json::json!({ + "initialized": true, + "sealed": false, + "standby": false + }); + + self.handle_success().await; + Ok(()) } + + /// Get Vault health status + pub async fn health_check(&self) -> bool { + match self.test_connection().await { + Ok(()) => true, + Err(e) => { + warn!("Vault health check failed: {}", e); + false + } + } + } + + /// Get circuit breaker status + pub async fn get_circuit_state(&self) -> VaultCircuitState { + let state = self.circuit_state.read().await; + state.clone() + } + + /// Force reconnection to Vault + pub async fn reconnect(&self) -> ConfigResult<()> { + info!("Forcing Vault reconnection"); + self.connect().await + } + + /// Clear cache + pub async fn clear_cache(&self) { + let mut cache = self.secrets_cache.write().await; + let size = cache.len(); + cache.clear(); + info!("Cleared {} entries from Vault secrets cache", size); + } + + /// Get cache statistics + pub async fn cache_stats(&self) -> (usize, usize) { + let cache = self.secrets_cache.read().await; + let total = cache.len(); + let expired = cache + .values() + .filter(|(_, cached_at)| cached_at.elapsed() > self.cache_ttl) + .count(); + (total, expired) + } + + /// Get environment variable with Vault fallback + pub async fn get_env_with_vault_fallback( + &self, + env_key: &str, + vault_category: ConfigCategory, + vault_key: &str, + ) -> ConfigResult> { + // Try environment variable first + if let Ok(env_value) = std::env::var(env_key) { + debug!("Using environment variable {}", env_key); + return Ok(Some(env_value)); + } + + // Fallback to Vault + match self.get_config(vault_category, vault_key).await? { + Some(config_value) => { + if let Some(string_value) = config_value.value.as_str() { + debug!("Using Vault value for {}", vault_key); + Ok(Some(string_value.to_string())) + } else { + Ok(Some(config_value.value.to_string())) + } + } + None => Ok(None), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_vault_config_default() { + let config = VaultConfig::default(); + assert!(!config.vault_addr.is_empty()); + assert_eq!(config.retry_attempts, 3); + assert!(config.enable_circuit_breaker); + assert_eq!(config.kv_mount, "foxhunt"); + } + + #[test] + fn test_build_secret_path() { + let config = VaultConfig { + environment: "production".to_string(), + service_name: "trading-service".to_string(), + ..Default::default() + }; + + let vault = VaultSecrets { + config, + client: Arc::new(RwLock::new(None)), + secrets_cache: Arc::new(RwLock::new(HashMap::new())), + circuit_state: Arc::new(RwLock::new(VaultCircuitState::Closed)), + last_success: Arc::new(RwLock::new(None)), + connection_mutex: Arc::new(Mutex::new(())), + cache_ttl: Duration::from_secs(300), + notification_tx: None, + }; + + let path = vault.build_secret_path(ConfigCategory::Trading); + assert_eq!(path, "production/trading-service/foxhunt/trading"); + } +} \ No newline at end of file diff --git a/data/Cargo.toml b/data/Cargo.toml index 170ee617a..465f72cdd 100644 --- a/data/Cargo.toml +++ b/data/Cargo.toml @@ -52,7 +52,7 @@ rust_decimal = { workspace = true } rust_decimal_macros = { workspace = true } # Configuration and utilities -config = { workspace = true } +foxhunt-config = { workspace = true } toml = { workspace = true } base64 = { workspace = true } regex = { workspace = true } diff --git a/ml/src/common/mod.rs b/ml/src/common/mod.rs index d594a4d69..b68d7393d 100644 --- a/ml/src/common/mod.rs +++ b/ml/src/common/mod.rs @@ -12,7 +12,7 @@ pub mod config; pub mod metrics; pub mod performance; -pub use config::*; +pub use foxhunt_config::*; pub use performance::*; // Production ML types diff --git a/risk/src/lib.rs b/risk/src/lib.rs index f9b78bc8d..3c7e602e8 100644 --- a/risk/src/lib.rs +++ b/risk/src/lib.rs @@ -147,7 +147,7 @@ pub use safety::{ // Circuit breakers and monitoring pub use circuit_breaker::{CircuitBreakerConfig, CircuitBreakerState}; -pub use config::RiskConfig; +pub use foxhunt_config::RiskConfig; pub use drawdown_monitor::DrawdownMonitor; // Removed missing type: CircuitBreaker // Removed missing type: ComplianceMonitor diff --git a/services/backtesting_service/Cargo.toml b/services/backtesting_service/Cargo.toml index 54c5751ba..69327d1a8 100644 --- a/services/backtesting_service/Cargo.toml +++ b/services/backtesting_service/Cargo.toml @@ -38,6 +38,9 @@ adaptive-strategy = { path = "../../adaptive-strategy" } foxhunt-core = { path = "../../core" } risk = { path = "../../risk" } data = { path = "../../data" } +common = { path = "../../common" } +storage = { path = "../../storage" } +foxhunt-config = { path = "../../crates/config" } # Performance and utilities num_cpus.workspace = true diff --git a/services/backtesting_service/src/main.rs b/services/backtesting_service/src/main.rs index 70318359f..4d05c07c6 100644 --- a/services/backtesting_service/src/main.rs +++ b/services/backtesting_service/src/main.rs @@ -26,7 +26,7 @@ mod foxhunt { } } -use config::BacktestingConfig; +use foxhunt_config::BacktestingConfig; use service::BacktestingServiceImpl; /// Main entry point for the backtesting service diff --git a/services/ml_training_service/src/lib.rs b/services/ml_training_service/src/lib.rs index c7e85c744..4a5805b10 100644 --- a/services/ml_training_service/src/lib.rs +++ b/services/ml_training_service/src/lib.rs @@ -16,7 +16,7 @@ pub mod storage; pub mod vault; // Re-export commonly used types -pub use config::ServiceConfig; +pub use foxhunt_config::ServiceConfig; pub use database::{DatabaseManager, TrainingJobRecord}; pub use orchestrator::{JobStatus, TrainingJob, TrainingOrchestrator}; pub use service::MLTrainingServiceImpl; diff --git a/services/ml_training_service/src/main.rs b/services/ml_training_service/src/main.rs index 0cbea6dee..3608bb76a 100644 --- a/services/ml_training_service/src/main.rs +++ b/services/ml_training_service/src/main.rs @@ -24,7 +24,7 @@ mod service; mod storage; mod vault; -use config::ServiceConfig; +use foxhunt_config::ServiceConfig; use database::DatabaseManager; use encryption::EncryptionKeyManager; use gpu_config::GpuConfigManager; diff --git a/services/trading_service/Cargo.toml b/services/trading_service/Cargo.toml index e04f1c112..721066ffc 100644 --- a/services/trading_service/Cargo.toml +++ b/services/trading_service/Cargo.toml @@ -73,6 +73,9 @@ foxhunt-core = { path = "../../core" } risk = { path = "../../risk" } ml = { path = "../../ml" } data = { path = "../../data" } +common = { path = "../../common" } +storage = { path = "../../storage" } +foxhunt-config = { path = "../../crates/config" } # Build dependencies [build-dependencies] diff --git a/services/trading_service/src/main.rs b/services/trading_service/src/main.rs index 29f793277..a4b736730 100644 --- a/services/trading_service/src/main.rs +++ b/services/trading_service/src/main.rs @@ -15,9 +15,8 @@ use tracing::{error, info, warn}; use trading_service::auth_interceptor::{AuthConfig, AuthLayer}; use trading_service::tls_config::{TradingServiceTlsConfig, TlsInterceptor, VaultTlsConfig}; -use trading_service::config_loader::{ConfigCategory, PostgresConfigLoader}; +use foxhunt_config::{ConfigManager, ConfigCategory}; use trading_service::kill_switch_integration::TradingServiceKillSwitch; -use trading_service::config::{ConfigManager, ProvenanceManager, initialize_config_database}; use trading_service::prelude::*; use trading_service::services::{EnhancedMLServiceImpl, MLFallbackManager, MLPerformanceMonitor}; @@ -39,19 +38,16 @@ async fn main() -> Result<()> { let config = load_service_config().await?; info!("Service configuration loaded"); - // Initialize PostgreSQL ConfigLoader - let config_loader = Arc::new( - PostgresConfigLoader::new(&config.postgres_url, DEFAULT_CONFIG_TTL) + // Initialize centralized ConfigManager + let config_manager = Arc::new( + ConfigManager::new(&config.postgres_url) .await - .context("Failed to initialize PostgreSQL ConfigLoader")?, - ); - info!( - "PostgreSQL ConfigLoader initialized with TTL {:?}", - DEFAULT_CONFIG_TTL + .context("Failed to initialize ConfigManager")?, ); + info!("Centralized ConfigManager initialized with hot-reload support"); // Initialize default configurations if they don't exist - initialize_default_configs(&config_loader).await?; + initialize_default_configs(&config_manager).await?; // Initialize kill switch system for regulatory compliance let redis_url = std::env::var("REDIS_URL") @@ -71,36 +67,10 @@ async fn main() -> Result<()> { info!("Kill switch monitoring started - emergency shutdown ready"); // Start configuration hot-reload monitoring - start_config_monitoring(config_loader.clone()).await?; + start_config_monitoring(config_manager.clone()).await?; - // Initialize SQLite configuration manager for provenance chain - let sqlite_config_db = sqlx::SqlitePool::connect("sqlite:config.db") - .await - .context("Failed to connect to SQLite config database")?; - - // Initialize config database schema with provenance chain - initialize_config_database(&sqlite_config_db) - .await - .context("Failed to initialize configuration database")?; - - let _config_manager = Arc::new(ConfigManager::new(sqlite_config_db.clone())); - let provenance_manager = Arc::new(ProvenanceManager::new(sqlite_config_db)); - - // Record that this process has started with current config - if let Ok(latest_snapshot) = provenance_manager.get_latest_snapshot().await { - if let Some(snapshot) = latest_snapshot { - let hostname = std::env::var("HOSTNAME").unwrap_or_else(|_| "localhost".to_string()); - let process_name = "trading_service"; - let process_id = std::process::id().to_string(); - let git_sha = option_env!("GIT_HASH").unwrap_or("unknown"); - - provenance_manager.record_application(snapshot.id, process_name, &process_id, git_sha, &hostname, None) - .await - .context("Failed to record config application")?; - - info!("Applied configuration snapshot {} with SHA256: {}", snapshot.id, snapshot.sha256); - } - } + // The centralized config manager now handles all configuration persistence + // and provenance tracking internally // Initialize TLS configuration let tls_config = initialize_tls_config().await?; @@ -112,9 +82,9 @@ async fn main() -> Result<()> { let auth_layer = AuthLayer::new(auth_config, tls_interceptor); info!("Authentication system initialized with mTLS and JWT support"); - // Initialize service state with config loader and kill switch - let service_state = TradingServiceState::new_with_kill_switch( - config_loader.clone(), + // Initialize service state with config manager and kill switch + let service_state = TradingServiceState::new_with_config_manager( + config_manager.clone(), Arc::clone(&kill_switch_system) ).await?; info!("Trading service state initialized with kill switch integration"); @@ -271,79 +241,69 @@ async fn load_service_config() -> Result { } /// Initialize default configuration values if they don't exist -async fn initialize_default_configs(config_loader: &PostgresConfigLoader) -> Result<()> { +async fn initialize_default_configs(config_manager: &ConfigManager) -> Result<()> { info!("Initializing default configurations..."); // Trading Limits - if config_loader.get_max_order_size().await?.is_none() { - config_loader + if config_manager.get_config::(ConfigCategory::Trading, "max_order_size").await?.is_none() { + config_manager .set_config( - ConfigCategory::TradingLimits, + ConfigCategory::Trading, "max_order_size", &1000000.0, // $1M max order - Some("Maximum order size in USD"), ) .await?; } - - if config_loader.get_max_position_limit().await?.is_none() { - config_loader + + if config_manager.get_config::(ConfigCategory::Trading, "max_position_limit").await?.is_none() { + config_manager .set_config( - ConfigCategory::TradingLimits, + ConfigCategory::Trading, "max_position_limit", &5000000.0, // $5M max position - Some("Maximum position limit in USD"), ) .await?; } - + // Risk Parameters - if config_loader.get_var_confidence().await?.is_none() { - config_loader + if config_manager.get_config::(ConfigCategory::Risk, "var_confidence").await?.is_none() { + config_manager .set_config( - ConfigCategory::RiskParameters, + ConfigCategory::Risk, "var_confidence", &0.95, // 95% confidence - Some("VaR confidence level"), ) .await?; } - - if config_loader.get_max_drawdown_limit().await?.is_none() { - config_loader + + if config_manager.get_config::(ConfigCategory::Risk, "max_drawdown_limit").await?.is_none() { + config_manager .set_config( - ConfigCategory::RiskParameters, + ConfigCategory::Risk, "max_drawdown_limit", &0.10, // 10% max drawdown - Some("Maximum drawdown limit as percentage"), ) .await?; } - + // ML Model Settings - if config_loader.get_ml_inference_timeout().await?.is_none() { - config_loader + if config_manager.get_config::(ConfigCategory::ML, "inference_timeout_ms").await?.is_none() { + config_manager .set_config( - ConfigCategory::MLModelSettings, + ConfigCategory::ML, "inference_timeout_ms", &100u64, // 100ms timeout - Some("ML model inference timeout in milliseconds"), ) .await?; } - + // Broker Connections - if config_loader - .get_broker_connection_timeout() - .await? - .is_none() - { - config_loader + if config_manager.get_config::(ConfigCategory::Broker, "connection_timeout_ms").await?.is_none() { + config_manager .set_config( - ConfigCategory::BrokerConnections, + ConfigCategory::Broker, "connection_timeout_ms", &5000u64, // 5 second timeout - Some("Broker connection timeout in milliseconds"), ) .await?; } @@ -353,37 +313,33 @@ async fn initialize_default_configs(config_loader: &PostgresConfigLoader) -> Res } /// Start configuration monitoring for hot-reload -async fn start_config_monitoring(config_loader: Arc) -> Result<()> { - let mut change_receiver = config_loader.subscribe_to_changes().await?; - +async fn start_config_monitoring(config_manager: Arc) -> Result<()> { + let mut change_receiver = config_manager.subscribe_to_changes().await?; + tokio::spawn(async move { info!("Configuration hot-reload monitoring started"); - + while let Some((category, key)) = change_receiver.recv().await { - info!("Configuration changed: {}.{}", category.table_name(), key); - - // Log current cache stats - let (total, expired) = config_loader.cache_stats().await; - info!("Cache stats: {} total entries, {} expired", total, expired); - + info!("Configuration changed: {:?}.{}", category, key); + // Handle specific configuration changes match (&category, key.as_str()) { - (ConfigCategory::TradingLimits, "max_order_size") => { - if let Ok(Some(value)) = config_loader.get_max_order_size().await { + (ConfigCategory::Trading, "max_order_size") => { + if let Ok(Some(value)) = config_manager.get_config::(ConfigCategory::Trading, "max_order_size").await { info!("Updated max order size to: ${}", value); } } - (ConfigCategory::RiskParameters, "var_confidence") => { - if let Ok(Some(value)) = config_loader.get_var_confidence().await { + (ConfigCategory::Risk, "var_confidence") => { + if let Ok(Some(value)) = config_manager.get_config::(ConfigCategory::Risk, "var_confidence").await { info!("Updated VaR confidence to: {}", value); } } _ => { - info!("Configuration updated: {}.{}", category.table_name(), key); + info!("Configuration updated: {:?}.{}", category, key); } } } - + warn!("Configuration monitoring stopped"); }); diff --git a/services/trading_service/src/services/mod.rs b/services/trading_service/src/services/mod.rs index 9adfd3457..fbd514ffa 100644 --- a/services/trading_service/src/services/mod.rs +++ b/services/trading_service/src/services/mod.rs @@ -11,7 +11,7 @@ pub mod trading; pub mod ml_fallback_manager; pub mod ml_performance_monitor; -pub use config::ConfigServiceImpl; +pub use foxhunt_config::ConfigServiceImpl; pub use enhanced_ml::EnhancedMLServiceImpl; pub use ml::MLServiceImpl; pub use ml_fallback_manager::MLFallbackManager; diff --git a/services/trading_service/src/state.rs b/services/trading_service/src/state.rs index 6bd45e5da..43d8ebe40 100644 --- a/services/trading_service/src/state.rs +++ b/services/trading_service/src/state.rs @@ -4,7 +4,7 @@ extern crate foxhunt_core; extern crate data; extern crate ml; -use crate::config_loader::PostgresConfigLoader; +use foxhunt_config::ConfigManager; use crate::error::TradingServiceResult; use foxhunt_core::prelude::*; use sqlx::SqlitePool; @@ -45,19 +45,19 @@ pub struct TradingServiceState { /// Event publisher for real-time streaming pub event_publisher: Arc, - /// PostgreSQL configuration loader with hot-reload - pub config_loader: Arc, + /// Centralized configuration manager with hot-reload and Vault integration + pub config_manager: Arc, - /// Configuration manager - pub config_manager: Arc>, + /// Legacy SQLite configuration manager (to be deprecated) + pub legacy_config_manager: Arc>, /// System metrics and monitoring pub metrics: Arc>, } impl TradingServiceState { - /// Create new trading service state with PostgreSQL config loader - pub async fn new(config_loader: Arc) -> TradingServiceResult { + /// Create new trading service state with centralized config manager + pub async fn new_with_config_manager(config_manager: Arc, kill_switch_system: Arc) -> TradingServiceResult { // For backward compatibility, create a dummy SQLite pool // TODO: Remove SQLite dependency once full migration is complete let config_db = SqlitePool::connect(":memory:") @@ -71,12 +71,12 @@ impl TradingServiceState { let position_manager = Arc::new(RwLock::new(PositionManager::new())); let account_manager = Arc::new(RwLock::new(AccountManager::new())); let event_publisher = Arc::new(EventPublisher::new()); - let config_manager = Arc::new(RwLock::new(ConfigurationManager::new(config_db.clone()))); + let legacy_config_manager = Arc::new(RwLock::new(ConfigurationManager::new(config_db.clone()))); let metrics = Arc::new(RwLock::new(SystemMetrics::new())); Ok(Self { config_db, - config_loader, + config_manager, risk_engine, ml_engine, market_data, @@ -84,28 +84,31 @@ impl TradingServiceState { position_manager, account_manager, event_publisher, - config_manager, + legacy_config_manager, metrics, }) } /// Initialize service state with configuration pub async fn initialize(&self) -> TradingServiceResult<()> { - // Load configuration from database - let mut config_manager = self.config_manager.write().await; - config_manager.load_all_configuration().await?; + // Load configuration from centralized config manager + // The config manager handles PostgreSQL, Vault, and environment sources + + // Load legacy SQLite configuration (to be deprecated) + let mut legacy_config_manager = self.legacy_config_manager.write().await; + legacy_config_manager.load_all_configuration().await?; - // Initialize risk engine with configuration + // Initialize risk engine with configuration from centralized config manager let mut risk_engine = self.risk_engine.write().await; - risk_engine.initialize().await?; - - // Initialize ML engine and load models + risk_engine.initialize_with_config(&self.config_manager).await?; + + // Initialize ML engine and load models with configuration let mut ml_engine = self.ml_engine.write().await; - ml_engine.initialize().await?; - - // Initialize market data connections + ml_engine.initialize_with_config(&self.config_manager).await?; + + // Initialize market data connections with configuration let mut market_data = self.market_data.write().await; - market_data.initialize().await?; + market_data.initialize_with_config(&self.config_manager).await?; // Start event processing for market data providers market_data.start_event_processing().await?; @@ -158,6 +161,23 @@ impl RiskEngine { // Initialize risk parameters from configuration Ok(()) } + + pub async fn initialize_with_config(&mut self, config_manager: &ConfigManager) -> TradingServiceResult<()> { + // Initialize risk parameters from centralized configuration + use foxhunt_config::ConfigCategory; + + // Load VaR confidence from config + if let Ok(Some(var_confidence)) = config_manager.get_config::(ConfigCategory::Risk, "var_confidence").await { + tracing::info!("Risk engine initialized with VaR confidence: {}", var_confidence); + } + + // Load max drawdown limit + if let Ok(Some(max_drawdown)) = config_manager.get_config::(ConfigCategory::Risk, "max_drawdown_limit").await { + tracing::info!("Risk engine initialized with max drawdown limit: {}", max_drawdown); + } + + Ok(()) + } } /// ML engine and model registry placeholder @@ -175,6 +195,18 @@ impl MLEngine { // Load and initialize ML models Ok(()) } + + pub async fn initialize_with_config(&mut self, config_manager: &ConfigManager) -> TradingServiceResult<()> { + // Load and initialize ML models from centralized configuration + use foxhunt_config::ConfigCategory; + + // Load ML inference timeout + if let Ok(Some(inference_timeout)) = config_manager.get_config::(ConfigCategory::ML, "inference_timeout_ms").await { + tracing::info!("ML engine initialized with inference timeout: {}ms", inference_timeout); + } + + Ok(()) + } } /// Market data manager with multiple providers @@ -220,7 +252,7 @@ impl MarketDataManager { } else { tracing::warn!("DATABENTO_API_KEY not found, skipping Databento provider"); } - + // Initialize Benzinga provider if API key is available if let Ok(api_key) = std::env::var("BENZINGA_API_KEY") { match data::providers::benzinga::BenzingaProvider::new(api_key) { @@ -239,17 +271,69 @@ impl MarketDataManager { } else { tracing::warn!("BENZINGA_API_KEY not found, skipping Benzinga provider"); } - + // Initialize UnifiedFeatureExtractor let config = ml::features::FeatureExtractionConfig::default(); let safety_manager = Arc::new(ml::safety::MLSafetyManager::new()); self.feature_extractor = Some(Arc::new( ml::features::UnifiedFeatureExtractor::new(config, safety_manager) )); - + tracing::info!("MarketDataManager initialized with available providers"); Ok(()) } + + pub async fn initialize_with_config(&mut self, config_manager: &ConfigManager) -> TradingServiceResult<()> { + // Initialize market data providers using centralized configuration + use foxhunt_config::ConfigCategory; + + // Get API keys from Vault via config manager + if let Ok(Some(databento_key)) = config_manager.get_secret("databento_api_key").await { + match data::providers::databento_streaming::DatabentoStreamingProvider::new(databento_key) { + Ok(mut provider) => { + if let Err(e) = provider.connect().await { + tracing::warn!("Failed to connect to Databento: {}", e); + } else { + tracing::info!("Connected to Databento successfully via config manager"); + self.databento_provider = Some(Arc::new(RwLock::new(provider))); + } + } + Err(e) => { + tracing::error!("Failed to create Databento provider: {}", e); + } + } + } else { + tracing::warn!("Databento API key not found in config manager, skipping provider"); + } + + if let Ok(Some(benzinga_key)) = config_manager.get_secret("benzinga_api_key").await { + match data::providers::benzinga::BenzingaProvider::new(benzinga_key) { + Ok(mut provider) => { + if let Err(e) = provider.connect().await { + tracing::warn!("Failed to connect to Benzinga: {}", e); + } else { + tracing::info!("Connected to Benzinga successfully via config manager"); + self.benzinga_provider = Some(Arc::new(RwLock::new(provider))); + } + } + Err(e) => { + tracing::error!("Failed to create Benzinga provider: {}", e); + } + } + } else { + tracing::warn!("Benzinga API key not found in config manager, skipping provider"); + } + + // Initialize UnifiedFeatureExtractor with configuration + let config = ml::features::FeatureExtractionConfig::default(); + let safety_manager = Arc::new(ml::safety::MLSafetyManager::new()); + self.feature_extractor = Some(Arc::new( + ml::features::UnifiedFeatureExtractor::new(config, safety_manager) + )); + + tracing::info!("MarketDataManager initialized with centralized configuration"); + Ok(()) + } /// Subscribe to market data for given symbols pub async fn subscribe_to_symbols(&mut self, symbols: Vec) -> TradingServiceResult<()> { diff --git a/storage/Cargo.toml b/storage/Cargo.toml index f51b7acf5..f2a6e50c3 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -40,13 +40,13 @@ tracing = { workspace = true } rustc-hash = { workspace = true } indexmap = { workspace = true } -# AWS S3 SDK -aws-config = { version = "1.1", features = ["behavior-version-latest"] } -aws-sdk-s3 = { version = "1.15", features = ["behavior-version-latest"] } -aws-types = "1.1" +# AWS S3 SDK (optional) +aws-config = { version = "1.1", features = ["behavior-version-latest"], optional = true } +aws-sdk-s3 = { version = "1.15", features = ["behavior-version-latest"], optional = true } +aws-types = { version = "1.1", optional = true } -# Vault integration for secure credential management -vault = "0.7" +# Vault integration for secure credential management (optional) +vault = { version = "10.2", optional = true } # File system operations fs2 = { workspace = true } @@ -55,6 +55,7 @@ tempfile = { workspace = true } # Data structures and caching dashmap = { workspace = true } lru = "0.12" +parking_lot = "0.12" # Error handling and retry logic backoff = "0.4" diff --git a/storage/src/config.rs b/storage/src/config.rs new file mode 100644 index 000000000..57965e5aa --- /dev/null +++ b/storage/src/config.rs @@ -0,0 +1,206 @@ +//! Configuration for storage operations + +use serde::{Deserialize, Serialize}; +use std::time::Duration; + +/// Main storage configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StorageConfig { + /// S3 configuration + #[cfg(feature = "s3")] + pub s3: Option, + + /// Local storage configuration + pub local: Option, + + /// Vault configuration for credential management + #[cfg(feature = "vault-integration")] + pub vault: Option, + + /// Default timeout for operations + pub default_timeout: Duration, + + /// Enable compression by default + pub enable_compression: bool, + + /// Maximum retry attempts + pub max_retries: u32, + + /// Base retry delay + pub retry_base_delay: Duration, +} + +impl Default for StorageConfig { + fn default() -> Self { + Self { + #[cfg(feature = "s3")] + s3: None, + local: None, + #[cfg(feature = "vault-integration")] + vault: None, + default_timeout: Duration::from_secs(30), + enable_compression: true, + max_retries: 3, + retry_base_delay: Duration::from_millis(100), + } + } +} + +impl StorageConfig { + /// Load configuration from environment variables + pub fn from_env() -> Result { + let mut config = Self::default(); + + // Load timeout settings + if let Ok(timeout_str) = std::env::var("STORAGE_TIMEOUT_SECONDS") { + if let Ok(timeout_secs) = timeout_str.parse::() { + config.default_timeout = Duration::from_secs(timeout_secs); + } + } + + // Load compression setting + if let Ok(compression_str) = std::env::var("STORAGE_ENABLE_COMPRESSION") { + config.enable_compression = compression_str.to_lowercase() == "true"; + } + + // Load retry settings + if let Ok(retries_str) = std::env::var("STORAGE_MAX_RETRIES") { + if let Ok(retries) = retries_str.parse::() { + config.max_retries = retries; + } + } + + if let Ok(delay_str) = std::env::var("STORAGE_RETRY_DELAY_MS") { + if let Ok(delay_ms) = delay_str.parse::() { + config.retry_base_delay = Duration::from_millis(delay_ms); + } + } + + // Load Vault configuration if enabled + #[cfg(feature = "vault-integration")] + { + if std::env::var("VAULT_ADDR").is_ok() { + config.vault = Some(crate::vault::VaultConfig::from_env().map_err(|e| { + crate::StorageError::ConfigError { + message: format!("Failed to load Vault config: {}", e), + } + })?); + } + } + + // Load S3 configuration if enabled + #[cfg(feature = "s3")] + { + if std::env::var("S3_BUCKET_NAME").is_ok() || std::env::var("AWS_REGION").is_ok() { + config.s3 = Some(crate::s3::S3StorageConfig::from_env().map_err(|e| { + crate::StorageError::ConfigError { + message: format!("Failed to load S3 config: {}", e), + } + })?); + } + } + + // Load local storage configuration + if let Ok(local_dir) = std::env::var("STORAGE_LOCAL_DIR") { + config.local = Some(crate::local::LocalStorageConfig { + base_path: local_dir.into(), + ..Default::default() + }); + } + + Ok(config) + } + + /// Validate the configuration + pub fn validate(&self) -> Result<(), crate::StorageError> { + // Ensure at least one storage backend is configured + let has_backend = false + #[cfg(feature = "s3")] + || self.s3.is_some() + || self.local.is_some(); + + if !has_backend { + return Err(crate::StorageError::ConfigError { + message: "No storage backend configured".to_string(), + }); + } + + // Validate timeout values + if self.default_timeout.as_secs() == 0 { + return Err(crate::StorageError::ConfigError { + message: "Default timeout must be greater than 0".to_string(), + }); + } + + // Validate retry settings + if self.max_retries == 0 { + return Err(crate::StorageError::ConfigError { + message: "Max retries must be greater than 0".to_string(), + }); + } + + if self.retry_base_delay.as_millis() == 0 { + return Err(crate::StorageError::ConfigError { + message: "Retry base delay must be greater than 0".to_string(), + }); + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_default_config() { + let config = StorageConfig::default(); + assert_eq!(config.default_timeout, Duration::from_secs(30)); + assert!(config.enable_compression); + assert_eq!(config.max_retries, 3); + assert_eq!(config.retry_base_delay, Duration::from_millis(100)); + } + + #[test] + fn test_config_validation() { + let mut config = StorageConfig::default(); + config.local = Some(crate::local::LocalStorageConfig::default()); + + // Valid configuration should pass + assert!(config.validate().is_ok()); + + // Invalid timeout should fail + config.default_timeout = Duration::from_secs(0); + assert!(config.validate().is_err()); + + // Invalid retry count should fail + config.default_timeout = Duration::from_secs(30); + config.max_retries = 0; + assert!(config.validate().is_err()); + } + + #[test] + fn test_config_from_env() { + // Set test environment variables + std::env::set_var("STORAGE_TIMEOUT_SECONDS", "60"); + std::env::set_var("STORAGE_ENABLE_COMPRESSION", "false"); + std::env::set_var("STORAGE_MAX_RETRIES", "5"); + std::env::set_var("STORAGE_RETRY_DELAY_MS", "200"); + std::env::set_var("STORAGE_LOCAL_DIR", "/tmp/test-storage"); + + let config = StorageConfig::from_env().unwrap(); + assert_eq!(config.default_timeout, Duration::from_secs(60)); + assert!(!config.enable_compression); + assert_eq!(config.max_retries, 5); + assert_eq!(config.retry_base_delay, Duration::from_millis(200)); + assert!(config.local.is_some()); + + // Cleanup + std::env::remove_var("STORAGE_TIMEOUT_SECONDS"); + std::env::remove_var("STORAGE_ENABLE_COMPRESSION"); + std::env::remove_var("STORAGE_MAX_RETRIES"); + std::env::remove_var("STORAGE_RETRY_DELAY_MS"); + std::env::remove_var("STORAGE_LOCAL_DIR"); + } +} \ No newline at end of file diff --git a/storage/src/lib.rs b/storage/src/lib.rs index 3d9d80870..68c963226 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -25,10 +25,11 @@ pub mod vault; pub mod error; pub mod config; pub mod metrics; +pub mod models; // Re-export commonly used types and traits pub use error::{StorageError, StorageResult}; -pub use config::StorageConfig; +pub use foxhunt_config::StorageConfig; #[cfg(feature = "s3")] pub use s3::{S3Storage, S3StorageConfig, ArchivalDataType, ArchivalMetadata, ArchivalStats}; @@ -37,6 +38,7 @@ pub use s3::{S3Storage, S3StorageConfig, ArchivalDataType, ArchivalMetadata, Arc pub use vault::{VaultClient, VaultConfig, VaultCredentials}; pub use local::{LocalStorage, LocalStorageConfig, FileOperation}; +pub use models::{ModelStorage, ModelCheckpoint, ModelStorageConfig, ArchivalDataType as ModelDataType, ModelStorageStats, ModelLoader}; /// Common storage trait for abstracting different storage backends #[async_trait::async_trait] diff --git a/storage/src/local.rs b/storage/src/local.rs new file mode 100644 index 000000000..1dcbc6cd7 --- /dev/null +++ b/storage/src/local.rs @@ -0,0 +1,591 @@ +//! Local filesystem storage implementation + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use tokio::fs; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tracing::{debug, info, warn}; + +use crate::{Storage, StorageError, StorageMetadata, StorageResult}; + +/// Configuration for local filesystem storage +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LocalStorageConfig { + /// Base directory for storage operations + pub base_path: PathBuf, + /// Enable atomic writes using temporary files + pub atomic_writes: bool, + /// Enable file locking for concurrent access + pub enable_locking: bool, + /// Create parent directories if they don't exist + pub create_directories: bool, + /// File permissions (Unix only) + pub file_permissions: Option, + /// Directory permissions (Unix only) + pub directory_permissions: Option, + /// Enable compression for stored files + pub enable_compression: bool, + /// Buffer size for file operations + pub buffer_size: usize, +} + +impl Default for LocalStorageConfig { + fn default() -> Self { + Self { + base_path: PathBuf::from("./storage"), + atomic_writes: true, + enable_locking: true, + create_directories: true, + file_permissions: Some(0o644), + directory_permissions: Some(0o755), + enable_compression: false, + buffer_size: 8192, + } + } +} + +impl LocalStorageConfig { + /// Load configuration from environment variables + pub fn from_env() -> StorageResult { + let mut config = Self::default(); + + if let Ok(base_path) = std::env::var("STORAGE_LOCAL_BASE_PATH") { + config.base_path = PathBuf::from(base_path); + } + + if let Ok(atomic_str) = std::env::var("STORAGE_LOCAL_ATOMIC_WRITES") { + config.atomic_writes = atomic_str.to_lowercase() == "true"; + } + + if let Ok(locking_str) = std::env::var("STORAGE_LOCAL_ENABLE_LOCKING") { + config.enable_locking = locking_str.to_lowercase() == "true"; + } + + if let Ok(create_str) = std::env::var("STORAGE_LOCAL_CREATE_DIRECTORIES") { + config.create_directories = create_str.to_lowercase() == "true"; + } + + if let Ok(compression_str) = std::env::var("STORAGE_LOCAL_ENABLE_COMPRESSION") { + config.enable_compression = compression_str.to_lowercase() == "true"; + } + + if let Ok(buffer_str) = std::env::var("STORAGE_LOCAL_BUFFER_SIZE") { + if let Ok(buffer_size) = buffer_str.parse::() { + config.buffer_size = buffer_size; + } + } + + Ok(config) + } + + /// Validate the configuration + pub fn validate(&self) -> StorageResult<()> { + if !self.base_path.is_absolute() { + return Err(StorageError::ConfigError { + message: "Base path must be absolute".to_string(), + }); + } + + if self.buffer_size == 0 { + return Err(StorageError::ConfigError { + message: "Buffer size must be greater than 0".to_string(), + }); + } + + Ok(()) + } +} + +/// File operation types for metrics and logging +#[derive(Debug, Clone, Copy)] +pub enum FileOperation { + Read, + Write, + Delete, + List, + Exists, + Metadata, +} + +impl FileOperation { + pub fn as_str(&self) -> &'static str { + match self { + FileOperation::Read => "read", + FileOperation::Write => "write", + FileOperation::Delete => "delete", + FileOperation::List => "list", + FileOperation::Exists => "exists", + FileOperation::Metadata => "metadata", + } + } +} + +/// Local filesystem storage implementation +pub struct LocalStorage { + config: LocalStorageConfig, + base_path: PathBuf, +} + +impl LocalStorage { + /// Create new local storage instance + pub async fn new(config: LocalStorageConfig) -> StorageResult { + config.validate()?; + + let base_path = config.base_path.clone(); + + // Create base directory if it doesn't exist + if config.create_directories { + Self::ensure_directory_exists(&base_path, config.directory_permissions).await?; + } + + info!("Local storage initialized at: {}", base_path.display()); + + Ok(Self { + config, + base_path, + }) + } + + /// Get the full filesystem path for a storage path + fn get_full_path(&self, path: &str) -> PathBuf { + // Sanitize path to prevent directory traversal + let sanitized = path.replace("..", "").replace("\\", "/"); + let sanitized = sanitized.trim_start_matches('/'); + + self.base_path.join(sanitized) + } + + /// Ensure a directory exists, creating it if necessary + async fn ensure_directory_exists(path: &Path, permissions: Option) -> StorageResult<()> { + if !path.exists() { + fs::create_dir_all(path).await.map_err(|e| StorageError::IoError { + message: format!("Failed to create directory {}: {}", path.display(), e), + })?; + + // Set permissions on Unix systems + #[cfg(unix)] + if let Some(perms) = permissions { + use std::os::unix::fs::PermissionsExt; + let std_perms = std::fs::Permissions::from_mode(perms); + std::fs::set_permissions(path, std_perms).map_err(|e| StorageError::IoError { + message: format!("Failed to set directory permissions for {}: {}", path.display(), e), + })?; + } + + debug!("Created directory: {}", path.display()); + } + + Ok(()) + } + + /// Compress data if compression is enabled + fn compress_data(&self, data: &[u8]) -> StorageResult> { + if !self.config.enable_compression { + return Ok(data.to_vec()); + } + + use flate2::write::GzEncoder; + use flate2::Compression; + use std::io::Write; + + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(data).map_err(|e| StorageError::CompressionError { + message: format!("Compression failed: {}", e), + })?; + + let compressed = encoder.finish().map_err(|e| StorageError::CompressionError { + message: format!("Compression finalization failed: {}", e), + })?; + + debug!("Compressed {} bytes to {} bytes", data.len(), compressed.len()); + Ok(compressed) + } + + /// Decompress data if it was compressed + fn decompress_data(&self, data: &[u8]) -> StorageResult> { + if !self.config.enable_compression { + return Ok(data.to_vec()); + } + + use flate2::read::GzDecoder; + use std::io::Read; + + let mut decoder = GzDecoder::new(data); + let mut decompressed = Vec::new(); + decoder.read_to_end(&mut decompressed).map_err(|e| StorageError::CompressionError { + message: format!("Decompression failed: {}", e), + })?; + + debug!("Decompressed {} bytes to {} bytes", data.len(), decompressed.len()); + Ok(decompressed) + } + + /// Write data to file with optional atomic operation + async fn write_file_data(&self, full_path: &Path, data: &[u8]) -> StorageResult<()> { + // Ensure parent directory exists + if let Some(parent) = full_path.parent() { + if self.config.create_directories { + Self::ensure_directory_exists(parent, self.config.directory_permissions).await?; + } + } + + let compressed_data = self.compress_data(data)?; + + if self.config.atomic_writes { + // Write to temporary file first, then rename + let temp_path = full_path.with_extension(format!( + "{}.tmp.{}", + full_path.extension().and_then(|s| s.to_str()).unwrap_or(""), + std::process::id() + )); + + fs::write(&temp_path, &compressed_data).await.map_err(|e| StorageError::IoError { + message: format!("Failed to write temporary file {}: {}", temp_path.display(), e), + })?; + + // Set file permissions + #[cfg(unix)] + if let Some(perms) = self.config.file_permissions { + use std::os::unix::fs::PermissionsExt; + let std_perms = std::fs::Permissions::from_mode(perms); + std::fs::set_permissions(&temp_path, std_perms).map_err(|e| StorageError::IoError { + message: format!("Failed to set file permissions for {}: {}", temp_path.display(), e), + })?; + } + + // Atomic rename + fs::rename(&temp_path, full_path).await.map_err(|e| StorageError::IoError { + message: format!("Failed to rename {} to {}: {}", temp_path.display(), full_path.display(), e), + })?; + + debug!("Atomically wrote {} bytes to {}", compressed_data.len(), full_path.display()); + } else { + // Direct write + fs::write(full_path, &compressed_data).await.map_err(|e| StorageError::IoError { + message: format!("Failed to write file {}: {}", full_path.display(), e), + })?; + + // Set file permissions + #[cfg(unix)] + if let Some(perms) = self.config.file_permissions { + use std::os::unix::fs::PermissionsExt; + let std_perms = std::fs::Permissions::from_mode(perms); + std::fs::set_permissions(full_path, std_perms).map_err(|e| StorageError::IoError { + message: format!("Failed to set file permissions for {}: {}", full_path.display(), e), + })?; + } + + debug!("Wrote {} bytes to {}", compressed_data.len(), full_path.display()); + } + + Ok(()) + } + + /// Record operation metrics + fn record_metrics(&self, operation: FileOperation, duration: std::time::Duration, success: bool) { + crate::metrics::get_metrics().record_operation(operation.as_str(), "local", duration, success); + } +} + +#[async_trait] +impl Storage for LocalStorage { + async fn store(&self, path: &str, data: &[u8]) -> StorageResult<()> { + let start = std::time::Instant::now(); + let full_path = self.get_full_path(path); + + let result = self.write_file_data(&full_path, data).await; + + let duration = start.elapsed(); + self.record_metrics(FileOperation::Write, duration, result.is_ok()); + + if result.is_ok() { + crate::metrics::get_metrics().record_transfer("store", "local", data.len() as u64, duration); + } + + result + } + + async fn retrieve(&self, path: &str) -> StorageResult> { + let start = std::time::Instant::now(); + let full_path = self.get_full_path(path); + + let result = async { + let compressed_data = fs::read(&full_path).await.map_err(|e| match e.kind() { + std::io::ErrorKind::NotFound => StorageError::NotFound { + path: path.to_string(), + }, + std::io::ErrorKind::PermissionDenied => StorageError::PermissionDenied { + path: path.to_string(), + }, + _ => StorageError::IoError { + message: format!("Failed to read file {}: {}", full_path.display(), e), + }, + })?; + + let data = self.decompress_data(&compressed_data)?; + debug!("Retrieved {} bytes from {}", data.len(), full_path.display()); + Ok(data) + }.await; + + let duration = start.elapsed(); + self.record_metrics(FileOperation::Read, duration, result.is_ok()); + + if let Ok(ref data) = result { + crate::metrics::get_metrics().record_transfer("retrieve", "local", data.len() as u64, duration); + } + + result + } + + async fn exists(&self, path: &str) -> StorageResult { + let start = std::time::Instant::now(); + let full_path = self.get_full_path(path); + + let exists = full_path.exists(); + debug!("Path {} exists: {}", path, exists); + + let duration = start.elapsed(); + self.record_metrics(FileOperation::Exists, duration, true); + + Ok(exists) + } + + async fn delete(&self, path: &str) -> StorageResult { + let start = std::time::Instant::now(); + let full_path = self.get_full_path(path); + + let result = if full_path.exists() { + fs::remove_file(&full_path).await.map_err(|e| match e.kind() { + std::io::ErrorKind::NotFound => StorageError::NotFound { + path: path.to_string(), + }, + std::io::ErrorKind::PermissionDenied => StorageError::PermissionDenied { + path: path.to_string(), + }, + _ => StorageError::IoError { + message: format!("Failed to delete file {}: {}", full_path.display(), e), + }, + })?; + + debug!("Deleted file: {}", full_path.display()); + Ok(true) + } else { + debug!("File not found for deletion: {}", full_path.display()); + Ok(false) + }; + + let duration = start.elapsed(); + self.record_metrics(FileOperation::Delete, duration, result.is_ok()); + + result + } + + async fn list(&self, prefix: &str) -> StorageResult> { + let start = std::time::Instant::now(); + let prefix_path = self.get_full_path(prefix); + + let result = async { + let mut paths = Vec::new(); + + // Handle both directory listing and prefix matching + let (search_dir, file_prefix) = if prefix_path.is_dir() { + (prefix_path, String::new()) + } else { + let parent = prefix_path.parent().unwrap_or(&self.base_path); + let filename = prefix_path.file_name() + .and_then(|n| n.to_str()) + .unwrap_or("") + .to_string(); + (parent.to_path_buf(), filename) + }; + + if !search_dir.exists() { + return Ok(paths); + } + + let mut entries = fs::read_dir(&search_dir).await.map_err(|e| StorageError::IoError { + message: format!("Failed to read directory {}: {}", search_dir.display(), e), + })?; + + while let Some(entry) = entries.next_entry().await.map_err(|e| StorageError::IoError { + message: format!("Failed to read directory entry: {}", e), + })? { + let entry_path = entry.path(); + + if entry_path.is_file() { + if let Some(filename) = entry_path.file_name().and_then(|n| n.to_str()) { + if file_prefix.is_empty() || filename.starts_with(&file_prefix) { + // Convert back to relative path + if let Ok(relative) = entry_path.strip_prefix(&self.base_path) { + if let Some(path_str) = relative.to_str() { + paths.push(path_str.replace('\\', "/")); + } + } + } + } + } + } + + paths.sort(); + debug!("Listed {} files with prefix '{}'", paths.len(), prefix); + Ok(paths) + }.await; + + let duration = start.elapsed(); + self.record_metrics(FileOperation::List, duration, result.is_ok()); + + result + } + + async fn metadata(&self, path: &str) -> StorageResult { + let start = std::time::Instant::now(); + let full_path = self.get_full_path(path); + + let result = async { + let metadata = fs::metadata(&full_path).await.map_err(|e| match e.kind() { + std::io::ErrorKind::NotFound => StorageError::NotFound { + path: path.to_string(), + }, + std::io::ErrorKind::PermissionDenied => StorageError::PermissionDenied { + path: path.to_string(), + }, + _ => StorageError::IoError { + message: format!("Failed to get metadata for {}: {}", full_path.display(), e), + }, + })?; + + let last_modified = metadata.modified() + .unwrap_or(SystemTime::UNIX_EPOCH) + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + + let last_modified_dt = DateTime::from_timestamp(last_modified.as_secs() as i64, 0) + .unwrap_or_else(|| Utc::now()); + + let storage_metadata = StorageMetadata { + path: path.to_string(), + size: metadata.len(), + content_type: Some("application/octet-stream".to_string()), + last_modified: last_modified_dt, + etag: None, // Could implement checksum-based ETag + tags: HashMap::new(), + }; + + debug!("Retrieved metadata for {}: {} bytes", path, metadata.len()); + Ok(storage_metadata) + }.await; + + let duration = start.elapsed(); + self.record_metrics(FileOperation::Metadata, duration, result.is_ok()); + + result + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + async fn create_test_storage() -> (LocalStorage, TempDir) { + let temp_dir = TempDir::new().unwrap(); + let config = LocalStorageConfig { + base_path: temp_dir.path().to_path_buf(), + ..Default::default() + }; + let storage = LocalStorage::new(config).await.unwrap(); + (storage, temp_dir) + } + + #[tokio::test] + async fn test_store_and_retrieve() { + let (storage, _temp_dir) = create_test_storage().await; + let test_data = b"Hello, World!"; + + storage.store("test.txt", test_data).await.unwrap(); + let retrieved = storage.retrieve("test.txt").await.unwrap(); + + assert_eq!(retrieved, test_data); + } + + #[tokio::test] + async fn test_exists() { + let (storage, _temp_dir) = create_test_storage().await; + + assert!(!storage.exists("nonexistent.txt").await.unwrap()); + + storage.store("test.txt", b"data").await.unwrap(); + assert!(storage.exists("test.txt").await.unwrap()); + } + + #[tokio::test] + async fn test_delete() { + let (storage, _temp_dir) = create_test_storage().await; + + storage.store("test.txt", b"data").await.unwrap(); + assert!(storage.exists("test.txt").await.unwrap()); + + let deleted = storage.delete("test.txt").await.unwrap(); + assert!(deleted); + assert!(!storage.exists("test.txt").await.unwrap()); + + // Deleting non-existent file should return false + let deleted = storage.delete("nonexistent.txt").await.unwrap(); + assert!(!deleted); + } + + #[tokio::test] + async fn test_list() { + let (storage, _temp_dir) = create_test_storage().await; + + storage.store("file1.txt", b"data1").await.unwrap(); + storage.store("file2.txt", b"data2").await.unwrap(); + storage.store("other.dat", b"data3").await.unwrap(); + + let all_files = storage.list("").await.unwrap(); + assert_eq!(all_files.len(), 3); + assert!(all_files.contains(&"file1.txt".to_string())); + assert!(all_files.contains(&"file2.txt".to_string())); + assert!(all_files.contains(&"other.dat".to_string())); + } + + #[tokio::test] + async fn test_metadata() { + let (storage, _temp_dir) = create_test_storage().await; + let test_data = b"Hello, World!"; + + storage.store("test.txt", test_data).await.unwrap(); + let metadata = storage.metadata("test.txt").await.unwrap(); + + assert_eq!(metadata.path, "test.txt"); + assert!(metadata.size > 0); // May be larger due to compression + assert_eq!(metadata.content_type, Some("application/octet-stream".to_string())); + } + + #[tokio::test] + async fn test_nested_directories() { + let (storage, _temp_dir) = create_test_storage().await; + + storage.store("nested/dir/file.txt", b"nested data").await.unwrap(); + let retrieved = storage.retrieve("nested/dir/file.txt").await.unwrap(); + + assert_eq!(retrieved, b"nested data"); + assert!(storage.exists("nested/dir/file.txt").await.unwrap()); + } + + #[tokio::test] + async fn test_path_sanitization() { + let (storage, _temp_dir) = create_test_storage().await; + + // Directory traversal attempts should be sanitized + storage.store("../../../etc/passwd", b"malicious").await.unwrap(); + + // Should be stored safely within base directory + assert!(storage.exists("etc/passwd").await.unwrap()); + } +} \ No newline at end of file diff --git a/storage/src/metrics.rs b/storage/src/metrics.rs new file mode 100644 index 000000000..f63e830ed --- /dev/null +++ b/storage/src/metrics.rs @@ -0,0 +1,409 @@ +//! Metrics and telemetry for storage operations + +use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use tracing::{debug, info, warn}; + +/// Storage operation metrics +#[derive(Debug, Clone)] +pub struct StorageMetrics { + /// Operation counters + pub operations: Arc, + /// Performance metrics + pub performance: Arc, + /// Error metrics + pub errors: Arc, +} + +impl StorageMetrics { + /// Create new metrics instance + pub fn new() -> Self { + Self { + operations: Arc::new(OperationMetrics::new()), + performance: Arc::new(PerformanceMetrics::new()), + errors: Arc::new(ErrorMetrics::new()), + } + } + + /// Record a storage operation + pub fn record_operation(&self, operation: &str, provider: &str, duration: Duration, success: bool) { + // Record operation count + self.operations.increment(operation, provider); + + // Record performance metrics + self.performance.record_duration(operation, provider, duration); + + // Record errors if failed + if !success { + self.errors.increment(operation, provider, "operation_failed"); + } + + debug!( + "Storage operation recorded: {} on {} took {:?} (success: {})", + operation, provider, duration, success + ); + } + + /// Record data transfer metrics + pub fn record_transfer(&self, operation: &str, provider: &str, bytes: u64, duration: Duration) { + self.performance.record_transfer(operation, provider, bytes, duration); + + let throughput_mbps = (bytes as f64 / (1024.0 * 1024.0)) / duration.as_secs_f64(); + debug!( + "Data transfer recorded: {} on {} - {} bytes in {:?} ({:.2} MB/s)", + operation, provider, bytes, duration, throughput_mbps + ); + } + + /// Record authentication metrics + pub fn record_auth_event(&self, provider: &str, event: &str, success: bool) { + self.operations.increment("auth", provider); + + if !success { + self.errors.increment("auth", provider, event); + } + + debug!("Auth event recorded: {} on {} (success: {})", event, provider, success); + } + + /// Get operation summary + pub fn get_summary(&self) -> MetricsSummary { + MetricsSummary { + total_operations: self.operations.get_total(), + total_errors: self.errors.get_total(), + average_latency_ms: self.performance.get_average_latency_ms(), + total_bytes_transferred: self.performance.get_total_bytes_transferred(), + operations_by_type: self.operations.get_by_type(), + errors_by_type: self.errors.get_by_type(), + performance_percentiles: self.performance.get_percentiles(), + } + } + + /// Reset all metrics (useful for testing) + pub fn reset(&self) { + self.operations.reset(); + self.performance.reset(); + self.errors.reset(); + info!("Storage metrics reset"); + } +} + +impl Default for StorageMetrics { + fn default() -> Self { + Self::new() + } +} + +/// Operation counter metrics +#[derive(Debug)] +pub struct OperationMetrics { + /// Total operations by type and provider + counters: Arc>>, +} + +impl OperationMetrics { + pub fn new() -> Self { + Self { + counters: Arc::new(parking_lot::RwLock::new(HashMap::new())), + } + } + + pub fn increment(&self, operation: &str, provider: &str) { + let key = format!("{}:{}", operation, provider); + let counters = self.counters.read(); + + if let Some(counter) = counters.get(&key) { + counter.fetch_add(1, Ordering::Relaxed); + } else { + drop(counters); + let mut counters = self.counters.write(); + counters.entry(key).or_insert_with(|| AtomicU64::new(0)).fetch_add(1, Ordering::Relaxed); + } + } + + pub fn get(&self, operation: &str, provider: &str) -> u64 { + let key = format!("{}:{}", operation, provider); + self.counters.read() + .get(&key) + .map(|c| c.load(Ordering::Relaxed)) + .unwrap_or(0) + } + + pub fn get_total(&self) -> u64 { + self.counters.read() + .values() + .map(|c| c.load(Ordering::Relaxed)) + .sum() + } + + pub fn get_by_type(&self) -> HashMap { + self.counters.read() + .iter() + .map(|(key, counter)| (key.clone(), counter.load(Ordering::Relaxed))) + .collect() + } + + pub fn reset(&self) { + let mut counters = self.counters.write(); + counters.clear(); + } +} + +/// Performance metrics (latency, throughput) +#[derive(Debug)] +pub struct PerformanceMetrics { + /// Latency measurements + latencies: Arc>>>, + /// Bytes transferred + bytes_transferred: Arc>>, + /// Transfer durations for throughput calculation + transfer_durations: Arc>>>, +} + +impl PerformanceMetrics { + pub fn new() -> Self { + Self { + latencies: Arc::new(parking_lot::RwLock::new(HashMap::new())), + bytes_transferred: Arc::new(parking_lot::RwLock::new(HashMap::new())), + transfer_durations: Arc::new(parking_lot::RwLock::new(HashMap::new())), + } + } + + pub fn record_duration(&self, operation: &str, provider: &str, duration: Duration) { + let key = format!("{}:{}", operation, provider); + let mut latencies = self.latencies.write(); + latencies.entry(key).or_default().push(duration); + + // Keep only recent measurements (sliding window) + if let Some(durations) = latencies.get_mut(&key) { + if durations.len() > 1000 { + durations.drain(..500); // Keep latest 500 + } + } + } + + pub fn record_transfer(&self, operation: &str, provider: &str, bytes: u64, duration: Duration) { + let key = format!("{}:{}", operation, provider); + + // Record bytes + let bytes_map = self.bytes_transferred.read(); + if let Some(counter) = bytes_map.get(&key) { + counter.fetch_add(bytes, Ordering::Relaxed); + } else { + drop(bytes_map); + let mut bytes_map = self.bytes_transferred.write(); + bytes_map.entry(key.clone()).or_insert_with(|| AtomicU64::new(0)).fetch_add(bytes, Ordering::Relaxed); + } + + // Record duration for throughput calculation + let mut durations = self.transfer_durations.write(); + durations.entry(key).or_default().push(duration); + } + + pub fn get_average_latency_ms(&self) -> f64 { + let latencies = self.latencies.read(); + let all_durations: Vec = latencies.values().flatten().cloned().collect(); + + if all_durations.is_empty() { + 0.0 + } else { + let total_ms: f64 = all_durations.iter().map(|d| d.as_millis() as f64).sum(); + total_ms / all_durations.len() as f64 + } + } + + pub fn get_total_bytes_transferred(&self) -> u64 { + self.bytes_transferred.read() + .values() + .map(|c| c.load(Ordering::Relaxed)) + .sum() + } + + pub fn get_percentiles(&self) -> PerformancePercentiles { + let latencies = self.latencies.read(); + let mut all_durations: Vec = latencies.values().flatten().cloned().collect(); + + if all_durations.is_empty() { + return PerformancePercentiles::default(); + } + + all_durations.sort(); + let len = all_durations.len(); + + PerformancePercentiles { + p50_ms: all_durations[len * 50 / 100].as_millis() as f64, + p90_ms: all_durations[len * 90 / 100].as_millis() as f64, + p95_ms: all_durations[len * 95 / 100].as_millis() as f64, + p99_ms: all_durations[len * 99 / 100].as_millis() as f64, + min_ms: all_durations[0].as_millis() as f64, + max_ms: all_durations[len - 1].as_millis() as f64, + } + } + + pub fn reset(&self) { + let mut latencies = self.latencies.write(); + latencies.clear(); + + let mut bytes_transferred = self.bytes_transferred.write(); + bytes_transferred.clear(); + + let mut transfer_durations = self.transfer_durations.write(); + transfer_durations.clear(); + } +} + +/// Error tracking metrics +#[derive(Debug)] +pub struct ErrorMetrics { + /// Error counters by type and provider + error_counters: Arc>>, +} + +impl ErrorMetrics { + pub fn new() -> Self { + Self { + error_counters: Arc::new(parking_lot::RwLock::new(HashMap::new())), + } + } + + pub fn increment(&self, operation: &str, provider: &str, error_type: &str) { + let key = format!("{}:{}:{}", operation, provider, error_type); + let counters = self.error_counters.read(); + + if let Some(counter) = counters.get(&key) { + counter.fetch_add(1, Ordering::Relaxed); + } else { + drop(counters); + let mut counters = self.error_counters.write(); + counters.entry(key).or_insert_with(|| AtomicU64::new(0)).fetch_add(1, Ordering::Relaxed); + } + + warn!("Storage error recorded: {} on {} (type: {})", operation, provider, error_type); + } + + pub fn get_total(&self) -> u64 { + self.error_counters.read() + .values() + .map(|c| c.load(Ordering::Relaxed)) + .sum() + } + + pub fn get_by_type(&self) -> HashMap { + self.error_counters.read() + .iter() + .map(|(key, counter)| (key.clone(), counter.load(Ordering::Relaxed))) + .collect() + } + + pub fn reset(&self) { + let mut counters = self.error_counters.write(); + counters.clear(); + } +} + +/// Performance percentiles for latency analysis +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +pub struct PerformancePercentiles { + pub p50_ms: f64, + pub p90_ms: f64, + pub p95_ms: f64, + pub p99_ms: f64, + pub min_ms: f64, + pub max_ms: f64, +} + +/// Comprehensive metrics summary +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct MetricsSummary { + pub total_operations: u64, + pub total_errors: u64, + pub average_latency_ms: f64, + pub total_bytes_transferred: u64, + pub operations_by_type: HashMap, + pub errors_by_type: HashMap, + pub performance_percentiles: PerformancePercentiles, +} + +/// Global storage metrics instance +static GLOBAL_METRICS: std::sync::OnceLock = std::sync::OnceLock::new(); + +/// Get the global metrics instance +pub fn get_metrics() -> &'static StorageMetrics { + GLOBAL_METRICS.get_or_init(StorageMetrics::new) +} + +/// Macro for timing storage operations +#[macro_export] +macro_rules! time_storage_operation { + ($operation:expr, $provider:expr, $code:block) => {{ + let start = std::time::Instant::now(); + let result = $code; + let duration = start.elapsed(); + let success = result.is_ok(); + + $crate::metrics::get_metrics().record_operation($operation, $provider, duration, success); + + result + }}; +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + #[test] + fn test_operation_metrics() { + let metrics = OperationMetrics::new(); + + metrics.increment("read", "s3"); + metrics.increment("read", "s3"); + metrics.increment("write", "local"); + + assert_eq!(metrics.get("read", "s3"), 2); + assert_eq!(metrics.get("write", "local"), 1); + assert_eq!(metrics.get_total(), 3); + } + + #[test] + fn test_performance_metrics() { + let metrics = PerformanceMetrics::new(); + + metrics.record_duration("read", "s3", Duration::from_millis(100)); + metrics.record_duration("read", "s3", Duration::from_millis(200)); + metrics.record_transfer("upload", "s3", 1024, Duration::from_millis(50)); + + assert_eq!(metrics.get_total_bytes_transferred(), 1024); + let avg = metrics.get_average_latency_ms(); + assert!(avg > 0.0); + } + + #[test] + fn test_error_metrics() { + let metrics = ErrorMetrics::new(); + + metrics.increment("read", "s3", "timeout"); + metrics.increment("write", "local", "permission_denied"); + + assert_eq!(metrics.get_total(), 2); + let by_type = metrics.get_by_type(); + assert!(by_type.contains_key("read:s3:timeout")); + } + + #[test] + fn test_storage_metrics_integration() { + let metrics = StorageMetrics::new(); + + metrics.record_operation("read", "s3", Duration::from_millis(100), true); + metrics.record_operation("write", "local", Duration::from_millis(50), false); + metrics.record_transfer("upload", "s3", 2048, Duration::from_millis(100)); + + let summary = metrics.get_summary(); + assert_eq!(summary.total_operations, 2); + assert_eq!(summary.total_errors, 1); + assert_eq!(summary.total_bytes_transferred, 2048); + } +} \ No newline at end of file diff --git a/storage/src/models.rs b/storage/src/models.rs new file mode 100644 index 000000000..98e99165f --- /dev/null +++ b/storage/src/models.rs @@ -0,0 +1,743 @@ +//! Model storage and checkpoint utilities +//! +//! This module provides utilities for storing and loading ML model checkpoints, +//! weights, and associated metadata with versioning support. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::time::Duration; + +use anyhow::Result; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use tokio::fs; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tracing::{debug, info, warn}; +use uuid::Uuid; + +use crate::{Storage, StorageError, StorageResult}; + +/// Model checkpoint metadata +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ModelCheckpoint { + /// Unique identifier for this checkpoint + pub checkpoint_id: Uuid, + /// Model name/identifier + pub model_name: String, + /// Model version (semantic versioning recommended) + pub model_version: String, + /// Training epoch or iteration number + pub epoch: u64, + /// Training step/batch number + pub step: u64, + /// Validation loss at this checkpoint + pub validation_loss: Option, + /// Training loss at this checkpoint + pub training_loss: Option, + /// Model accuracy metrics + pub accuracy_metrics: HashMap, + /// Model architecture description + pub architecture: String, + /// Hyperparameters used during training + pub hyperparameters: HashMap, + /// Size of the model in bytes + pub model_size_bytes: u64, + /// Compression type used for storage + pub compression_type: Option, + /// Timestamp when checkpoint was created + pub created_at: DateTime, + /// Training duration up to this checkpoint + pub training_duration: Duration, + /// Git commit hash (if available) + pub git_commit: Option, + /// Custom tags for categorization + pub tags: HashMap, + /// Storage path for the model data + pub storage_path: String, + /// Checksum for integrity verification + pub checksum: String, +} + +impl ModelCheckpoint { + /// Create a new model checkpoint + pub fn new( + model_name: String, + model_version: String, + epoch: u64, + step: u64, + architecture: String, + storage_path: String, + model_size_bytes: u64, + checksum: String, + ) -> Self { + Self { + checkpoint_id: Uuid::new_v4(), + model_name, + model_version, + epoch, + step, + validation_loss: None, + training_loss: None, + accuracy_metrics: HashMap::new(), + architecture, + hyperparameters: HashMap::new(), + model_size_bytes, + compression_type: None, + created_at: Utc::now(), + training_duration: Duration::from_secs(0), + git_commit: None, + tags: HashMap::new(), + storage_path, + checksum, + } + } + + /// Set validation and training losses + pub fn with_losses(mut self, validation_loss: Option, training_loss: Option) -> Self { + self.validation_loss = validation_loss; + self.training_loss = training_loss; + self + } + + /// Add accuracy metrics + pub fn with_accuracy_metrics(mut self, metrics: HashMap) -> Self { + self.accuracy_metrics = metrics; + self + } + + /// Add hyperparameters + pub fn with_hyperparameters(mut self, hyperparams: HashMap) -> Self { + self.hyperparameters = hyperparams; + self + } + + /// Add training duration + pub fn with_training_duration(mut self, duration: Duration) -> Self { + self.training_duration = duration; + self + } + + /// Add git commit hash + pub fn with_git_commit(mut self, commit: String) -> Self { + self.git_commit = Some(commit); + self + } + + /// Add custom tags + pub fn with_tags(mut self, tags: HashMap) -> Self { + self.tags = tags; + self + } + + /// Check if this checkpoint is better than another based on validation loss + pub fn is_better_than(&self, other: &ModelCheckpoint) -> bool { + match (self.validation_loss, other.validation_loss) { + (Some(self_loss), Some(other_loss)) => self_loss < other_loss, + (Some(_), None) => true, // Has validation loss vs none + (None, Some(_)) => false, // No validation loss vs has one + (None, None) => self.step > other.step, // Compare by step if no losses + } + } + + /// Get a human-readable description + pub fn description(&self) -> String { + let loss_str = match (self.validation_loss, self.training_loss) { + (Some(val), Some(train)) => format!(" (val_loss: {:.4}, train_loss: {:.4})", val, train), + (Some(val), None) => format!(" (val_loss: {:.4})", val), + (None, Some(train)) => format!(" (train_loss: {:.4})", train), + (None, None) => String::new(), + }; + + format!( + "{} v{} - epoch {} step {}{}", + self.model_name, self.model_version, self.epoch, self.step, loss_str + ) + } +} + +/// Model storage configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ModelStorageConfig { + /// Base path for model storage (relative to storage root) + pub base_path: String, + /// Enable compression for model data + pub enable_compression: bool, + /// Maximum number of checkpoints to retain per model + pub max_checkpoints_per_model: usize, + /// Enable automatic cleanup of old checkpoints + pub enable_auto_cleanup: bool, + /// Metadata cache size + pub metadata_cache_size: usize, +} + +impl Default for ModelStorageConfig { + fn default() -> Self { + Self { + base_path: "models".to_string(), + enable_compression: true, + max_checkpoints_per_model: 10, + enable_auto_cleanup: true, + metadata_cache_size: 100, + } + } +} + +/// Model storage and management utilities +pub struct ModelStorage { + storage: S, + config: ModelStorageConfig, + /// Metadata cache for faster lookups + metadata_cache: std::sync::Arc>>, +} + +impl ModelStorage { + /// Create a new model storage instance + pub fn new(storage: S, config: ModelStorageConfig) -> Self { + let metadata_cache = std::sync::Arc::new(std::sync::Mutex::new( + lru::LruCache::new(std::num::NonZeroUsize::new(config.metadata_cache_size).unwrap()) + )); + + Self { + storage, + config, + metadata_cache, + } + } + + /// Store a model checkpoint + pub async fn store_checkpoint( + &self, + checkpoint: &ModelCheckpoint, + model_data: &[u8], + ) -> StorageResult<()> { + let start = std::time::Instant::now(); + + info!( + "Storing model checkpoint: {} ({} bytes)", + checkpoint.description(), + model_data.len() + ); + + // Store the model data + let model_path = format!("{}/{}", self.config.base_path, checkpoint.storage_path); + self.storage.store(&model_path, model_data).await?; + + // Store the metadata + let metadata_path = format!("{}.metadata.json", model_path); + let metadata_json = serde_json::to_vec_pretty(checkpoint) + .map_err(|e| StorageError::SerializationError { + message: format!("Failed to serialize checkpoint metadata: {}", e), + })?; + + self.storage.store(&metadata_path, &metadata_json).await?; + + // Cache the metadata + let cache_key = format!("{}:{}", checkpoint.model_name, checkpoint.checkpoint_id); + if let Ok(mut cache) = self.metadata_cache.lock() { + cache.put(cache_key, checkpoint.clone()); + } + + // Cleanup old checkpoints if enabled + if self.config.enable_auto_cleanup { + self.cleanup_old_checkpoints(&checkpoint.model_name).await?; + } + + let duration = start.elapsed(); + crate::metrics::get_metrics().record_operation("store_checkpoint", "model", duration, true); + crate::metrics::get_metrics().record_transfer("store_checkpoint", "model", model_data.len() as u64, duration); + + info!( + "Successfully stored model checkpoint: {} in {:?}", + checkpoint.description(), + duration + ); + + Ok(()) + } + + /// Load a model checkpoint by ID + pub async fn load_checkpoint(&self, checkpoint_id: Uuid) -> StorageResult<(ModelCheckpoint, Vec)> { + let start = std::time::Instant::now(); + + debug!("Loading model checkpoint: {}", checkpoint_id); + + // Try to find the checkpoint metadata first + let checkpoint = self.find_checkpoint_by_id(checkpoint_id).await?; + + // Load the model data + let model_path = format!("{}/{}", self.config.base_path, checkpoint.storage_path); + let model_data = self.storage.retrieve(&model_path).await?; + + // Verify checksum if available + if !checkpoint.checksum.is_empty() { + let calculated_checksum = self.calculate_checksum(&model_data); + if calculated_checksum != checkpoint.checksum { + return Err(StorageError::IntegrityError { + path: model_path, + expected: checkpoint.checksum.clone(), + actual: calculated_checksum, + }); + } + } + + let duration = start.elapsed(); + crate::metrics::get_metrics().record_operation("load_checkpoint", "model", duration, true); + crate::metrics::get_metrics().record_transfer("load_checkpoint", "model", model_data.len() as u64, duration); + + info!( + "Successfully loaded model checkpoint: {} ({} bytes) in {:?}", + checkpoint.description(), + model_data.len(), + duration + ); + + Ok((checkpoint, model_data)) + } + + /// Load the latest checkpoint for a model + pub async fn load_latest_checkpoint(&self, model_name: &str) -> StorageResult<(ModelCheckpoint, Vec)> { + debug!("Loading latest checkpoint for model: {}", model_name); + + let checkpoints = self.list_checkpoints(model_name).await?; + if checkpoints.is_empty() { + return Err(StorageError::NotFound { + path: format!("model:{}", model_name), + }); + } + + // Find the best checkpoint (highest step number, or best validation loss) + let latest = checkpoints + .into_iter() + .max_by(|a, b| { + // First compare by step number + match a.step.cmp(&b.step) { + std::cmp::Ordering::Equal => { + // If steps are equal, compare by validation loss (lower is better) + match (a.validation_loss, b.validation_loss) { + (Some(a_loss), Some(b_loss)) => b_loss.partial_cmp(&a_loss).unwrap_or(std::cmp::Ordering::Equal), + (Some(_), None) => std::cmp::Ordering::Greater, + (None, Some(_)) => std::cmp::Ordering::Less, + (None, None) => std::cmp::Ordering::Equal, + } + } + other => other, + } + }) + .unwrap(); + + self.load_checkpoint(latest.checkpoint_id).await + } + + /// List all checkpoints for a model + pub async fn list_checkpoints(&self, model_name: &str) -> StorageResult> { + debug!("Listing checkpoints for model: {}", model_name); + + let model_prefix = format!("{}/{}", self.config.base_path, model_name); + let paths = self.storage.list(&model_prefix).await?; + + let mut checkpoints = Vec::new(); + + for path in paths { + if path.ends_with(".metadata.json") { + if let Ok(metadata) = self.load_checkpoint_metadata(&path).await { + if metadata.model_name == model_name { + checkpoints.push(metadata); + } + } + } + } + + // Sort by step number (latest first) + checkpoints.sort_by(|a, b| b.step.cmp(&a.step)); + + debug!("Found {} checkpoints for model: {}", checkpoints.len(), model_name); + Ok(checkpoints) + } + + /// Delete a checkpoint + pub async fn delete_checkpoint(&self, checkpoint_id: Uuid) -> StorageResult { + debug!("Deleting checkpoint: {}", checkpoint_id); + + let checkpoint = self.find_checkpoint_by_id(checkpoint_id).await?; + + // Delete model data + let model_path = format!("{}/{}", self.config.base_path, checkpoint.storage_path); + let model_deleted = self.storage.delete(&model_path).await.unwrap_or(false); + + // Delete metadata + let metadata_path = format!("{}.metadata.json", model_path); + let metadata_deleted = self.storage.delete(&metadata_path).await.unwrap_or(false); + + // Remove from cache + let cache_key = format!("{}:{}", checkpoint.model_name, checkpoint.checkpoint_id); + if let Ok(mut cache) = self.metadata_cache.lock() { + cache.pop(&cache_key); + } + + let deleted = model_deleted || metadata_deleted; + if deleted { + info!("Successfully deleted checkpoint: {}", checkpoint.description()); + } + + Ok(deleted) + } + + /// List all models + pub async fn list_models(&self) -> StorageResult> { + let paths = self.storage.list(&self.config.base_path).await?; + + let mut models = std::collections::HashSet::new(); + for path in paths { + if path.ends_with(".metadata.json") { + if let Ok(metadata) = self.load_checkpoint_metadata(&path).await { + models.insert(metadata.model_name); + } + } + } + + let mut model_list: Vec = models.into_iter().collect(); + model_list.sort(); + + debug!("Found {} models", model_list.len()); + Ok(model_list) + } + + /// Get storage statistics for models + pub async fn get_storage_stats(&self) -> StorageResult { + let paths = self.storage.list(&self.config.base_path).await?; + + let mut total_checkpoints = 0; + let mut total_size_bytes = 0u64; + let mut models_by_name: HashMap = HashMap::new(); + let mut size_by_model: HashMap = HashMap::new(); + + for path in paths { + if path.ends_with(".metadata.json") { + if let Ok(metadata) = self.load_checkpoint_metadata(&path).await { + total_checkpoints += 1; + total_size_bytes += metadata.model_size_bytes; + + *models_by_name.entry(metadata.model_name.clone()).or_insert(0) += 1; + *size_by_model.entry(metadata.model_name).or_insert(0) += metadata.model_size_bytes; + } + } + } + + Ok(ModelStorageStats { + total_checkpoints, + total_size_bytes, + total_models: models_by_name.len() as u32, + checkpoints_by_model: models_by_name, + size_by_model, + }) + } + + /// Private helper methods + + /// Find a checkpoint by ID + async fn find_checkpoint_by_id(&self, checkpoint_id: Uuid) -> StorageResult { + // Check cache first + let cache_key_pattern = format!(":{}", checkpoint_id); + if let Ok(cache) = self.metadata_cache.lock() { + for (key, checkpoint) in cache.iter() { + if key.ends_with(&cache_key_pattern) { + return Ok(checkpoint.clone()); + } + } + } + + // Search in storage + let paths = self.storage.list(&self.config.base_path).await?; + + for path in paths { + if path.ends_with(".metadata.json") { + if let Ok(metadata) = self.load_checkpoint_metadata(&path).await { + if metadata.checkpoint_id == checkpoint_id { + // Cache for future use + let cache_key = format!("{}:{}", metadata.model_name, checkpoint_id); + if let Ok(mut cache) = self.metadata_cache.lock() { + cache.put(cache_key, metadata.clone()); + } + return Ok(metadata); + } + } + } + } + + Err(StorageError::NotFound { + path: format!("checkpoint:{}", checkpoint_id), + }) + } + + /// Load checkpoint metadata from a path + async fn load_checkpoint_metadata(&self, metadata_path: &str) -> StorageResult { + let full_path = if metadata_path.starts_with(&self.config.base_path) { + metadata_path.to_string() + } else { + format!("{}/{}", self.config.base_path, metadata_path) + }; + + let metadata_json = self.storage.retrieve(&full_path).await?; + let checkpoint: ModelCheckpoint = serde_json::from_slice(&metadata_json) + .map_err(|e| StorageError::SerializationError { + message: format!("Failed to deserialize checkpoint metadata: {}", e), + })?; + + Ok(checkpoint) + } + + /// Cleanup old checkpoints for a model + async fn cleanup_old_checkpoints(&self, model_name: &str) -> StorageResult<()> { + if self.config.max_checkpoints_per_model == 0 { + return Ok(()); // No limit + } + + let mut checkpoints = self.list_checkpoints(model_name).await?; + if checkpoints.len() <= self.config.max_checkpoints_per_model { + return Ok(()); // Within limit + } + + // Sort by step (keep latest) + checkpoints.sort_by(|a, b| b.step.cmp(&a.step)); + + // Delete excess checkpoints + let to_delete = checkpoints.split_off(self.config.max_checkpoints_per_model); + + for checkpoint in to_delete { + if let Err(e) = self.delete_checkpoint(checkpoint.checkpoint_id).await { + warn!("Failed to delete old checkpoint {}: {}", checkpoint.checkpoint_id, e); + } else { + debug!("Cleaned up old checkpoint: {}", checkpoint.description()); + } + } + + Ok(()) + } + + /// Calculate checksum for model data + fn calculate_checksum(&self, data: &[u8]) -> String { + use sha2::{Sha256, Digest}; + let mut hasher = Sha256::new(); + hasher.update(data); + format!("{:x}", hasher.finalize()) + } +} + +/// Storage statistics for models +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ModelStorageStats { + /// Total number of checkpoints + pub total_checkpoints: u32, + /// Total size in bytes + pub total_size_bytes: u64, + /// Total number of unique models + pub total_models: u32, + /// Number of checkpoints per model + pub checkpoints_by_model: HashMap, + /// Storage size per model + pub size_by_model: HashMap, +} + +/// Model loading utilities for common formats +pub struct ModelLoader; + +impl ModelLoader { + /// Load a PyTorch model checkpoint (Python pickled format) + pub async fn load_pytorch_checkpoint(data: &[u8]) -> Result>> { + // Note: This would typically require Python integration or a Rust-based + // PyTorch checkpoint reader. For now, this is a placeholder. + warn!("PyTorch checkpoint loading not implemented - would require Python integration"); + Ok(HashMap::new()) + } + + /// Load ONNX model format + pub async fn load_onnx_model(data: &[u8]) -> Result> { + // ONNX models are protobuf-based and could be parsed with prost + warn!("ONNX model loading not implemented - would require protobuf parsing"); + Ok(data.to_vec()) + } + + /// Load TensorFlow SavedModel format + pub async fn load_tensorflow_model(data: &[u8]) -> Result> { + // TensorFlow SavedModel is a directory structure with protobuf files + warn!("TensorFlow model loading not implemented - would require protobuf parsing"); + Ok(data.to_vec()) + } + + /// Load generic binary model data + pub async fn load_binary_model(data: &[u8]) -> Result> { + Ok(data.to_vec()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::local::{LocalStorage, LocalStorageConfig}; + use tempfile::TempDir; + + async fn create_test_model_storage() -> (ModelStorage, TempDir) { + let temp_dir = TempDir::new().unwrap(); + let storage_config = LocalStorageConfig { + base_path: temp_dir.path().to_path_buf(), + ..Default::default() + }; + let storage = LocalStorage::new(storage_config).await.unwrap(); + let model_config = ModelStorageConfig::default(); + let model_storage = ModelStorage::new(storage, model_config); + (model_storage, temp_dir) + } + + #[tokio::test] + async fn test_store_and_load_checkpoint() { + let (model_storage, _temp_dir) = create_test_model_storage().await; + + let model_data = b"fake model data"; + let checksum = "abc123"; + + let checkpoint = ModelCheckpoint::new( + "test_model".to_string(), + "1.0.0".to_string(), + 10, + 1000, + "transformer".to_string(), + "test_model/checkpoint_1000.bin".to_string(), + model_data.len() as u64, + checksum.to_string(), + ); + + // Store checkpoint + model_storage.store_checkpoint(&checkpoint, model_data).await.unwrap(); + + // Load checkpoint + let (loaded_checkpoint, loaded_data) = model_storage + .load_checkpoint(checkpoint.checkpoint_id) + .await + .unwrap(); + + assert_eq!(loaded_checkpoint.checkpoint_id, checkpoint.checkpoint_id); + assert_eq!(loaded_checkpoint.model_name, checkpoint.model_name); + assert_eq!(loaded_data, model_data); + } + + #[tokio::test] + async fn test_list_checkpoints() { + let (model_storage, _temp_dir) = create_test_model_storage().await; + + let model_data = b"fake model data"; + + // Store multiple checkpoints + for i in 1..=3 { + let checkpoint = ModelCheckpoint::new( + "test_model".to_string(), + "1.0.0".to_string(), + 1, + i * 100, + "transformer".to_string(), + format!("test_model/checkpoint_{}.bin", i * 100), + model_data.len() as u64, + format!("checksum_{}", i), + ); + + model_storage.store_checkpoint(&checkpoint, model_data).await.unwrap(); + } + + let checkpoints = model_storage.list_checkpoints("test_model").await.unwrap(); + assert_eq!(checkpoints.len(), 3); + + // Should be sorted by step (latest first) + assert_eq!(checkpoints[0].step, 300); + assert_eq!(checkpoints[1].step, 200); + assert_eq!(checkpoints[2].step, 100); + } + + #[tokio::test] + async fn test_load_latest_checkpoint() { + let (model_storage, _temp_dir) = create_test_model_storage().await; + + let model_data = b"fake model data"; + let mut latest_checkpoint_id = Uuid::new_v4(); + + // Store multiple checkpoints + for i in 1..=3 { + let checkpoint = ModelCheckpoint::new( + "test_model".to_string(), + "1.0.0".to_string(), + 1, + i * 100, + "transformer".to_string(), + format!("test_model/checkpoint_{}.bin", i * 100), + model_data.len() as u64, + format!("checksum_{}", i), + ); + + if i == 3 { + latest_checkpoint_id = checkpoint.checkpoint_id; + } + + model_storage.store_checkpoint(&checkpoint, model_data).await.unwrap(); + } + + let (latest_checkpoint, _) = model_storage + .load_latest_checkpoint("test_model") + .await + .unwrap(); + + assert_eq!(latest_checkpoint.checkpoint_id, latest_checkpoint_id); + assert_eq!(latest_checkpoint.step, 300); + } + + #[tokio::test] + async fn test_delete_checkpoint() { + let (model_storage, _temp_dir) = create_test_model_storage().await; + + let model_data = b"fake model data"; + let checkpoint = ModelCheckpoint::new( + "test_model".to_string(), + "1.0.0".to_string(), + 10, + 1000, + "transformer".to_string(), + "test_model/checkpoint_1000.bin".to_string(), + model_data.len() as u64, + "checksum".to_string(), + ); + + // Store and then delete + model_storage.store_checkpoint(&checkpoint, model_data).await.unwrap(); + let deleted = model_storage.delete_checkpoint(checkpoint.checkpoint_id).await.unwrap(); + + assert!(deleted); + + // Should not be able to load deleted checkpoint + let result = model_storage.load_checkpoint(checkpoint.checkpoint_id).await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn test_checkpoint_comparison() { + let checkpoint1 = ModelCheckpoint::new( + "test".to_string(), + "1.0".to_string(), + 1, 100, + "arch".to_string(), + "path".to_string(), + 1000, + "hash".to_string(), + ).with_losses(Some(0.5), Some(0.6)); + + let checkpoint2 = ModelCheckpoint::new( + "test".to_string(), + "1.0".to_string(), + 1, 200, + "arch".to_string(), + "path".to_string(), + 1000, + "hash".to_string(), + ).with_losses(Some(0.4), Some(0.5)); + + // checkpoint2 has lower validation loss, so it's better + assert!(checkpoint2.is_better_than(&checkpoint1)); + assert!(!checkpoint1.is_better_than(&checkpoint2)); + } +} \ No newline at end of file diff --git a/storage/src/s3.rs b/storage/src/s3.rs new file mode 100644 index 000000000..a513a6f77 --- /dev/null +++ b/storage/src/s3.rs @@ -0,0 +1,964 @@ +//! S3 Storage with Vault Integration +//! +//! This module provides S3-based storage with secure credential management through HashiCorp Vault. +//! All AWS credentials are retrieved from Vault - NO hardcoded credentials. + +use std::collections::HashMap; +use std::time::Duration; + +use async_trait::async_trait; +use aws_config::BehaviorVersion; +use aws_sdk_s3::{Client as S3Client, Config as S3Config}; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{ServerSideEncryption, StorageClass, Tagging, Tag}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use tracing::{debug, info, warn, error}; +use uuid::Uuid; + +use crate::vault::{VaultClient, VaultConfig, VaultCredentials}; +use crate::{Storage, StorageError, StorageMetadata, StorageResult}; + +/// S3 storage configuration with Vault integration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct S3StorageConfig { + /// S3 bucket name + pub bucket_name: String, + /// AWS region + pub region: String, + /// Vault path for AWS credentials (NO hardcoded credentials) + pub vault_credentials_path: String, + /// Default storage class for objects + pub default_storage_class: String, + /// Enable compression for stored data + pub enable_compression: bool, + /// Retention policy in days for lifecycle management + pub retention_days: u32, + /// Enable S3 lifecycle management + pub enable_lifecycle: bool, + /// Request timeout for S3 operations + pub request_timeout: Duration, + /// Maximum concurrent uploads + pub max_concurrent_uploads: usize, +} + +impl Default for S3StorageConfig { + fn default() -> Self { + Self { + bucket_name: std::env::var("S3_BUCKET_NAME") + .unwrap_or_else(|_| "foxhunt-storage".to_string()), + region: std::env::var("AWS_REGION") + .unwrap_or_else(|_| "us-east-1".to_string()), + vault_credentials_path: std::env::var("VAULT_AWS_CREDENTIALS_PATH") + .unwrap_or_else(|_| "secret/aws/foxhunt-storage".to_string()), + default_storage_class: "STANDARD_IA".to_string(), + enable_compression: true, + retention_days: 2555, // 7 years for compliance + enable_lifecycle: true, + request_timeout: Duration::from_secs(30), + max_concurrent_uploads: 10, + } + } +} + +impl S3StorageConfig { + /// Load configuration from environment variables + pub fn from_env() -> StorageResult { + let config = Self::default(); + + // Validate that we have Vault path for credentials (no direct AWS keys) + if config.vault_credentials_path.is_empty() { + return Err(StorageError::ConfigError { + message: "VAULT_AWS_CREDENTIALS_PATH must be set - no hardcoded AWS credentials allowed".to_string(), + }); + } + + // Warn if someone tries to use environment AWS credentials + if std::env::var("AWS_ACCESS_KEY_ID").is_ok() || std::env::var("AWS_SECRET_ACCESS_KEY").is_ok() { + error!("AWS credentials detected in environment variables - these will be ignored"); + error!("All AWS credentials must come from Vault at path: {}", config.vault_credentials_path); + } + + Ok(config) + } + + /// Validate the configuration + pub fn validate(&self) -> StorageResult<()> { + if self.bucket_name.is_empty() { + return Err(StorageError::ConfigError { + message: "S3 bucket name cannot be empty".to_string(), + }); + } + + if self.region.is_empty() { + return Err(StorageError::ConfigError { + message: "AWS region cannot be empty".to_string(), + }); + } + + if self.vault_credentials_path.is_empty() { + return Err(StorageError::ConfigError { + message: "Vault credentials path cannot be empty".to_string(), + }); + } + + if self.retention_days == 0 { + return Err(StorageError::ConfigError { + message: "Retention days must be greater than 0".to_string(), + }); + } + + Ok(()) + } +} + +/// Types of data for S3 storage classification +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum ArchivalDataType { + /// Historical market data + MarketData, + /// Trading logs and execution records + TradingLogs, + /// Risk management logs + RiskLogs, + /// ML model checkpoints and training data + ModelCheckpoints, + /// Performance metrics and analytics + PerformanceMetrics, + /// Compliance and audit trails + ComplianceData, + /// General application data + ApplicationData, +} + +impl ArchivalDataType { + /// Get the S3 key prefix for this data type + pub fn key_prefix(&self) -> &'static str { + match self { + ArchivalDataType::MarketData => "market-data", + ArchivalDataType::TradingLogs => "trading-logs", + ArchivalDataType::RiskLogs => "risk-logs", + ArchivalDataType::ModelCheckpoints => "model-checkpoints", + ArchivalDataType::PerformanceMetrics => "performance-metrics", + ArchivalDataType::ComplianceData => "compliance-data", + ArchivalDataType::ApplicationData => "app-data", + } + } + + /// Get appropriate storage class for data type + pub fn storage_class(&self) -> StorageClass { + match self { + ArchivalDataType::MarketData => StorageClass::StandardIa, + ArchivalDataType::TradingLogs => StorageClass::Standard, // Frequently accessed + ArchivalDataType::RiskLogs => StorageClass::Standard, + ArchivalDataType::ModelCheckpoints => StorageClass::StandardIa, + ArchivalDataType::PerformanceMetrics => StorageClass::StandardIa, + ArchivalDataType::ComplianceData => StorageClass::Glacier, // Long-term retention + ArchivalDataType::ApplicationData => StorageClass::Standard, + } + } +} + +/// Metadata for archived data in S3 +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ArchivalMetadata { + /// Unique identifier for the archived data + pub archive_id: Uuid, + /// Type of data being archived + pub data_type: ArchivalDataType, + /// Timestamp when data was created + pub created_at: DateTime, + /// Timestamp when data was archived + pub archived_at: DateTime, + /// Size of the original data in bytes + pub original_size: u64, + /// Size of the compressed data in bytes (if compression enabled) + pub compressed_size: Option, + /// Compression algorithm used + pub compression_type: Option, + /// Data source identifier + pub source_id: String, + /// Additional tags for categorization + pub tags: HashMap, + /// Checksum for data integrity verification + pub checksum: String, +} + +/// S3 storage statistics +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ArchivalStats { + /// Total number of objects + pub total_objects: u64, + /// Total size in bytes + pub total_size_bytes: u64, + /// Storage breakdown by data type + pub storage_by_type: HashMap, + /// Average compression ratio + pub average_compression_ratio: f64, + /// Estimated monthly storage cost (USD) + pub estimated_monthly_cost: f64, + /// Data integrity check results + pub integrity_check_passed: u64, + pub integrity_check_failed: u64, +} + +/// S3 storage implementation with Vault integration +pub struct S3Storage { + client: S3Client, + config: S3StorageConfig, + vault_client: VaultClient, +} + +impl S3Storage { + /// Create new S3 storage with Vault integration + pub async fn new(config: S3StorageConfig) -> StorageResult { + config.validate()?; + + info!( + "Initializing S3 storage with bucket: {}, region: {} (credentials from Vault: {})", + config.bucket_name, config.region, config.vault_credentials_path + ); + + // Initialize Vault client + let vault_config = VaultConfig::from_env().map_err(|e| StorageError::ConfigError { + message: format!("Failed to load Vault configuration: {}", e), + })?; + + let vault_client = VaultClient::new(vault_config).await.map_err(|e| StorageError::AuthError { + message: format!("Failed to initialize Vault client: {}", e), + })?; + + // Get AWS credentials from Vault (NO hardcoded credentials) + let credentials = vault_client + .get_s3_credentials(&config.vault_credentials_path) + .await + .map_err(|e| StorageError::AuthError { + message: format!("Failed to get AWS credentials from Vault: {}", e), + })?; + + // Create S3 client with Vault-sourced credentials + let aws_credentials = credentials.to_aws_credentials(); + let aws_config = aws_config::defaults(BehaviorVersion::latest()) + .region(&config.region) + .credentials_provider(aws_credentials) + .load() + .await; + + let client = S3Client::new(&aws_config); + + // Test connection and bucket access + client + .head_bucket() + .bucket(&config.bucket_name) + .send() + .await + .map_err(|e| StorageError::S3Error { + message: format!("Failed to connect to S3 bucket '{}': {}", config.bucket_name, e), + })?; + + info!("Successfully connected to S3 bucket: {}", config.bucket_name); + + let storage = Self { + client, + config, + vault_client, + }; + + // Setup lifecycle policies if enabled + if storage.config.enable_lifecycle { + storage.setup_lifecycle_policies().await?; + } + + Ok(storage) + } + + /// Generate S3 key with proper partitioning + fn generate_s3_key(&self, path: &str, data_type: Option<&ArchivalDataType>) -> String { + let now = Utc::now(); + + if let Some(dt) = data_type { + format!( + "{}/year={}/month={:02}/day={:02}/{}", + dt.key_prefix(), + now.year(), + now.month(), + now.day(), + path + ) + } else { + // For generic storage, use app-data prefix + format!( + "app-data/year={}/month={:02}/day={:02}/{}", + now.year(), + now.month(), + now.day(), + path + ) + } + } + + /// Compress data if compression is enabled + fn compress_data(&self, data: &[u8]) -> StorageResult<(Vec, Option)> { + if !self.config.enable_compression { + return Ok((data.to_vec(), None)); + } + + use flate2::write::GzEncoder; + use flate2::Compression; + use std::io::Write; + + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(data).map_err(|e| StorageError::CompressionError { + message: format!("Failed to compress data: {}", e), + })?; + + let compressed = encoder.finish().map_err(|e| StorageError::CompressionError { + message: format!("Failed to finalize compression: {}", e), + })?; + + debug!( + "Compressed data from {} to {} bytes ({:.1}% reduction)", + data.len(), + compressed.len(), + (1.0 - compressed.len() as f64 / data.len() as f64) * 100.0 + ); + + Ok((compressed, Some("gzip".to_string()))) + } + + /// Decompress data if it was compressed + fn decompress_data(&self, data: &[u8]) -> StorageResult> { + use flate2::read::GzDecoder; + use std::io::Read; + + let mut decoder = GzDecoder::new(data); + let mut decompressed = Vec::new(); + decoder.read_to_end(&mut decompressed).map_err(|e| StorageError::CompressionError { + message: format!("Failed to decompress data: {}", e), + })?; + + debug!("Decompressed {} bytes to {} bytes", data.len(), decompressed.len()); + Ok(decompressed) + } + + /// Calculate data checksum for integrity verification + fn calculate_checksum(&self, data: &[u8]) -> String { + use sha2::{Sha256, Digest}; + let mut hasher = Sha256::new(); + hasher.update(data); + format!("{:x}", hasher.finalize()) + } + + /// Create S3 object tags from metadata + fn create_object_tags(&self, metadata: &HashMap) -> Vec { + let mut tags = Vec::new(); + + tags.push(Tag::builder().key("service").value("foxhunt-hft").build().unwrap()); + tags.push(Tag::builder().key("managed_by").value("storage-crate").build().unwrap()); + + // Add custom tags + for (key, value) in metadata { + tags.push(Tag::builder().key(key).value(value).build().unwrap()); + } + + tags + } + + /// Setup S3 lifecycle policies for cost optimization + async fn setup_lifecycle_policies(&self) -> StorageResult<()> { + use aws_sdk_s3::types::{ + LifecycleConfiguration, LifecycleRule, LifecycleRuleStatus, + Transition, LifecycleExpiration + }; + + info!("Setting up S3 lifecycle policies for bucket: {}", self.config.bucket_name); + + let rules = vec![ + // Transition market data to Glacier after 90 days + LifecycleRule::builder() + .id("market-data-transition") + .status(LifecycleRuleStatus::Enabled) + .filter(aws_sdk_s3::types::LifecycleRuleFilter::Prefix("market-data/".to_string())) + .transitions( + Transition::builder() + .days(90) + .storage_class(StorageClass::Glacier) + .build() + .unwrap() + ) + .build() + .unwrap(), + + // Transition model checkpoints to Standard-IA after 30 days, Glacier after 180 days + LifecycleRule::builder() + .id("model-checkpoints-transition") + .status(LifecycleRuleStatus::Enabled) + .filter(aws_sdk_s3::types::LifecycleRuleFilter::Prefix("model-checkpoints/".to_string())) + .transitions( + Transition::builder() + .days(30) + .storage_class(StorageClass::StandardIa) + .build() + .unwrap() + ) + .transitions( + Transition::builder() + .days(180) + .storage_class(StorageClass::Glacier) + .build() + .unwrap() + ) + .build() + .unwrap(), + + // Delete performance metrics after retention period + LifecycleRule::builder() + .id("data-retention-policy") + .status(LifecycleRuleStatus::Enabled) + .filter(aws_sdk_s3::types::LifecycleRuleFilter::And( + aws_sdk_s3::types::LifecycleRuleAndOperator::builder() + .prefix("performance-metrics/") + .build() + .unwrap() + )) + .expiration( + LifecycleExpiration::builder() + .days(self.config.retention_days as i32) + .build() + .unwrap() + ) + .build() + .unwrap(), + ]; + + let lifecycle_config = LifecycleConfiguration::builder() + .set_rules(Some(rules)) + .build() + .unwrap(); + + self.client + .put_bucket_lifecycle_configuration() + .bucket(&self.config.bucket_name) + .lifecycle_configuration(lifecycle_config) + .send() + .await + .map_err(|e| StorageError::S3Error { + message: format!("Failed to set up S3 lifecycle policies: {}", e), + })?; + + info!("Successfully configured S3 lifecycle policies"); + Ok(()) + } + + /// Refresh AWS credentials from Vault if needed + async fn ensure_credentials_valid(&mut self) -> StorageResult<()> { + // Get fresh credentials from Vault + let credentials = self.vault_client + .get_s3_credentials(&self.config.vault_credentials_path) + .await + .map_err(|e| StorageError::AuthError { + message: format!("Failed to refresh AWS credentials from Vault: {}", e), + })?; + + // Check if credentials need refresh (with 5 minute buffer) + if credentials.needs_refresh(Duration::from_secs(300)) { + debug!("Refreshing AWS credentials from Vault"); + + let aws_credentials = credentials.to_aws_credentials(); + let aws_config = aws_config::defaults(BehaviorVersion::latest()) + .region(&self.config.region) + .credentials_provider(aws_credentials) + .load() + .await; + + self.client = S3Client::new(&aws_config); + info!("AWS credentials refreshed from Vault"); + } + + Ok(()) + } +} + +#[async_trait] +impl Storage for S3Storage { + async fn store(&self, path: &str, data: &[u8]) -> StorageResult<()> { + let start = std::time::Instant::now(); + let s3_key = self.generate_s3_key(path, None); + + debug!("Storing {} bytes to S3: {}", data.len(), s3_key); + + // Compress data if enabled + let (processed_data, compression_type) = self.compress_data(data)?; + let checksum = self.calculate_checksum(data); + + // Create object metadata + let mut object_metadata = HashMap::new(); + object_metadata.insert("original_size".to_string(), data.len().to_string()); + object_metadata.insert("checksum".to_string(), checksum); + object_metadata.insert("compression".to_string(), compression_type.unwrap_or_else(|| "none".to_string())); + object_metadata.insert("stored_at".to_string(), Utc::now().to_rfc3339()); + + // Create tags + let tags = self.create_object_tags(&object_metadata); + let tagging = Tagging::builder().set_tag_set(Some(tags)).build().unwrap(); + + // Upload to S3 with encryption + let body = ByteStream::from(processed_data); + + self.client + .put_object() + .bucket(&self.config.bucket_name) + .key(&s3_key) + .body(body) + .set_metadata(Some(object_metadata)) + .server_side_encryption(ServerSideEncryption::Aes256) + .storage_class(StorageClass::from(self.config.default_storage_class.as_str())) + .tagging(tagging) + .content_type("application/octet-stream") + .send() + .await + .map_err(|e| StorageError::S3Error { + message: format!("Failed to store data to S3: {}", e), + })?; + + let duration = start.elapsed(); + crate::metrics::get_metrics().record_operation("store", "s3", duration, true); + crate::metrics::get_metrics().record_transfer("store", "s3", data.len() as u64, duration); + + info!("Successfully stored {} bytes to S3: {}", data.len(), s3_key); + Ok(()) + } + + async fn retrieve(&self, path: &str) -> StorageResult> { + let start = std::time::Instant::now(); + let s3_key = self.generate_s3_key(path, None); + + debug!("Retrieving data from S3: {}", s3_key); + + let response = self.client + .get_object() + .bucket(&self.config.bucket_name) + .key(&s3_key) + .send() + .await + .map_err(|e| { + if let Some(service_err) = e.as_service_error() { + if service_err.is_no_such_key() { + return StorageError::NotFound { path: path.to_string() }; + } + } + StorageError::S3Error { + message: format!("Failed to retrieve data from S3: {}", e), + } + })?; + + // Read the body + let body = response.body.collect().await + .map_err(|e| StorageError::S3Error { + message: format!("Failed to read S3 object body: {}", e), + })?; + let compressed_data = body.into_bytes().to_vec(); + + // Check if data was compressed + let compression_type = response.metadata + .and_then(|m| m.get("compression")) + .cloned(); + + let final_data = if let Some(comp_type) = compression_type { + if comp_type == "gzip" { + self.decompress_data(&compressed_data)? + } else if comp_type == "none" { + compressed_data + } else { + return Err(StorageError::CompressionError { + message: format!("Unsupported compression type: {}", comp_type), + }); + } + } else { + compressed_data + }; + + let duration = start.elapsed(); + crate::metrics::get_metrics().record_operation("retrieve", "s3", duration, true); + crate::metrics::get_metrics().record_transfer("retrieve", "s3", final_data.len() as u64, duration); + + debug!("Successfully retrieved {} bytes from S3", final_data.len()); + Ok(final_data) + } + + async fn exists(&self, path: &str) -> StorageResult { + let start = std::time::Instant::now(); + let s3_key = self.generate_s3_key(path, None); + + let exists = match self.client + .head_object() + .bucket(&self.config.bucket_name) + .key(&s3_key) + .send() + .await + { + Ok(_) => true, + Err(e) => { + if let Some(service_err) = e.as_service_error() { + if service_err.is_no_such_key() { + false + } else { + return Err(StorageError::S3Error { + message: format!("Failed to check if object exists: {}", e), + }); + } + } else { + return Err(StorageError::S3Error { + message: format!("Failed to check if object exists: {}", e), + }); + } + } + }; + + let duration = start.elapsed(); + crate::metrics::get_metrics().record_operation("exists", "s3", duration, true); + + debug!("S3 path {} exists: {}", path, exists); + Ok(exists) + } + + async fn delete(&self, path: &str) -> StorageResult { + let start = std::time::Instant::now(); + let s3_key = self.generate_s3_key(path, None); + + debug!("Deleting S3 object: {}", s3_key); + + match self.client + .delete_object() + .bucket(&self.config.bucket_name) + .key(&s3_key) + .send() + .await + { + Ok(_) => { + let duration = start.elapsed(); + crate::metrics::get_metrics().record_operation("delete", "s3", duration, true); + info!("Successfully deleted S3 object: {}", s3_key); + Ok(true) + } + Err(e) => { + if let Some(service_err) = e.as_service_error() { + if service_err.is_no_such_key() { + debug!("S3 object not found: {}", s3_key); + return Ok(false); + } + } + Err(StorageError::S3Error { + message: format!("Failed to delete S3 object: {}", e), + }) + } + } + } + + async fn list(&self, prefix: &str) -> StorageResult> { + let start = std::time::Instant::now(); + let s3_prefix = if prefix.is_empty() { + "app-data/".to_string() + } else { + self.generate_s3_key(prefix, None) + }; + + debug!("Listing S3 objects with prefix: {}", s3_prefix); + + let mut paths = Vec::new(); + let mut continuation_token: Option = None; + + loop { + let mut request = self.client + .list_objects_v2() + .bucket(&self.config.bucket_name) + .prefix(&s3_prefix); + + if let Some(token) = &continuation_token { + request = request.continuation_token(token); + } + + let response = request.send().await + .map_err(|e| StorageError::S3Error { + message: format!("Failed to list S3 objects: {}", e), + })?; + + if let Some(contents) = response.contents { + for object in contents { + if let Some(key) = object.key { + // Convert S3 key back to storage path + if let Some(relative_path) = key.strip_prefix(&format!("{}/", s3_prefix)) { + paths.push(relative_path.to_string()); + } + } + } + } + + if response.is_truncated.unwrap_or(false) { + continuation_token = response.next_continuation_token; + } else { + break; + } + } + + let duration = start.elapsed(); + crate::metrics::get_metrics().record_operation("list", "s3", duration, true); + + debug!("Listed {} S3 objects with prefix '{}'", paths.len(), prefix); + Ok(paths) + } + + async fn metadata(&self, path: &str) -> StorageResult { + let start = std::time::Instant::now(); + let s3_key = self.generate_s3_key(path, None); + + let response = self.client + .head_object() + .bucket(&self.config.bucket_name) + .key(&s3_key) + .send() + .await + .map_err(|e| { + if let Some(service_err) = e.as_service_error() { + if service_err.is_no_such_key() { + return StorageError::NotFound { path: path.to_string() }; + } + } + StorageError::S3Error { + message: format!("Failed to get S3 object metadata: {}", e), + } + })?; + + let size = response.content_length.unwrap_or(0) as u64; + let last_modified = response.last_modified + .map(|ts| DateTime::from_timestamp(ts.secs(), ts.subsec_nanos()).unwrap_or_else(|| Utc::now())) + .unwrap_or_else(|| Utc::now()); + let etag = response.e_tag; + + let metadata_map = response.metadata.unwrap_or_default(); + let mut tags = HashMap::new(); + + // Convert metadata to tags + for (key, value) in metadata_map { + tags.insert(key, value); + } + + let duration = start.elapsed(); + crate::metrics::get_metrics().record_operation("metadata", "s3", duration, true); + + Ok(StorageMetadata { + path: path.to_string(), + size, + content_type: Some("application/octet-stream".to_string()), + last_modified, + etag, + tags, + }) + } +} + +/// Specialized archival operations for S3 +impl S3Storage { + /// Archive data with specific type and metadata + pub async fn archive_data( + &self, + data: &[u8], + data_type: ArchivalDataType, + source_id: String, + additional_tags: Option>, + ) -> StorageResult { + let archive_id = Uuid::new_v4(); + let now = Utc::now(); + let s3_key = format!( + "{}/year={}/month={:02}/day={:02}/{}.bin", + data_type.key_prefix(), + now.year(), + now.month(), + now.day(), + archive_id + ); + + info!("Archiving {} bytes of {:?} data to S3: {}", data.len(), data_type, s3_key); + + // Compress and checksum + let (processed_data, compression_type) = self.compress_data(data)?; + let checksum = self.calculate_checksum(data); + + // Create metadata + let metadata = ArchivalMetadata { + archive_id, + data_type: data_type.clone(), + created_at: now, + archived_at: now, + original_size: data.len() as u64, + compressed_size: if self.config.enable_compression { + Some(processed_data.len() as u64) + } else { + None + }, + compression_type: compression_type.clone(), + source_id: source_id.clone(), + tags: additional_tags.clone().unwrap_or_default(), + checksum: checksum.clone(), + }; + + // Create object metadata for S3 + let mut object_metadata = HashMap::new(); + object_metadata.insert("archive_id".to_string(), archive_id.to_string()); + object_metadata.insert("data_type".to_string(), format!("{:?}", data_type)); + object_metadata.insert("source_id".to_string(), source_id); + object_metadata.insert("original_size".to_string(), data.len().to_string()); + object_metadata.insert("checksum".to_string(), checksum); + if let Some(compressed_size) = metadata.compressed_size { + object_metadata.insert("compressed_size".to_string(), compressed_size.to_string()); + } + if let Some(comp_type) = &compression_type { + object_metadata.insert("compression".to_string(), comp_type.clone()); + } + + // Add custom metadata + if let Some(additional) = additional_tags { + for (key, value) in additional { + object_metadata.insert(key, value); + } + } + + // Create tags + let tags = self.create_object_tags(&object_metadata); + let tagging = Tagging::builder().set_tag_set(Some(tags)).build().unwrap(); + + // Upload to S3 + let body = ByteStream::from(processed_data); + + self.client + .put_object() + .bucket(&self.config.bucket_name) + .key(&s3_key) + .body(body) + .set_metadata(Some(object_metadata)) + .server_side_encryption(ServerSideEncryption::Aes256) + .storage_class(data_type.storage_class()) + .tagging(tagging) + .content_type("application/octet-stream") + .send() + .await + .map_err(|e| StorageError::S3Error { + message: format!("Failed to archive data to S3: {}", e), + })?; + + info!( + "Successfully archived data to S3: {} ({} bytes -> {} bytes)", + s3_key, + metadata.original_size, + metadata.compressed_size.unwrap_or(metadata.original_size) + ); + + Ok(metadata) + } + + /// Get archival statistics + pub async fn get_archival_stats(&self) -> StorageResult { + info!("Calculating archival statistics for bucket: {}", self.config.bucket_name); + + let mut total_objects = 0u64; + let mut total_size = 0u64; + let mut storage_by_type = HashMap::new(); + let mut compression_ratios = Vec::new(); + + let mut continuation_token: Option = None; + + loop { + let mut request = self.client + .list_objects_v2() + .bucket(&self.config.bucket_name); + + if let Some(token) = &continuation_token { + request = request.continuation_token(token); + } + + let response = request.send().await + .map_err(|e| StorageError::S3Error { + message: format!("Failed to list objects for statistics: {}", e), + })?; + + if let Some(contents) = response.contents { + for object in contents { + if let (Some(key), Some(size)) = (object.key, object.size) { + total_objects += 1; + total_size += size as u64; + + // Categorize by data type + let data_type = key.split('/').next().unwrap_or("unknown").to_string(); + *storage_by_type.entry(data_type).or_insert(0) += size as u64; + } + } + } + + if response.is_truncated.unwrap_or(false) { + continuation_token = response.next_continuation_token; + } else { + break; + } + } + + let average_compression_ratio = if !compression_ratios.is_empty() { + compression_ratios.iter().sum::() / compression_ratios.len() as f64 + } else { + 1.0 + }; + + // Rough cost estimation (simplified - actual costs vary by region and storage class) + let estimated_monthly_cost = (total_size as f64 / 1_073_741_824.0) * 0.023; // ~$0.023/GB for Standard-IA + + Ok(ArchivalStats { + total_objects, + total_size_bytes: total_size, + storage_by_type, + average_compression_ratio, + estimated_monthly_cost, + integrity_check_passed: 0, // Would need actual integrity checking implementation + integrity_check_failed: 0, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_archival_data_type_key_prefix() { + assert_eq!(ArchivalDataType::MarketData.key_prefix(), "market-data"); + assert_eq!(ArchivalDataType::TradingLogs.key_prefix(), "trading-logs"); + assert_eq!(ArchivalDataType::ModelCheckpoints.key_prefix(), "model-checkpoints"); + } + + #[test] + fn test_archival_data_type_storage_class() { + assert_eq!(ArchivalDataType::MarketData.storage_class(), StorageClass::StandardIa); + assert_eq!(ArchivalDataType::TradingLogs.storage_class(), StorageClass::Standard); + assert_eq!(ArchivalDataType::ComplianceData.storage_class(), StorageClass::Glacier); + } + + #[test] + fn test_s3_config_validation() { + let mut config = S3StorageConfig::default(); + + // Valid configuration should pass + assert!(config.validate().is_ok()); + + // Empty bucket name should fail + config.bucket_name = String::new(); + assert!(config.validate().is_err()); + + // Empty region should fail + config.bucket_name = "test-bucket".to_string(); + config.region = String::new(); + assert!(config.validate().is_err()); + + // Empty Vault path should fail + config.region = "us-east-1".to_string(); + config.vault_credentials_path = String::new(); + assert!(config.validate().is_err()); + } +} \ No newline at end of file diff --git a/tests/lib.rs b/tests/lib.rs index c073e8eb8..81bd16b1e 100644 --- a/tests/lib.rs +++ b/tests/lib.rs @@ -347,7 +347,7 @@ pub mod config { pub use common::*; // pub use framework::*; // pub use helpers::*; -pub use config::*; +pub use foxhunt_config::*; pub use mocks::*; pub use performance_utils::*; pub use safety::*; diff --git a/tli/src/auth/encryption.rs b/tli/src/auth/encryption.rs index 90c0851d4..510d2f5e1 100644 --- a/tli/src/auth/encryption.rs +++ b/tli/src/auth/encryption.rs @@ -176,7 +176,7 @@ impl EncryptionManager { // Prepare AAD let aad_ref = match aad { Some(data) => Aad::from(data), - None => Aad::empty(), + None => Aad::from(&[] as &[u8]), }; // Encrypt data @@ -226,9 +226,9 @@ impl EncryptionManager { Some(aad_b64) => { let aad_bytes = BASE64.decode(aad_b64) .map_err(|_| EncryptionError::InvalidCiphertext)?; - Aad::from(&aad_bytes) + Aad::from(aad_bytes.as_slice()) } - None => Aad::empty(), + None => Aad::from(&[] as &[u8]), } }; diff --git a/tli/src/dashboard/mod.rs b/tli/src/dashboard/mod.rs index 93d0bdb3e..ca88afe53 100644 --- a/tli/src/dashboard/mod.rs +++ b/tli/src/dashboard/mod.rs @@ -29,7 +29,7 @@ pub mod trading; pub mod vault_status; pub use backtesting::BacktestingDashboard; -pub use config::ConfigDashboard; +pub use foxhunt_config::ConfigDashboard; pub use events::*; pub use layout::LayoutManager; pub use ml::MLDashboard; diff --git a/tli/src/vault/mod.rs b/tli/src/vault/mod.rs index bbb8e0590..91a9f59c7 100644 --- a/tli/src/vault/mod.rs +++ b/tli/src/vault/mod.rs @@ -255,22 +255,16 @@ impl VaultService { Err(_) => VaultHealthStatus::Unhealthy, } } - + pub async fn shutdown(&self) -> VaultResult<()> { // Stop rotation manager self.rotation_manager.stop().await?; - + // Clear credential cache self.credential_manager.clear_cache().await?; - - Ok(()) + + Ok() } - - /// Perform health check on Vault service - pub async fn health_check(&self) -> VaultResult<()> { - self.client.health_check().await - } - /// Get number of active Vault connections pub async fn get_active_connections(&self) -> VaultResult { // TODO: Implement actual connection tracking