foxhunt/storage/src/lib.rs

//! Storage Library for Foxhunt HFT Trading System
//!
//! This crate provides comprehensive storage solutions for the HFT system including:
//! - S3 archival with lifecycle management and compression
//! - Local file operations with atomic writes and locking
//! - Secure credential management through config crate
//! - Model storage and retrieval utilities
//! - Backup and disaster recovery operations
//!
//! # Features
//!
//! - **S3 Integration**: High-performance S3 operations with automatic retry, compression, and lifecycle policies
//! - **Security**: Secure credential retrieval through config crate
//! - **Local Storage**: Thread-safe local file operations with atomic writes and file locking
//! - **Data Integrity**: Checksums and verification for all storage operations
//! - **Performance Monitoring**: Built-in metrics and telemetry for storage operations

#![warn(missing_docs)]
#![warn(clippy::unwrap_used)]
#![warn(clippy::expect_used)]

// pub mod s3; // Removed - we use object_store_backend now
pub mod error;

// Re-export critical error types for external crate compatibility
pub use error::{StorageError, StorageResult};
pub mod local;
pub mod metrics;
pub mod model_helpers;
pub mod models;
pub mod object_store_backend;

// Export ObjectStoreBackend for external use
pub use object_store_backend::ObjectStoreBackend;
use async_trait::async_trait;

/// Common storage trait for abstracting different storage backends
#[async_trait]
pub trait Storage: Send + Sync {
    /// Store data at the specified path
    async fn store(&self, path: &str, data: &[u8]) -> crate::error::StorageResult<()>;

    /// Retrieve data from the specified path
    async fn retrieve(&self, path: &str) -> crate::error::StorageResult<Vec<u8>>;

    /// Check if data exists at the specified path
    async fn exists(&self, path: &str) -> crate::error::StorageResult<bool>;

    /// Delete data at the specified path
    async fn delete(&self, path: &str) -> crate::error::StorageResult<bool>;

    /// List all paths with the given prefix
    async fn list(&self, prefix: &str) -> crate::error::StorageResult<Vec<String>>;

    /// Get metadata for the data at the specified path
    async fn metadata(&self, path: &str) -> crate::error::StorageResult<StorageMetadata>;
}

/// Metadata for stored data
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct StorageMetadata {
    /// Path where the data is stored
    pub path: String,
    /// Size of the data in bytes
    pub size: u64,
    /// Content type/MIME type
    pub content_type: Option<String>,
    /// Last modified timestamp
    pub last_modified: chrono::DateTime<chrono::Utc>,
    /// ETag or checksum for data integrity
    pub etag: Option<String>,
    /// Custom metadata tags
    pub tags: std::collections::HashMap<String, String>,
}

/// Storage provider enumeration for configuration
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub enum StorageProvider {
    /// Local filesystem storage
    Local(local::LocalStorageConfig),
    /// AWS S3 storage
    #[cfg(feature = "s3")]
    S3(config::schemas::S3Config),
    /// Multi-tier storage (e.g., local cache + S3 archival)
    MultiTier {
        /// Primary fast storage
        primary: Box<StorageProvider>,
        /// Secondary archival storage
        secondary: Box<StorageProvider>,
    },
}

/// Factory for creating storage instances from configuration
pub struct StorageFactory;

impl StorageFactory {
    /// Create a storage instance from the provided configuration
    pub async fn create(
        provider: StorageProvider,
        config_manager: Option<std::sync::Arc<config::manager::ConfigManager>>,
    ) -> crate::error::StorageResult<Box<dyn Storage>> {
        match provider {
            StorageProvider::Local(config) => {
                let storage = local::LocalStorage::new(config).await?;
                Ok(Box::new(storage))
            }
            #[cfg(feature = "s3")]
            StorageProvider::S3(config) => {
                let storage = crate::object_store_backend::ObjectStoreBackend::new(
                    config,
                    config_manager.clone(),
                )
                .await?;
                Ok(Box::new(storage))
            }
            StorageProvider::MultiTier { primary, secondary } => {
                let primary_storage =
                    Box::pin(Self::create(*primary, config_manager.clone())).await?;
                let secondary_storage = Box::pin(Self::create(*secondary, config_manager)).await?;
                let storage = MultiTierStorage::new(primary_storage, secondary_storage);
                Ok(Box::new(storage))
            }
        }
    }
}

/// Multi-tier storage implementation that uses primary storage for fast access
/// and secondary storage for archival/backup
pub struct MultiTierStorage {
    primary: Box<dyn Storage>,
    secondary: Box<dyn Storage>,
}

impl MultiTierStorage {
    /// Create a new multi-tier storage with primary and secondary backends
    pub fn new(primary: Box<dyn Storage>, secondary: Box<dyn Storage>) -> Self {
        Self { primary, secondary }
    }
}

#[async_trait::async_trait]
impl Storage for MultiTierStorage {
    async fn store(&self, path: &str, data: &[u8]) -> crate::error::StorageResult<()> {
        // Store in primary first
        self.primary.store(path, data).await?;

        // Store in secondary storage in background
        let _secondary_storage = self.secondary.as_ref();
        let path_clone = path.to_string();
        let data_clone = data.to_vec();

        // For now, we'll store synchronously to avoid lifetime issues
        // In a production system, you'd want proper background task management
        if let Err(e) = self.secondary.store(&path_clone, &data_clone).await {
            tracing::warn!("Failed to store in secondary storage: {}", e);
        }

        Ok(())
    }

    async fn retrieve(&self, path: &str) -> crate::error::StorageResult<Vec<u8>> {
        // Try primary first
        match self.primary.retrieve(path).await {
            Ok(data) => Ok(data),
            Err(_) => {
                // Fallback to secondary
                tracing::debug!(
                    "Primary storage failed, trying secondary for path: {}",
                    path
                );
                self.secondary.retrieve(path).await
            }
        }
    }

    async fn exists(&self, path: &str) -> crate::error::StorageResult<bool> {
        // Check primary first, then secondary
        if self.primary.exists(path).await.unwrap_or(false) {
            Ok(true)
        } else {
            self.secondary.exists(path).await
        }
    }

    async fn delete(&self, path: &str) -> crate::error::StorageResult<bool> {
        // Delete from both storages
        let primary_result = self.primary.delete(path).await.unwrap_or(false);
        let secondary_result = self.secondary.delete(path).await.unwrap_or(false);

        Ok(primary_result || secondary_result)
    }

    async fn list(&self, prefix: &str) -> crate::error::StorageResult<Vec<String>> {
        // Combine results from both storages
        let mut paths = std::collections::HashSet::new();

        if let Ok(primary_paths) = self.primary.list(prefix).await {
            paths.extend(primary_paths);
        }

        if let Ok(secondary_paths) = self.secondary.list(prefix).await {
            paths.extend(secondary_paths);
        }

        Ok(paths.into_iter().collect())
    }

    async fn metadata(&self, path: &str) -> crate::error::StorageResult<StorageMetadata> {
        // Try primary first, fallback to secondary
        match self.primary.metadata(path).await {
            Ok(metadata) => Ok(metadata),
            Err(_) => self.secondary.metadata(path).await,
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_storage_metadata_serialization() {
        let metadata = StorageMetadata {
            path: "test/path".to_string(),
            size: 1024,
            content_type: Some("application/json".to_string()),
            last_modified: chrono::Utc::now(),
            etag: Some("abc123".to_string()),
            tags: std::collections::HashMap::new(),
        };

        let serialized = serde_json::to_string(&metadata).unwrap();
        let deserialized: StorageMetadata = serde_json::from_str(&serialized).unwrap();

        assert_eq!(metadata.path, deserialized.path);
        assert_eq!(metadata.size, deserialized.size);
    }
}