foxhunt/storage/src/lib.rs

//! Storage Library for Foxhunt HFT Trading System
//!
//! This crate provides comprehensive storage solutions for the HFT system including:
//! - S3 archival with lifecycle management and compression
//! - Local file operations with atomic writes and locking
//! - Secure credential management through config crate
//! - Model storage and retrieval utilities
//! - Backup and disaster recovery operations
//!
//! # Features
//!
//! - **S3 Integration**: High-performance S3 operations with automatic retry, compression, and lifecycle policies
//! - **Security**: Secure credential retrieval through config crate
//! - **Local Storage**: Thread-safe local file operations with atomic writes and file locking
//! - **Data Integrity**: Checksums and verification for all storage operations
//! - **Performance Monitoring**: Built-in metrics and telemetry for storage operations

#![allow(missing_docs)] // Internal implementation details
#![warn(clippy::unwrap_used)]
#![warn(clippy::expect_used)]

// pub mod s3; // Removed - we use object_store_backend now
pub mod error;

// Re-export critical error types for external crate compatibility
pub use error::{StorageError, StorageResult};
pub mod local;
pub mod metrics;
pub mod model_helpers;
pub mod models;
pub mod object_store_backend;

// Export ObjectStoreBackend for external use
use async_trait::async_trait;
use chrono::{DateTime, Utc};
pub use object_store_backend::ObjectStoreBackend;

/// Common storage trait for abstracting different storage backends
#[async_trait]
pub trait Storage: Send + Sync {
    /// Store data at the specified path
    async fn store(&self, path: &str, data: &[u8]) -> crate::error::StorageResult<()>;

    /// Retrieve data from the specified path
    async fn retrieve(&self, path: &str) -> crate::error::StorageResult<Vec<u8>>;

    /// Check if data exists at the specified path
    async fn exists(&self, path: &str) -> crate::error::StorageResult<bool>;

    /// Delete data at the specified path
    async fn delete(&self, path: &str) -> crate::error::StorageResult<bool>;

    /// List all paths with the given prefix
    async fn list(&self, prefix: &str) -> crate::error::StorageResult<Vec<String>>;

    /// Get metadata for the data at the specified path
    async fn metadata(&self, path: &str) -> crate::error::StorageResult<StorageMetadata>;
}

/// Metadata for stored data
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct StorageMetadata {
    /// Path where the data is stored
    pub path: String,
    /// Size of the data in bytes
    pub size: u64,
    /// Content type/MIME type
    pub content_type: Option<String>,
    /// Last modified timestamp
    pub last_modified: DateTime<Utc>,
    /// ETag or checksum for data integrity
    pub etag: Option<String>,
    /// Custom metadata tags
    pub tags: std::collections::HashMap<String, String>,
}

/// Storage provider enumeration for configuration
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub enum StorageProvider {
    /// Local filesystem storage
    Local(local::LocalStorageConfig),
    /// AWS S3 storage
    #[cfg(feature = "s3")]
    S3(config::schemas::S3Config),
    /// Multi-tier storage (e.g., local cache + S3 archival)
    MultiTier {
        /// Primary fast storage
        primary: Box<StorageProvider>,
        /// Secondary archival storage
        secondary: Box<StorageProvider>,
    },
}

/// Factory for creating storage instances from configuration
pub struct StorageFactory;

impl StorageFactory {
    /// Create a storage instance from the provided configuration
    ///
    /// # Errors
    /// Returns error if the operation fails
    ///
    /// # Errors
    /// Returns error if the operation fails
    pub async fn create(
        provider: StorageProvider,
        config_manager: Option<std::sync::Arc<config::manager::ConfigManager>>,
    ) -> crate::error::StorageResult<Box<dyn Storage>> {
        match provider {
            StorageProvider::Local(config) => {
                let storage = local::LocalStorage::new(config).await?;
                Ok(Box::new(storage))
            },
            #[cfg(feature = "s3")]
            StorageProvider::S3(config) => {
                let storage = crate::object_store_backend::ObjectStoreBackend::new(
                    config,
                    config_manager.clone(),
                )
                .await?;
                Ok(Box::new(storage))
            },
            StorageProvider::MultiTier { primary, secondary } => {
                let primary_storage =
                    Box::pin(Self::create(*primary, config_manager.clone())).await?;
                let secondary_storage = Box::pin(Self::create(*secondary, config_manager)).await?;
                let storage = MultiTierStorage::new(primary_storage, secondary_storage);
                Ok(Box::new(storage))
            },
        }
    }
}

/// Multi-tier storage implementation that uses primary storage for fast access
/// and secondary storage for archival/backup
pub struct MultiTierStorage {
    primary: Box<dyn Storage>,
    secondary: Box<dyn Storage>,
}

impl MultiTierStorage {
    /// Create a new multi-tier storage with primary and secondary backends
    pub fn new(primary: Box<dyn Storage>, secondary: Box<dyn Storage>) -> Self {
        Self { primary, secondary }
    }
}

#[async_trait::async_trait]
impl Storage for MultiTierStorage {
    async fn store(&self, path: &str, data: &[u8]) -> crate::error::StorageResult<()> {
        // Store in primary first
        self.primary.store(path, data).await?;

        // Store in secondary storage in background
        let _secondary_storage = self.secondary.as_ref();
        let path_clone = path.to_string();
        let data_clone = data.to_vec();

        // For now, we'll store synchronously to avoid lifetime issues
        // In a production system, you'd want proper background task management
        if let Err(e) = self.secondary.store(&path_clone, &data_clone).await {
            tracing::warn!("Failed to store in secondary storage: {}", e);
        }

        Ok(())
    }

    async fn retrieve(&self, path: &str) -> crate::error::StorageResult<Vec<u8>> {
        // Try primary first
        match self.primary.retrieve(path).await {
            Ok(data) => Ok(data),
            Err(_) => {
                // Fallback to secondary
                tracing::debug!(
                    "Primary storage failed, trying secondary for path: {}",
                    path
                );
                self.secondary.retrieve(path).await
            },
        }
    }

    async fn exists(&self, path: &str) -> crate::error::StorageResult<bool> {
        // Check primary first, then secondary
        match self.primary.exists(path).await {
            Ok(true) => Ok(true),
            Ok(false) | Err(_) => {
                // If primary says no or errors, check secondary
                self.secondary.exists(path).await
            },
        }
    }

    async fn delete(&self, path: &str) -> crate::error::StorageResult<bool> {
        // Delete from both storages
        let primary_result = match self.primary.delete(path).await {
            Ok(deleted) => deleted,
            Err(e) => {
                tracing::warn!("Failed to delete from primary storage: {}", e);
                false
            },
        };
        let secondary_result = match self.secondary.delete(path).await {
            Ok(deleted) => deleted,
            Err(e) => {
                tracing::warn!("Failed to delete from secondary storage: {}", e);
                false
            },
        };

        Ok(primary_result || secondary_result)
    }

    async fn list(&self, prefix: &str) -> crate::error::StorageResult<Vec<String>> {
        // Combine results from both storages
        let mut paths = std::collections::HashSet::new();

        if let Ok(primary_paths) = self.primary.list(prefix).await {
            paths.extend(primary_paths);
        }

        if let Ok(secondary_paths) = self.secondary.list(prefix).await {
            paths.extend(secondary_paths);
        }

        Ok(paths.into_iter().collect())
    }

    async fn metadata(&self, path: &str) -> crate::error::StorageResult<StorageMetadata> {
        // Try primary first, fallback to secondary
        match self.primary.metadata(path).await {
            Ok(metadata) => Ok(metadata),
            Err(_) => self.secondary.metadata(path).await,
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_storage_metadata_serialization() {
        let metadata = StorageMetadata {
            path: "test/path".to_owned(),
            size: 1024,
            content_type: Some("application/json".to_owned()),
            last_modified: Utc::now(),
            etag: Some("abc123".to_owned()),
            tags: std::collections::HashMap::new(),
        };

        let serialized = serde_json::to_string(&metadata).expect("Failed to serialize metadata");
        let deserialized: StorageMetadata =
            serde_json::from_str(&serialized).expect("Failed to deserialize metadata");

        assert_eq!(metadata.path, deserialized.path);
        assert_eq!(metadata.size, deserialized.size);
    }

    #[tokio::test]
    async fn test_multi_tier_storage() {
        use crate::local::{LocalStorage, LocalStorageConfig};
        use tempfile::TempDir;

        let temp_dir1 = TempDir::new().expect("Failed to create temp dir 1");
        let temp_dir2 = TempDir::new().expect("Failed to create temp dir 2");

        let config1 = LocalStorageConfig {
            base_path: temp_dir1.path().to_path_buf(),
            ..Default::default()
        };
        let config2 = LocalStorageConfig {
            base_path: temp_dir2.path().to_path_buf(),
            ..Default::default()
        };

        let storage1 = LocalStorage::new(config1)
            .await
            .expect("Failed to create storage 1");
        let storage2 = LocalStorage::new(config2)
            .await
            .expect("Failed to create storage 2");

        let multi_tier = MultiTierStorage::new(Box::new(storage1), Box::new(storage2));

        // Store data (should go to both)
        multi_tier
            .store("test.txt", b"test data")
            .await
            .expect("Failed to store");

        // Should be able to retrieve from primary
        let data = multi_tier
            .retrieve("test.txt")
            .await
            .expect("Failed to retrieve");
        assert_eq!(data, b"test data");
    }

    #[tokio::test]
    async fn test_multi_tier_fallback() {
        use crate::local::{LocalStorage, LocalStorageConfig};
        use tempfile::TempDir;

        let temp_dir1 = TempDir::new().expect("Failed to create temp dir 1");
        let temp_dir2 = TempDir::new().expect("Failed to create temp dir 2");

        let config1 = LocalStorageConfig {
            base_path: temp_dir1.path().to_path_buf(),
            ..Default::default()
        };
        let config2 = LocalStorageConfig {
            base_path: temp_dir2.path().to_path_buf(),
            ..Default::default()
        };

        let storage1 = LocalStorage::new(config1)
            .await
            .expect("Failed to create storage 1");
        let storage2 = LocalStorage::new(config2)
            .await
            .expect("Failed to create storage 2");

        // Store only in secondary
        storage2
            .store("secondary_only.txt", b"secondary data")
            .await
            .expect("Failed to store");

        let multi_tier = MultiTierStorage::new(Box::new(storage1), Box::new(storage2));

        // Should fallback to secondary
        let data = multi_tier
            .retrieve("secondary_only.txt")
            .await
            .expect("Failed to retrieve");
        assert_eq!(data, b"secondary data");
    }
}