//! S3 Archival System
//!
//! This module provides S3-based cold storage for historical data, trading logs,
//! and backup functionality for the HFT system. Designed for long-term retention
//! and cost-effective storage of large datasets.

use std::collections::HashMap;
use std::time::Duration;

use anyhow::{Context, Result};
use async_trait::async_trait;
use futures::StreamExt;
use std::sync::Arc;
// Using storage crate's object_store backend instead of AWS SDK
use storage::{Storage, StorageResult};
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use tracing::{debug, info, warn, error};
use uuid::Uuid;

/// S3 archival configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
/// S3ArchivalConfig
/// 
/// Auto-generated documentation placeholder - enhance with specifics
pub struct S3ArchivalConfig {
    /// S3 bucket name for archival
    pub bucket_name: String,
    /// AWS region
    pub region: String,
    /// Access key ID (use environment variable in production)
    pub access_key_id: String,
    /// Secret access key (use environment variable in production)
    pub secret_access_key: String,
    /// Default storage class for archived data
    pub storage_class: String,
    /// Enable compression for archived data
    pub enable_compression: bool,
    /// Retention policy in days
    pub retention_days: u32,
    /// Enable lifecycle management
    pub enable_lifecycle: bool,
}

impl Default for S3ArchivalConfig {
    fn default() -> Self {
        Self {
            bucket_name: std::env::var("S3_ARCHIVAL_BUCKET")
                .unwrap_or_else(|_| "foxhunt-archives".to_string()),
            region: std::env::var("AWS_REGION")
                .unwrap_or_else(|_| "us-east-1".to_string()),
            access_key_id: std::env::var("AWS_ACCESS_KEY_ID")
                .unwrap_or_else(|_| {
                    warn!("AWS_ACCESS_KEY_ID not set - using default AWS credential chain");
                    String::new()
                }),
            secret_access_key: std::env::var("AWS_SECRET_ACCESS_KEY")
                .unwrap_or_else(|_| {
                    warn!("AWS_SECRET_ACCESS_KEY not set - using default AWS credential chain");
                    String::new()
                }),
            storage_class: std::env::var("S3_DEFAULT_STORAGE_CLASS")
                .unwrap_or_else(|_| "STANDARD_IA".to_string()), // Infrequent Access for cost optimization
            enable_compression: std::env::var("S3_ENABLE_COMPRESSION")
                .map(|v| v.to_lowercase() == "true")
                .unwrap_or(true),
            retention_days: std::env::var("S3_ARCHIVAL_RETENTION_DAYS")
                .and_then(|v| v.parse().ok())
                .unwrap_or(2555), // 7 years for regulatory compliance
            enable_lifecycle: std::env::var("S3_ENABLE_LIFECYCLE")
                .map(|v| v.to_lowercase() == "true")
                .unwrap_or(true),
        }
    }
}

/// Types of data that can be archived
#[derive(Debug, Clone, Serialize, Deserialize)]
/// ArchivalDataType
/// 
/// Auto-generated documentation placeholder - enhance with specifics
pub enum ArchivalDataType {
    /// Historical market data
    MarketData,
    /// Trading logs and execution records
    TradingLogs,
    /// Risk management logs
    RiskLogs,
    /// Model checkpoints and training data
    ModelCheckpoints,
    /// Performance metrics and analytics
    PerformanceMetrics,
    /// Compliance and audit trails
    ComplianceData,
}

impl ArchivalDataType {
    /// Get the S3 key prefix for this data type
    pub fn key_prefix(&self) -> &'static str {
        match self {
            ArchivalDataType::MarketData => "market-data",
            ArchivalDataType::TradingLogs => "trading-logs",
            ArchivalDataType::RiskLogs => "risk-logs",
            ArchivalDataType::ModelCheckpoints => "model-checkpoints",
            ArchivalDataType::PerformanceMetrics => "performance-metrics",
            ArchivalDataType::ComplianceData => "compliance-data",
        }
    }

    /// Get appropriate storage class for data type
    pub fn storage_class(&self) -> StorageClass {
        match self {
            ArchivalDataType::MarketData => StorageClass::StandardIa,
            ArchivalDataType::TradingLogs => StorageClass::Standard, // Frequently accessed for analysis
            ArchivalDataType::RiskLogs => StorageClass::Standard,
            ArchivalDataType::ModelCheckpoints => StorageClass::StandardIa,
            ArchivalDataType::PerformanceMetrics => StorageClass::StandardIa,
            ArchivalDataType::ComplianceData => StorageClass::Glacier, // Long-term retention
        }
    }
}

/// Archival metadata for tracking and organizing data
#[derive(Debug, Clone, Serialize, Deserialize)]
/// ArchivalMetadata
/// 
/// Auto-generated documentation placeholder - enhance with specifics
pub struct ArchivalMetadata {
    /// Unique identifier for the archived data
    pub archive_id: Uuid,
    /// Type of data being archived
    pub data_type: ArchivalDataType,
    /// Timestamp when data was created
    pub created_at: DateTime<Utc>,
    /// Timestamp when data was archived
    pub archived_at: DateTime<Utc>,
    /// Size of the original data in bytes
    pub original_size: u64,
    /// Size of the compressed data in bytes (if compression enabled)
    pub compressed_size: Option<u64>,
    /// Compression algorithm used
    pub compression_type: Option<String>,
    /// Data source identifier (e.g., trading session ID, model ID)
    pub source_id: String,
    /// Additional tags for categorization
    pub tags: HashMap<String, String>,
    /// Checksum for data integrity verification
    pub checksum: String,
}

/// Trait for S3 archival operations
#[async_trait]
/// S3Archival
/// 
/// Auto-generated documentation placeholder - enhance with specifics
pub trait S3Archival: Send + Sync {
    /// Archive data to S3 with appropriate lifecycle policies
    async fn archive_data(
        &self,
        data: &[u8],
        data_type: ArchivalDataType,
        source_id: String,
        metadata: Option<HashMap<String, String>>,
    ) -> Result<ArchivalMetadata>;

    /// Retrieve archived data from S3
    async fn retrieve_archived_data(&self, archive_id: Uuid) -> Result<Vec<u8>>;

    /// List archived data by type and date range
    async fn list_archived_data(
        &self,
        data_type: Option<ArchivalDataType>,
        from_date: Option<DateTime<Utc>>,
        to_date: Option<DateTime<Utc>>,
    ) -> Result<Vec<ArchivalMetadata>>;

    /// Delete archived data (with compliance checks)
    async fn delete_archived_data(&self, archive_id: Uuid) -> Result<bool>;

    /// Get storage statistics and costs
    async fn get_archival_stats(&self) -> Result<ArchivalStats>;

    /// Verify data integrity using checksums
    async fn verify_data_integrity(&self, archive_id: Uuid) -> Result<bool>;
}

/// Statistics about archived data
#[derive(Debug, Clone, Serialize, Deserialize)]
/// ArchivalStats
/// 
/// Auto-generated documentation placeholder - enhance with specifics
pub struct ArchivalStats {
    /// Total number of archived objects
    pub total_objects: u64,
    /// Total size in bytes
    pub total_size_bytes: u64,
    /// Storage breakdown by data type
    pub storage_by_type: HashMap<String, u64>,
    /// Average compression ratio
    pub average_compression_ratio: f64,
    /// Estimated monthly storage cost (USD)
    pub estimated_monthly_cost: f64,
    /// Data integrity check results
    pub integrity_check_passed: u64,
    /// Integrity Check Failed
    pub integrity_check_failed: u64,
}

/// S3 archival service for cold storage
pub struct S3ArchivalService {
    client: S3Client,
    config: S3ArchivalConfig,
}

/// Create AWS S3 client using environment variables or AWS credential chain
async fn create_s3_client_from_env() -> Result<S3Client> {
    // Try to use explicit credentials first, then fall back to AWS credential chain
    let config = if let (Ok(access_key), Ok(secret_key)) = (
        std::env::var("AWS_ACCESS_KEY_ID"),
        std::env::var("AWS_SECRET_ACCESS_KEY"),
    ) {
        info!("Using explicit AWS credentials from environment variables");
        let creds = aws_types::Credentials::new(
            access_key,
            secret_key,
            std::env::var("AWS_SESSION_TOKEN").ok(),
    // None variant
            None,
            "environment",
        );

        aws_config::defaults(BehaviorVersion::latest())
            .region(std::env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".to_string()))
            .credentials_provider(creds)
            .load()
            .await
    } else {
        info!("Using AWS default credential chain (IAM roles, profiles, etc.)");
        aws_config::defaults(BehaviorVersion::latest())
            .region(std::env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".to_string()))
            .load()
            .await
    };

    Ok(S3Client::new(&config))
}
impl S3ArchivalService {
    /// Create a new S3 archival service using environment configuration
    pub async fn new() -> Result<Self> {
        let config = S3ArchivalConfig::default();
        Self::with_config(config).await
    }

    /// Create a new S3 archival service with custom configuration
    pub async fn with_config(config: S3ArchivalConfig) -> Result<Self> {
        info!(
            "Initializing S3 archival service with bucket: {}, region: {}",
            config.bucket_name, config.region
        );

        let client = create_s3_client_from_env().await
            .context("Failed to create S3 client from environment")?;

        // Test connection and bucket access
        client
            .head_bucket()
            .bucket(&config.bucket_name)
            .send()
            .await
            .context("Failed to connect to S3 archival bucket. Check credentials and bucket permissions.")?;

        info!("Successfully connected to S3 archival bucket: {}", config.bucket_name);

        let service = Self {
            client,
            config,
        };

        // Setup lifecycle policies if enabled
        if service.config.enable_lifecycle {
            service.setup_lifecycle_policies().await?;
        }

    // Ok variant
        Ok(service)
    }

    /// Generate S3 key for archived data
    fn generate_archive_key(&self, data_type: &ArchivalDataType, archive_id: Uuid, timestamp: DateTime<Utc>) -> String {
        format!(
            "{}/year={}/month={:02}/day={:02}/{}.bin",
            data_type.key_prefix(),
            timestamp.year(),
            timestamp.month(),
            timestamp.day(),
            archive_id
        )
    }

    /// Compress data if compression is enabled
    fn compress_data(&self, data: &[u8]) -> Result<(Vec<u8>, Option<String>)> {
        if !self.config.enable_compression {
            return Ok((data.to_vec(), None));
        }

        use flate2::write::GzEncoder;
        use flate2::Compression;
        use std::io::Write;

        let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
        encoder.write_all(data)
            .context("Failed to compress data")?;
        let compressed = encoder.finish()
            .context("Failed to finalize compression")?;

        debug!(
            "Compressed data from {} to {} bytes ({:.1}% reduction)",
            data.len(),
            compressed.len(),
            (1.0 - compressed.len() as f64 / data.len() as f64) * 100.0
        );

        Ok((compressed, Some("gzip".to_string())))
    }

    /// Calculate data checksum for integrity verification
    fn calculate_checksum(&self, data: &[u8]) -> String {
        use sha2::{Sha256, Digest};
        let mut hasher = Sha256::new();
        hasher.update(data);
        format!("{:x}", hasher.finalize())
    }

    /// Create S3 object tags from metadata
    fn create_object_tags(&self, metadata: &ArchivalMetadata, additional_tags: Option<HashMap<String, String>>) -> Result<Vec<Tag>> {
        let mut tags = Vec::new();

        tags.push(Tag::builder().key("archive_id").value(metadata.archive_id.to_string()).build()
            .map_err(|e| Error::Archival(format!("Failed to build archive_id tag: {}", e)))?);
        tags.push(Tag::builder().key("data_type").value(format!("{:?}", metadata.data_type)).build()
            .map_err(|e| Error::Archival(format!("Failed to build data_type tag: {}", e)))?);
        tags.push(Tag::builder().key("source_id").value(&metadata.source_id).build()
            .map_err(|e| Error::Archival(format!("Failed to build source_id tag: {}", e)))?);
        tags.push(Tag::builder().key("created_at").value(metadata.created_at.to_rfc3339()).build()
            .map_err(|e| Error::Archival(format!("Failed to build created_at tag: {}", e)))?);
        tags.push(Tag::builder().key("service").value("foxhunt-hft").build()
            .map_err(|e| Error::Archival(format!("Failed to build service tag: {}", e)))?);

        // Add custom tags
        if let Some(additional) = additional_tags {
            for (key, value) in additional {
                tags.push(Tag::builder().key(key).value(value).build()
                    .map_err(|e| Error::Archival(format!("Failed to build custom tag: {}", e)))?);
            }
        }

        Ok(tags)
    }

    /// Setup S3 lifecycle policies for automatic transitions and deletion
    async fn setup_lifecycle_policies(&self) -> Result<()> {
        use aws_sdk_s3::types::{
            LifecycleConfiguration, LifecycleRule, LifecycleRuleStatus, 
            Transition, ExpirationStatus, LifecycleExpiration
        };

        info!("Setting up S3 lifecycle policies for archival bucket");

        let rules = vec![
            // Transition market data to Glacier after 90 days
            LifecycleRule::builder()
                .id("market-data-transition")
                .status(LifecycleRuleStatus::Enabled)
                .filter(aws_sdk_s3::types::LifecycleRuleFilter::Prefix("market-data/".to_string()))
                .transitions(
                    Transition::builder()
                        .days(90)
                        .storage_class(StorageClass::Glacier)
                        .build()
                        .map_err(|e| Error::Archival(format!("Failed to build transition: {}", e)))?
                )
                .build()
                .map_err(|e| Error::Archival(format!("Failed to build lifecycle rule: {}", e)))?,
            
            // Transition model checkpoints to Standard-IA after 30 days, Glacier after 180 days
            LifecycleRule::builder()
                .id("model-checkpoints-transition")
                .status(LifecycleRuleStatus::Enabled)
                .filter(aws_sdk_s3::types::LifecycleRuleFilter::Prefix("model-checkpoints/".to_string()))
                .transitions(
                    Transition::builder()
                        .days(30)
                        .storage_class(StorageClass::StandardIa)
                        .build()
                        .map_err(|e| Error::Archival(format!("Failed to build transition: {}", e)))?
                )
                .transitions(
                    Transition::builder()
                        .days(180)
                        .storage_class(StorageClass::Glacier)
                        .build()
                        .map_err(|e| Error::Archival(format!("Failed to build transition: {}", e)))?
                )
                .build()
                .map_err(|e| Error::Archival(format!("Failed to build lifecycle rule: {}", e)))?,
            
            // Delete non-compliance data after retention period
            LifecycleRule::builder()
                .id("data-retention-policy")
                .status(LifecycleRuleStatus::Enabled)
                .filter(aws_sdk_s3::types::LifecycleRuleFilter::And(
                    aws_sdk_s3::types::LifecycleRuleAndOperator::builder()
                        .prefix("performance-metrics/")
                        .build()
                        .map_err(|e| Error::Archival(format!("Failed to build lifecycle filter: {}", e)))?
                ))
                .expiration(
                    LifecycleExpiration::builder()
                        .days(self.config.retention_days as i32)
                        .build()
                        .map_err(|e| Error::Archival(format!("Failed to build expiration: {}", e)))?
                )
                .build()
                .map_err(|e| Error::Archival(format!("Failed to build lifecycle rule: {}", e)))?,
        ];

        let lifecycle_config = LifecycleConfiguration::builder()
            .set_rules(Some(rules))
            .build()
            .unwrap();

        self.client
            .put_bucket_lifecycle_configuration()
            .bucket(&self.config.bucket_name)
            .lifecycle_configuration(lifecycle_config)
            .send()
            .await
            .context("Failed to set up S3 lifecycle policies")?;

        info!("Successfully configured S3 lifecycle policies");
        Ok(())
    }
}

#[async_trait]
impl S3Archival for S3ArchivalService {
    async fn archive_data(
        &self,
        data: &[u8],
        data_type: ArchivalDataType,
        source_id: String,
        additional_tags: Option<HashMap<String, String>>,
    ) -> Result<ArchivalMetadata> {
        let archive_id = Uuid::new_v4();
        let now = Utc::now();
        let key = self.generate_archive_key(&data_type, archive_id, now);
        
        info!("Archiving {} bytes of {:?} data to S3: {}", data.len(), data_type, key);

        // Compress data if enabled
        let (processed_data, compression_type) = self.compress_data(data)?;
        let checksum = self.calculate_checksum(data);

        // Create metadata
        let metadata = ArchivalMetadata {
            archive_id,
            data_type: data_type.clone(),
            created_at: now,
            archived_at: now,
            original_size: data.len() as u64,
            compressed_size: if self.config.enable_compression {
                Some(processed_data.len() as u64)
            } else {
    // None variant
                None
            },
            compression_type,
            source_id: source_id.clone(),
            tags: additional_tags.clone().unwrap_or_default(),
            checksum: checksum.clone(),
        };

        // Create object metadata for S3
        let mut object_metadata = HashMap::new();
        object_metadata.insert("archive_id".to_string(), archive_id.to_string());
        object_metadata.insert("data_type".to_string(), format!("{:?}", data_type));
        object_metadata.insert("source_id".to_string(), source_id);
        object_metadata.insert("original_size".to_string(), data.len().to_string());
        object_metadata.insert("checksum".to_string(), checksum);
        if let Some(compressed_size) = metadata.compressed_size {
            object_metadata.insert("compressed_size".to_string(), compressed_size.to_string());
        }

        // Create tags
        let tags = self.create_object_tags(&metadata, additional_tags)?;
        let tagging = Tagging::builder().set_tag_set(Some(tags)).build()
            .map_err(|e| Error::Archival(format!("Failed to build tagging: {}", e)))?;

        // Upload to S3
        let body = ByteStream::from(processed_data);
        
        self.client
            .put_object()
            .bucket(&self.config.bucket_name)
            .key(&key)
            .body(body)
            .set_metadata(Some(object_metadata))
            .server_side_encryption(ServerSideEncryption::Aes256)
            .storage_class(data_type.storage_class())
            .tagging(tagging)
            .content_type("application/octet-stream")
            .send()
            .await
            .context("Failed to archive data to S3")?;

        info!(
            "Successfully archived data to S3: {} ({} bytes -> {} bytes)",
            key,
            metadata.original_size,
            metadata.compressed_size.unwrap_or(metadata.original_size)
        );

    // Ok variant
        Ok(metadata)
    }

    async fn retrieve_archived_data(&self, archive_id: Uuid) -> Result<Vec<u8>> {
        // First, find the object by searching with the archive_id tag
        let objects = self.list_objects_by_tag("archive_id", &archive_id.to_string()).await?;
        
        if objects.is_empty() {
            return Err(anyhow::anyhow!("Archived data not found: {}", archive_id));
        }

        let key = &objects[0];
        info!("Retrieving archived data: {}", key);

        let response = self.client
            .get_object()
            .bucket(&self.config.bucket_name)
            .key(key)
            .send()
            .await
            .context("Failed to retrieve archived data from S3")?;

        // Read the body
        let body = response.body.collect().await
            .context("Failed to read S3 object body")?;
        let data = body.into_bytes().to_vec();

        // Check if data was compressed
        let compression_type = response.metadata
            .and_then(|m| m.get("compression_type"))
            .cloned();

        let final_data = if let Some(comp_type) = compression_type {
            if comp_type == "gzip" {
                self.decompress_gzip_data(&data)?
            } else {
                return Err(anyhow::anyhow!("Unsupported compression type: {}", comp_type));
            }
        } else {
            data
        };

        info!("Successfully retrieved {} bytes of archived data", final_data.len());
    // Ok variant
        Ok(final_data)
    }

    async fn list_archived_data(
        &self,
        data_type: Option<ArchivalDataType>,
        from_date: Option<DateTime<Utc>>,
        to_date: Option<DateTime<Utc>>,
    ) -> Result<Vec<ArchivalMetadata>> {
        let prefix = data_type.as_ref().map(|dt| dt.key_prefix().to_string());
        
        debug!("Listing archived data with prefix: {:?}, date range: {:?} - {:?}", 
               prefix, from_date, to_date);

        let mut metadata_list = Vec::new();
        let mut continuation_token: Option<String> = None;

        loop {
            let mut request = self.client
                .list_objects_v2()
                .bucket(&self.config.bucket_name);

            if let Some(p) = &prefix {
                request = request.prefix(p);
            }

            if let Some(token) = &continuation_token {
                request = request.continuation_token(token);
            }

            let response = request.send().await
                .context("Failed to list archived objects")?;

            if let Some(contents) = response.contents {
                for object in contents {
                    if let Some(key) = object.key {
                        // Get object metadata to construct ArchivalMetadata
                        if let Ok(metadata) = self.get_object_metadata(&key).await {
                            // Filter by date range if specified
                            if let Some(from) = from_date {
                                if metadata.created_at < from {
                                    continue;
                                }
                            }
                            if let Some(to) = to_date {
                                if metadata.created_at > to {
                                    continue;
                                }
                            }
                            metadata_list.push(metadata);
                        }
                    }
                }
            }

            if response.is_truncated.unwrap_or(false) {
                continuation_token = response.next_continuation_token;
            } else {
                break;
            }
        }

        debug!("Found {} archived data entries", metadata_list.len());
    // Ok variant
        Ok(metadata_list)
    }

    async fn delete_archived_data(&self, archive_id: Uuid) -> Result<bool> {
        // Find the object by archive_id tag
        let objects = self.list_objects_by_tag("archive_id", &archive_id.to_string()).await?;
        
        if objects.is_empty() {
            debug!("Archived data not found for deletion: {}", archive_id);
            return Ok(false);
        }

        let key = &objects[0];
        info!("Deleting archived data: {}", key);

        // Check compliance requirements before deletion
        if let Ok(metadata) = self.get_object_metadata(key).await {
            let age_days = (Utc::now() - metadata.created_at).num_days();
            if age_days < self.config.retention_days as i64 {
                warn!("Attempted to delete data before retention period: {} (age: {} days, required: {} days)",
                      archive_id, age_days, self.config.retention_days);
                return Err(anyhow::anyhow!(
                    "Cannot delete data before retention period expires ({} days remaining)",
                    self.config.retention_days as i64 - age_days
                ));
            }
        }

        match self.client
            .delete_object()
            .bucket(&self.config.bucket_name)
            .key(key)
            .send()
            .await
        {
            Ok(_) => {
                info!("Successfully deleted archived data: {}", archive_id);
    // Ok variant
                Ok(true)
            }
            Err(e) => {
                if let Some(service_err) = e.as_service_error() {
                    if service_err.is_no_such_key() {
                        debug!("Archived data not found: {}", archive_id);
                        return Ok(false);
                    }
                }
                Err(e).context("Failed to delete archived data from S3")
            }
        }
    }

    async fn get_archival_stats(&self) -> Result<ArchivalStats> {
        info!("Calculating archival statistics for bucket: {}", self.config.bucket_name);

        let mut total_objects = 0_u64;
        let mut total_size = 0_u64;
        let mut storage_by_type = HashMap::new();
        let mut compression_ratios = Vec::new();
        let mut integrity_passed = 0_u64;
        let mut integrity_failed = 0_u64;

        let mut continuation_token: Option<String> = None;

        loop {
            let mut request = self.client
                .list_objects_v2()
                .bucket(&self.config.bucket_name);

            if let Some(token) = &continuation_token {
                request = request.continuation_token(token);
            }

            let response = request.send().await
                .context("Failed to list objects for statistics")?;

            if let Some(contents) = response.contents {
                for object in contents {
                    if let (Some(key), Some(size)) = (object.key, object.size) {
                        total_objects += 1;
                        total_size += size as u64;

                        // Categorize by data type
                        let data_type = key.split('/').next().unwrap_or("unknown").to_string();
                        *storage_by_type.entry(data_type).or_insert(0) += size as u64;

                        // Get compression ratio if available
                        if let Ok(metadata) = self.get_object_metadata(&key).await {
                            if let (Some(original), Some(compressed)) = (
    // Some variant
                                Some(metadata.original_size),
                                metadata.compressed_size
                            ) {
                                let ratio = original as f64 / compressed as f64;
                                compression_ratios.push(ratio);
                            }

                            // Perform integrity check (sampling for performance)
                            if total_objects % 100 == 0 {  // Check every 100th object
                                match self.verify_data_integrity(metadata.archive_id).await {
                                    Ok(true) => integrity_passed += 1,
                                    Ok(false) => integrity_failed += 1,
                                    Err(_) => integrity_failed += 1,
                                }
                            }
                        }
                    }
                }
            }

            if response.is_truncated.unwrap_or(false) {
                continuation_token = response.next_continuation_token;
            } else {
                break;
            }
        }

        let average_compression_ratio = if !compression_ratios.is_empty() {
            compression_ratios.iter().sum::<f64>() / compression_ratios.len() as f64
        } else {
            1.0
        };

        // Rough cost estimation (simplified)
        let estimated_monthly_cost = (total_size as f64 / 1_073_741_824.0) * 0.023; // ~$0.023/GB for Standard-IA

        Ok(ArchivalStats {
            total_objects,
            total_size_bytes: total_size,
            storage_by_type,
            average_compression_ratio,
            estimated_monthly_cost,
            integrity_check_passed: integrity_passed,
            integrity_check_failed: integrity_failed,
        })
    }

    async fn verify_data_integrity(&self, archive_id: Uuid) -> Result<bool> {
        // Retrieve stored checksum
        let objects = self.list_objects_by_tag("archive_id", &archive_id.to_string()).await?;
        
        if objects.is_empty() {
            return Err(anyhow::anyhow!("Archive not found: {}", archive_id));
        }

        let key = &objects[0];
        let metadata = self.get_object_metadata(key).await?;
        
        // Retrieve and verify data
        let data = self.retrieve_archived_data(archive_id).await?;
        let calculated_checksum = self.calculate_checksum(&data);
        
        let integrity_valid = calculated_checksum == metadata.checksum;
        
        if integrity_valid {
            debug!("Data integrity verified for archive: {}", archive_id);
        } else {
            warn!("Data integrity check failed for archive: {} (expected: {}, calculated: {})",
                  archive_id, metadata.checksum, calculated_checksum);
        }
        
    // Ok variant
        Ok(integrity_valid)
    }
}

impl S3ArchivalService {
    /// Helper method to decompress gzip data
    fn decompress_gzip_data(&self, compressed_data: &[u8]) -> Result<Vec<u8>> {
        use flate2::read::GzDecoder;
        use std::io::Read;

        let mut decoder = GzDecoder::new(compressed_data);
        let mut decompressed = Vec::new();
        decoder.read_to_end(&mut decompressed)
            .context("Failed to decompress gzip data")?;

    // Ok variant
        Ok(decompressed)
    }

    /// Helper method to find objects by tag
    async fn list_objects_by_tag(&self, tag_key: &str, tag_value: &str) -> Result<Vec<String>> {
        // Note: This is a simplified implementation. In production, you might want to
        // use S3 Select or maintain an index for better performance
        let mut matching_objects = Vec::new();
        let mut continuation_token: Option<String> = None;

        loop {
            let mut request = self.client
                .list_objects_v2()
                .bucket(&self.config.bucket_name);

            if let Some(token) = &continuation_token {
                request = request.continuation_token(token);
            }

            let response = request.send().await?;

            if let Some(contents) = response.contents {
                for object in contents {
                    if let Some(key) = object.key {
                        // Get object tagging to check for matching tag
                        if let Ok(tagging_response) = self.client
                            .get_object_tagging()
                            .bucket(&self.config.bucket_name)
                            .key(&key)
                            .send()
                            .await
                        {
                            if let Some(tag_set) = tagging_response.tag_set {
                                for tag in tag_set {
                                    if tag.key() == Some(tag_key) && tag.value() == Some(tag_value) {
                                        matching_objects.push(key.clone());
                                        break;
                                    }
                                }
                            }
                        }
                    }
                }
            }

            if response.is_truncated.unwrap_or(false) {
                continuation_token = response.next_continuation_token;
            } else {
                break;
            }
        }

    // Ok variant
        Ok(matching_objects)
    }

    /// Helper method to get object metadata
    async fn get_object_metadata(&self, key: &str) -> Result<ArchivalMetadata> {
        let response = self.client
            .head_object()
            .bucket(&self.config.bucket_name)
            .key(key)
            .send()
            .await
            .context("Failed to get object metadata")?;

        let metadata_map = response.metadata.unwrap_or_default();
        
        let archive_id = metadata_map.get("archive_id")
            .and_then(|s| Uuid::parse_str(s).ok())
            .ok_or_else(|| anyhow::anyhow!("Invalid or missing archive_id in metadata"))?;

        let data_type_str = metadata_map.get("data_type")
            .ok_or_else(|| anyhow::anyhow!("Missing data_type in metadata"))?;
        
        let data_type = match data_type_str.as_str() {
            "MarketData" => ArchivalDataType::MarketData,
            "TradingLogs" => ArchivalDataType::TradingLogs,
            "RiskLogs" => ArchivalDataType::RiskLogs,
            "ModelCheckpoints" => ArchivalDataType::ModelCheckpoints,
            "PerformanceMetrics" => ArchivalDataType::PerformanceMetrics,
            "ComplianceData" => ArchivalDataType::ComplianceData,
            _ => return Err(anyhow::anyhow!("Unknown data type: {}", data_type_str)),
        };

        let source_id = metadata_map.get("source_id")
            .cloned()
            .unwrap_or_else(|| "unknown".to_string());

        let original_size = metadata_map.get("original_size")
            .and_then(|s| s.parse().ok())
            .unwrap_or(0);

        let compressed_size = metadata_map.get("compressed_size")
            .and_then(|s| s.parse().ok());

        let checksum = metadata_map.get("checksum")
            .cloned()
            .unwrap_or_else(|| "unknown".to_string());

        // Use last_modified as both created_at and archived_at if not available in metadata
        let timestamp = response.last_modified
            .map(|ts| DateTime::from_timestamp(ts.secs(), ts.subsec_nanos()).unwrap_or_else(|| Utc::now()))
            .unwrap_or_else(|| Utc::now());

        Ok(ArchivalMetadata {
            archive_id,
            data_type,
            created_at: timestamp,
            archived_at: timestamp,
            original_size,
            compressed_size,
            compression_type: if compressed_size.is_some() { Some("gzip".to_string()) } else { None },
            source_id,
            tags: HashMap::new(), // Would need separate call to get tags
            checksum,
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::collections::HashMap;

    #[test]
    fn test_archival_data_type_key_prefix() {
        assert_eq!(ArchivalDataType::MarketData.key_prefix(), "market-data");
        assert_eq!(ArchivalDataType::TradingLogs.key_prefix(), "trading-logs");
        assert_eq!(ArchivalDataType::ModelCheckpoints.key_prefix(), "model-checkpoints");
    }

    #[test]
    fn test_archival_data_type_storage_class() {
        assert_eq!(ArchivalDataType::MarketData.storage_class(), StorageClass::StandardIa);
        assert_eq!(ArchivalDataType::TradingLogs.storage_class(), StorageClass::Standard);
        assert_eq!(ArchivalDataType::ComplianceData.storage_class(), StorageClass::Glacier);
    }

    #[test]
    fn test_generate_archive_key() {
        let config = S3ArchivalConfig::default();
        let service = S3ArchivalService {
            client: S3Client::new(&aws_config::load_from_env().await),
            config,
        };

        let archive_id = Uuid::new_v4();
        let timestamp = DateTime::parse_from_rfc3339("2025-01-23T10:30:00Z").unwrap().with_timezone(&Utc);
        let key = service.generate_archive_key(&ArchivalDataType::MarketData, archive_id, timestamp);
        
        assert!(key.starts_with("market-data/year=2025/month=01/day=23/"));
        assert!(key.ends_with(".bin"));
    }

    #[test]
    fn test_checksum_calculation() {
        let config = S3ArchivalConfig::default();
        let service = S3ArchivalService {
            client: S3Client::new(&aws_config::load_from_env().await),
            config,
        };

        let data = b"test data for checksum";
        let checksum1 = service.calculate_checksum(data);
        let checksum2 = service.calculate_checksum(data);
        
        assert_eq!(checksum1, checksum2);
        assert!(!checksum1.is_empty());
        assert_eq!(checksum1.len(), 64); // SHA-256 hex string length
    }
}