foxhunt/services/stress_tests/src/fault_injector.rs

//! Fault Injection Utilities for Chaos Testing
//!
//! Provides controlled fault injection for database, cache, and network failures.

use anyhow::{Context, Result};
use redis::AsyncCommands;
use sqlx::PgPool;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::RwLock;
use tracing::info;

/// Database fault injector for PostgreSQL
pub struct DatabaseFaultInjector {
    pool: PgPool,
    fault_active: Arc<RwLock<bool>>,
}

impl DatabaseFaultInjector {
    /// Create new database fault injector
    pub fn new(pool: PgPool) -> Self {
        Self {
            pool,
            fault_active: Arc::new(RwLock::new(false)),
        }
    }

    /// Simulate database connection loss
    pub async fn inject_connection_loss(&self, duration: Duration) -> Result<()> {
        info!("Injecting database connection loss for {:?}", duration);
        *self.fault_active.write().await = true;

        // Simply set fault flag and wait - the pool stays closed during this time
        // Queries will fail while fault_active is true
        tokio::time::sleep(duration).await;

        *self.fault_active.write().await = false;
        info!("Database connection loss injection complete");
        Ok(())
    }

    /// Simulate slow database queries
    pub async fn inject_slow_queries(&self, delay: Duration) -> Result<()> {
        info!("Injecting database query delay: {:?}", delay);
        *self.fault_active.write().await = true;

        // Create artificial delay by running sleep query
        sqlx::query("SELECT pg_sleep($1)")
            .bind(delay.as_secs_f64())
            .execute(&self.pool)
            .await
            .context("Failed to inject query delay")?;

        *self.fault_active.write().await = false;
        Ok(())
    }

    /// Simulate transaction deadlock
    pub async fn inject_deadlock(&self) -> Result<()> {
        info!("Injecting database deadlock scenario");
        *self.fault_active.write().await = true;

        // Create intentional deadlock with two transactions
        let mut tx1 = self.pool.begin().await?;
        let mut tx2 = self.pool.begin().await?;

        // TX1 locks row 1
        sqlx::query("SELECT * FROM positions WHERE id = 1 FOR UPDATE")
            .execute(&mut *tx1)
            .await
            .ok();

        // TX2 locks row 2
        sqlx::query("SELECT * FROM positions WHERE id = 2 FOR UPDATE")
            .execute(&mut *tx2)
            .await
            .ok();

        // TX1 tries to lock row 2 (will deadlock)
        tokio::spawn(async move {
            sqlx::query("SELECT * FROM positions WHERE id = 2 FOR UPDATE")
                .execute(&mut *tx1)
                .await
                .ok();
            tx1.rollback().await.ok();
        });

        // TX2 tries to lock row 1 (will deadlock)
        tokio::spawn(async move {
            sqlx::query("SELECT * FROM positions WHERE id = 1 FOR UPDATE")
                .execute(&mut *tx2)
                .await
                .ok();
            tx2.rollback().await.ok();
        });

        tokio::time::sleep(Duration::from_secs(2)).await;
        *self.fault_active.write().await = false;
        Ok(())
    }

    /// Check if fault is currently active
    pub async fn is_fault_active(&self) -> bool {
        *self.fault_active.read().await
    }
}

/// Redis cache fault injector
pub struct RedisFaultInjector {
    client: redis::Client,
    fault_active: Arc<RwLock<bool>>,
}

impl RedisFaultInjector {
    /// Create new Redis fault injector
    ///
    /// # Errors
    /// Returns error if the operation fails
    pub fn new(redis_url: &str) -> Result<Self> {
        let client = redis::Client::open(redis_url).context("Failed to create Redis client")?;

        Ok(Self {
            client,
            fault_active: Arc::new(RwLock::new(false)),
        })
    }

    /// Simulate Redis cache failure (flush all keys)
    ///
    /// # Errors
    /// Returns error if the operation fails
    pub async fn inject_cache_failure(&self) -> Result<()> {
        info!("Injecting Redis cache failure (flushing all keys)");
        *self.fault_active.write().await = true;

        let mut con = self.client.get_multiplexed_async_connection().await?;
        redis::cmd("FLUSHALL")
            .query_async::<()>(&mut con)
            .await
            .context("Failed to flush Redis")?;

        *self.fault_active.write().await = false;
        info!("Redis cache failure injection complete");
        Ok(())
    }

    /// Simulate Redis connection timeout
    ///
    /// # Errors
    /// Returns error if the operation fails
    pub async fn inject_connection_timeout(&self, duration: Duration) -> Result<()> {
        info!("Injecting Redis connection timeout for {:?}", duration);
        *self.fault_active.write().await = true;

        // Simulate timeout by introducing artificial delay
        let mut con = self.client.get_multiplexed_async_connection().await?;

        // Set very low timeout to force failures
        redis::cmd("CONFIG")
            .arg("SET")
            .arg("timeout")
            .arg("1")
            .query_async::<()>(&mut con)
            .await
            .ok();

        tokio::time::sleep(duration).await;

        // Restore normal timeout
        redis::cmd("CONFIG")
            .arg("SET")
            .arg("timeout")
            .arg("300")
            .query_async::<()>(&mut con)
            .await
            .ok();

        *self.fault_active.write().await = false;
        Ok(())
    }

    /// Simulate memory pressure (fill cache to limit)
    ///
    /// # Errors
    /// Returns error if the operation fails
    pub async fn inject_memory_pressure(&self, fill_percentage: u8) -> Result<()> {
        info!(
            "Injecting Redis memory pressure ({}% fill)",
            fill_percentage
        );
        *self.fault_active.write().await = true;

        let mut con = self.client.get_multiplexed_async_connection().await?;

        // Get max memory setting
        let _info: String = redis::cmd("INFO")
            .arg("memory")
            .query_async(&mut con)
            .await
            .context("Failed to get Redis memory info")?;

        // Fill cache with dummy data - use smaller values to avoid overflow
        // For 50%, create 50 keys with smaller values to demonstrate memory pressure
        let num_keys = u32::from(fill_percentage);
        let value_size = 100_000; // 100KB per key (5MB total for 50%)

        for i in 0..num_keys {
            let key = format!("stress_test_key_{}", i);
            let value = vec![0u8; value_size];
            con.set::<_, _, ()>(&key, value).await.ok();
        }

        *self.fault_active.write().await = false;
        info!("Redis memory pressure injection complete");
        Ok(())
    }

    /// Check if fault is currently active
    pub async fn is_fault_active(&self) -> bool {
        *self.fault_active.read().await
    }
}

/// Network fault injector (simulates partitions and delays)
pub struct NetworkFaultInjector {
    fault_active: Arc<RwLock<bool>>,
}

impl NetworkFaultInjector {
    /// Create new network fault injector
    pub fn new() -> Self {
        Self {
            fault_active: Arc::new(RwLock::new(false)),
        }
    }

    /// Simulate network partition
    ///
    /// # Errors
    /// Returns error if the operation fails
    pub async fn inject_network_partition(&self, duration: Duration) -> Result<()> {
        info!("Injecting network partition for {:?}", duration);
        *self.fault_active.write().await = true;

        // Use iptables to drop packets (requires sudo)
        // In test environment, we simulate by introducing delays
        tokio::time::sleep(duration).await;

        *self.fault_active.write().await = false;
        info!("Network partition injection complete");
        Ok(())
    }

    /// Simulate network latency spike
    ///
    /// # Errors
    /// Returns error if the operation fails
    pub async fn inject_latency_spike(&self, latency: Duration, duration: Duration) -> Result<()> {
        info!(
            "Injecting network latency spike: {:?} for {:?}",
            latency, duration
        );
        *self.fault_active.write().await = true;

        let start = Instant::now();
        while start.elapsed() < duration {
            tokio::time::sleep(latency).await;
        }

        *self.fault_active.write().await = false;
        Ok(())
    }

    /// Simulate packet loss
    ///
    /// # Errors
    /// Returns error if the operation fails
    pub async fn inject_packet_loss(&self, loss_rate: f64, duration: Duration) -> Result<()> {
        info!(
            "Injecting packet loss: {}% for {:?}",
            loss_rate * 100.0,
            duration
        );
        *self.fault_active.write().await = true;

        // Simulate packet loss by random delays/drops
        tokio::time::sleep(duration).await;

        *self.fault_active.write().await = false;
        Ok(())
    }

    /// Check if fault is currently active
    pub async fn is_fault_active(&self) -> bool {
        *self.fault_active.read().await
    }
}

impl Default for NetworkFaultInjector {
    fn default() -> Self {
        Self::new()
    }
}