foxhunt/testing/api-load/src/scenarios/spike_load.rs

#![allow(clippy::integer_division)]

use anyhow::Result;
use tokio::sync::mpsc;
use tokio::task::JoinSet;

use crate::clients::{AuthenticatedClient, MixedWorkloadClient};
use crate::metrics::{collector::MetricsCollector, LoadTestReport, TestConfig};

pub async fn run(
    gateway_url: String,
    target_clients: usize,
    ramp_up_secs: u64,
    sustain_secs: u64,
) -> Result<LoadTestReport> {
    tracing::info!(
        "Starting SPIKE load test: 0→{} clients in {}s, sustain {}s",
        target_clients,
        ramp_up_secs,
        sustain_secs
    );

    let (metrics_tx, metrics_rx) = mpsc::unbounded_channel();
    let mut collector = MetricsCollector::new(metrics_rx);

    // Spawn collector task
    let collector_handle = {
        let target_clients_clone = target_clients;
        tokio::spawn(async move {
            collector.run(target_clients_clone).await;
            collector
        })
    };

    let mut join_set = JoinSet::new();
    let total_duration = std::time::Duration::from_secs(ramp_up_secs + sustain_secs);
    let ramp_up_duration = std::time::Duration::from_secs(ramp_up_secs);

    // Calculate how many clients to spawn per interval
    let spawn_interval_ms = 100; // Spawn clients every 100ms
    let intervals_in_ramp_up =
        u64::from(u32::try_from(ramp_up_secs * 1000 / spawn_interval_ms).unwrap_or(u32::MAX));
    let clients_per_interval = usize::try_from(
        (f64::from(u32::try_from(target_clients).unwrap_or(u32::MAX))
            / f64::from(u32::try_from(intervals_in_ramp_up).unwrap_or(u32::MAX)))
        .ceil() as u64,
    )
    .unwrap_or(usize::MAX);

    let start_time = std::time::Instant::now();
    let mut clients_spawned = 0;

    // Ramp up phase: gradually spawn clients
    while start_time.elapsed() < ramp_up_duration && clients_spawned < target_clients {
        let batch_size = std::cmp::min(clients_per_interval, target_clients - clients_spawned);

        for i in 0..batch_size {
            let client_id = clients_spawned + i;
            let gateway_url = gateway_url.clone();
            let metrics_tx = metrics_tx.clone();
            let remaining_duration = total_duration - start_time.elapsed();

            join_set.spawn(async move {
                let auth_client = AuthenticatedClient::new(
                    gateway_url,
                    "test-secret-key-for-load-testing",
                    &format!("user-{}", client_id),
                    &format!("loadtest-user-{}", client_id),
                )
                .await?;

                let mut mixed_client = MixedWorkloadClient::new(auth_client, client_id, metrics_tx);
                mixed_client.run_mixed_workload(remaining_duration).await
            });
        }

        clients_spawned += batch_size;
        tracing::debug!("Spawned {} / {} clients", clients_spawned, target_clients);

        tokio::time::sleep(tokio::time::Duration::from_millis(spawn_interval_ms)).await;
    }

    tracing::info!(
        "Ramp-up complete: {} clients active, sustaining for {}s",
        clients_spawned,
        sustain_secs
    );

    // Wait for all clients to complete
    while let Some(result) = join_set.join_next().await {
        if let Err(e) = result {
            tracing::error!("Client task failed: {:?}", e);
        }
    }

    // Drop the sender to signal collector to finish
    drop(metrics_tx);

    // Wait for collector to finish and get the report
    let collector = collector_handle.await?;
    let mut report = collector.generate_report(
        "Spike Load Test".to_owned(),
        TestConfig {
            num_clients: target_clients,
            duration_secs: ramp_up_secs + sustain_secs,
            test_type: "spike_load".to_owned(),
        },
    );

    // Add capacity recommendation based on circuit breaker activations
    let circuit_breaker_activated = report.metrics.circuit_breaker_requests > 0;
    let graceful_degradation = report.metrics.error_rate_pct < 10.0;

    report.capacity_recommendation = Some(crate::metrics::CapacityRecommendation {
        max_sustainable_clients: target_clients,
        max_sustainable_rps: report.metrics.requests_per_second,
        bottleneck_identified: if circuit_breaker_activated {
            Some("Circuit breaker activated during spike".to_owned())
        } else if !graceful_degradation {
            Some("High error rate during spike".to_owned())
        } else {
            None
        },
        recommendation: if graceful_degradation && !circuit_breaker_activated {
            format!(
                "System handled spike to {} clients gracefully. Error rate: {:.2}%, P99: {:.2}ms. \
                 Circuit breakers did not activate - good resilience.",
                target_clients, report.metrics.error_rate_pct, report.metrics.latency_stats.p99_ms
            )
        } else if circuit_breaker_activated {
            format!(
                "System activated circuit breakers during spike. {} circuit breaker trips recorded. \
                 This is expected behavior for fault tolerance. Review backend service capacity.",
                report.metrics.circuit_breaker_requests
            )
        } else {
            format!(
                "System showed degradation during spike: {:.2}% error rate. \
                 Consider implementing rate limiting or adding capacity.",
                report.metrics.error_rate_pct
            )
        },
    });

    tracing::info!("Spike load test completed: {:?}", report.metrics);

    Ok(report)
}