foxhunt/ml/tests/wave_d_latency_profiling_test.rs

//! Wave D: 225-Feature Pipeline Latency Profiling Test
//!
//! This test measures end-to-end latency for the complete 225-feature extraction pipeline
//! under realistic production workloads. It profiles the latency breakdown across Wave C
//! features (201 features) and Wave D regime features (24 features).
//!
//! ## Test Strategy
//! 1. Load ES.FUT data (1000 bars)
//! 2. Run 1000 iterations to get stable P50/P90/P99 latencies
//! 3. Profile latency breakdown:
//!    - Wave C features (201 features): Target <40μs/bar
//!    - Wave D CUSUM (10 features): Target <10μs/bar
//!    - Wave D ADX (5 features): Target <5μs/bar
//!    - Wave D Transition (5 features): Target <5μs/bar
//!    - Wave D Adaptive (4 features): Target <5μs/bar
//!    - **Total 225 features**: Target <65μs/bar
//! 4. Validate P99 latency <100μs (production SLA)
//!
//! ## Performance Targets
//! - P50 latency: <50μs/bar
//! - P99 latency: <100μs/bar
//! - No outliers >500μs
//!
//! ## TDD Status
//! - RED: Test written, awaiting implementation

use anyhow::Result;
use chrono::{DateTime, Utc};
use std::collections::HashMap;
use std::time::Instant;

/// Latency histogram bucket for profiling
#[derive(Debug, Clone)]
struct LatencyHistogram {
    buckets: HashMap<u64, u64>, // bucket_us -> count
    samples: Vec<u64>,           // all samples in microseconds
}

impl LatencyHistogram {
    fn new() -> Self {
        Self {
            buckets: HashMap::new(),
            samples: Vec::with_capacity(1000),
        }
    }

    fn record(&mut self, latency_us: u64) {
        // Record in histogram buckets (10μs buckets)
        let bucket = (latency_us / 10) * 10;
        *self.buckets.entry(bucket).or_insert(0) += 1;

        // Record raw sample
        self.samples.push(latency_us);
    }

    fn p50(&self) -> u64 {
        self.percentile(0.50)
    }

    fn p90(&self) -> u64 {
        self.percentile(0.90)
    }

    fn p99(&self) -> u64 {
        self.percentile(0.99)
    }

    fn percentile(&self, p: f64) -> u64 {
        if self.samples.is_empty() {
            return 0;
        }

        let mut sorted = self.samples.clone();
        sorted.sort_unstable();

        let index = ((sorted.len() as f64 - 1.0) * p) as usize;
        sorted[index.min(sorted.len() - 1)]
    }

    fn mean(&self) -> u64 {
        if self.samples.is_empty() {
            return 0;
        }
        self.samples.iter().sum::<u64>() / self.samples.len() as u64
    }

    fn max(&self) -> u64 {
        self.samples.iter().copied().max().unwrap_or(0)
    }
}

/// Latency profiler for 225-feature pipeline
#[derive(Debug)]
struct FeatureLatencyProfiler {
    wave_c_latencies: LatencyHistogram,          // 201 features
    wave_d_cusum_latencies: LatencyHistogram,    // 10 features
    wave_d_adx_latencies: LatencyHistogram,      // 5 features
    wave_d_transition_latencies: LatencyHistogram, // 5 features
    wave_d_adaptive_latencies: LatencyHistogram, // 4 features
    total_latencies: LatencyHistogram,           // 225 features total
}

impl FeatureLatencyProfiler {
    fn new() -> Self {
        Self {
            wave_c_latencies: LatencyHistogram::new(),
            wave_d_cusum_latencies: LatencyHistogram::new(),
            wave_d_adx_latencies: LatencyHistogram::new(),
            wave_d_transition_latencies: LatencyHistogram::new(),
            wave_d_adaptive_latencies: LatencyHistogram::new(),
            total_latencies: LatencyHistogram::new(),
        }
    }

    /// Profile complete 225-feature extraction for a single bar
    fn profile_bar(&mut self, bar: &OHLCVBar) -> Result<()> {
        let total_start = Instant::now();

        // Stage 1: Wave C features (201 features)
        let wave_c_start = Instant::now();
        let _wave_c_features = self.extract_wave_c_features(bar)?;
        let wave_c_latency_us = wave_c_start.elapsed().as_micros() as u64;
        self.wave_c_latencies.record(wave_c_latency_us);

        // Stage 2: Wave D CUSUM features (10 features, indices 201-210)
        let cusum_start = Instant::now();
        let _cusum_features = self.extract_cusum_features(bar)?;
        let cusum_latency_us = cusum_start.elapsed().as_micros() as u64;
        self.wave_d_cusum_latencies.record(cusum_latency_us);

        // Stage 3: Wave D ADX features (5 features, indices 211-215)
        let adx_start = Instant::now();
        let _adx_features = self.extract_adx_features(bar)?;
        let adx_latency_us = adx_start.elapsed().as_micros() as u64;
        self.wave_d_adx_latencies.record(adx_latency_us);

        // Stage 4: Wave D Transition features (5 features, indices 216-220)
        let transition_start = Instant::now();
        let _transition_features = self.extract_transition_features(bar)?;
        let transition_latency_us = transition_start.elapsed().as_micros() as u64;
        self.wave_d_transition_latencies.record(transition_latency_us);

        // Stage 5: Wave D Adaptive features (4 features, indices 221-224)
        let adaptive_start = Instant::now();
        let _adaptive_features = self.extract_adaptive_features(bar)?;
        let adaptive_latency_us = adaptive_start.elapsed().as_micros() as u64;
        self.wave_d_adaptive_latencies.record(adaptive_latency_us);

        // Total pipeline latency
        let total_latency_us = total_start.elapsed().as_micros() as u64;
        self.total_latencies.record(total_latency_us);

        Ok(())
    }

    /// Extract Wave C features (201 features)
    /// PLACEHOLDER: Will be replaced with actual Wave C pipeline
    fn extract_wave_c_features(&self, _bar: &OHLCVBar) -> Result<Vec<f64>> {
        // Simulate Wave C feature extraction (201 features)
        // Target: <40μs/bar
        let mut features = vec![0.0; 201];

        // Perform minimal computation to ensure non-zero latency
        for i in 0..201 {
            features[i] = (i as f64 * 0.01).sin();
        }

        Ok(features)
    }

    /// Extract Wave D CUSUM features (10 features)
    /// PLACEHOLDER: Will be replaced with actual CUSUM statistics extractor
    fn extract_cusum_features(&self, _bar: &OHLCVBar) -> Result<Vec<f64>> {
        // Simulate CUSUM feature extraction (10 features)
        // Target: <10μs/bar
        let mut features = vec![0.0; 10];

        // Perform minimal computation
        for i in 0..10 {
            features[i] = (i as f64 * 0.02).cos();
        }

        Ok(features)
    }

    /// Extract Wave D ADX features (5 features)
    /// PLACEHOLDER: Will be replaced with actual ADX extractor
    fn extract_adx_features(&self, _bar: &OHLCVBar) -> Result<Vec<f64>> {
        // Simulate ADX feature extraction (5 features)
        // Target: <5μs/bar
        let mut features = vec![0.0; 5];

        // Perform minimal computation
        for i in 0..5 {
            features[i] = (i as f64 * 0.03).tan();
        }

        Ok(features)
    }

    /// Extract Wave D Transition features (5 features)
    /// PLACEHOLDER: Will be replaced with actual transition probability extractor
    fn extract_transition_features(&self, _bar: &OHLCVBar) -> Result<Vec<f64>> {
        // Simulate transition feature extraction (5 features)
        // Target: <5μs/bar
        let mut features = vec![0.0; 5];

        // Perform minimal computation
        for i in 0..5 {
            features[i] = (i as f64 * 0.04).sqrt();
        }

        Ok(features)
    }

    /// Extract Wave D Adaptive features (4 features)
    /// PLACEHOLDER: Will be replaced with actual adaptive strategy metrics extractor
    fn extract_adaptive_features(&self, _bar: &OHLCVBar) -> Result<Vec<f64>> {
        // Simulate adaptive feature extraction (4 features)
        // Target: <5μs/bar
        let mut features = vec![0.0; 4];

        // Perform minimal computation
        for i in 0..4 {
            features[i] = (i as f64 * 0.05).exp() / 10.0;
        }

        Ok(features)
    }

    /// Generate latency report
    fn generate_report(&self) -> LatencyReport {
        LatencyReport {
            wave_c: LatencyStats::from_histogram(&self.wave_c_latencies),
            wave_d_cusum: LatencyStats::from_histogram(&self.wave_d_cusum_latencies),
            wave_d_adx: LatencyStats::from_histogram(&self.wave_d_adx_latencies),
            wave_d_transition: LatencyStats::from_histogram(&self.wave_d_transition_latencies),
            wave_d_adaptive: LatencyStats::from_histogram(&self.wave_d_adaptive_latencies),
            total: LatencyStats::from_histogram(&self.total_latencies),
        }
    }
}

/// Latency statistics summary
#[derive(Debug, Clone)]
struct LatencyStats {
    p50_us: u64,
    p90_us: u64,
    p99_us: u64,
    mean_us: u64,
    max_us: u64,
    sample_count: usize,
}

impl LatencyStats {
    fn from_histogram(hist: &LatencyHistogram) -> Self {
        Self {
            p50_us: hist.p50(),
            p90_us: hist.p90(),
            p99_us: hist.p99(),
            mean_us: hist.mean(),
            max_us: hist.max(),
            sample_count: hist.samples.len(),
        }
    }

    fn meets_target(&self, target_p99_us: u64) -> bool {
        self.p99_us <= target_p99_us
    }
}

/// Complete latency profiling report
#[derive(Debug)]
struct LatencyReport {
    wave_c: LatencyStats,
    wave_d_cusum: LatencyStats,
    wave_d_adx: LatencyStats,
    wave_d_transition: LatencyStats,
    wave_d_adaptive: LatencyStats,
    total: LatencyStats,
}

impl LatencyReport {
    fn print_summary(&self) {
        println!("\n=== 225-Feature Pipeline Latency Profiling Report ===\n");

        println!("Wave C Features (201 features):");
        self.print_stats(&self.wave_c, 40);

        println!("\nWave D CUSUM Features (10 features):");
        self.print_stats(&self.wave_d_cusum, 10);

        println!("\nWave D ADX Features (5 features):");
        self.print_stats(&self.wave_d_adx, 5);

        println!("\nWave D Transition Features (5 features):");
        self.print_stats(&self.wave_d_transition, 5);

        println!("\nWave D Adaptive Features (4 features):");
        self.print_stats(&self.wave_d_adaptive, 5);

        println!("\n=== TOTAL PIPELINE (225 features) ===");
        self.print_stats(&self.total, 65);

        // Overall assessment
        println!("\n=== Production Readiness Assessment ===");
        let p50_ok = self.total.p50_us <= 50;
        let p99_ok = self.total.p99_us <= 100;
        let outliers_ok = self.total.max_us <= 500;

        println!("  P50 latency: {} (target: <50μs)", if p50_ok { "✅ PASS" } else { "❌ FAIL" });
        println!("  P99 latency: {} (target: <100μs)", if p99_ok { "✅ PASS" } else { "❌ FAIL" });
        println!("  Max latency: {} (target: <500μs)", if outliers_ok { "✅ PASS" } else { "❌ FAIL" });

        let overall_pass = p50_ok && p99_ok && outliers_ok;
        println!("\n  Overall: {}", if overall_pass { "✅ PRODUCTION READY" } else { "❌ NEEDS OPTIMIZATION" });
    }

    fn print_stats(&self, stats: &LatencyStats, target_p99_us: u64) {
        let target_met = if stats.meets_target(target_p99_us) { "✅" } else { "❌" };
        println!("  Sample count: {}", stats.sample_count);
        println!("  P50: {}μs", stats.p50_us);
        println!("  P90: {}μs", stats.p90_us);
        println!("  P99: {}μs {} (target: <{}μs)", stats.p99_us, target_met, target_p99_us);
        println!("  Mean: {}μs", stats.mean_us);
        println!("  Max: {}μs", stats.max_us);
    }

    fn assert_production_ready(&self) {
        // P50 latency target
        assert!(
            self.total.p50_us <= 50,
            "P50 latency {}μs exceeds target 50μs",
            self.total.p50_us
        );

        // P99 latency target
        assert!(
            self.total.p99_us <= 100,
            "P99 latency {}μs exceeds target 100μs",
            self.total.p99_us
        );

        // No outliers >500μs
        assert!(
            self.total.max_us <= 500,
            "Max latency {}μs exceeds target 500μs",
            self.total.max_us
        );

        // Component-level targets
        assert!(
            self.wave_c.meets_target(40),
            "Wave C P99 latency {}μs exceeds target 40μs",
            self.wave_c.p99_us
        );

        assert!(
            self.wave_d_cusum.meets_target(10),
            "Wave D CUSUM P99 latency {}μs exceeds target 10μs",
            self.wave_d_cusum.p99_us
        );

        assert!(
            self.wave_d_adx.meets_target(5),
            "Wave D ADX P99 latency {}μs exceeds target 5μs",
            self.wave_d_adx.p99_us
        );

        assert!(
            self.wave_d_transition.meets_target(5),
            "Wave D Transition P99 latency {}μs exceeds target 5μs",
            self.wave_d_transition.p99_us
        );

        assert!(
            self.wave_d_adaptive.meets_target(5),
            "Wave D Adaptive P99 latency {}μs exceeds target 5μs",
            self.wave_d_adaptive.p99_us
        );
    }
}

/// Simplified OHLCV bar for testing
#[derive(Debug, Clone)]
struct OHLCVBar {
    timestamp: DateTime<Utc>,
    open: f64,
    high: f64,
    low: f64,
    close: f64,
    volume: f64,
}

/// Generate synthetic ES.FUT-like data
fn generate_test_bars(count: usize) -> Vec<OHLCVBar> {
    let mut bars = Vec::with_capacity(count);
    let mut price = 4500.0; // ES.FUT typical price level

    for i in 0..count {
        let timestamp = Utc::now() + chrono::Duration::seconds(i as i64);

        // Simulate realistic price movement
        let volatility = 0.05; // 5% volatility
        let change = (i as f64 * 0.01).sin() * volatility * price / 100.0;
        price += change;

        let open = price;
        let high = price * 1.001; // 0.1% spread
        let low = price * 0.999;
        let close = price + (i as f64 * 0.001).cos() * price * 0.0005;
        let volume = 1000.0 + (i as f64 * 10.0);

        bars.push(OHLCVBar {
            timestamp,
            open,
            high,
            low,
            close,
            volume,
        });
    }

    bars
}

#[test]
#[ignore] // Run explicitly with: cargo test -p ml --test wave_d_latency_profiling_test -- --ignored --nocapture
fn test_wave_d_225_feature_latency_profiling() -> Result<()> {
    println!("\n🔍 Starting 225-Feature Pipeline Latency Profiling...\n");

    // Generate test data (1000 bars)
    let bars = generate_test_bars(1000);
    println!("Generated {} ES.FUT-like test bars", bars.len());

    // Initialize profiler
    let mut profiler = FeatureLatencyProfiler::new();

    // Warmup phase (10 iterations)
    println!("Running warmup phase (10 iterations)...");
    for bar in bars.iter().take(10) {
        profiler.profile_bar(bar)?;
    }

    // Reset profiler after warmup
    profiler = FeatureLatencyProfiler::new();

    // Profiling phase (1000 iterations)
    println!("Running profiling phase (1000 iterations)...");
    for (i, bar) in bars.iter().enumerate() {
        profiler.profile_bar(bar)?;

        if (i + 1) % 100 == 0 {
            println!("  Processed {}/{} bars...", i + 1, bars.len());
        }
    }

    // Generate and print report
    let report = profiler.generate_report();
    report.print_summary();

    // Assert production readiness
    report.assert_production_ready();

    println!("\n✅ Latency profiling complete - all targets met!\n");

    Ok(())
}

#[test]
fn test_latency_histogram_basic() {
    let mut hist = LatencyHistogram::new();

    // Record test latencies
    hist.record(10);
    hist.record(20);
    hist.record(30);
    hist.record(40);
    hist.record(50);

    assert_eq!(hist.p50(), 30);
    assert_eq!(hist.mean(), 30);
    assert_eq!(hist.max(), 50);
}

#[test]
fn test_latency_stats_target_checking() {
    let mut hist = LatencyHistogram::new();

    for i in 1..=100 {
        hist.record(i);
    }

    let stats = LatencyStats::from_histogram(&hist);

    // P99 should be 99μs
    assert_eq!(stats.p99_us, 99);

    // Should meet 100μs target
    assert!(stats.meets_target(100));

    // Should not meet 50μs target
    assert!(!stats.meets_target(50));
}

#[test]
fn test_feature_extractor_placeholder() -> Result<()> {
    let profiler = FeatureLatencyProfiler::new();

    let bar = OHLCVBar {
        timestamp: Utc::now(),
        open: 4500.0,
        high: 4505.0,
        low: 4495.0,
        close: 4502.0,
        volume: 1000.0,
    };

    // Test all extractors return correct dimensions
    let wave_c = profiler.extract_wave_c_features(&bar)?;
    assert_eq!(wave_c.len(), 201);

    let cusum = profiler.extract_cusum_features(&bar)?;
    assert_eq!(cusum.len(), 10);

    let adx = profiler.extract_adx_features(&bar)?;
    assert_eq!(adx.len(), 5);

    let transition = profiler.extract_transition_features(&bar)?;
    assert_eq!(transition.len(), 5);

    let adaptive = profiler.extract_adaptive_features(&bar)?;
    assert_eq!(adaptive.len(), 4);

    Ok(())
}