foxhunt/data/tests/parquet_persistence_tests.rs

//! Comprehensive tests for Parquet persistence functionality
//! Target: 35+ test functions for full coverage

use anyhow::Result;
use chrono::{DateTime, Utc};
use data::parquet_persistence::{
    MarketDataEvent, ParquetConfig, ParquetMarketDataReader, ParquetMarketDataWriter,
};
use parquet::basic::Compression;
use parquet::file::properties::EnabledStatistics;
use std::collections::HashMap;
use std::fs;
use std::path::Path;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use tempfile::TempDir;
use tokio::time::{sleep, Duration};
use tracing_subscriber;

// Test utilities and setup
struct TestSetup {
    temp_dir: TempDir,
    config: ParquetConfig,
}

impl TestSetup {
    fn new() -> Self {
        let temp_dir = tempfile::tempdir().expect("Failed to create temp directory");
        let config = ParquetConfig {
            base_path: temp_dir.path().to_string_lossy().to_string(),
            batch_size: 100,
            flush_interval_ms: 1000,
            compression: Compression::SNAPPY,
            enable_dictionary: true,
            enable_statistics: EnabledStatistics::Page,
        };
        Self { temp_dir, config }
    }

    fn custom_config(batch_size: usize, flush_interval_ms: u64) -> Self {
        let temp_dir = tempfile::tempdir().expect("Failed to create temp directory");
        let config = ParquetConfig {
            base_path: temp_dir.path().to_string_lossy().to_string(),
            batch_size,
            flush_interval_ms,
            compression: Compression::SNAPPY,
            enable_dictionary: true,
            enable_statistics: EnabledStatistics::Page,
        };
        Self { temp_dir, config }
    }

    fn with_compression(compression: Compression) -> Self {
        let temp_dir = tempfile::tempdir().expect("Failed to create temp directory");
        let config = ParquetConfig {
            base_path: temp_dir.path().to_string_lossy().to_string(),
            batch_size: 100,
            flush_interval_ms: 1000,
            compression,
            enable_dictionary: true,
            enable_statistics: EnabledStatistics::Page,
        };
        Self { temp_dir, config }
    }
}

fn create_test_event(timestamp_ns: u64, symbol: &str, sequence: u64) -> MarketDataEvent {
    MarketDataEvent {
        timestamp_ns,
        symbol: symbol.to_string(),
        venue: "test_venue".to_string(),
        event_type: "trade".to_string(),
        price: Some(100.0 + sequence as f64),
        quantity: Some(1.0),
        bid_price: Some(99.5),
        ask_price: Some(100.5),
        bid_size: Some(10.0),
        ask_size: Some(15.0),
        sequence,
        latency_ns: Some(1000),
    }
}

fn create_quote_event(timestamp_ns: u64, symbol: &str, sequence: u64) -> MarketDataEvent {
    MarketDataEvent {
        timestamp_ns,
        symbol: symbol.to_string(),
        venue: "test_venue".to_string(),
        event_type: "quote".to_string(),
        price: None,
        quantity: None,
        bid_price: Some(99.0 + sequence as f64 * 0.1),
        ask_price: Some(101.0 + sequence as f64 * 0.1),
        bid_size: Some(100.0),
        ask_size: Some(150.0),
        sequence,
        latency_ns: Some(500),
    }
}

// Initialize test logging (call once per test process)
fn init_logging() {
    let _ = tracing_subscriber::fmt().with_test_writer().try_init();
}

// === BASIC FUNCTIONALITY TESTS ===

#[tokio::test]
async fn test_parquet_config_default() {
    let config = ParquetConfig::default();
    assert_eq!(config.base_path, "./market_data");
    assert_eq!(config.batch_size, 10000);
    assert_eq!(config.flush_interval_ms, 5000);
    assert_eq!(config.compression, Compression::SNAPPY);
    assert!(config.enable_dictionary);
    assert_eq!(config.enable_statistics, EnabledStatistics::Page);
}

#[tokio::test]
async fn test_market_data_event_creation() {
    let event = create_test_event(1234567890000000000, "BTCUSD", 1);
    assert_eq!(event.timestamp_ns, 1234567890000000000);
    assert_eq!(event.symbol, "BTCUSD");
    assert_eq!(event.venue, "test_venue");
    assert_eq!(event.event_type, "trade");
    assert_eq!(event.price, Some(101.0));
    assert_eq!(event.sequence, 1);
}

#[tokio::test]
async fn test_parquet_writer_creation() {
    init_logging();
    let setup = TestSetup::new();
    
    let writer = ParquetMarketDataWriter::new(setup.config).await;
    assert!(writer.is_ok(), "Failed to create ParquetMarketDataWriter");
}

#[tokio::test]
async fn test_parquet_writer_creation_invalid_path() {
    init_logging();
    let config = ParquetConfig {
        base_path: "/invalid/path/that/cannot/be/created".to_string(),
        ..Default::default()
    };
    
    let writer = ParquetMarketDataWriter::new(config).await;
    assert!(writer.is_err(), "Should fail with invalid path");
}

#[tokio::test]
async fn test_single_event_recording() {
    init_logging();
    let setup = TestSetup::custom_config(1, 100); // Immediate flush
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    let event = create_test_event(1234567890000000000, "ETHUSD", 1);
    
    let result = writer.record(event);
    assert!(result.is_ok(), "Failed to record event");
    
    // Wait for background processing
    sleep(Duration::from_millis(200)).await;
    
    // Check that file was created
    let files: Vec<_> = fs::read_dir(setup.temp_dir.path())
        .unwrap()
        .filter_map(|entry| {
            let entry = entry.ok()?;
            let path = entry.path();
            if path.extension()?.to_str()? == "parquet" {
                Some(path)
            } else {
                None
            }
        })
        .collect();
    
    assert_eq!(files.len(), 1, "Expected exactly one parquet file");
}

// === BATCH PROCESSING TESTS ===

#[tokio::test]
async fn test_batch_size_flush() {
    init_logging();
    let setup = TestSetup::custom_config(5, 10000); // Large flush interval
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    
    // Send exactly batch_size events
    for i in 0..5 {
        let event = create_test_event(1234567890000000000 + i * 1000, "BTCUSD", i);
        writer.record(event).unwrap();
    }
    
    // Wait for batch flush
    sleep(Duration::from_millis(200)).await;
    
    let files: Vec<_> = fs::read_dir(setup.temp_dir.path()).unwrap()
        .filter_map(|entry| {
            let entry = entry.ok()?;
            let path = entry.path();
            if path.extension()?.to_str()? == "parquet" {
                Some(path)
            } else {
                None
            }
        })
        .collect();
    
    assert_eq!(files.len(), 1, "Expected one parquet file after batch flush");
}

#[tokio::test]
async fn test_time_based_flush() {
    init_logging();
    let setup = TestSetup::custom_config(1000, 100); // Small flush interval
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    
    // Send fewer events than batch size
    for i in 0..3 {
        let event = create_test_event(1234567890000000000 + i * 1000, "ETHUSD", i);
        writer.record(event).unwrap();
    }
    
    // Wait for time-based flush
    sleep(Duration::from_millis(300)).await;
    
    let files: Vec<_> = fs::read_dir(setup.temp_dir.path()).unwrap()
        .filter_map(|entry| {
            let entry = entry.ok()?;
            let path = entry.path();
            if path.extension()?.to_str()? == "parquet" {
                Some(path)
            } else {
                None
            }
        })
        .collect();
    
    assert_eq!(files.len(), 1, "Expected one parquet file after time flush");
}

#[tokio::test]
async fn test_multiple_batches() {
    init_logging();
    let setup = TestSetup::custom_config(3, 10000);
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    
    // Send two full batches
    for i in 0..6 {
        let event = create_test_event(1234567890000000000 + i * 1000, "BTCUSD", i);
        writer.record(event).unwrap();
    }
    
    sleep(Duration::from_millis(300)).await;
    
    let files: Vec<_> = fs::read_dir(setup.temp_dir.path()).unwrap()
        .filter_map(|entry| {
            let entry = entry.ok()?;
            let path = entry.path();
            if path.extension()?.to_str()? == "parquet" {
                Some(path)
            } else {
                None
            }
        })
        .collect();
    
    assert_eq!(files.len(), 2, "Expected two parquet files for two batches");
}

// === COMPRESSION TESTS ===

#[tokio::test]
async fn test_snappy_compression() {
    init_logging();
    let setup = TestSetup::with_compression(Compression::SNAPPY);
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    let event = create_test_event(1234567890000000000, "BTCUSD", 1);
    
    writer.record(event).unwrap();
    sleep(Duration::from_millis(200)).await;
    
    let files: Vec<_> = fs::read_dir(setup.temp_dir.path()).unwrap()
        .filter_map(|entry| {
            let entry = entry.ok()?;
            let path = entry.path();
            if path.extension()?.to_str()? == "parquet" {
                Some(path)
            } else {
                None
            }
        })
        .collect();
    
    assert_eq!(files.len(), 1);
    assert!(files[0].metadata().unwrap().len() > 0);
}

#[tokio::test]
async fn test_gzip_compression() {
    init_logging();
    let setup = TestSetup::with_compression(Compression::GZIP(parquet::basic::GzipLevel::default()));
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    let event = create_test_event(1234567890000000000, "ETHUSD", 1);
    
    writer.record(event).unwrap();
    sleep(Duration::from_millis(200)).await;
    
    let files: Vec<_> = fs::read_dir(setup.temp_dir.path()).unwrap()
        .filter_map(|entry| {
            let entry = entry.ok()?;
            let path = entry.path();
            if path.extension()?.to_str()? == "parquet" {
                Some(path)
            } else {
                None
            }
        })
        .collect();
    
    assert_eq!(files.len(), 1);
}

#[tokio::test]
async fn test_lz4_compression() {
    init_logging();
    let setup = TestSetup::with_compression(Compression::LZ4);
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    let event = create_test_event(1234567890000000000, "ADAUSD", 1);
    
    writer.record(event).unwrap();
    sleep(Duration::from_millis(200)).await;
    
    let files: Vec<_> = fs::read_dir(setup.temp_dir.path()).unwrap()
        .filter_map(|entry| {
            let entry = entry.ok()?;
            let path = entry.path();
            if path.extension()?.to_str()? == "parquet" {
                Some(path)
            } else {
                None
            }
        })
        .collect();
    
    assert_eq!(files.len(), 1);
}

#[tokio::test]
async fn test_uncompressed() {
    init_logging();
    let setup = TestSetup::with_compression(Compression::UNCOMPRESSED);
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    let event = create_test_event(1234567890000000000, "SOLUSD", 1);
    
    writer.record(event).unwrap();
    sleep(Duration::from_millis(200)).await;
    
    let files: Vec<_> = fs::read_dir(setup.temp_dir.path()).unwrap()
        .filter_map(|entry| {
            let entry = entry.ok()?;
            let path = entry.path();
            if path.extension()?.to_str()? == "parquet" {
                Some(path)
            } else {
                None
            }
        })
        .collect();
    
    assert_eq!(files.len(), 1);
}

// === DATA TYPE TESTS ===

#[tokio::test]
async fn test_trade_events() {
    init_logging();
    let setup = TestSetup::custom_config(1, 100);
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    let event = MarketDataEvent {
        timestamp_ns: 1234567890000000000,
        symbol: "BTCUSD".to_string(),
        venue: "binance".to_string(),
        event_type: "trade".to_string(),
        price: Some(50000.0),
        quantity: Some(0.1),
        bid_price: None,
        ask_price: None,
        bid_size: None,
        ask_size: None,
        sequence: 1,
        latency_ns: Some(1000),
    };
    
    writer.record(event).unwrap();
    sleep(Duration::from_millis(200)).await;
    
    let files: Vec<_> = fs::read_dir(setup.temp_dir.path()).unwrap()
        .filter_map(|entry| {
            let entry = entry.ok()?;
            let path = entry.path();
            if path.extension()?.to_str()? == "parquet" {
                Some(path)
            } else {
                None
            }
        })
        .collect();
    
    assert_eq!(files.len(), 1);
}

#[tokio::test]
async fn test_quote_events() {
    init_logging();
    let setup = TestSetup::custom_config(1, 100);
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    let event = create_quote_event(1234567890000000000, "ETHUSD", 1);
    
    writer.record(event).unwrap();
    sleep(Duration::from_millis(200)).await;
    
    let files: Vec<_> = fs::read_dir(setup.temp_dir.path()).unwrap()
        .filter_map(|entry| {
            let entry = entry.ok()?;
            let path = entry.path();
            if path.extension()?.to_str()? == "parquet" {
                Some(path)
            } else {
                None
            }
        })
        .collect();
    
    assert_eq!(files.len(), 1);
}

#[tokio::test]
async fn test_orderbook_events() {
    init_logging();
    let setup = TestSetup::custom_config(1, 100);
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    let event = MarketDataEvent {
        timestamp_ns: 1234567890000000000,
        symbol: "ADAUSD".to_string(),
        venue: "coinbase".to_string(),
        event_type: "orderbook".to_string(),
        price: None,
        quantity: None,
        bid_price: Some(1.25),
        ask_price: Some(1.26),
        bid_size: Some(1000.0),
        ask_size: Some(1500.0),
        sequence: 1,
        latency_ns: Some(2000),
    };
    
    writer.record(event).unwrap();
    sleep(Duration::from_millis(200)).await;
    
    let files: Vec<_> = fs::read_dir(setup.temp_dir.path()).unwrap()
        .filter_map(|entry| {
            let entry = entry.ok()?;
            let path = entry.path();
            if path.extension()?.to_str()? == "parquet" {
                Some(path)
            } else {
                None
            }
        })
        .collect();
    
    assert_eq!(files.len(), 1);
}

#[tokio::test]
async fn test_mixed_event_types() {
    init_logging();
    let setup = TestSetup::custom_config(10, 10000);
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    
    // Mix of different event types
    let events = vec![
        create_test_event(1234567890000000000, "BTCUSD", 1),
        create_quote_event(1234567890000001000, "BTCUSD", 2),
        MarketDataEvent {
            timestamp_ns: 1234567890000002000,
            symbol: "BTCUSD".to_string(),
            venue: "binance".to_string(),
            event_type: "status".to_string(),
            price: None,
            quantity: None,
            bid_price: None,
            ask_price: None,
            bid_size: None,
            ask_size: None,
            sequence: 3,
            latency_ns: None,
        },
    ];
    
    for event in events {
        writer.record(event).unwrap();
    }
    
    sleep(Duration::from_millis(200)).await;
}

// === LARGE DATASET TESTS ===

#[tokio::test]
async fn test_large_batch_processing() {
    init_logging();
    let setup = TestSetup::custom_config(1000, 5000);
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    
    // Send 2500 events (2.5 batches)
    for i in 0..2500 {
        let event = create_test_event(1234567890000000000 + i * 1000, "BTCUSD", i);
        writer.record(event).unwrap();
    }
    
    sleep(Duration::from_millis(1000)).await;
    
    let files: Vec<_> = fs::read_dir(setup.temp_dir.path()).unwrap()
        .filter_map(|entry| {
            let entry = entry.ok()?;
            let path = entry.path();
            if path.extension()?.to_str()? == "parquet" {
                Some(path)
            } else {
                None
            }
        })
        .collect();
    
    assert!(files.len() >= 2, "Expected at least 2 files for large dataset");
}

#[tokio::test]
async fn test_high_frequency_events() {
    init_logging();
    let setup = TestSetup::custom_config(100, 1000);
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    
    // Simulate high-frequency trading events
    let start_time = 1234567890000000000u64;
    for i in 0..500 {
        let event = create_test_event(start_time + i * 1000, "ETHUSD", i);
        writer.record(event).unwrap();
    }
    
    sleep(Duration::from_millis(2000)).await;
    
    let files: Vec<_> = fs::read_dir(setup.temp_dir.path()).unwrap()
        .filter_map(|entry| {
            let entry = entry.ok()?;
            let path = entry.path();
            if path.extension()?.to_str()? == "parquet" {
                Some(path)
            } else {
                None
            }
        })
        .collect();
    
    assert!(files.len() >= 1, "Expected at least 1 file for high-frequency data");
}

// === BUFFER MANAGEMENT TESTS ===

#[tokio::test]
async fn test_buffer_stats() {
    init_logging();
    let setup = TestSetup::custom_config(1000, 10000); // Large batch, long interval
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    
    // Initial stats should show empty buffer
    let stats = writer.get_buffer_stats().await;
    assert_eq!(stats.buffered_events, 0);
    assert!(stats.buffer_capacity > 0);
    assert_eq!(stats.utilization_percent, 0.0);
    
    // Add some events
    for i in 0..10 {
        let event = create_test_event(1234567890000000000 + i * 1000, "BTCUSD", i);
        writer.record(event).unwrap();
    }
    
    // Give time for events to be queued
    sleep(Duration::from_millis(50)).await;
    
    let stats = writer.get_buffer_stats().await;
    // Note: Events might be processed quickly, so we can't guarantee exact count
    assert!(stats.buffer_capacity > 0);
}

#[tokio::test]
async fn test_buffer_utilization() {
    init_logging();
    let setup = TestSetup::custom_config(100, 10000); // Medium batch, long interval
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    
    // Fill buffer partially
    for i in 0..50 {
        let event = create_test_event(1234567890000000000 + i * 1000, "ETHUSD", i);
        writer.record(event).unwrap();
    }
    
    sleep(Duration::from_millis(100)).await;
    
    let stats = writer.get_buffer_stats().await;
    assert!(stats.buffer_capacity >= 100); // At least batch_size * 2
}

// === MULTITHREADING AND CONCURRENCY TESTS ===

#[tokio::test]
async fn test_concurrent_writes() {
    init_logging();
    let setup = TestSetup::custom_config(50, 1000);
    
    let writer = Arc::new(ParquetMarketDataWriter::new(setup.config).await.unwrap());
    let sequence_counter = Arc::new(AtomicU64::new(0));
    
    // Spawn multiple concurrent tasks
    let mut handles = Vec::new();
    for task_id in 0..5 {
        let writer_clone = writer.clone();
        let counter_clone = sequence_counter.clone();
        
        let handle = tokio::spawn(async move {
            for i in 0..20 {
                let seq = counter_clone.fetch_add(1, Ordering::SeqCst);
                let event = create_test_event(
                    1234567890000000000 + seq * 1000,
                    &format!("SYM{}", task_id),
                    seq,
                );
                writer_clone.record(event).unwrap();
            }
        });
        handles.push(handle);
    }
    
    // Wait for all tasks to complete
    for handle in handles {
        handle.await.unwrap();
    }
    
    sleep(Duration::from_millis(2000)).await;
    
    let files: Vec<_> = fs::read_dir(setup.temp_dir.path()).unwrap()
        .filter_map(|entry| {
            let entry = entry.ok()?;
            let path = entry.path();
            if path.extension()?.to_str()? == "parquet" {
                Some(path)
            } else {
                None
            }
        })
        .collect();
    
    assert!(files.len() >= 1, "Expected files from concurrent writes");
}

// === SCHEMA AND PARTITIONING TESTS ===

#[tokio::test]
async fn test_multiple_symbols() {
    init_logging();
    let setup = TestSetup::custom_config(10, 1000);
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    
    let symbols = vec!["BTCUSD", "ETHUSD", "ADAUSD", "SOLUSD", "DOTUSD"];
    
    for (i, symbol) in symbols.iter().enumerate() {
        let event = create_test_event(1234567890000000000 + i as u64 * 1000, symbol, i as u64);
        writer.record(event).unwrap();
    }
    
    sleep(Duration::from_millis(2000)).await;
    
    let files: Vec<_> = fs::read_dir(setup.temp_dir.path()).unwrap()
        .filter_map(|entry| {
            let entry = entry.ok()?;
            let path = entry.path();
            if path.extension()?.to_str()? == "parquet" {
                Some(path)
            } else {
                None
            }
        })
        .collect();
    
    assert!(files.len() >= 1);
}

#[tokio::test]
async fn test_multiple_venues() {
    init_logging();
    let setup = TestSetup::custom_config(10, 1000);
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    
    let venues = vec!["binance", "coinbase", "kraken", "bitstamp", "gemini"];
    
    for (i, venue) in venues.iter().enumerate() {
        let mut event = create_test_event(1234567890000000000 + i as u64 * 1000, "BTCUSD", i as u64);
        event.venue = venue.to_string();
        writer.record(event).unwrap();
    }
    
    sleep(Duration::from_millis(2000)).await;
}

// === FILE NAMING AND ORGANIZATION TESTS ===

#[tokio::test]
async fn test_file_naming_convention() {
    init_logging();
    let setup = TestSetup::custom_config(1, 100);
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    let event = create_test_event(1234567890000000000, "BTCUSD", 1);
    
    writer.record(event).unwrap();
    sleep(Duration::from_millis(200)).await;
    
    let files: Vec<_> = fs::read_dir(setup.temp_dir.path()).unwrap()
        .filter_map(|entry| {
            let entry = entry.ok()?;
            let path = entry.path();
            if path.extension()?.to_str()? == "parquet" {
                path.file_name()?.to_str().map(|s| s.to_string())
            } else {
                None
            }
        })
        .collect();
    
    assert_eq!(files.len(), 1);
    let filename = &files[0];
    assert!(filename.starts_with("market_data_"));
    assert!(filename.ends_with(".parquet"));
    assert!(filename.contains("_")); // Contains timestamp and UUID
}

#[tokio::test]
async fn test_directory_creation() {
    init_logging();
    let temp_dir = tempfile::tempdir().unwrap();
    let nested_path = temp_dir.path().join("nested").join("path");
    
    let config = ParquetConfig {
        base_path: nested_path.to_string_lossy().to_string(),
        batch_size: 1,
        flush_interval_ms: 100,
        compression: Compression::SNAPPY,
        enable_dictionary: true,
        enable_statistics: EnabledStatistics::Page,
    };
    
    let writer = ParquetMarketDataWriter::new(config).await;
    assert!(writer.is_ok(), "Should create nested directories");
    
    let event = create_test_event(1234567890000000000, "TESTCOIN", 1);
    writer.unwrap().record(event).unwrap();
    
    sleep(Duration::from_millis(200)).await;
    assert!(nested_path.exists());
}

// === ERROR HANDLING AND EDGE CASES ===

#[tokio::test]
async fn test_empty_symbol() {
    init_logging();
    let setup = TestSetup::custom_config(1, 100);
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    let mut event = create_test_event(1234567890000000000, "", 1);
    event.symbol = "".to_string();
    
    let result = writer.record(event);
    assert!(result.is_ok(), "Should handle empty symbol");
    
    sleep(Duration::from_millis(200)).await;
}

#[tokio::test]
async fn test_extreme_values() {
    init_logging();
    let setup = TestSetup::custom_config(1, 100);
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    let event = MarketDataEvent {
        timestamp_ns: u64::MAX,
        symbol: "EXTREME".to_string(),
        venue: "test".to_string(),
        event_type: "test".to_string(),
        price: Some(f64::MAX),
        quantity: Some(f64::MIN_POSITIVE),
        bid_price: Some(0.0),
        ask_price: Some(f64::INFINITY),
        bid_size: Some(f64::NEG_INFINITY),
        ask_size: Some(f64::NAN),
        sequence: u64::MAX,
        latency_ns: Some(u64::MAX),
    };
    
    let result = writer.record(event);
    assert!(result.is_ok(), "Should handle extreme values");
    
    sleep(Duration::from_millis(200)).await;
}

#[tokio::test]
async fn test_unicode_symbols() {
    init_logging();
    let setup = TestSetup::custom_config(1, 100);
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    let mut event = create_test_event(1234567890000000000, "测试币", 1);
    event.venue = "交易所".to_string();
    
    let result = writer.record(event);
    assert!(result.is_ok(), "Should handle Unicode strings");
    
    sleep(Duration::from_millis(200)).await;
}

#[tokio::test]
async fn test_null_optional_fields() {
    init_logging();
    let setup = TestSetup::custom_config(1, 100);
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    let event = MarketDataEvent {
        timestamp_ns: 1234567890000000000,
        symbol: "NULLTEST".to_string(),
        venue: "test".to_string(),
        event_type: "test".to_string(),
        price: None,
        quantity: None,
        bid_price: None,
        ask_price: None,
        bid_size: None,
        ask_size: None,
        sequence: 1,
        latency_ns: None,
    };
    
    let result = writer.record(event);
    assert!(result.is_ok(), "Should handle all null optional fields");
    
    sleep(Duration::from_millis(200)).await;
}

// === READER TESTS ===

#[tokio::test]
async fn test_reader_creation() {
    let temp_dir = tempfile::tempdir().unwrap();
    let reader = ParquetMarketDataReader::new(temp_dir.path().to_string_lossy().to_string());
    
    // Just test that reader can be created
    assert_eq!(reader.base_path, temp_dir.path().to_string_lossy().to_string());
}

#[tokio::test]
async fn test_reader_list_empty_directory() {
    let temp_dir = tempfile::tempdir().unwrap();
    let reader = ParquetMarketDataReader::new(temp_dir.path().to_string_lossy().to_string());
    
    let files = reader.list_available_files().await.unwrap();
    assert_eq!(files.len(), 0, "Empty directory should have no files");
}

#[tokio::test]
async fn test_reader_list_with_parquet_files() {
    let temp_dir = tempfile::tempdir().unwrap();
    
    // Create some test files
    std::fs::write(temp_dir.path().join("test1.parquet"), b"fake parquet").unwrap();
    std::fs::write(temp_dir.path().join("test2.parquet"), b"fake parquet").unwrap();
    std::fs::write(temp_dir.path().join("test.txt"), b"not parquet").unwrap();
    
    let reader = ParquetMarketDataReader::new(temp_dir.path().to_string_lossy().to_string());
    let files = reader.list_available_files().await.unwrap();
    
    assert_eq!(files.len(), 2, "Should find only parquet files");
    assert!(files.contains(&"test1.parquet".to_string()));
    assert!(files.contains(&"test2.parquet".to_string()));
    assert!(files[0] <= files[1], "Files should be sorted");
}

#[tokio::test]
async fn test_reader_invalid_directory() {
    let reader = ParquetMarketDataReader::new("/invalid/path/that/does/not/exist".to_string());
    
    let result = reader.list_available_files().await;
    assert!(result.is_err(), "Should fail for invalid directory");
}

#[tokio::test]
async fn test_reader_read_placeholder() {
    let temp_dir = tempfile::tempdir().unwrap();
    let reader = ParquetMarketDataReader::new(temp_dir.path().to_string_lossy().to_string());
    
    // Test placeholder implementation
    let events = reader.read_file("nonexistent.parquet").await.unwrap();
    assert_eq!(events.len(), 0, "Placeholder should return empty vec");
}

// === PERFORMANCE AND MONITORING TESTS ===

#[tokio::test]
async fn test_metrics_integration() {
    init_logging();
    let setup = TestSetup::custom_config(1, 100);
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    let event = create_test_event(1234567890000000000, "METRICSTEST", 1);
    
    writer.record(event).unwrap();
    sleep(Duration::from_millis(300)).await;
    
    // Metrics should be recorded, but we can't easily test them without
    // accessing the actual metrics registry
}

#[tokio::test]
async fn test_performance_timing() {
    init_logging();
    let setup = TestSetup::custom_config(100, 5000);
    
    let writer = ParquetMarketDataWriter::new(setup.config).await.unwrap();
    
    let start = std::time::Instant::now();
    
    // Write many events quickly
    for i in 0..1000 {
        let event = create_test_event(1234567890000000000 + i * 1000, "PERFTEST", i);
        writer.record(event).unwrap();
    }
    
    let record_duration = start.elapsed();
    println!("Recorded 1000 events in {:?}", record_duration);
    
    // Should be very fast for recording (just queuing)
    assert!(record_duration.as_millis() < 100, "Recording should be fast");
    
    sleep(Duration::from_millis(2000)).await;
}