foxhunt/crates/ml/tests/dqn_feature_cache_test.rs

#![allow(
    clippy::assertions_on_constants,
    clippy::assertions_on_result_states,
    clippy::clone_on_copy,
    clippy::decimal_literal_representation,
    clippy::doc_markdown,
    clippy::empty_line_after_doc_comments,
    clippy::field_reassign_with_default,
    clippy::get_unwrap,
    clippy::identity_op,
    clippy::inconsistent_digit_grouping,
    clippy::indexing_slicing,
    clippy::integer_division,
    clippy::len_zero,
    clippy::let_underscore_must_use,
    clippy::manual_div_ceil,
    clippy::manual_let_else,
    clippy::manual_range_contains,
    clippy::modulo_arithmetic,
    clippy::needless_range_loop,
    clippy::non_ascii_literal,
    clippy::redundant_clone,
    clippy::shadow_reuse,
    clippy::shadow_same,
    clippy::shadow_unrelated,
    clippy::single_match_else,
    clippy::str_to_string,
    clippy::string_slice,
    clippy::tests_outside_test_module,
    clippy::too_many_lines,
    clippy::unnecessary_wraps,
    clippy::unseparated_literal_suffix,
    clippy::use_debug,
    clippy::useless_vec,
    clippy::wildcard_enum_match_arm,
    clippy::else_if_without_else,
    clippy::expect_used,
    clippy::missing_const_for_fn,
    clippy::similar_names,
    clippy::type_complexity,
    clippy::collapsible_else_if,
    clippy::doc_lazy_continuation,
    clippy::items_after_test_module,
    clippy::map_clone,
    clippy::multiple_unsafe_ops_per_block,
    clippy::unwrap_or_default,
    clippy::assign_op_pattern,
    clippy::needless_borrow,
    clippy::println_empty_string,
    clippy::unnecessary_cast,
    clippy::used_underscore_binding,
    clippy::create_dir,
    clippy::implicit_saturating_sub,
    clippy::exit,
    clippy::expect_fun_call,
    clippy::too_many_arguments,
    clippy::unnecessary_map_or,
    clippy::unwrap_used,
    dead_code,
    unused_imports,
    unused_variables,
    clippy::cloned_ref_to_slice_refs,
    clippy::neg_multiply,
    clippy::while_let_loop,
    clippy::bool_assert_comparison,
    clippy::excessive_precision,
    clippy::trivially_copy_pass_by_ref,
    clippy::op_ref,
    clippy::redundant_closure,
    clippy::unnecessary_lazy_evaluations,
    clippy::if_then_some_else_none,
    clippy::unnecessary_to_owned,
    clippy::single_component_path_imports,
)]
//! # DQN Feature Cache Integration Tests
//!
//! Comprehensive test suite for the DQN feature caching system that pre-computes
//! 51-feature vectors (43 base + 8 OFI) once and reuses them across hyperopt trials.
//!
//! ## Test Coverage
//!
//! 1. **Cache Creation Success**: Verify cache file creation with correct metadata
//! 2. **Cache Loading Matches Original**: Ensure cached features match non-cached computation
//! 3. **Cache Invalidation**: Test cache key changes when input data changes
//! 4. **Hyperopt Integration**: Multi-trial cache reuse verification
//! 5. **Performance Benchmark**: Validate 20s → <1s loading speedup
//! 6. **Edge Cases**: Missing cache, corrupted cache, multiple datasets
//!
//! ## Expected Performance Gains
//!
//! - Cache creation: ~20-30s (one-time setup)
//! - Cache loading: <1s per trial
//! - Time savings: 87-96% for 50-trial hyperopt
//! - Cache size: 3-5 MB compressed Parquet per dataset
//!
//! Design reference: `/tmp/MBP10_OFI_CACHING_DESIGN.md` section 8

use anyhow::{Context, Result};
use std::fs;
use std::path::Path;
use std::time::{Duration, Instant};
use tempfile::TempDir;
use std::ffi::OsStr;
use tracing::{info, warn};


// Cache functions (to be implemented based on design doc)
// These will initially fail compilation until implemented

/// Calculate SHA256-based cache key from input data
///
/// Components:
/// - Parquet file path + modification time
/// - MBP-10 directory path + file list + modification times
/// - Feature extraction version (code hash)
/// - Warmup period constant
fn calculate_cache_key(
    parquet_path: &Path,
    mbp10_dir: &Path,
    warmup_period: usize,
) -> Result<String> {
    use sha2::{Digest, Sha256};
    use std::time::UNIX_EPOCH;

    let mut hasher = Sha256::new();

    // 1. Parquet file path + mtime
    hasher.update(parquet_path.to_string_lossy().as_bytes());
    let parquet_mtime = parquet_path
        .metadata()
        .context("Failed to read parquet metadata")?
        .modified()
        .context("Failed to get parquet mtime")?
        .duration_since(UNIX_EPOCH)
        .context("System time error")?
        .as_secs();
    hasher.update(&parquet_mtime.to_le_bytes());

    // 2. MBP-10 directory + all .dbn files + mtimes
    hasher.update(mbp10_dir.to_string_lossy().as_bytes());
    
    if mbp10_dir.exists() {
        let mut dbn_files: Vec<_> = fs::read_dir(mbp10_dir)
            .context("Failed to read mbp10 directory")?
            .filter_map(|e| e.ok())
            .filter(|e| e.path().extension() == Some(OsStr::new("dbn")))
            .collect();
        
        dbn_files.sort_by_key(|e| e.path());
        
        for entry in &dbn_files {
            hasher.update(entry.path().to_string_lossy().as_bytes());
            let mtime = entry
                .metadata()
                .context("Failed to read dbn metadata")?
                .modified()
                .context("Failed to get dbn mtime")?
                .duration_since(UNIX_EPOCH)
                .context("System time error")?
                .as_secs();
            hasher.update(&mtime.to_le_bytes());
        }
    }

    // 3. Feature extraction version (simplified - use constant for now)
    // In production, this would hash the extraction.rs source file
    let feature_version = "v1.0.0";
    hasher.update(feature_version.as_bytes());

    // 4. Warmup period
    hasher.update(&warmup_period.to_le_bytes());

    Ok(format!("{:x}", hasher.finalize()))
}

/// Create feature cache from parquet and MBP-10 data
///
/// This is a placeholder that will be implemented based on the design doc.
/// Expected behavior:
/// - Load data from parquet_path and mbp10_dir
/// - Extract 51-feature vectors
/// - Save to cache_dir as Parquet file
/// - Create metadata JSON file
async fn create_feature_cache(
    _parquet_path: &Path,
    _mbp10_dir: &Path,
    _cache_dir: &Path,
    _warmup_period: usize,
) -> Result<String> {
    // NOTE: This is a stub that will fail until implemented
    // Implementation should follow design doc section 4 (Phase 1)
    
    anyhow::bail!(
        "create_feature_cache not yet implemented - see /tmp/MBP10_OFI_CACHING_DESIGN.md Phase 1"
    )
}

/// Load features from cache
///
/// Returns (train_data, val_data) tuples matching DQN trainer format.
/// Each sample is (FeatureVector, Vec<f64>) where Vec<f64> contains target prices.
async fn load_features_from_cache(
    _cache_dir: &Path,
    _parquet_path: &Path,
    _mbp10_dir: &Path,
    _warmup_period: usize,
) -> Result<Option<(Vec<([f64; 42], Vec<f64>)>, Vec<([f64; 42], Vec<f64>)>)>> {
    // NOTE: This is a stub that will fail until implemented
    // Implementation should follow design doc section 4 (Phase 2)
    
    anyhow::bail!(
        "load_features_from_cache not yet implemented - see /tmp/MBP10_OFI_CACHING_DESIGN.md Phase 2"
    )
}

// ============================================================================
// TEST 1: Cache Creation Success
// ============================================================================

#[tokio::test]
#[ignore] // Ignore until cache creation is implemented
async fn test_cache_creation_success() -> Result<()> {
    // Setup
    let temp_dir = TempDir::new().context("Failed to create temp directory")?;
    let cache_dir = temp_dir.path().join("cache");
    fs::create_dir_all(&cache_dir).context("Failed to create cache directory")?;
    
    // Create cache
    let result = create_feature_cache(
        Path::new("test_data/ES_FUT_180d.parquet"),
        Path::new("test_data/mbp10"),
        &cache_dir,
        50,
    ).await;
    
    assert!(
        result.is_ok(),
        "Cache creation should succeed: {:?}",
        result.err()
    );
    
    // Verify cache file exists
    let cache_files: Vec<_> = fs::read_dir(&cache_dir)
        .context("Failed to read cache directory")?
        .filter_map(|e| e.ok())
        .filter(|e| e.path().extension() == Some(OsStr::new("parquet")))
        .collect();
    
    assert_eq!(
        cache_files.len(),
        1,
        "Should create exactly one cache file"
    );
    
    // Verify metadata exists
    let metadata_path = cache_dir.join("cache_metadata.json");
    assert!(
        metadata_path.exists(),
        "Metadata file should exist at {:?}",
        metadata_path
    );
    
    // Verify file size (should be 3-5 MB compressed)
    let file_size = fs::metadata(cache_files[0].path())
        .context("Failed to read cache file metadata")?
        .len();
    
    assert!(
        file_size > 2_000_000,
        "Cache file too small: {} bytes (expected >2MB)",
        file_size
    );
    assert!(
        file_size < 10_000_000,
        "Cache file too large: {} bytes (expected <10MB)",
        file_size
    );
    
    info!(file_size_kb = file_size / 1024, "Cache created successfully");
    Ok(())
}

// ============================================================================
// TEST 2: Cache Loading Matches Original
// ============================================================================

#[tokio::test]
#[ignore] // Ignore until cache loading is implemented
async fn test_cache_loading_matches_original() -> Result<()> {
    let temp_dir = TempDir::new().context("Failed to create temp directory")?;
    let cache_dir = temp_dir.path().join("cache");
    fs::create_dir_all(&cache_dir).context("Failed to create cache directory")?;
    
    // Create cache
    create_feature_cache(
        Path::new("test_data/ES_FUT_180d.parquet"),
        Path::new("test_data/mbp10"),
        &cache_dir,
        50,
    )
    .await
    .context("Failed to create cache")?;
    
    // Load from cache
    let (cached_train, cached_val) = load_features_from_cache(
        &cache_dir,
        Path::new("test_data/ES_FUT_180d.parquet"),
        Path::new("test_data/mbp10"),
        50,
    )
    .await
    .context("Failed to load from cache")?
    .expect("Cache should exist");
    
    // Load original way (no cache) - this requires DQN trainer modification
    // For now, we'll verify the cached data structure is correct
    
    assert!(
        !cached_train.is_empty(),
        "Training data should not be empty"
    );
    assert!(
        !cached_val.is_empty(),
        "Validation data should not be empty"
    );
    
    // Verify feature dimensions
    let (features, targets) = &cached_train[0];
    assert_eq!(
        features.len(),
        51,
        "Each sample should have 51 features"
    );
    assert_eq!(
        targets.len(),
        4,
        "Each sample should have 4 target prices"
    );
    
    // Verify train/val split is approximately 80/20
    let total = cached_train.len() + cached_val.len();
    let train_ratio = cached_train.len() as f64 / total as f64;
    assert!(
        (train_ratio - 0.8).abs() < 0.05,
        "Train/val split should be ~80/20, got {:.2}",
        train_ratio
    );
    
    // Compare first 10 samples (features should be deterministic)
    for i in 0..10.min(cached_train.len()) {
        let features = &cached_train[i].0;
        
        // Verify no NaN or Inf values
        for (j, &value) in features.iter().enumerate() {
            assert!(
                value.is_finite(),
                "Feature {} in sample {} is not finite: {}",
                j,
                i,
                value
            );
        }
    }
    
    info!(train_samples = cached_train.len(), val_samples = cached_val.len(), "Cache loaded successfully");
    Ok(())
}

// ============================================================================
// TEST 3: Cache Invalidation on Data Change
// ============================================================================

#[tokio::test]
async fn test_cache_invalidation_on_data_change() -> Result<()> {
    let temp_dir = TempDir::new().context("Failed to create temp directory")?;
    
    // Create a temporary parquet file
    let parquet_path = temp_dir.path().join("test.parquet");
    fs::write(&parquet_path, b"mock parquet data v1")
        .context("Failed to write parquet file")?;
    
    let mbp10_dir = temp_dir.path().join("mbp10");
    fs::create_dir_all(&mbp10_dir).context("Failed to create mbp10 directory")?;
    
    // Calculate initial cache key
    let cache_key1 = calculate_cache_key(&parquet_path, &mbp10_dir, 50)
        .context("Failed to calculate initial cache key")?;
    
    // Wait to ensure mtime changes (Linux has second-level granularity in most filesystems)
    std::thread::sleep(Duration::from_secs(2));
    
    // Modify file content to trigger mtime update
    fs::write(&parquet_path, b"mock parquet data v2 - MODIFIED CONTENT")
        .context("Failed to modify parquet file")?;
    
    // Calculate new cache key
    let cache_key2 = calculate_cache_key(&parquet_path, &mbp10_dir, 50)
        .context("Failed to calculate new cache key")?;
    
    info!(cache_key1 = &cache_key1[..16], "Cache key 1 (original)");
    info!(cache_key2 = &cache_key2[..16], "Cache key 2 (after modification)");
    
    // Keys should differ (cache invalidated)
    assert_ne!(
        cache_key1, cache_key2,
        "Cache key should change when data file is modified.\nKey1: {}\nKey2: {}",
        cache_key1, cache_key2
    );
    
    info!(old_key = &cache_key1[..16], new_key = &cache_key2[..16], "Cache invalidation works");
    Ok(())
}

// ============================================================================
// TEST 4: Cache Key Stability
// ============================================================================

#[tokio::test]
async fn test_cache_key_stability() -> Result<()> {
    let temp_dir = TempDir::new().context("Failed to create temp directory")?;
    
    // Create test files
    let parquet_path = temp_dir.path().join("test.parquet");
    fs::write(&parquet_path, b"mock parquet data")
        .context("Failed to write parquet file")?;
    
    let mbp10_dir = temp_dir.path().join("mbp10");
    fs::create_dir_all(&mbp10_dir).context("Failed to create mbp10 directory")?;
    
    // Calculate cache key multiple times without modification
    let key1 = calculate_cache_key(&parquet_path, &mbp10_dir, 50)?;
    let key2 = calculate_cache_key(&parquet_path, &mbp10_dir, 50)?;
    let key3 = calculate_cache_key(&parquet_path, &mbp10_dir, 50)?;
    
    // Keys should be identical (deterministic)
    assert_eq!(
        key1, key2,
        "Cache key should be deterministic"
    );
    assert_eq!(
        key2, key3,
        "Cache key should be deterministic"
    );
    
    info!(key = %key1, "Cache key is stable");
    Ok(())
}

// ============================================================================
// TEST 5: Cache Key with Different Warmup Periods
// ============================================================================

#[tokio::test]
async fn test_cache_key_different_warmup() -> Result<()> {
    let temp_dir = TempDir::new().context("Failed to create temp directory")?;
    
    // Create test files
    let parquet_path = temp_dir.path().join("test.parquet");
    fs::write(&parquet_path, b"mock parquet data")
        .context("Failed to write parquet file")?;
    
    let mbp10_dir = temp_dir.path().join("mbp10");
    fs::create_dir_all(&mbp10_dir).context("Failed to create mbp10 directory")?;
    
    // Calculate cache keys with different warmup periods
    let key_warmup_50 = calculate_cache_key(&parquet_path, &mbp10_dir, 50)?;
    let key_warmup_100 = calculate_cache_key(&parquet_path, &mbp10_dir, 100)?;
    
    // Keys should differ (different warmup periods)
    assert_ne!(
        key_warmup_50, key_warmup_100,
        "Cache key should change when warmup period changes"
    );
    
    info!(key_warmup_50 = &key_warmup_50[..16], key_warmup_100 = &key_warmup_100[..16], "Different warmup periods produce different keys");
    Ok(())
}

// ============================================================================
// TEST 6: Hyperopt Integration (Multi-Trial Cache Reuse)
// ============================================================================

#[tokio::test]
#[ignore] // Ignore until hyperopt integration is implemented
async fn test_hyperopt_with_cache() -> Result<()> {
    let temp_dir = TempDir::new().context("Failed to create temp directory")?;
    let cache_dir = temp_dir.path().join("cache");
    fs::create_dir_all(&cache_dir).context("Failed to create cache directory")?;
    
    // Pre-create cache
    create_feature_cache(
        Path::new("test_data/ES_FUT_180d.parquet"),
        Path::new("test_data/mbp10"),
        &cache_dir,
        50,
    )
    .await
    .context("Failed to create cache")?;
    
    // Simulate 3 hyperopt trials (in production, this would use DQNHyperoptAdapter)
    let start = Instant::now();
    
    let mut trial_durations = Vec::new();
    
    for trial in 1..=3 {
        let trial_start = Instant::now();
        
        // Load features from cache
        let (_train, _val) = load_features_from_cache(
            &cache_dir,
            Path::new("test_data/ES_FUT_180d.parquet"),
            Path::new("test_data/mbp10"),
            50,
        )
        .await
        .context("Failed to load from cache")?
        .expect("Cache should exist");
        
        let trial_duration = trial_start.elapsed();
        trial_durations.push(trial_duration);
        
        info!(trial, duration = ?trial_duration, "Loaded features from cache");
    }
    
    let total_duration = start.elapsed();
    
    // Verify results
    assert_eq!(trial_durations.len(), 3, "Should complete all 3 trials");
    
    // Each trial should load in <2s (with cache)
    for (i, duration) in trial_durations.iter().enumerate() {
        assert!(
            duration < &Duration::from_secs(2),
            "Trial {} took too long: {:?} (expected <2s with cache)",
            i + 1,
            duration
        );
    }
    
    // Total should be <10s (generous, in practice should be ~3s)
    assert!(
        total_duration < Duration::from_secs(10),
        "Total hyperopt time with cache took too long: {:?} (expected <10s)",
        total_duration
    );
    
    info!(total_duration = ?total_duration, avg_per_trial = ?(total_duration / 3), "Hyperopt with cache completed");
    Ok(())
}

// ============================================================================
// TEST 7: Performance Benchmark (Cache vs No Cache)
// ============================================================================

#[tokio::test]
#[ignore] // Ignore by default - only run for performance validation
async fn test_cache_performance_benchmark() -> Result<()> {
    let temp_dir = TempDir::new().context("Failed to create temp directory")?;
    let cache_dir = temp_dir.path().join("cache");
    fs::create_dir_all(&cache_dir).context("Failed to create cache directory")?;
    
    info!("DQN Feature Cache Performance Benchmark");

    // Benchmark: Cache creation
    info!("Step 1: Cache creation (one-time setup)");
    let start = Instant::now();
    create_feature_cache(
        Path::new("test_data/ES_FUT_180d.parquet"),
        Path::new("test_data/mbp10"),
        &cache_dir,
        50,
    )
    .await
    .context("Failed to create cache")?;
    let creation_time = start.elapsed();
    info!(creation_time = ?creation_time, "Cache creation time");
    
    // Verify creation time is reasonable
    assert!(
        creation_time < Duration::from_secs(60),
        "Cache creation too slow: {:?} (expected <60s)",
        creation_time
    );
    
    // Benchmark: Cache loading (3 trials)
    info!("Step 2: Cache loading (simulating hyperopt trials)");
    let mut load_times = Vec::new();
    for trial in 1..=3 {
        let start = Instant::now();
        let (_train, _val) = load_features_from_cache(
            &cache_dir,
            Path::new("test_data/ES_FUT_180d.parquet"),
            Path::new("test_data/mbp10"),
            50,
        )
        .await
        .context("Failed to load from cache")?
        .expect("Cache should exist");
        let load_time = start.elapsed();
        load_times.push(load_time);
        info!(trial, load_time = ?load_time, "Cache load time for trial");
    }

    let avg_load_time = load_times.iter().sum::<Duration>() / load_times.len() as u32;

    // Summary
    let without_cache = Duration::from_secs(20 * 50); // 20s per trial
    let with_cache = creation_time + (avg_load_time * 50);
    let savings = without_cache.as_secs() - with_cache.as_secs();
    let savings_pct = (savings as f64 / without_cache.as_secs() as f64) * 100.0;

    info!(creation_time = ?creation_time, avg_load_time = ?avg_load_time, savings_pct, "Performance summary");
    info!(without_cache_secs = without_cache.as_secs(), with_cache_secs = with_cache.as_secs(), savings_secs = savings, "Hyperopt impact (50 trials)");
    
    // Assertions
    assert!(
        avg_load_time < Duration::from_secs(2),
        "Cache loading too slow: {:?} (expected <2s)",
        avg_load_time
    );
    
    assert!(
        savings_pct > 80.0,
        "Cache savings insufficient: {:.1}% (expected >80%)",
        savings_pct
    );
    
    info!("Performance benchmark passed");
    Ok(())
}

// ============================================================================
// TEST 8: Edge Case - Missing Cache Directory
// ============================================================================

#[tokio::test]
#[ignore] // Ignore until cache loading is implemented
async fn test_missing_cache_graceful_fallback() -> Result<()> {
    let temp_dir = TempDir::new().context("Failed to create temp directory")?;
    let nonexistent_cache = temp_dir.path().join("nonexistent_cache");
    
    // Try to load from non-existent cache
    let result = load_features_from_cache(
        &nonexistent_cache,
        Path::new("test_data/ES_FUT_180d.parquet"),
        Path::new("test_data/mbp10"),
        50,
    )
    .await;
    
    // Should return None (cache miss) rather than error
    match result {
        Ok(None) => {
            info!("Missing cache handled gracefully (returns None)");
            Ok(())
        }
        Ok(Some(_)) => {
            anyhow::bail!("Should not find cache in non-existent directory")
        }
        Err(e) => {
            anyhow::bail!("Should return None for missing cache, got error: {}", e)
        }
    }
}

// ============================================================================
// TEST 9: Multiple Datasets (Separate Cache Files)
// ============================================================================

#[tokio::test]
#[ignore] // Ignore until cache creation is implemented
async fn test_multiple_datasets_separate_caches() -> Result<()> {
    let temp_dir = TempDir::new().context("Failed to create temp directory")?;
    let cache_dir = temp_dir.path().join("cache");
    fs::create_dir_all(&cache_dir).context("Failed to create cache directory")?;
    
    // Create test files for two datasets
    let dataset1 = temp_dir.path().join("ES_FUT_180d.parquet");
    let dataset2 = temp_dir.path().join("NQ_FUT_180d.parquet");
    
    fs::write(&dataset1, b"ES futures data").context("Failed to write dataset1")?;
    fs::write(&dataset2, b"NQ futures data").context("Failed to write dataset2")?;
    
    let mbp10_dir = temp_dir.path().join("mbp10");
    fs::create_dir_all(&mbp10_dir).context("Failed to create mbp10 directory")?;
    
    // Calculate cache keys for both datasets
    let key1 = calculate_cache_key(&dataset1, &mbp10_dir, 50)?;
    let key2 = calculate_cache_key(&dataset2, &mbp10_dir, 50)?;
    
    // Keys should differ (different datasets)
    assert_ne!(
        key1, key2,
        "Different datasets should produce different cache keys"
    );
    
    info!(es_key = &key1[..16], nq_key = &key2[..16], "Multiple datasets produce separate cache keys");
    Ok(())
}