foxhunt/crates/ml/tests/dqn_feature_cache_test.rs

//! # DQN Feature Cache Integration Tests
//!
//! Comprehensive test suite for the DQN feature caching system that pre-computes
//! 51-feature vectors (43 base + 8 OFI) once and reuses them across hyperopt trials.
//!
//! ## Test Coverage
//!
//! 1. **Cache Creation Success**: Verify cache file creation with correct metadata
//! 2. **Cache Loading Matches Original**: Ensure cached features match non-cached computation
//! 3. **Cache Invalidation**: Test cache key changes when input data changes
//! 4. **Hyperopt Integration**: Multi-trial cache reuse verification
//! 5. **Performance Benchmark**: Validate 20s → <1s loading speedup
//! 6. **Edge Cases**: Missing cache, corrupted cache, multiple datasets
//!
//! ## Expected Performance Gains
//!
//! - Cache creation: ~20-30s (one-time setup)
//! - Cache loading: <1s per trial
//! - Time savings: 87-96% for 50-trial hyperopt
//! - Cache size: 3-5 MB compressed Parquet per dataset
//!
//! Design reference: `/tmp/MBP10_OFI_CACHING_DESIGN.md` section 8

use anyhow::{Context, Result};
use std::fs;
use std::path::Path;
use std::time::{Duration, Instant};
use tempfile::TempDir;
use std::ffi::OsStr;


// Cache functions (to be implemented based on design doc)
// These will initially fail compilation until implemented

/// Calculate SHA256-based cache key from input data
///
/// Components:
/// - Parquet file path + modification time
/// - MBP-10 directory path + file list + modification times
/// - Feature extraction version (code hash)
/// - Warmup period constant
fn calculate_cache_key(
    parquet_path: &Path,
    mbp10_dir: &Path,
    warmup_period: usize,
) -> Result<String> {
    use sha2::{Digest, Sha256};
    use std::time::UNIX_EPOCH;

    let mut hasher = Sha256::new();

    // 1. Parquet file path + mtime
    hasher.update(parquet_path.to_string_lossy().as_bytes());
    let parquet_mtime = parquet_path
        .metadata()
        .context("Failed to read parquet metadata")?
        .modified()
        .context("Failed to get parquet mtime")?
        .duration_since(UNIX_EPOCH)
        .context("System time error")?
        .as_secs();
    hasher.update(&parquet_mtime.to_le_bytes());

    // 2. MBP-10 directory + all .dbn files + mtimes
    hasher.update(mbp10_dir.to_string_lossy().as_bytes());
    
    if mbp10_dir.exists() {
        let mut dbn_files: Vec<_> = fs::read_dir(mbp10_dir)
            .context("Failed to read mbp10 directory")?
            .filter_map(|e| e.ok())
            .filter(|e| e.path().extension() == Some(OsStr::new("dbn")))
            .collect();
        
        dbn_files.sort_by_key(|e| e.path());
        
        for entry in &dbn_files {
            hasher.update(entry.path().to_string_lossy().as_bytes());
            let mtime = entry
                .metadata()
                .context("Failed to read dbn metadata")?
                .modified()
                .context("Failed to get dbn mtime")?
                .duration_since(UNIX_EPOCH)
                .context("System time error")?
                .as_secs();
            hasher.update(&mtime.to_le_bytes());
        }
    }

    // 3. Feature extraction version (simplified - use constant for now)
    // In production, this would hash the extraction.rs source file
    let feature_version = "v1.0.0";
    hasher.update(feature_version.as_bytes());

    // 4. Warmup period
    hasher.update(&warmup_period.to_le_bytes());

    Ok(format!("{:x}", hasher.finalize()))
}

/// Create feature cache from parquet and MBP-10 data
///
/// This is a placeholder that will be implemented based on the design doc.
/// Expected behavior:
/// - Load data from parquet_path and mbp10_dir
/// - Extract 51-feature vectors
/// - Save to cache_dir as Parquet file
/// - Create metadata JSON file
async fn create_feature_cache(
    _parquet_path: &Path,
    _mbp10_dir: &Path,
    _cache_dir: &Path,
    _warmup_period: usize,
) -> Result<String> {
    // NOTE: This is a stub that will fail until implemented
    // Implementation should follow design doc section 4 (Phase 1)
    
    anyhow::bail!(
        "create_feature_cache not yet implemented - see /tmp/MBP10_OFI_CACHING_DESIGN.md Phase 1"
    )
}

/// Load features from cache
///
/// Returns (train_data, val_data) tuples matching DQN trainer format.
/// Each sample is (FeatureVector, Vec<f64>) where Vec<f64> contains target prices.
async fn load_features_from_cache(
    _cache_dir: &Path,
    _parquet_path: &Path,
    _mbp10_dir: &Path,
    _warmup_period: usize,
) -> Result<Option<(Vec<([f64; 42], Vec<f64>)>, Vec<([f64; 42], Vec<f64>)>)>> {
    // NOTE: This is a stub that will fail until implemented
    // Implementation should follow design doc section 4 (Phase 2)
    
    anyhow::bail!(
        "load_features_from_cache not yet implemented - see /tmp/MBP10_OFI_CACHING_DESIGN.md Phase 2"
    )
}

// ============================================================================
// TEST 1: Cache Creation Success
// ============================================================================

#[tokio::test]
#[ignore] // Ignore until cache creation is implemented
async fn test_cache_creation_success() -> Result<()> {
    // Setup
    let temp_dir = TempDir::new().context("Failed to create temp directory")?;
    let cache_dir = temp_dir.path().join("cache");
    fs::create_dir_all(&cache_dir).context("Failed to create cache directory")?;
    
    // Create cache
    let result = create_feature_cache(
        Path::new("test_data/ES_FUT_180d.parquet"),
        Path::new("test_data/mbp10"),
        &cache_dir,
        50,
    ).await;
    
    assert!(
        result.is_ok(),
        "Cache creation should succeed: {:?}",
        result.err()
    );
    
    // Verify cache file exists
    let cache_files: Vec<_> = fs::read_dir(&cache_dir)
        .context("Failed to read cache directory")?
        .filter_map(|e| e.ok())
        .filter(|e| e.path().extension() == Some(OsStr::new("parquet")))
        .collect();
    
    assert_eq!(
        cache_files.len(),
        1,
        "Should create exactly one cache file"
    );
    
    // Verify metadata exists
    let metadata_path = cache_dir.join("cache_metadata.json");
    assert!(
        metadata_path.exists(),
        "Metadata file should exist at {:?}",
        metadata_path
    );
    
    // Verify file size (should be 3-5 MB compressed)
    let file_size = fs::metadata(cache_files[0].path())
        .context("Failed to read cache file metadata")?
        .len();
    
    assert!(
        file_size > 2_000_000,
        "Cache file too small: {} bytes (expected >2MB)",
        file_size
    );
    assert!(
        file_size < 10_000_000,
        "Cache file too large: {} bytes (expected <10MB)",
        file_size
    );
    
    println!("✅ Cache created successfully: {} KB", file_size / 1024);
    Ok(())
}

// ============================================================================
// TEST 2: Cache Loading Matches Original
// ============================================================================

#[tokio::test]
#[ignore] // Ignore until cache loading is implemented
async fn test_cache_loading_matches_original() -> Result<()> {
    let temp_dir = TempDir::new().context("Failed to create temp directory")?;
    let cache_dir = temp_dir.path().join("cache");
    fs::create_dir_all(&cache_dir).context("Failed to create cache directory")?;
    
    // Create cache
    create_feature_cache(
        Path::new("test_data/ES_FUT_180d.parquet"),
        Path::new("test_data/mbp10"),
        &cache_dir,
        50,
    )
    .await
    .context("Failed to create cache")?;
    
    // Load from cache
    let (cached_train, cached_val) = load_features_from_cache(
        &cache_dir,
        Path::new("test_data/ES_FUT_180d.parquet"),
        Path::new("test_data/mbp10"),
        50,
    )
    .await
    .context("Failed to load from cache")?
    .expect("Cache should exist");
    
    // Load original way (no cache) - this requires DQN trainer modification
    // For now, we'll verify the cached data structure is correct
    
    assert!(
        !cached_train.is_empty(),
        "Training data should not be empty"
    );
    assert!(
        !cached_val.is_empty(),
        "Validation data should not be empty"
    );
    
    // Verify feature dimensions
    let (features, targets) = &cached_train[0];
    assert_eq!(
        features.len(),
        51,
        "Each sample should have 51 features"
    );
    assert_eq!(
        targets.len(),
        4,
        "Each sample should have 4 target prices"
    );
    
    // Verify train/val split is approximately 80/20
    let total = cached_train.len() + cached_val.len();
    let train_ratio = cached_train.len() as f64 / total as f64;
    assert!(
        (train_ratio - 0.8).abs() < 0.05,
        "Train/val split should be ~80/20, got {:.2}",
        train_ratio
    );
    
    // Compare first 10 samples (features should be deterministic)
    for i in 0..10.min(cached_train.len()) {
        let features = &cached_train[i].0;
        
        // Verify no NaN or Inf values
        for (j, &value) in features.iter().enumerate() {
            assert!(
                value.is_finite(),
                "Feature {} in sample {} is not finite: {}",
                j,
                i,
                value
            );
        }
    }
    
    println!(
        "✅ Cache loaded successfully: {} train + {} val samples",
        cached_train.len(),
        cached_val.len()
    );
    Ok(())
}

// ============================================================================
// TEST 3: Cache Invalidation on Data Change
// ============================================================================

#[tokio::test]
async fn test_cache_invalidation_on_data_change() -> Result<()> {
    let temp_dir = TempDir::new().context("Failed to create temp directory")?;
    
    // Create a temporary parquet file
    let parquet_path = temp_dir.path().join("test.parquet");
    fs::write(&parquet_path, b"mock parquet data v1")
        .context("Failed to write parquet file")?;
    
    let mbp10_dir = temp_dir.path().join("mbp10");
    fs::create_dir_all(&mbp10_dir).context("Failed to create mbp10 directory")?;
    
    // Calculate initial cache key
    let cache_key1 = calculate_cache_key(&parquet_path, &mbp10_dir, 50)
        .context("Failed to calculate initial cache key")?;
    
    // Wait to ensure mtime changes (Linux has second-level granularity in most filesystems)
    std::thread::sleep(Duration::from_secs(2));
    
    // Modify file content to trigger mtime update
    fs::write(&parquet_path, b"mock parquet data v2 - MODIFIED CONTENT")
        .context("Failed to modify parquet file")?;
    
    // Calculate new cache key
    let cache_key2 = calculate_cache_key(&parquet_path, &mbp10_dir, 50)
        .context("Failed to calculate new cache key")?;
    
    println!("  Cache key 1 (original): {}...", &cache_key1[..16]);
    println!("  Cache key 2 (after modification): {}...", &cache_key2[..16]);
    
    // Keys should differ (cache invalidated)
    assert_ne!(
        cache_key1, cache_key2,
        "Cache key should change when data file is modified.\nKey1: {}\nKey2: {}",
        cache_key1, cache_key2
    );
    
    println!(
        "✅ Cache invalidation works:\n  Old key: {}...\n  New key: {}...",
        &cache_key1[..16],
        &cache_key2[..16]
    );
    Ok(())
}

// ============================================================================
// TEST 4: Cache Key Stability
// ============================================================================

#[tokio::test]
async fn test_cache_key_stability() -> Result<()> {
    let temp_dir = TempDir::new().context("Failed to create temp directory")?;
    
    // Create test files
    let parquet_path = temp_dir.path().join("test.parquet");
    fs::write(&parquet_path, b"mock parquet data")
        .context("Failed to write parquet file")?;
    
    let mbp10_dir = temp_dir.path().join("mbp10");
    fs::create_dir_all(&mbp10_dir).context("Failed to create mbp10 directory")?;
    
    // Calculate cache key multiple times without modification
    let key1 = calculate_cache_key(&parquet_path, &mbp10_dir, 50)?;
    let key2 = calculate_cache_key(&parquet_path, &mbp10_dir, 50)?;
    let key3 = calculate_cache_key(&parquet_path, &mbp10_dir, 50)?;
    
    // Keys should be identical (deterministic)
    assert_eq!(
        key1, key2,
        "Cache key should be deterministic"
    );
    assert_eq!(
        key2, key3,
        "Cache key should be deterministic"
    );
    
    println!("✅ Cache key is stable: {}", key1);
    Ok(())
}

// ============================================================================
// TEST 5: Cache Key with Different Warmup Periods
// ============================================================================

#[tokio::test]
async fn test_cache_key_different_warmup() -> Result<()> {
    let temp_dir = TempDir::new().context("Failed to create temp directory")?;
    
    // Create test files
    let parquet_path = temp_dir.path().join("test.parquet");
    fs::write(&parquet_path, b"mock parquet data")
        .context("Failed to write parquet file")?;
    
    let mbp10_dir = temp_dir.path().join("mbp10");
    fs::create_dir_all(&mbp10_dir).context("Failed to create mbp10 directory")?;
    
    // Calculate cache keys with different warmup periods
    let key_warmup_50 = calculate_cache_key(&parquet_path, &mbp10_dir, 50)?;
    let key_warmup_100 = calculate_cache_key(&parquet_path, &mbp10_dir, 100)?;
    
    // Keys should differ (different warmup periods)
    assert_ne!(
        key_warmup_50, key_warmup_100,
        "Cache key should change when warmup period changes"
    );
    
    println!(
        "✅ Different warmup periods produce different keys:\n  warmup=50:  {}...\n  warmup=100: {}...",
        &key_warmup_50[..16],
        &key_warmup_100[..16]
    );
    Ok(())
}

// ============================================================================
// TEST 6: Hyperopt Integration (Multi-Trial Cache Reuse)
// ============================================================================

#[tokio::test]
#[ignore] // Ignore until hyperopt integration is implemented
async fn test_hyperopt_with_cache() -> Result<()> {
    let temp_dir = TempDir::new().context("Failed to create temp directory")?;
    let cache_dir = temp_dir.path().join("cache");
    fs::create_dir_all(&cache_dir).context("Failed to create cache directory")?;
    
    // Pre-create cache
    create_feature_cache(
        Path::new("test_data/ES_FUT_180d.parquet"),
        Path::new("test_data/mbp10"),
        &cache_dir,
        50,
    )
    .await
    .context("Failed to create cache")?;
    
    // Simulate 3 hyperopt trials (in production, this would use DQNHyperoptAdapter)
    let start = Instant::now();
    
    let mut trial_durations = Vec::new();
    
    for trial in 1..=3 {
        let trial_start = Instant::now();
        
        // Load features from cache
        let (_train, _val) = load_features_from_cache(
            &cache_dir,
            Path::new("test_data/ES_FUT_180d.parquet"),
            Path::new("test_data/mbp10"),
            50,
        )
        .await
        .context("Failed to load from cache")?
        .expect("Cache should exist");
        
        let trial_duration = trial_start.elapsed();
        trial_durations.push(trial_duration);
        
        println!("  Trial {}: loaded features in {:?}", trial, trial_duration);
    }
    
    let total_duration = start.elapsed();
    
    // Verify results
    assert_eq!(trial_durations.len(), 3, "Should complete all 3 trials");
    
    // Each trial should load in <2s (with cache)
    for (i, duration) in trial_durations.iter().enumerate() {
        assert!(
            duration < &Duration::from_secs(2),
            "Trial {} took too long: {:?} (expected <2s with cache)",
            i + 1,
            duration
        );
    }
    
    // Total should be <10s (generous, in practice should be ~3s)
    assert!(
        total_duration < Duration::from_secs(10),
        "Total hyperopt time with cache took too long: {:?} (expected <10s)",
        total_duration
    );
    
    println!(
        "✅ Hyperopt with cache completed in {:?} ({:?} avg per trial)",
        total_duration,
        total_duration / 3
    );
    Ok(())
}

// ============================================================================
// TEST 7: Performance Benchmark (Cache vs No Cache)
// ============================================================================

#[tokio::test]
#[ignore] // Ignore by default - only run for performance validation
async fn test_cache_performance_benchmark() -> Result<()> {
    let temp_dir = TempDir::new().context("Failed to create temp directory")?;
    let cache_dir = temp_dir.path().join("cache");
    fs::create_dir_all(&cache_dir).context("Failed to create cache directory")?;
    
    println!("\n📊 DQN Feature Cache Performance Benchmark");
    println!("{}", "=".repeat(60));
    
    // Benchmark: Cache creation
    println!("\n1. Cache Creation (one-time setup):");
    let start = Instant::now();
    create_feature_cache(
        Path::new("test_data/ES_FUT_180d.parquet"),
        Path::new("test_data/mbp10"),
        &cache_dir,
        50,
    )
    .await
    .context("Failed to create cache")?;
    let creation_time = start.elapsed();
    println!("   Time: {:?}", creation_time);
    
    // Verify creation time is reasonable
    assert!(
        creation_time < Duration::from_secs(60),
        "Cache creation too slow: {:?} (expected <60s)",
        creation_time
    );
    
    // Benchmark: Cache loading (3 trials)
    println!("\n2. Cache Loading (simulating hyperopt trials):");
    let mut load_times = Vec::new();
    for trial in 1..=3 {
        let start = Instant::now();
        let (_train, _val) = load_features_from_cache(
            &cache_dir,
            Path::new("test_data/ES_FUT_180d.parquet"),
            Path::new("test_data/mbp10"),
            50,
        )
        .await
        .context("Failed to load from cache")?
        .expect("Cache should exist");
        let load_time = start.elapsed();
        load_times.push(load_time);
        println!("   Trial {}: {:?}", trial, load_time);
    }
    
    let avg_load_time = load_times.iter().sum::<Duration>() / load_times.len() as u32;
    
    // Summary
    println!("\n3. Performance Summary:");
    println!("   • Cache creation: {:?} (one-time)", creation_time);
    println!("   • Cache loading (avg): {:?} per trial", avg_load_time);
    println!("   • Expected speedup: 20s → {:?} (~{}x faster)", avg_load_time, 20 / avg_load_time.as_secs().max(1));
    
    // For 50-trial hyperopt
    let without_cache = Duration::from_secs(20 * 50); // 20s per trial
    let with_cache = creation_time + (avg_load_time * 50);
    let savings = without_cache.as_secs() - with_cache.as_secs();
    let savings_pct = (savings as f64 / without_cache.as_secs() as f64) * 100.0;
    
    println!("\n4. Hyperopt Impact (50 trials):");
    println!("   • Without cache: {:?} (~16.7 min)", without_cache);
    println!("   • With cache: {:?} (~{:.1} min)", with_cache, with_cache.as_secs_f64() / 60.0);
    println!("   • Time saved: {}s ({:.1}%)", savings, savings_pct);
    
    // Assertions
    assert!(
        avg_load_time < Duration::from_secs(2),
        "Cache loading too slow: {:?} (expected <2s)",
        avg_load_time
    );
    
    assert!(
        savings_pct > 80.0,
        "Cache savings insufficient: {:.1}% (expected >80%)",
        savings_pct
    );
    
    println!("{}", "=".repeat(60));
    println!("✅ Performance benchmark passed!\n");
    Ok(())
}

// ============================================================================
// TEST 8: Edge Case - Missing Cache Directory
// ============================================================================

#[tokio::test]
#[ignore] // Ignore until cache loading is implemented
async fn test_missing_cache_graceful_fallback() -> Result<()> {
    let temp_dir = TempDir::new().context("Failed to create temp directory")?;
    let nonexistent_cache = temp_dir.path().join("nonexistent_cache");
    
    // Try to load from non-existent cache
    let result = load_features_from_cache(
        &nonexistent_cache,
        Path::new("test_data/ES_FUT_180d.parquet"),
        Path::new("test_data/mbp10"),
        50,
    )
    .await;
    
    // Should return None (cache miss) rather than error
    match result {
        Ok(None) => {
            println!("✅ Missing cache handled gracefully (returns None)");
            Ok(())
        }
        Ok(Some(_)) => {
            anyhow::bail!("Should not find cache in non-existent directory")
        }
        Err(e) => {
            anyhow::bail!("Should return None for missing cache, got error: {}", e)
        }
    }
}

// ============================================================================
// TEST 9: Multiple Datasets (Separate Cache Files)
// ============================================================================

#[tokio::test]
#[ignore] // Ignore until cache creation is implemented
async fn test_multiple_datasets_separate_caches() -> Result<()> {
    let temp_dir = TempDir::new().context("Failed to create temp directory")?;
    let cache_dir = temp_dir.path().join("cache");
    fs::create_dir_all(&cache_dir).context("Failed to create cache directory")?;
    
    // Create test files for two datasets
    let dataset1 = temp_dir.path().join("ES_FUT_180d.parquet");
    let dataset2 = temp_dir.path().join("NQ_FUT_180d.parquet");
    
    fs::write(&dataset1, b"ES futures data").context("Failed to write dataset1")?;
    fs::write(&dataset2, b"NQ futures data").context("Failed to write dataset2")?;
    
    let mbp10_dir = temp_dir.path().join("mbp10");
    fs::create_dir_all(&mbp10_dir).context("Failed to create mbp10 directory")?;
    
    // Calculate cache keys for both datasets
    let key1 = calculate_cache_key(&dataset1, &mbp10_dir, 50)?;
    let key2 = calculate_cache_key(&dataset2, &mbp10_dir, 50)?;
    
    // Keys should differ (different datasets)
    assert_ne!(
        key1, key2,
        "Different datasets should produce different cache keys"
    );
    
    println!(
        "✅ Multiple datasets produce separate cache keys:\n  ES: {}...\n  NQ: {}...",
        &key1[..16],
        &key2[..16]
    );
    Ok(())
}