foxhunt/crates/ml/src/feature_cache.rs

//! Feature Cache Key Utilities
//!
//! Provides cache key generation for `.fxcache` auto-discovery.
//! The flat-binary `.fxcache` format is the only cache path; Parquet
//! and old binary caches have been removed.

use anyhow::{Context, Result};
use sha2::{Digest, Sha256};
use std::path::{Path, PathBuf};

/// Calculate a cache key for a DBN data directory.
///
/// Hashes every `.dbn` and `.dbn.zst` file found recursively under `data_dir`
/// by (canonical path, size, mtime). Any change to the directory contents
/// (add / remove / modify) produces a different key.
pub fn calculate_dbn_cache_key(data_dir: &Path) -> Result<String> {
    calculate_dbn_cache_key_full(data_dir, None, None, "ES.FUT", "ohlcv")
}

/// Extended cache key that includes MBP-10 and trades directories.
/// The cache is invalidated when ANY data source changes — OHLCV, MBP-10, or trades.
/// Without this, a cache built without trades would serve stale features
/// even after trades data is added, silently dropping VPIN/Kyle's Lambda enrichment.
pub fn calculate_dbn_cache_key_full(
    data_dir: &Path,
    mbp10_dir: Option<&Path>,
    trades_dir: Option<&Path>,
    symbol: &str,
    data_source: &str,
) -> Result<String> {
    let mut hasher = Sha256::new();

    // Include symbol and data_source in hash — different instruments/modes get different keys
    hasher.update(symbol.as_bytes());
    hasher.update(b"|");
    hasher.update(data_source.as_bytes());
    hasher.update(b"|");

    // Hash only file contents (size + mtime), not paths.
    // This makes the key independent of CWD, relative/absolute paths,
    // and directory naming — only actual data changes invalidate it.
    let mut files: Vec<_> = collect_dbn_files_for_hash(data_dir);
    if let Some(dir) = mbp10_dir {
        files.extend(collect_dbn_files_for_hash(dir));
    }
    if let Some(dir) = trades_dir {
        files.extend(collect_dbn_files_for_hash(dir));
    }
    // Sort by filename only (strip parent dirs) for deterministic ordering
    files.sort_by(|a, b| {
        a.file_name().cmp(&b.file_name())
    });

    if files.is_empty() {
        return Err(anyhow::anyhow!(
            "No .dbn or .dbn.zst files found in {:?} — cannot build cache key",
            data_dir
        ));
    }

    for path in &files {
        // Hash filename (not full path) + size.
        // NOT mtime: PVC remounts and node reboots change file timestamps
        // without changing data, causing cache misses on every new pod.
        if let Some(name) = path.file_name() {
            hasher.update(name.to_string_lossy().as_bytes());
        }
        let meta = path
            .metadata()
            .with_context(|| format!("Failed to stat {:?}", path))?;
        hasher.update(&meta.len().to_le_bytes());
    }

    Ok(format!("{:x}", hasher.finalize()))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_cache_key_includes_symbol() {
        let dir = Path::new("test_data/futures-baseline");
        if !dir.exists() { return; }
        let key_es = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "ohlcv").unwrap();
        let key_nq = calculate_dbn_cache_key_full(dir, None, None, "NQ.FUT", "ohlcv").unwrap();
        assert_ne!(key_es, key_nq, "Different symbols must produce different cache keys");
    }

    #[test]
    fn test_cache_key_includes_data_source() {
        let dir = Path::new("test_data/futures-baseline");
        if !dir.exists() { return; }
        let key_ohlcv = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "ohlcv").unwrap();
        let key_mbp10 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10").unwrap();
        assert_ne!(key_ohlcv, key_mbp10, "Different data sources must produce different cache keys");
    }

    #[test]
    fn test_cache_key_stable() {
        let dir = Path::new("test_data/futures-baseline");
        if !dir.exists() { return; }
        let key1 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "ohlcv").unwrap();
        let key2 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "ohlcv").unwrap();
        assert_eq!(key1, key2, "Same inputs must produce same key");
    }
}

/// Collect all .dbn and .dbn.zst paths recursively (for hashing purposes).
fn collect_dbn_files_for_hash(dir: &Path) -> Vec<PathBuf> {
    let mut out = Vec::new();
    if let Ok(entries) = std::fs::read_dir(dir) {
        for entry in entries.flatten() {
            let path = entry.path();
            if path.is_dir() {
                out.extend(collect_dbn_files_for_hash(&path));
            } else if path.extension().and_then(|s| s.to_str()) == Some("dbn")
                || path.to_string_lossy().ends_with(".dbn.zst")
            {
                out.push(path);
            }
        }
    }
    out
}