foxhunt/crates/ml/src/feature_cache.rs

//! Feature Cache Key Utilities
//!
//! Provides cache key generation for `.fxcache` auto-discovery.
//! The flat-binary `.fxcache` format is the only cache path; Parquet
//! and old binary caches have been removed.

use anyhow::{Context, Result};
use sha2::{Digest, Sha256};
use std::path::{Path, PathBuf};

/// Calculate a cache key for a DBN data directory.
///
/// Hashes every `.dbn` and `.dbn.zst` file found recursively under `data_dir`
/// by (canonical path, size, mtime). Any change to the directory contents
/// (add / remove / modify) produces a different key.
///
/// Production defaults for the bar-formation params (matches `dqn-production.toml`):
/// `imbalance_bar_threshold = 0.5`, `imbalance_bar_ewma_alpha = 0.1`,
/// `volume_bar_size = 100` (matches `DEFAULT_VOLUME_BAR_SIZE`).
pub fn calculate_dbn_cache_key(data_dir: &Path) -> Result<String> {
    calculate_dbn_cache_key_full(data_dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1, 100)
}

/// Extended cache key that includes MBP-10 and trades directories AND bar
/// formation parameters. The cache is invalidated when ANY data source
/// changes — OHLCV, MBP-10, trades — OR when bar-formation params change.
///
/// Bar-formation params (`imbalance_bar_threshold`, `imbalance_bar_ewma_alpha`,
/// `volume_bar_size`) are hashed in regardless of `data_source` because:
/// (a) they uniquely identify the bar stream the cache contains, (b) downstream
/// features (e.g. log-returns, targets) depend on bar boundaries, and
/// (c) without this, two runs configured with different params will silently
/// share the same cache file (canonical 2026-05-09 audit: 14 prior production
/// runs all collided on `a3f933aa...` / `c07c960a...` keys regardless of TOML
/// threshold value because the bar params were not in the hash).
pub fn calculate_dbn_cache_key_full(
    data_dir: &Path,
    mbp10_dir: Option<&Path>,
    trades_dir: Option<&Path>,
    symbol: &str,
    data_source: &str,
    imbalance_bar_threshold: f64,
    imbalance_bar_ewma_alpha: f64,
    volume_bar_size: u64,
) -> Result<String> {
    let mut hasher = Sha256::new();

    // Include symbol and data_source in hash — different instruments/modes get different keys
    hasher.update(symbol.as_bytes());
    hasher.update(b"|");
    hasher.update(data_source.as_bytes());
    hasher.update(b"|");

    // Include bar-formation params — different bar streams get different keys
    hasher.update(&imbalance_bar_threshold.to_le_bytes());
    hasher.update(b"|");
    hasher.update(&imbalance_bar_ewma_alpha.to_le_bytes());
    hasher.update(b"|");
    hasher.update(&volume_bar_size.to_le_bytes());
    hasher.update(b"|");

    // Hash only file contents (size + mtime), not paths.
    // This makes the key independent of CWD, relative/absolute paths,
    // and directory naming — only actual data changes invalidate it.
    let mut files: Vec<_> = collect_dbn_files_for_hash(data_dir);
    if let Some(dir) = mbp10_dir {
        files.extend(collect_dbn_files_for_hash(dir));
    }
    if let Some(dir) = trades_dir {
        files.extend(collect_dbn_files_for_hash(dir));
    }
    // Sort by filename only (strip parent dirs) for deterministic ordering
    files.sort_by(|a, b| {
        a.file_name().cmp(&b.file_name())
    });

    if files.is_empty() {
        return Err(anyhow::anyhow!(
            "No .dbn or .dbn.zst files found in {:?} — cannot build cache key",
            data_dir
        ));
    }

    for path in &files {
        // Hash filename (not full path) + size.
        // NOT mtime: PVC remounts and node reboots change file timestamps
        // without changing data, causing cache misses on every new pod.
        if let Some(name) = path.file_name() {
            hasher.update(name.to_string_lossy().as_bytes());
        }
        let meta = path
            .metadata()
            .with_context(|| format!("Failed to stat {:?}", path))?;
        hasher.update(&meta.len().to_le_bytes());
    }

    Ok(format!("{:x}", hasher.finalize()))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_cache_key_includes_symbol() {
        let dir = Path::new("test_data/futures-baseline");
        if !dir.exists() { return; }
        let key_es = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1, 100).unwrap();
        let key_nq = calculate_dbn_cache_key_full(dir, None, None, "NQ.FUT", "mbp10", 0.5, 0.1, 100).unwrap();
        assert_ne!(key_es, key_nq, "Different symbols must produce different cache keys");
    }

    #[test]
    fn test_cache_key_includes_data_source() {
        let dir = Path::new("test_data/futures-baseline");
        if !dir.exists() { return; }
        let key_ohlcv = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "ohlcv", 0.5, 0.1, 100).unwrap();
        let key_mbp10 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1, 100).unwrap();
        assert_ne!(key_ohlcv, key_mbp10, "Different data sources must produce different cache keys");
    }

    #[test]
    fn test_cache_key_includes_bar_threshold() {
        let dir = Path::new("test_data/futures-baseline");
        if !dir.exists() { return; }
        let key_a = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1, 100).unwrap();
        let key_b = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 2.5, 0.1, 100).unwrap();
        assert_ne!(key_a, key_b, "Different imbalance_bar_threshold must produce different cache keys");
    }

    #[test]
    fn test_cache_key_includes_bar_alpha() {
        let dir = Path::new("test_data/futures-baseline");
        if !dir.exists() { return; }
        let key_a = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1, 100).unwrap();
        let key_b = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 1.0, 100).unwrap();
        assert_ne!(key_a, key_b, "Different imbalance_bar_ewma_alpha must produce different cache keys");
    }

    #[test]
    fn test_cache_key_includes_volume_bar_size() {
        let dir = Path::new("test_data/futures-baseline");
        if !dir.exists() { return; }
        let key_a = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "ohlcv", 0.5, 0.1, 100).unwrap();
        let key_b = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "ohlcv", 0.5, 0.1, 500).unwrap();
        assert_ne!(key_a, key_b, "Different volume_bar_size must produce different cache keys");
    }

    #[test]
    fn test_cache_key_stable() {
        let dir = Path::new("test_data/futures-baseline");
        if !dir.exists() { return; }
        let key1 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1, 100).unwrap();
        let key2 = calculate_dbn_cache_key_full(dir, None, None, "ES.FUT", "mbp10", 0.5, 0.1, 100).unwrap();
        assert_eq!(key1, key2, "Same inputs must produce same key");
    }
}

/// Collect all .dbn and .dbn.zst paths recursively (for hashing purposes).
fn collect_dbn_files_for_hash(dir: &Path) -> Vec<PathBuf> {
    let mut out = Vec::new();
    if let Ok(entries) = std::fs::read_dir(dir) {
        for entry in entries.flatten() {
            let path = entry.path();
            if path.is_dir() {
                out.extend(collect_dbn_files_for_hash(&path));
            } else if path.extension().and_then(|s| s.to_str()) == Some("dbn")
                || path.to_string_lossy().ends_with(".dbn.zst")
            {
                out.push(path);
            }
        }
    }
    out
}