93 lines
3.2 KiB
Rust
93 lines
3.2 KiB
Rust
//! Feature Cache Key Utilities
|
|
//!
|
|
//! Provides cache key generation for `.fxcache` auto-discovery.
|
|
//! The flat-binary `.fxcache` format is the only cache path; Parquet
|
|
//! and old binary caches have been removed.
|
|
|
|
use anyhow::{Context, Result};
|
|
use sha2::{Digest, Sha256};
|
|
use std::path::{Path, PathBuf};
|
|
use std::time::SystemTime;
|
|
|
|
/// Calculate a cache key for a DBN data directory.
|
|
///
|
|
/// Hashes every `.dbn` and `.dbn.zst` file found recursively under `data_dir`
|
|
/// by (canonical path, size, mtime). Any change to the directory contents
|
|
/// (add / remove / modify) produces a different key.
|
|
pub fn calculate_dbn_cache_key(data_dir: &Path) -> Result<String> {
|
|
calculate_dbn_cache_key_full(data_dir, None, None)
|
|
}
|
|
|
|
/// Extended cache key that includes MBP-10 and trades directories.
|
|
/// The cache is invalidated when ANY data source changes — OHLCV, MBP-10, or trades.
|
|
/// Without this, a cache built without trades would serve stale features
|
|
/// even after trades data is added, silently dropping VPIN/Kyle's Lambda enrichment.
|
|
pub fn calculate_dbn_cache_key_full(
|
|
data_dir: &Path,
|
|
mbp10_dir: Option<&Path>,
|
|
trades_dir: Option<&Path>,
|
|
) -> Result<String> {
|
|
let mut hasher = Sha256::new();
|
|
hasher.update(data_dir.to_string_lossy().as_bytes());
|
|
|
|
let mut files: Vec<_> = collect_dbn_files_for_hash(data_dir);
|
|
// Also include MBP-10 and trades files in the hash
|
|
if let Some(dir) = mbp10_dir {
|
|
hasher.update(b"mbp10:");
|
|
hasher.update(dir.to_string_lossy().as_bytes());
|
|
files.extend(collect_dbn_files_for_hash(dir));
|
|
} else {
|
|
hasher.update(b"mbp10:none");
|
|
}
|
|
if let Some(dir) = trades_dir {
|
|
hasher.update(b"trades:");
|
|
hasher.update(dir.to_string_lossy().as_bytes());
|
|
files.extend(collect_dbn_files_for_hash(dir));
|
|
} else {
|
|
hasher.update(b"trades:none");
|
|
}
|
|
files.sort(); // deterministic ordering
|
|
|
|
if files.is_empty() {
|
|
return Err(anyhow::anyhow!(
|
|
"No .dbn or .dbn.zst files found in {:?} — cannot build cache key",
|
|
data_dir
|
|
));
|
|
}
|
|
|
|
for path in &files {
|
|
hasher.update(path.to_string_lossy().as_bytes());
|
|
let meta = path
|
|
.metadata()
|
|
.with_context(|| format!("Failed to stat {:?}", path))?;
|
|
hasher.update(&meta.len().to_le_bytes());
|
|
let mtime = meta
|
|
.modified()
|
|
.context("mtime unavailable")?
|
|
.duration_since(SystemTime::UNIX_EPOCH)
|
|
.context("mtime before epoch")?
|
|
.as_secs();
|
|
hasher.update(&mtime.to_le_bytes());
|
|
}
|
|
|
|
Ok(format!("{:x}", hasher.finalize()))
|
|
}
|
|
|
|
/// Collect all .dbn and .dbn.zst paths recursively (for hashing purposes).
|
|
fn collect_dbn_files_for_hash(dir: &Path) -> Vec<PathBuf> {
|
|
let mut out = Vec::new();
|
|
if let Ok(entries) = std::fs::read_dir(dir) {
|
|
for entry in entries.flatten() {
|
|
let path = entry.path();
|
|
if path.is_dir() {
|
|
out.extend(collect_dbn_files_for_hash(&path));
|
|
} else if path.extension().and_then(|s| s.to_str()) == Some("dbn")
|
|
|| path.to_string_lossy().ends_with(".dbn.zst")
|
|
{
|
|
out.push(path);
|
|
}
|
|
}
|
|
}
|
|
out
|
|
}
|