fix: include MBP-10 + trades dirs in feature cache key

The DBN feature cache was keyed ONLY on OHLCV .dbn files. If a cache was
created without trades data (VPIN/Kyle's Lambda), subsequent runs with trades
would silently serve stale features from cache, dropping VPIN enrichment.

Now cache key hashes OHLCV + MBP-10 + trades dirs together. Adding or removing
data sources invalidates the cache correctly.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-22 23:31:05 +01:00
parent d9e896fa50
commit de21ea514d
2 changed files with 61 additions and 4 deletions

View File

@@ -337,10 +337,37 @@ pub async fn save_features_to_cache(
/// by (canonical path, size, mtime). Any change to the directory contents
/// (add / remove / modify) produces a different key.
pub fn calculate_dbn_cache_key(data_dir: &Path) -> Result<String> {
calculate_dbn_cache_key_full(data_dir, None, None)
}
/// Extended cache key that includes MBP-10 and trades directories.
/// The cache is invalidated when ANY data source changes — OHLCV, MBP-10, or trades.
/// Without this, a cache built without trades would serve stale features
/// even after trades data is added, silently dropping VPIN/Kyle's Lambda enrichment.
pub fn calculate_dbn_cache_key_full(
data_dir: &Path,
mbp10_dir: Option<&Path>,
trades_dir: Option<&Path>,
) -> Result<String> {
let mut hasher = Sha256::new();
hasher.update(data_dir.to_string_lossy().as_bytes());
let mut files: Vec<_> = collect_dbn_files_for_hash(data_dir);
// Also include MBP-10 and trades files in the hash
if let Some(dir) = mbp10_dir {
hasher.update(b"mbp10:");
hasher.update(dir.to_string_lossy().as_bytes());
files.extend(collect_dbn_files_for_hash(dir));
} else {
hasher.update(b"mbp10:none");
}
if let Some(dir) = trades_dir {
hasher.update(b"trades:");
hasher.update(dir.to_string_lossy().as_bytes());
files.extend(collect_dbn_files_for_hash(dir));
} else {
hasher.update(b"trades:none");
}
files.sort(); // deterministic ordering
if files.is_empty() {
@@ -423,6 +450,15 @@ fn dbn_cache_dir() -> Option<PathBuf> {
/// ```
pub fn load_dbn_training_cache(
data_dir: &Path,
) -> Result<Option<(Vec<([f64; 42], Vec<f64>)>, Vec<([f64; 42], Vec<f64>)>)>> {
load_dbn_training_cache_full(data_dir, None, None)
}
/// Load cached features with full data source awareness (OHLCV + MBP-10 + trades).
pub fn load_dbn_training_cache_full(
data_dir: &Path,
mbp10_dir: Option<&Path>,
trades_dir: Option<&Path>,
) -> Result<Option<(Vec<([f64; 42], Vec<f64>)>, Vec<([f64; 42], Vec<f64>)>)>> {
use std::io::{BufReader, Read};
@@ -431,7 +467,7 @@ pub fn load_dbn_training_cache(
None => return Ok(None),
};
let key = match calculate_dbn_cache_key(data_dir) {
let key = match calculate_dbn_cache_key_full(data_dir, mbp10_dir, trades_dir) {
Ok(k) => k,
Err(e) => {
debug!("DBN cache key error (skipping cache): {e}");
@@ -514,6 +550,17 @@ pub fn save_dbn_training_cache(
data_dir: &Path,
train_data: &[([f64; 42], Vec<f64>)],
val_data: &[([f64; 42], Vec<f64>)],
) {
save_dbn_training_cache_full(data_dir, None, None, train_data, val_data);
}
/// Save cached features with full data source awareness.
pub fn save_dbn_training_cache_full(
data_dir: &Path,
mbp10_dir: Option<&Path>,
trades_dir: Option<&Path>,
train_data: &[([f64; 42], Vec<f64>)],
val_data: &[([f64; 42], Vec<f64>)],
) {
use std::io::{BufWriter, Write};
@@ -522,7 +569,7 @@ pub fn save_dbn_training_cache(
None => return,
};
let key = match calculate_dbn_cache_key(data_dir) {
let key = match calculate_dbn_cache_key_full(data_dir, mbp10_dir, trades_dir) {
Ok(k) => k,
Err(e) => {
warn!("DBN cache key error (skipping save): {e}");

View File

@@ -601,8 +601,16 @@ impl DQNTrainer {
// any data change automatically invalidates it.
// Disabled by FOXHUNT_FEATURE_CACHE=0.
let dir_path_for_cache = Path::new(dbn_data_dir);
let mbp10_owned = self.hyperparams.mbp10_data_dir.clone();
let trades_owned = self.hyperparams.trades_data_dir.clone();
let mbp10_path = mbp10_owned.as_ref().map(|s| Path::new(s.as_str()));
let trades_path = trades_owned.as_ref().map(|s| Path::new(s.as_str()));
if dir_path_for_cache.exists() {
match crate::feature_cache::load_dbn_training_cache(dir_path_for_cache) {
match crate::feature_cache::load_dbn_training_cache_full(
dir_path_for_cache,
mbp10_path,
trades_path,
) {
Ok(Some((train, val))) => {
info!(
"DBN feature cache hit: {} train + {} val samples (skipped ~60s load)",
@@ -852,8 +860,10 @@ impl DQNTrainer {
);
// ── Persist to cache for future runs ─────────────────────────────────
crate::feature_cache::save_dbn_training_cache(
crate::feature_cache::save_dbn_training_cache_full(
Path::new(dbn_data_dir),
mbp10_path,
trades_path,
&train_data,
&val_data,
);