From de21ea514d59d476b22aab8551dff809176c05db Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 22 Mar 2026 23:31:05 +0100 Subject: [PATCH] fix: include MBP-10 + trades dirs in feature cache key The DBN feature cache was keyed ONLY on OHLCV .dbn files. If a cache was created without trades data (VPIN/Kyle's Lambda), subsequent runs with trades would silently serve stale features from cache, dropping VPIN enrichment. Now cache key hashes OHLCV + MBP-10 + trades dirs together. Adding or removing data sources invalidates the cache correctly. Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/ml/src/feature_cache.rs | 51 +++++++++++++++++++++- crates/ml/src/trainers/dqn/data_loading.rs | 14 +++++- 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/crates/ml/src/feature_cache.rs b/crates/ml/src/feature_cache.rs index 52f0d9e2c..8cc20197a 100644 --- a/crates/ml/src/feature_cache.rs +++ b/crates/ml/src/feature_cache.rs @@ -337,10 +337,37 @@ pub async fn save_features_to_cache( /// by (canonical path, size, mtime). Any change to the directory contents /// (add / remove / modify) produces a different key. pub fn calculate_dbn_cache_key(data_dir: &Path) -> Result { + calculate_dbn_cache_key_full(data_dir, None, None) +} + +/// Extended cache key that includes MBP-10 and trades directories. +/// The cache is invalidated when ANY data source changes — OHLCV, MBP-10, or trades. +/// Without this, a cache built without trades would serve stale features +/// even after trades data is added, silently dropping VPIN/Kyle's Lambda enrichment. +pub fn calculate_dbn_cache_key_full( + data_dir: &Path, + mbp10_dir: Option<&Path>, + trades_dir: Option<&Path>, +) -> Result { let mut hasher = Sha256::new(); hasher.update(data_dir.to_string_lossy().as_bytes()); let mut files: Vec<_> = collect_dbn_files_for_hash(data_dir); + // Also include MBP-10 and trades files in the hash + if let Some(dir) = mbp10_dir { + hasher.update(b"mbp10:"); + hasher.update(dir.to_string_lossy().as_bytes()); + files.extend(collect_dbn_files_for_hash(dir)); + } else { + hasher.update(b"mbp10:none"); + } + if let Some(dir) = trades_dir { + hasher.update(b"trades:"); + hasher.update(dir.to_string_lossy().as_bytes()); + files.extend(collect_dbn_files_for_hash(dir)); + } else { + hasher.update(b"trades:none"); + } files.sort(); // deterministic ordering if files.is_empty() { @@ -423,6 +450,15 @@ fn dbn_cache_dir() -> Option { /// ``` pub fn load_dbn_training_cache( data_dir: &Path, +) -> Result)>, Vec<([f64; 42], Vec)>)>> { + load_dbn_training_cache_full(data_dir, None, None) +} + +/// Load cached features with full data source awareness (OHLCV + MBP-10 + trades). +pub fn load_dbn_training_cache_full( + data_dir: &Path, + mbp10_dir: Option<&Path>, + trades_dir: Option<&Path>, ) -> Result)>, Vec<([f64; 42], Vec)>)>> { use std::io::{BufReader, Read}; @@ -431,7 +467,7 @@ pub fn load_dbn_training_cache( None => return Ok(None), }; - let key = match calculate_dbn_cache_key(data_dir) { + let key = match calculate_dbn_cache_key_full(data_dir, mbp10_dir, trades_dir) { Ok(k) => k, Err(e) => { debug!("DBN cache key error (skipping cache): {e}"); @@ -514,6 +550,17 @@ pub fn save_dbn_training_cache( data_dir: &Path, train_data: &[([f64; 42], Vec)], val_data: &[([f64; 42], Vec)], +) { + save_dbn_training_cache_full(data_dir, None, None, train_data, val_data); +} + +/// Save cached features with full data source awareness. +pub fn save_dbn_training_cache_full( + data_dir: &Path, + mbp10_dir: Option<&Path>, + trades_dir: Option<&Path>, + train_data: &[([f64; 42], Vec)], + val_data: &[([f64; 42], Vec)], ) { use std::io::{BufWriter, Write}; @@ -522,7 +569,7 @@ pub fn save_dbn_training_cache( None => return, }; - let key = match calculate_dbn_cache_key(data_dir) { + let key = match calculate_dbn_cache_key_full(data_dir, mbp10_dir, trades_dir) { Ok(k) => k, Err(e) => { warn!("DBN cache key error (skipping save): {e}"); diff --git a/crates/ml/src/trainers/dqn/data_loading.rs b/crates/ml/src/trainers/dqn/data_loading.rs index e1a9eff33..d28804ab0 100644 --- a/crates/ml/src/trainers/dqn/data_loading.rs +++ b/crates/ml/src/trainers/dqn/data_loading.rs @@ -601,8 +601,16 @@ impl DQNTrainer { // any data change automatically invalidates it. // Disabled by FOXHUNT_FEATURE_CACHE=0. let dir_path_for_cache = Path::new(dbn_data_dir); + let mbp10_owned = self.hyperparams.mbp10_data_dir.clone(); + let trades_owned = self.hyperparams.trades_data_dir.clone(); + let mbp10_path = mbp10_owned.as_ref().map(|s| Path::new(s.as_str())); + let trades_path = trades_owned.as_ref().map(|s| Path::new(s.as_str())); if dir_path_for_cache.exists() { - match crate::feature_cache::load_dbn_training_cache(dir_path_for_cache) { + match crate::feature_cache::load_dbn_training_cache_full( + dir_path_for_cache, + mbp10_path, + trades_path, + ) { Ok(Some((train, val))) => { info!( "DBN feature cache hit: {} train + {} val samples (skipped ~60s load)", @@ -852,8 +860,10 @@ impl DQNTrainer { ); // ── Persist to cache for future runs ───────────────────────────────── - crate::feature_cache::save_dbn_training_cache( + crate::feature_cache::save_dbn_training_cache_full( Path::new(dbn_data_dir), + mbp10_path, + trades_path, &train_data, &val_data, );