fix: include MBP-10 + trades dirs in feature cache key
The DBN feature cache was keyed ONLY on OHLCV .dbn files. If a cache was created without trades data (VPIN/Kyle's Lambda), subsequent runs with trades would silently serve stale features from cache, dropping VPIN enrichment. Now cache key hashes OHLCV + MBP-10 + trades dirs together. Adding or removing data sources invalidates the cache correctly. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -337,10 +337,37 @@ pub async fn save_features_to_cache(
|
||||
/// by (canonical path, size, mtime). Any change to the directory contents
|
||||
/// (add / remove / modify) produces a different key.
|
||||
pub fn calculate_dbn_cache_key(data_dir: &Path) -> Result<String> {
|
||||
calculate_dbn_cache_key_full(data_dir, None, None)
|
||||
}
|
||||
|
||||
/// Extended cache key that includes MBP-10 and trades directories.
|
||||
/// The cache is invalidated when ANY data source changes — OHLCV, MBP-10, or trades.
|
||||
/// Without this, a cache built without trades would serve stale features
|
||||
/// even after trades data is added, silently dropping VPIN/Kyle's Lambda enrichment.
|
||||
pub fn calculate_dbn_cache_key_full(
|
||||
data_dir: &Path,
|
||||
mbp10_dir: Option<&Path>,
|
||||
trades_dir: Option<&Path>,
|
||||
) -> Result<String> {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(data_dir.to_string_lossy().as_bytes());
|
||||
|
||||
let mut files: Vec<_> = collect_dbn_files_for_hash(data_dir);
|
||||
// Also include MBP-10 and trades files in the hash
|
||||
if let Some(dir) = mbp10_dir {
|
||||
hasher.update(b"mbp10:");
|
||||
hasher.update(dir.to_string_lossy().as_bytes());
|
||||
files.extend(collect_dbn_files_for_hash(dir));
|
||||
} else {
|
||||
hasher.update(b"mbp10:none");
|
||||
}
|
||||
if let Some(dir) = trades_dir {
|
||||
hasher.update(b"trades:");
|
||||
hasher.update(dir.to_string_lossy().as_bytes());
|
||||
files.extend(collect_dbn_files_for_hash(dir));
|
||||
} else {
|
||||
hasher.update(b"trades:none");
|
||||
}
|
||||
files.sort(); // deterministic ordering
|
||||
|
||||
if files.is_empty() {
|
||||
@@ -423,6 +450,15 @@ fn dbn_cache_dir() -> Option<PathBuf> {
|
||||
/// ```
|
||||
pub fn load_dbn_training_cache(
|
||||
data_dir: &Path,
|
||||
) -> Result<Option<(Vec<([f64; 42], Vec<f64>)>, Vec<([f64; 42], Vec<f64>)>)>> {
|
||||
load_dbn_training_cache_full(data_dir, None, None)
|
||||
}
|
||||
|
||||
/// Load cached features with full data source awareness (OHLCV + MBP-10 + trades).
|
||||
pub fn load_dbn_training_cache_full(
|
||||
data_dir: &Path,
|
||||
mbp10_dir: Option<&Path>,
|
||||
trades_dir: Option<&Path>,
|
||||
) -> Result<Option<(Vec<([f64; 42], Vec<f64>)>, Vec<([f64; 42], Vec<f64>)>)>> {
|
||||
use std::io::{BufReader, Read};
|
||||
|
||||
@@ -431,7 +467,7 @@ pub fn load_dbn_training_cache(
|
||||
None => return Ok(None),
|
||||
};
|
||||
|
||||
let key = match calculate_dbn_cache_key(data_dir) {
|
||||
let key = match calculate_dbn_cache_key_full(data_dir, mbp10_dir, trades_dir) {
|
||||
Ok(k) => k,
|
||||
Err(e) => {
|
||||
debug!("DBN cache key error (skipping cache): {e}");
|
||||
@@ -514,6 +550,17 @@ pub fn save_dbn_training_cache(
|
||||
data_dir: &Path,
|
||||
train_data: &[([f64; 42], Vec<f64>)],
|
||||
val_data: &[([f64; 42], Vec<f64>)],
|
||||
) {
|
||||
save_dbn_training_cache_full(data_dir, None, None, train_data, val_data);
|
||||
}
|
||||
|
||||
/// Save cached features with full data source awareness.
|
||||
pub fn save_dbn_training_cache_full(
|
||||
data_dir: &Path,
|
||||
mbp10_dir: Option<&Path>,
|
||||
trades_dir: Option<&Path>,
|
||||
train_data: &[([f64; 42], Vec<f64>)],
|
||||
val_data: &[([f64; 42], Vec<f64>)],
|
||||
) {
|
||||
use std::io::{BufWriter, Write};
|
||||
|
||||
@@ -522,7 +569,7 @@ pub fn save_dbn_training_cache(
|
||||
None => return,
|
||||
};
|
||||
|
||||
let key = match calculate_dbn_cache_key(data_dir) {
|
||||
let key = match calculate_dbn_cache_key_full(data_dir, mbp10_dir, trades_dir) {
|
||||
Ok(k) => k,
|
||||
Err(e) => {
|
||||
warn!("DBN cache key error (skipping save): {e}");
|
||||
|
||||
@@ -601,8 +601,16 @@ impl DQNTrainer {
|
||||
// any data change automatically invalidates it.
|
||||
// Disabled by FOXHUNT_FEATURE_CACHE=0.
|
||||
let dir_path_for_cache = Path::new(dbn_data_dir);
|
||||
let mbp10_owned = self.hyperparams.mbp10_data_dir.clone();
|
||||
let trades_owned = self.hyperparams.trades_data_dir.clone();
|
||||
let mbp10_path = mbp10_owned.as_ref().map(|s| Path::new(s.as_str()));
|
||||
let trades_path = trades_owned.as_ref().map(|s| Path::new(s.as_str()));
|
||||
if dir_path_for_cache.exists() {
|
||||
match crate::feature_cache::load_dbn_training_cache(dir_path_for_cache) {
|
||||
match crate::feature_cache::load_dbn_training_cache_full(
|
||||
dir_path_for_cache,
|
||||
mbp10_path,
|
||||
trades_path,
|
||||
) {
|
||||
Ok(Some((train, val))) => {
|
||||
info!(
|
||||
"DBN feature cache hit: {} train + {} val samples (skipped ~60s load)",
|
||||
@@ -852,8 +860,10 @@ impl DQNTrainer {
|
||||
);
|
||||
|
||||
// ── Persist to cache for future runs ─────────────────────────────────
|
||||
crate::feature_cache::save_dbn_training_cache(
|
||||
crate::feature_cache::save_dbn_training_cache_full(
|
||||
Path::new(dbn_data_dir),
|
||||
mbp10_path,
|
||||
trades_path,
|
||||
&train_data,
|
||||
&val_data,
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user