gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: cache key uses filenames only — CWD-independent

Browse Source 
Cache key hashes filename + size + mtime instead of full paths.
Resolves mbp10/trades relative paths by walking up from data_dir.
Ensures precompute binary and cargo test produce the same key
regardless of working directory.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-01 09:02:16 +02:00
parent 6893d75800
commit bb8e75c50c
2 changed files with 30 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
crates/ml/src/feature_cache.rs
View File

@@ -28,29 +28,21 @@ pub fn calculate_dbn_cache_key_full(
trades_dir: Option<&Path>,
) -> Result<String> {
let mut hasher = Sha256::new();
// Canonicalize paths so relative/absolute produce the same key
let data_dir = &data_dir.canonicalize().unwrap_or_else(|_| data_dir.to_path_buf());
hasher.update(data_dir.to_string_lossy().as_bytes());
// Hash only file contents (size + mtime), not paths.
// This makes the key independent of CWD, relative/absolute paths,
// and directory naming — only actual data changes invalidate it.
let mut files: Vec<_> = collect_dbn_files_for_hash(data_dir);
// Also include MBP-10 and trades files in the hash
if let Some(dir) = mbp10_dir {
let dir = &dir.canonicalize().unwrap_or_else(|_| dir.to_path_buf());
hasher.update(b"mbp10:");
hasher.update(dir.to_string_lossy().as_bytes());
files.extend(collect_dbn_files_for_hash(dir));
} else {
hasher.update(b"mbp10:none");
}
if let Some(dir) = trades_dir {
let dir = &dir.canonicalize().unwrap_or_else(|_| dir.to_path_buf());
hasher.update(b"trades:");
hasher.update(dir.to_string_lossy().as_bytes());
files.extend(collect_dbn_files_for_hash(dir));
} else {
hasher.update(b"trades:none");
}
files.sort(); // deterministic ordering
// Sort by filename only (strip parent dirs) for deterministic ordering
files.sort_by(|a, b| {
a.file_name().cmp(&b.file_name())
});
if files.is_empty() {
return Err(anyhow::anyhow!(
@@ -60,7 +52,10 @@ pub fn calculate_dbn_cache_key_full(
}
for path in &files {
hasher.update(path.to_string_lossy().as_bytes());
// Hash filename (not full path) + size + mtime
if let Some(name) = path.file_name() {
hasher.update(name.to_string_lossy().as_bytes());
}
let meta = path
.metadata()
.with_context(|| format!("Failed to stat {:?}", path))?;

22
crates/ml/src/trainers/dqn/data_loading.rs
View File

@@ -96,11 +96,27 @@ impl DQNTrainer {
if let Some(ref cache_dir) = fxcache_dir {
// Try exact key match. On miss, delete stale .fxcache files.
let data_dir_path = Path::new(dbn_data_dir);
let mbp10_dir = if self.hyperparams.mbp10_data_dir.is_empty() { None } else { Some(Path::new(self.hyperparams.mbp10_data_dir.as_str())) };
let trades_dir = if self.hyperparams.trades_data_dir.is_empty() { None } else { Some(Path::new(self.hyperparams.trades_data_dir.as_str())) };
// Resolve mbp10/trades paths: if relative and not found from CWD,
// try relative to data_dir ancestors (handles cargo test CWD = crates/ml/)
let resolve_sibling = |raw: &str| -> Option<std::path::PathBuf> {
if raw.is_empty() { return None; }
let p = Path::new(raw);
if p.exists() { return Some(p.to_path_buf()); }
// Walk up from data_dir looking for the relative path
let mut dir = data_dir_path;
while let Some(parent) = dir.parent() {
let candidate = parent.join(raw);
if candidate.exists() { return Some(candidate); }
if parent == dir { break; }
dir = parent;
}
None
};
let mbp10_dir = resolve_sibling(&self.hyperparams.mbp10_data_dir);
let trades_dir = resolve_sibling(&self.hyperparams.trades_data_dir);
let fxcache_path = crate::feature_cache::calculate_dbn_cache_key_full(
data_dir_path, mbp10_dir, trades_dir,
data_dir_path, mbp10_dir.as_deref(), trades_dir.as_deref(),
).ok()
.and_then(|hex| hex::decode(&hex).ok())
.and_then(|b| <[u8; 32]>::try_from(b).ok())