gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ml-features): imbalance bar cache on persistent PVC, not /tmp

Browse Source 
Cache path moved from /tmp/.foxhunt_imbalance_cache/ (ephemeral,
lost between Argo pods) to <mbp10_dir>/.cache/ (on the training-data
PVC). Saves ~4 minutes per Argo submission by avoiding recomputation
of 209M trade ticks → 17.8M imbalance bars.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-05-24 23:11:03 +02:00
parent 1f58fee924
commit ac11dddc92
1 changed files with 9 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
crates/ml-features/src/mbp10_loader.rs
View File

@@ -556,23 +556,27 @@ fn collect_dbn_files(dir: &Path) -> Vec<std::path::PathBuf> {
//
// Computing imbalance bars from MBP-10 data requires decompressing ~19GB of
// .dbn.zst files (~450s). This cache stores the resulting `Vec<OHLCVBar>` to
// `/tmp/.foxhunt_imbalance_cache/<hash>.bars.bin` using bincode.
// `<mbp10_dir>/.cache/<hash>.bars.bin` using bincode. Stored on the
// persistent training-data PVC so it survives across Argo pod restarts.
//
// Cache key: DefaultHasher of (mbp10_dir, symbol, threshold bits, alpha bits).
// Invalidation: cache mtime vs. newest .dbn/.dbn.zst file in the source dir.
// ══════════════════════════════════════════════════════════════════════════════
/// Compute a cache path for an imbalance bar set.
/// Compute a cache path for an imbalance bar set. Stores alongside
/// the source data on the persistent volume so the cache survives
/// across Argo pod restarts.
fn imbalance_cache_path(mbp10_dir: &Path, symbol: &str, threshold: f64, alpha: f64) -> std::path::PathBuf {
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
let mut hasher = DefaultHasher::new();
mbp10_dir.hash(&mut hasher);
symbol.hash(&mut hasher);
threshold.to_bits().hash(&mut hasher);
alpha.to_bits().hash(&mut hasher);
let hash = hasher.finish();
std::path::PathBuf::from(format!("/tmp/.foxhunt_imbalance_cache/{:016x}.bars.bin", hash))
let cache_dir = mbp10_dir.join(".cache");
let _ = std::fs::create_dir_all(&cache_dir);
cache_dir.join(format!("{:016x}.bars.bin", hash))
}
/// Check whether the cache file is newer than every `.dbn`/`.dbn.zst` file under `source_dir`.
@@ -856,7 +860,7 @@ pub fn imbalance_bars_sequential(trades: &[Mbp10Trade], threshold: f64) -> Vec<O
/// Scans `mbp10_dir/symbol/` for `.dbn` and `.dbn.zst` files, loads each,
/// extracts trades (action == 'T'), feeds into adaptive `ImbalanceBarSampler`.
///
/// Results are cached to `/tmp/.foxhunt_imbalance_cache/<hash>.bars.bin`.
/// Results are cached to `<mbp10_dir>/.cache/<hash>.bars.bin`.
/// Subsequent calls with the same parameters return in <1s if no source files
/// have changed.
///