feat: DQN data loader checks .fxcache before DBN streaming

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-31 23:59:49 +02:00
parent cba407b79d
commit 5ea7c51580

View File

@@ -596,6 +596,57 @@ impl DQNTrainer {
Vec<(FeatureVector, Vec<f64>)>,
Vec<(FeatureVector, Vec<f64>)>,
)> {
// ── .fxcache (flat binary feature cache) — fastest path ──────────────
if let Some(ref cache_dir) = self.feature_cache_dir {
let data_dir_path = Path::new(dbn_data_dir);
let mbp10_dir = self.hyperparams.mbp10_data_dir.as_ref().map(|s| Path::new(s.as_str()));
let trades_dir = self.hyperparams.trades_data_dir.as_ref().map(|s| Path::new(s.as_str()));
if let Ok(cache_key_hex) = crate::feature_cache::calculate_dbn_cache_key_full(
data_dir_path,
mbp10_dir,
trades_dir,
) {
if let Ok(key_bytes) = hex::decode(&cache_key_hex) {
if let Ok(key) = <[u8; 32]>::try_from(key_bytes) {
if let Some(fxcache_path) = crate::fxcache::find_fxcache(cache_dir, &key) {
match crate::fxcache::load_fxcache(&fxcache_path) {
Ok(cached) => {
info!(
"fxcache hit: {} bars from {:?}",
cached.bar_count, fxcache_path
);
// Set OFI features from cache
let has_ofi = cached.ofi.iter().any(|o| o.iter().any(|&v| v != 0.0));
if has_ofi {
self.ofi_features = Some(Arc::from(cached.ofi));
}
// Combine features + targets
let all_data: Vec<([f64; 42], Vec<f64>)> = cached
.features
.into_iter()
.zip(cached.targets.into_iter())
.map(|(f, t)| (f, t.to_vec()))
.collect();
// 80/20 split (same as other cache paths)
let split = (all_data.len() * 80) / 100;
let train = all_data[..split].to_vec();
let val = all_data[split..].to_vec();
return Ok((train, val));
}
Err(e) => {
debug!("fxcache load failed: {e}, falling through");
}
}
}
}
}
}
}
// ── DBN feature cache (dev/test speedup) ──────────────────────────────
// Check for a previously computed and serialised training dataset.
// The cache is keyed on all .dbn/.dbn.zst filenames + sizes + mtimes so