feat: DQN data loader checks .fxcache before DBN streaming
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -596,6 +596,57 @@ impl DQNTrainer {
|
||||
Vec<(FeatureVector, Vec<f64>)>,
|
||||
Vec<(FeatureVector, Vec<f64>)>,
|
||||
)> {
|
||||
// ── .fxcache (flat binary feature cache) — fastest path ──────────────
|
||||
if let Some(ref cache_dir) = self.feature_cache_dir {
|
||||
let data_dir_path = Path::new(dbn_data_dir);
|
||||
let mbp10_dir = self.hyperparams.mbp10_data_dir.as_ref().map(|s| Path::new(s.as_str()));
|
||||
let trades_dir = self.hyperparams.trades_data_dir.as_ref().map(|s| Path::new(s.as_str()));
|
||||
|
||||
if let Ok(cache_key_hex) = crate::feature_cache::calculate_dbn_cache_key_full(
|
||||
data_dir_path,
|
||||
mbp10_dir,
|
||||
trades_dir,
|
||||
) {
|
||||
if let Ok(key_bytes) = hex::decode(&cache_key_hex) {
|
||||
if let Ok(key) = <[u8; 32]>::try_from(key_bytes) {
|
||||
if let Some(fxcache_path) = crate::fxcache::find_fxcache(cache_dir, &key) {
|
||||
match crate::fxcache::load_fxcache(&fxcache_path) {
|
||||
Ok(cached) => {
|
||||
info!(
|
||||
"fxcache hit: {} bars from {:?}",
|
||||
cached.bar_count, fxcache_path
|
||||
);
|
||||
|
||||
// Set OFI features from cache
|
||||
let has_ofi = cached.ofi.iter().any(|o| o.iter().any(|&v| v != 0.0));
|
||||
if has_ofi {
|
||||
self.ofi_features = Some(Arc::from(cached.ofi));
|
||||
}
|
||||
|
||||
// Combine features + targets
|
||||
let all_data: Vec<([f64; 42], Vec<f64>)> = cached
|
||||
.features
|
||||
.into_iter()
|
||||
.zip(cached.targets.into_iter())
|
||||
.map(|(f, t)| (f, t.to_vec()))
|
||||
.collect();
|
||||
|
||||
// 80/20 split (same as other cache paths)
|
||||
let split = (all_data.len() * 80) / 100;
|
||||
let train = all_data[..split].to_vec();
|
||||
let val = all_data[split..].to_vec();
|
||||
return Ok((train, val));
|
||||
}
|
||||
Err(e) => {
|
||||
debug!("fxcache load failed: {e}, falling through");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── DBN feature cache (dev/test speedup) ──────────────────────────────
|
||||
// Check for a previously computed and serialised training dataset.
|
||||
// The cache is keyed on all .dbn/.dbn.zst filenames + sizes + mtimes so
|
||||
|
||||
Reference in New Issue
Block a user