gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(data-loading): truncate OFI in lockstep with bars on max_bars path

Browse Source 
Two truncation sites in data_loading.rs truncated feature/target vectors
to max_bars but left self.ofi_features at the original (cache-full)
length. Downstream DqnGpuData::upload_slices asserts OFI len == num_bars
and surfaced this as "OFI/market data length mismatch: 175874 OFI rows
vs 5000 bars — fxcache is corrupt".

The cache was not corrupt — the upstream truncation was the bug. Fixed
by moving OFI storage below the truncation block in the fxcache path and
by adding a parallel truncate in the DBN fallback path. Added
debug_assert_eq on equal lengths post-truncation so future regressions
hard-fail in dev.

Also corrected the cuda_pipeline/mod.rs error message to point at the
actual culprit (upstream truncation desync) instead of blaming the
on-disk cache.
This commit is contained in:
jgrusewski
2026-04-22 00:13:08 +02:00
parent 83d524f866
commit 2ac956298b
2 changed files with 24 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/ml/src/cuda_pipeline/mod.rs
View File

@@ -286,7 +286,7 @@ impl DqnGpuData {
}
if ofi.len() != num_bars {
return Err(MLError::ModelError(format!(
"OFI/market data length mismatch: {} OFI rows vs {} bars — fxcache is corrupt",
"OFI/market data length mismatch: {} OFI rows vs {} bars — upstream truncation desync (check data_loading.rs max_bars paths)",
ofi.len(), num_bars,
)));
}

28
crates/ml/src/trainers/dqn/data_loading.rs
View File

@@ -157,8 +157,8 @@ impl DQNTrainer {
}
let ofi_nonzero = cached.ofi.iter().filter(|r| r.iter().any(|&v| v != 0.0)).count();
tracing::info!("OFI loaded from fxcache: {} bars, {} non-zero rows", cached.ofi.len(), ofi_nonzero);
self.ofi_features = Some(Arc::from(cached.ofi));
let mut ofi_data = cached.ofi;
let mut all_data: Vec<(FeatureVector, Vec<f64>)> = cached.features
.into_iter()
.zip(cached.targets.into_iter())
@@ -167,11 +167,18 @@ impl DQNTrainer {
})
.collect();
// Truncate to max_bars if configured (0 = unlimited)
// Truncate to max_bars if configured (0 = unlimited).
// OFI must be truncated in lockstep — downstream GPU upload asserts
// ofi.len() == num_bars and the desync previously surfaced as a
// misleading "fxcache is corrupt" error.
if self.hyperparams.max_bars > 0 && all_data.len() > self.hyperparams.max_bars {
tracing::info!("max_bars={}: truncating {} → {} bars", self.hyperparams.max_bars, all_data.len(), self.hyperparams.max_bars);
tracing::info!("max_bars={}: truncating {} → {} bars (incl. OFI)", self.hyperparams.max_bars, all_data.len(), self.hyperparams.max_bars);
all_data.truncate(self.hyperparams.max_bars);
ofi_data.truncate(self.hyperparams.max_bars);
}
debug_assert_eq!(ofi_data.len(), all_data.len(),
"OFI and market-data lengths must match after truncation");
self.ofi_features = Some(Arc::from(ofi_data));
let split = (all_data.len() * 80) / 100;
let train = all_data[..split].to_vec();
@@ -497,11 +504,22 @@ impl DQNTrainer {
feature_dim
);
// Truncate to max_bars if configured (0 = unlimited)
// Truncate to max_bars if configured (0 = unlimited).
// OFI (if present from the DBN path above) must be truncated in lockstep —
// downstream GPU upload asserts ofi.len() == num_bars.
if self.hyperparams.max_bars > 0 && training_data.len() > self.hyperparams.max_bars {
tracing::info!("max_bars={}: truncating {} → {} bars", self.hyperparams.max_bars, training_data.len(), self.hyperparams.max_bars);
tracing::info!("max_bars={}: truncating {} → {} bars (incl. OFI)", self.hyperparams.max_bars, training_data.len(), self.hyperparams.max_bars);
training_data.truncate(self.hyperparams.max_bars);
if let Some(ofi_arc) = self.ofi_features.take() {
let mut ofi_vec: Vec<[f64; 20]> = ofi_arc.iter().copied().collect();
ofi_vec.truncate(self.hyperparams.max_bars);
self.ofi_features = Some(Arc::from(ofi_vec));
}
}
debug_assert!(
self.ofi_features.as_ref().map_or(true, |o| o.len() == training_data.len()),
"OFI and training-data lengths must match after truncation",
);
// Split training data 80/20 for train/validation
let split_idx = (training_data.len() * 80) / 100;