diff --git a/crates/ml/src/cuda_pipeline/mod.rs b/crates/ml/src/cuda_pipeline/mod.rs index 981ef4b26..1b28bbb26 100644 --- a/crates/ml/src/cuda_pipeline/mod.rs +++ b/crates/ml/src/cuda_pipeline/mod.rs @@ -286,7 +286,7 @@ impl DqnGpuData { } if ofi.len() != num_bars { return Err(MLError::ModelError(format!( - "OFI/market data length mismatch: {} OFI rows vs {} bars — fxcache is corrupt", + "OFI/market data length mismatch: {} OFI rows vs {} bars — upstream truncation desync (check data_loading.rs max_bars paths)", ofi.len(), num_bars, ))); } diff --git a/crates/ml/src/trainers/dqn/data_loading.rs b/crates/ml/src/trainers/dqn/data_loading.rs index 9207c6575..ced3da860 100644 --- a/crates/ml/src/trainers/dqn/data_loading.rs +++ b/crates/ml/src/trainers/dqn/data_loading.rs @@ -157,8 +157,8 @@ impl DQNTrainer { } let ofi_nonzero = cached.ofi.iter().filter(|r| r.iter().any(|&v| v != 0.0)).count(); tracing::info!("OFI loaded from fxcache: {} bars, {} non-zero rows", cached.ofi.len(), ofi_nonzero); - self.ofi_features = Some(Arc::from(cached.ofi)); + let mut ofi_data = cached.ofi; let mut all_data: Vec<(FeatureVector, Vec)> = cached.features .into_iter() .zip(cached.targets.into_iter()) @@ -167,11 +167,18 @@ impl DQNTrainer { }) .collect(); - // Truncate to max_bars if configured (0 = unlimited) + // Truncate to max_bars if configured (0 = unlimited). + // OFI must be truncated in lockstep — downstream GPU upload asserts + // ofi.len() == num_bars and the desync previously surfaced as a + // misleading "fxcache is corrupt" error. if self.hyperparams.max_bars > 0 && all_data.len() > self.hyperparams.max_bars { - tracing::info!("max_bars={}: truncating {} → {} bars", self.hyperparams.max_bars, all_data.len(), self.hyperparams.max_bars); + tracing::info!("max_bars={}: truncating {} → {} bars (incl. OFI)", self.hyperparams.max_bars, all_data.len(), self.hyperparams.max_bars); all_data.truncate(self.hyperparams.max_bars); + ofi_data.truncate(self.hyperparams.max_bars); } + debug_assert_eq!(ofi_data.len(), all_data.len(), + "OFI and market-data lengths must match after truncation"); + self.ofi_features = Some(Arc::from(ofi_data)); let split = (all_data.len() * 80) / 100; let train = all_data[..split].to_vec(); @@ -497,11 +504,22 @@ impl DQNTrainer { feature_dim ); - // Truncate to max_bars if configured (0 = unlimited) + // Truncate to max_bars if configured (0 = unlimited). + // OFI (if present from the DBN path above) must be truncated in lockstep — + // downstream GPU upload asserts ofi.len() == num_bars. if self.hyperparams.max_bars > 0 && training_data.len() > self.hyperparams.max_bars { - tracing::info!("max_bars={}: truncating {} → {} bars", self.hyperparams.max_bars, training_data.len(), self.hyperparams.max_bars); + tracing::info!("max_bars={}: truncating {} → {} bars (incl. OFI)", self.hyperparams.max_bars, training_data.len(), self.hyperparams.max_bars); training_data.truncate(self.hyperparams.max_bars); + if let Some(ofi_arc) = self.ofi_features.take() { + let mut ofi_vec: Vec<[f64; 20]> = ofi_arc.iter().copied().collect(); + ofi_vec.truncate(self.hyperparams.max_bars); + self.ofi_features = Some(Arc::from(ofi_vec)); + } } + debug_assert!( + self.ofi_features.as_ref().map_or(true, |o| o.len() == training_data.len()), + "OFI and training-data lengths must match after truncation", + ); // Split training data 80/20 for train/validation let split_idx = (training_data.len() * 80) / 100;