From d7061ca2ac4fee9b5ca1d359f195e6482d1aabdc Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 7 Mar 2026 20:43:47 +0100 Subject: [PATCH] fix(ml): preload OFI features in hyperopt adapter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hyperopt preload_data() created a loader with default hyperparams (mbp10_data_dir=None), so MBP-10/trades data was never loaded. Each trial then set mbp10_data_dir → state_dim=51, but the preloaded data had no OFI features → shape mismatch [128,43] vs [51,1024]. - Pass mbp10_data_dir/trades_data_dir to preload hyperparams - Extract ofi_features from loader after preload - Store as preloaded_ofi_features: Option>> - Inject into each trial's DQNTrainer before training Co-Authored-By: Claude Opus 4.6 --- crates/ml/src/hyperopt/adapters/dqn.rs | 27 ++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/crates/ml/src/hyperopt/adapters/dqn.rs b/crates/ml/src/hyperopt/adapters/dqn.rs index bf652e443..6d23c8340 100644 --- a/crates/ml/src/hyperopt/adapters/dqn.rs +++ b/crates/ml/src/hyperopt/adapters/dqn.rs @@ -901,6 +901,9 @@ pub struct DQNTrainer { mbp10_data_dir: Option, /// Trades data directory for VPIN / Kyle's Lambda (Schema::Trades) trades_data_dir: Option, + /// Preloaded OFI features from MBP-10 data (8 features per bar). + /// Loaded during `preload_data()` and injected into each trial's DQNTrainer. + preloaded_ofi_features: Option>>, } /// Decode OHLCV bars from an already-opened DBN decoder. @@ -1015,6 +1018,7 @@ impl DQNTrainer { preloaded_val_data: None, // Loaded on first call to preload_data() mbp10_data_dir: None, // No MBP-10 OFI by default trades_data_dir: None, // No trades (VPIN/Kyle) by default + preloaded_ofi_features: None, // Loaded during preload_data() }) } @@ -1175,10 +1179,12 @@ impl DQNTrainer { info!("Preloading training data from: {} ...", data_path_str); let preload_start = std::time::Instant::now(); - // Create a minimal internal trainer just for data loading. - // The hyperparameters don't affect data loading, only the feature extraction. - let default_hyperparams = DQNHyperparameters::default(); - let mut loader = InternalDQNTrainer::new_with_device(default_hyperparams, self.device.clone()) + // Create a minimal internal trainer for data loading. + // Pass OFI data dirs so MBP-10 + trades data are loaded during preload. + let mut preload_hyperparams = DQNHyperparameters::default(); + preload_hyperparams.mbp10_data_dir = self.mbp10_data_dir.clone(); + preload_hyperparams.trades_data_dir = self.trades_data_dir.clone(); + let mut loader = InternalDQNTrainer::new_with_device(preload_hyperparams, self.device.clone()) .map_err(|e| MLError::TrainingError(format!( "Failed to create data loader: {}", e )))?; @@ -1199,6 +1205,12 @@ impl DQNTrainer { } .map_err(|e| MLError::TrainingError(format!("Failed to preload data: {}", e)))?; + // Extract OFI features loaded by the trainer (from MBP-10 + trades data) + let ofi_features = loader.ofi_features.take(); + if let Some(ref ofi) = ofi_features { + info!("OFI features preloaded: {} bars x 8 dims (VPIN, Kyle's Lambda, OFI, trade imbalance)", ofi.len()); + } + let elapsed = preload_start.elapsed(); info!( "Data preloaded: {} train + {} val samples in {:.1}s (cached for all trials)", @@ -1209,6 +1221,7 @@ impl DQNTrainer { self.preloaded_training_data = Some(Arc::new(train_data)); self.preloaded_val_data = Some(Arc::new(val_data)); + self.preloaded_ofi_features = ofi_features.map(Arc::new); Ok(()) } @@ -2553,6 +2566,12 @@ impl HyperparameterOptimizable for DQNTrainer { internal_trainer.init_stress_tester() .map_err(|e| MLError::TrainingError(format!("Failed to init stress tester: {}", e)))?; + // Inject preloaded OFI features into the trial trainer (avoids reloading MBP-10 per trial) + if let Some(ref ofi) = self.preloaded_ofi_features { + internal_trainer.ofi_features = Some(ofi.as_ref().clone()); + info!("Injected preloaded OFI features: {} bars x 8 dims", ofi.len()); + } + // Cache reference for the closure (preloaded data lives on self, but // catch_unwind requires the closure to be UnwindSafe — Arc> is). let cached_train = self.preloaded_training_data.clone();