fix(ml): preload OFI features in hyperopt adapter
The hyperopt preload_data() created a loader with default hyperparams (mbp10_data_dir=None), so MBP-10/trades data was never loaded. Each trial then set mbp10_data_dir → state_dim=51, but the preloaded data had no OFI features → shape mismatch [128,43] vs [51,1024]. - Pass mbp10_data_dir/trades_data_dir to preload hyperparams - Extract ofi_features from loader after preload - Store as preloaded_ofi_features: Option<Arc<Vec<[f64;8]>>> - Inject into each trial's DQNTrainer before training Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -901,6 +901,9 @@ pub struct DQNTrainer {
|
||||
mbp10_data_dir: Option<String>,
|
||||
/// Trades data directory for VPIN / Kyle's Lambda (Schema::Trades)
|
||||
trades_data_dir: Option<String>,
|
||||
/// Preloaded OFI features from MBP-10 data (8 features per bar).
|
||||
/// Loaded during `preload_data()` and injected into each trial's DQNTrainer.
|
||||
preloaded_ofi_features: Option<Arc<Vec<[f64; 8]>>>,
|
||||
}
|
||||
|
||||
/// Decode OHLCV bars from an already-opened DBN decoder.
|
||||
@@ -1015,6 +1018,7 @@ impl DQNTrainer {
|
||||
preloaded_val_data: None, // Loaded on first call to preload_data()
|
||||
mbp10_data_dir: None, // No MBP-10 OFI by default
|
||||
trades_data_dir: None, // No trades (VPIN/Kyle) by default
|
||||
preloaded_ofi_features: None, // Loaded during preload_data()
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1175,10 +1179,12 @@ impl DQNTrainer {
|
||||
info!("Preloading training data from: {} ...", data_path_str);
|
||||
let preload_start = std::time::Instant::now();
|
||||
|
||||
// Create a minimal internal trainer just for data loading.
|
||||
// The hyperparameters don't affect data loading, only the feature extraction.
|
||||
let default_hyperparams = DQNHyperparameters::default();
|
||||
let mut loader = InternalDQNTrainer::new_with_device(default_hyperparams, self.device.clone())
|
||||
// Create a minimal internal trainer for data loading.
|
||||
// Pass OFI data dirs so MBP-10 + trades data are loaded during preload.
|
||||
let mut preload_hyperparams = DQNHyperparameters::default();
|
||||
preload_hyperparams.mbp10_data_dir = self.mbp10_data_dir.clone();
|
||||
preload_hyperparams.trades_data_dir = self.trades_data_dir.clone();
|
||||
let mut loader = InternalDQNTrainer::new_with_device(preload_hyperparams, self.device.clone())
|
||||
.map_err(|e| MLError::TrainingError(format!(
|
||||
"Failed to create data loader: {}", e
|
||||
)))?;
|
||||
@@ -1199,6 +1205,12 @@ impl DQNTrainer {
|
||||
}
|
||||
.map_err(|e| MLError::TrainingError(format!("Failed to preload data: {}", e)))?;
|
||||
|
||||
// Extract OFI features loaded by the trainer (from MBP-10 + trades data)
|
||||
let ofi_features = loader.ofi_features.take();
|
||||
if let Some(ref ofi) = ofi_features {
|
||||
info!("OFI features preloaded: {} bars x 8 dims (VPIN, Kyle's Lambda, OFI, trade imbalance)", ofi.len());
|
||||
}
|
||||
|
||||
let elapsed = preload_start.elapsed();
|
||||
info!(
|
||||
"Data preloaded: {} train + {} val samples in {:.1}s (cached for all trials)",
|
||||
@@ -1209,6 +1221,7 @@ impl DQNTrainer {
|
||||
|
||||
self.preloaded_training_data = Some(Arc::new(train_data));
|
||||
self.preloaded_val_data = Some(Arc::new(val_data));
|
||||
self.preloaded_ofi_features = ofi_features.map(Arc::new);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -2553,6 +2566,12 @@ impl HyperparameterOptimizable for DQNTrainer {
|
||||
internal_trainer.init_stress_tester()
|
||||
.map_err(|e| MLError::TrainingError(format!("Failed to init stress tester: {}", e)))?;
|
||||
|
||||
// Inject preloaded OFI features into the trial trainer (avoids reloading MBP-10 per trial)
|
||||
if let Some(ref ofi) = self.preloaded_ofi_features {
|
||||
internal_trainer.ofi_features = Some(ofi.as_ref().clone());
|
||||
info!("Injected preloaded OFI features: {} bars x 8 dims", ofi.len());
|
||||
}
|
||||
|
||||
// Cache reference for the closure (preloaded data lives on self, but
|
||||
// catch_unwind requires the closure to be UnwindSafe — Arc<Vec<_>> is).
|
||||
let cached_train = self.preloaded_training_data.clone();
|
||||
|
||||
Reference in New Issue
Block a user