fix(ml): preload OFI features in hyperopt adapter

The hyperopt preload_data() created a loader with default hyperparams
(mbp10_data_dir=None), so MBP-10/trades data was never loaded. Each
trial then set mbp10_data_dir → state_dim=51, but the preloaded data
had no OFI features → shape mismatch [128,43] vs [51,1024].

- Pass mbp10_data_dir/trades_data_dir to preload hyperparams
- Extract ofi_features from loader after preload
- Store as preloaded_ofi_features: Option<Arc<Vec<[f64;8]>>>
- Inject into each trial's DQNTrainer before training

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-07 20:43:47 +01:00
parent 7a9683d5fb
commit d7061ca2ac

View File

@@ -901,6 +901,9 @@ pub struct DQNTrainer {
mbp10_data_dir: Option<String>,
/// Trades data directory for VPIN / Kyle's Lambda (Schema::Trades)
trades_data_dir: Option<String>,
/// Preloaded OFI features from MBP-10 data (8 features per bar).
/// Loaded during `preload_data()` and injected into each trial's DQNTrainer.
preloaded_ofi_features: Option<Arc<Vec<[f64; 8]>>>,
}
/// Decode OHLCV bars from an already-opened DBN decoder.
@@ -1015,6 +1018,7 @@ impl DQNTrainer {
preloaded_val_data: None, // Loaded on first call to preload_data()
mbp10_data_dir: None, // No MBP-10 OFI by default
trades_data_dir: None, // No trades (VPIN/Kyle) by default
preloaded_ofi_features: None, // Loaded during preload_data()
})
}
@@ -1175,10 +1179,12 @@ impl DQNTrainer {
info!("Preloading training data from: {} ...", data_path_str);
let preload_start = std::time::Instant::now();
// Create a minimal internal trainer just for data loading.
// The hyperparameters don't affect data loading, only the feature extraction.
let default_hyperparams = DQNHyperparameters::default();
let mut loader = InternalDQNTrainer::new_with_device(default_hyperparams, self.device.clone())
// Create a minimal internal trainer for data loading.
// Pass OFI data dirs so MBP-10 + trades data are loaded during preload.
let mut preload_hyperparams = DQNHyperparameters::default();
preload_hyperparams.mbp10_data_dir = self.mbp10_data_dir.clone();
preload_hyperparams.trades_data_dir = self.trades_data_dir.clone();
let mut loader = InternalDQNTrainer::new_with_device(preload_hyperparams, self.device.clone())
.map_err(|e| MLError::TrainingError(format!(
"Failed to create data loader: {}", e
)))?;
@@ -1199,6 +1205,12 @@ impl DQNTrainer {
}
.map_err(|e| MLError::TrainingError(format!("Failed to preload data: {}", e)))?;
// Extract OFI features loaded by the trainer (from MBP-10 + trades data)
let ofi_features = loader.ofi_features.take();
if let Some(ref ofi) = ofi_features {
info!("OFI features preloaded: {} bars x 8 dims (VPIN, Kyle's Lambda, OFI, trade imbalance)", ofi.len());
}
let elapsed = preload_start.elapsed();
info!(
"Data preloaded: {} train + {} val samples in {:.1}s (cached for all trials)",
@@ -1209,6 +1221,7 @@ impl DQNTrainer {
self.preloaded_training_data = Some(Arc::new(train_data));
self.preloaded_val_data = Some(Arc::new(val_data));
self.preloaded_ofi_features = ofi_features.map(Arc::new);
Ok(())
}
@@ -2553,6 +2566,12 @@ impl HyperparameterOptimizable for DQNTrainer {
internal_trainer.init_stress_tester()
.map_err(|e| MLError::TrainingError(format!("Failed to init stress tester: {}", e)))?;
// Inject preloaded OFI features into the trial trainer (avoids reloading MBP-10 per trial)
if let Some(ref ofi) = self.preloaded_ofi_features {
internal_trainer.ofi_features = Some(ofi.as_ref().clone());
info!("Injected preloaded OFI features: {} bars x 8 dims", ofi.len());
}
// Cache reference for the closure (preloaded data lives on self, but
// catch_unwind requires the closure to be UnwindSafe — Arc<Vec<_>> is).
let cached_train = self.preloaded_training_data.clone();