feat(ml): wire OFI data dirs into hyperopt DQN adapter

Add --mbp10-data-dir and --trades-data-dir CLI args to
hyperopt_baseline_rl binary so hyperopt trials can use real
order book and trade data for VPIN/Kyle's Lambda features.

- DQNTrainer: add mbp10_data_dir/trades_data_dir fields + with_ofi_data_dirs() builder
- DQNHyperparameters: pipe through from trainer instead of hardcoded None
- download-trades-job: fix nodeSelector to ci-compile-cpu (platform pool full)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-03-07 19:36:42 +01:00
parent c6c550a2be
commit 3f4c39e035
3 changed files with 31 additions and 5 deletions

View File

@@ -118,6 +118,14 @@ struct Args {
/// Initial trading capital in dollars
#[arg(long, default_value = "35000")]
initial_capital: f64,
/// Path to MBP-10 order book data for OFI features (order flow imbalance)
#[arg(long)]
mbp10_data_dir: Option<PathBuf>,
/// Path to trades data for VPIN / Kyle's Lambda features
#[arg(long)]
trades_data_dir: Option<PathBuf>,
}
/// Result entry for one model's hyperopt run
@@ -157,7 +165,11 @@ fn run_dqn_hyperopt(args: &Args, parallel: usize, gpu_devices: &[candle_core::De
.with_training_paths(training_paths)
.with_initial_capital(args.initial_capital)
.with_costs(args.tx_cost_bps, args.tick_size, args.spread_ticks)
.with_devices(gpu_devices.to_vec());
.with_devices(gpu_devices.to_vec())
.with_ofi_data_dirs(
args.mbp10_data_dir.as_ref().map(|p| p.to_string_lossy().into_owned()),
args.trades_data_dir.as_ref().map(|p| p.to_string_lossy().into_owned()),
);
// Preload training data once — all trials reuse via Arc (no per-trial disk I/O)
if let Err(e) = trainer.preload_data() {

View File

@@ -896,6 +896,11 @@ pub struct DQNTrainer {
preloaded_training_data: Option<Arc<Vec<(FeatureVector, Vec<f64>)>>>,
/// Preloaded validation data (same lifecycle as training data)
preloaded_val_data: Option<Arc<Vec<(FeatureVector, Vec<f64>)>>>,
/// MBP-10 data directory for OFI features (order book snapshots)
mbp10_data_dir: Option<String>,
/// Trades data directory for VPIN / Kyle's Lambda (Schema::Trades)
trades_data_dir: Option<String>,
}
/// Decode OHLCV bars from an already-opened DBN decoder.
@@ -1008,6 +1013,8 @@ impl DQNTrainer {
spread_ticks: 1.0, // ES typical: 1 tick spread
preloaded_training_data: None, // No preloaded data by default
preloaded_val_data: None, // Loaded on first call to preload_data()
mbp10_data_dir: None, // No MBP-10 OFI by default
trades_data_dir: None, // No trades (VPIN/Kyle) by default
})
}
@@ -1130,6 +1137,13 @@ impl DQNTrainer {
self
}
/// Set MBP-10 and trades data directories for OFI features (VPIN, Kyle's Lambda)
pub fn with_ofi_data_dirs(mut self, mbp10_dir: Option<String>, trades_dir: Option<String>) -> Self {
self.mbp10_data_dir = mbp10_dir;
self.trades_data_dir = trades_dir;
self
}
/// Preload training data from DBN files once, caching it for reuse across
/// all hyperopt trials. This eliminates the per-trial cost of reading 36
/// `.dbn.zst` files, decompressing them, and extracting 40 features.
@@ -2490,9 +2504,9 @@ impl HyperparameterOptimizable for DQNTrainer {
use_dsr: true,
dsr_eta: params.dsr_eta,
// MBP-10 data directory for OFI features (None = no OFI in hyperopt)
mbp10_data_dir: None,
trades_data_dir: None,
// MBP-10 + trades data directories for OFI features (VPIN, Kyle's Lambda)
mbp10_data_dir: self.mbp10_data_dir.clone(),
trades_data_dir: self.trades_data_dir.clone(),
// Offline RL: disabled in hyperopt (online experience collection)
offline_mode: false,

View File

@@ -41,7 +41,7 @@ spec:
foxhunt/job-type: data-download
spec:
nodeSelector:
k8s.scaleway.com/pool-name: platform
k8s.scaleway.com/pool-name: ci-compile-cpu
tolerations:
- key: node.cilium.io/agent-not-ready
operator: Exists