fix(ml): OFI preload uses adapter's load_ofi_features, not trainer's empty field
Root cause: preload_data() called loader.ofi_features.take() on the internal DQN trainer, but load_training_data() only loads OHLCV bars — it never populates ofi_features. The OFI loading is done by the hyperopt adapter's own load_ofi_features() method. Fixes: - preload_data() now calls self.load_ofi_features() directly - load_ofi_features() uses self.mbp10_data_dir when set (was hardcoded ../mbp10) - input_dim pre-computation uses OFI-aware size (51 when enabled, 43 otherwise) - CI 'latest' package: delete-then-upload to avoid GitLab duplicate file issue - Warn when mbp10_data_dir is set but no OFI features loaded Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1205,10 +1205,14 @@ impl DQNTrainer {
|
||||
}
|
||||
.map_err(|e| MLError::TrainingError(format!("Failed to preload data: {}", e)))?;
|
||||
|
||||
// Extract OFI features loaded by the trainer (from MBP-10 + trades data)
|
||||
let ofi_features = loader.ofi_features.take();
|
||||
// Load OFI features from MBP-10 data (separate from OHLCV loading).
|
||||
// The internal trainer's load_training_data() only loads OHLCV — OFI must be loaded
|
||||
// explicitly via the adapter's load_ofi_features() which uses self.mbp10_data_dir.
|
||||
let ofi_features = self.load_ofi_features();
|
||||
if let Some(ref ofi) = ofi_features {
|
||||
info!("OFI features preloaded: {} bars x 8 dims (VPIN, Kyle's Lambda, OFI, trade imbalance)", ofi.len());
|
||||
} else if self.mbp10_data_dir.is_some() {
|
||||
warn!("MBP-10 data dir configured but no OFI features loaded — state_dim mismatch likely");
|
||||
}
|
||||
|
||||
let elapsed = preload_start.elapsed();
|
||||
@@ -1459,9 +1463,14 @@ impl DQNTrainer {
|
||||
use crate::features::mbp10_loader::load_mbp10_snapshots_sync;
|
||||
use crate::features::ofi_calculator::OFICalculator;
|
||||
|
||||
let mbp10_dir = std::path::Path::new(&self.dbn_data_dir)
|
||||
.parent()
|
||||
.map(|p| p.join("mbp10"))?;
|
||||
// Use explicit mbp10_data_dir if set, otherwise fall back to ../mbp10 relative to OHLCV dir
|
||||
let mbp10_dir = if let Some(ref dir) = self.mbp10_data_dir {
|
||||
std::path::PathBuf::from(dir)
|
||||
} else {
|
||||
std::path::Path::new(&self.dbn_data_dir)
|
||||
.parent()
|
||||
.map(|p| p.join("mbp10"))?
|
||||
};
|
||||
|
||||
if !mbp10_dir.exists() {
|
||||
info!(
|
||||
|
||||
@@ -289,7 +289,8 @@ impl DQNTrainer {
|
||||
}
|
||||
|
||||
// Pre-compute hidden dims to get accurate model size for batch sizing.
|
||||
let input_dim: usize = 43;
|
||||
let ofi_pre = hyperparams.mbp10_data_dir.is_some();
|
||||
let input_dim: usize = if ofi_pre { 51 } else { 43 };
|
||||
let output_dim: usize = 5;
|
||||
let hidden_dims: Vec<usize> = match hyperparams.hidden_dim_base {
|
||||
Some(base) => {
|
||||
|
||||
@@ -523,8 +523,16 @@ spec:
|
||||
"${GITLAB}/api/v4/projects/1/packages/generic/foxhunt-services/${TAG}/${BIN_NAME}"
|
||||
done
|
||||
|
||||
# Also publish under 'latest' so deployments always have a known version
|
||||
# Delete old 'latest' package then re-upload (GitLab doesn't overwrite files)
|
||||
echo "=== Tagging as 'latest' ==="
|
||||
OLD_PKG=$(curl -sf -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
||||
"${GITLAB}/api/v4/projects/1/packages?package_name=foxhunt-services&package_version=latest" \
|
||||
| grep -oP '"id":\K[0-9]+' | head -1)
|
||||
if [ -n "$OLD_PKG" ]; then
|
||||
curl -sf -X DELETE -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
||||
"${GITLAB}/api/v4/projects/1/packages/${OLD_PKG}" && \
|
||||
echo "Deleted old 'latest' package (ID ${OLD_PKG})" || true
|
||||
fi
|
||||
for bin in "$WORKSPACE/bin/services/"*; do
|
||||
BIN_NAME=$(basename "$bin")
|
||||
curl -f --upload-file "$bin" \
|
||||
@@ -652,8 +660,16 @@ spec:
|
||||
"${GITLAB}/api/v4/projects/1/packages/generic/foxhunt-training/${TAG}/${BIN_NAME}"
|
||||
done
|
||||
|
||||
# Also publish under 'latest' so training workflows always have a known version
|
||||
# Delete old 'latest' package then re-upload (GitLab doesn't overwrite files)
|
||||
echo "=== Tagging as 'latest' ==="
|
||||
OLD_PKG=$(curl -sf -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
||||
"${GITLAB}/api/v4/projects/1/packages?package_name=foxhunt-training&package_version=latest" \
|
||||
| grep -oP '"id":\K[0-9]+' | head -1)
|
||||
if [ -n "$OLD_PKG" ]; then
|
||||
curl -sf -X DELETE -H "PRIVATE-TOKEN: ${GITLAB_PAT}" \
|
||||
"${GITLAB}/api/v4/projects/1/packages/${OLD_PKG}" && \
|
||||
echo "Deleted old 'latest' package (ID ${OLD_PKG})" || true
|
||||
fi
|
||||
for bin in "$WORKSPACE/bin/training/"*; do
|
||||
BIN_NAME=$(basename "$bin")
|
||||
curl -f --upload-file "$bin" \
|
||||
|
||||
Reference in New Issue
Block a user