feat: train_baseline_rl loads from fxcache, skipping DBN extraction
Auto-discovers fxcache via env var or sibling directory. Falls back to DBN loading if no cache found. Reconstructs aligned bars from cached targets for walk-forward windowing. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -808,27 +808,98 @@ fn run_training(args: &Args) -> Result<Vec<RlTrainingResult>> {
|
||||
info!(" Ensemble top-K: {} (training multiple models per fold)", args.ensemble_top_k);
|
||||
}
|
||||
|
||||
// 1. Load all OHLCV bars from DBN files
|
||||
info!("Step 1/5: Loading OHLCV bars from DBN files...");
|
||||
// 1. Try fxcache first, fall back to DBN loading + feature extraction
|
||||
info!("Step 1/5: Loading data...");
|
||||
let data_load_start = std::time::Instant::now();
|
||||
let bars = load_all_bars(&args.data_dir, &args.symbol)?;
|
||||
if bars.is_empty() {
|
||||
anyhow::bail!("No bars loaded from {}", args.data_dir.display());
|
||||
}
|
||||
info!(" Loaded {} bars ({} to {})",
|
||||
bars.len(),
|
||||
bars.first().map(|b| b.timestamp.to_string()).unwrap_or_default(),
|
||||
bars.last().map(|b| b.timestamp.to_string()).unwrap_or_default(),
|
||||
);
|
||||
|
||||
// 2. Extract features
|
||||
info!("Step 2/5: Extracting {}-dimensional features...", args.feature_dim);
|
||||
let all_features = extract_ml_features(&bars)
|
||||
.context("Feature extraction failed")?;
|
||||
info!(" Extracted {} feature vectors (warmup period consumed {} bars)",
|
||||
all_features.len(),
|
||||
bars.len().saturating_sub(all_features.len()),
|
||||
);
|
||||
// Auto-discover fxcache: env var > sibling feature-cache/ dir
|
||||
let fxcache_dir = std::env::var("FOXHUNT_FEATURE_CACHE_DIR").ok().map(PathBuf::from)
|
||||
.or_else(|| {
|
||||
let symbol_dir = args.data_dir.join(&args.symbol);
|
||||
let mut dir = symbol_dir.as_path();
|
||||
loop {
|
||||
if let Some(parent) = dir.parent() {
|
||||
let candidate = parent.join("feature-cache");
|
||||
if candidate.exists() { return Some(candidate); }
|
||||
if parent == dir { break; }
|
||||
dir = parent;
|
||||
} else { break; }
|
||||
}
|
||||
None
|
||||
});
|
||||
|
||||
// Try loading from fxcache
|
||||
let fxcache_data = fxcache_dir.and_then(|cache_dir| {
|
||||
let symbol_dir = args.data_dir.join(&args.symbol);
|
||||
let default_mbp10 = PathBuf::from("test_data/futures-baseline-mbp10");
|
||||
let default_trades = PathBuf::from("test_data/futures-baseline-trades");
|
||||
let mbp10 = args.mbp10_data_dir.as_deref().unwrap_or(&default_mbp10);
|
||||
let trades = args.trades_data_dir.as_deref().unwrap_or(&default_trades);
|
||||
let mbp10: Option<&Path> = if mbp10.exists() { Some(mbp10) } else { None };
|
||||
let trades: Option<&Path> = if trades.exists() { Some(trades) } else { None };
|
||||
|
||||
let key_hex = ml::feature_cache::calculate_dbn_cache_key_full(
|
||||
&symbol_dir, mbp10, trades,
|
||||
).ok()?;
|
||||
let key: [u8; 32] = hex::decode(&key_hex).ok()?.try_into().ok()?;
|
||||
let path = ml::fxcache::find_fxcache(&cache_dir, &key)?;
|
||||
match ml::fxcache::load_fxcache(&path) {
|
||||
Ok(data) => {
|
||||
info!("fxcache hit: {} bars from {:?}", data.bar_count, path);
|
||||
Some(data)
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("fxcache load failed: {e}");
|
||||
None
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let (bars, all_features) = if let Some(cached) = fxcache_data {
|
||||
// Reconstruct bars from cached targets (raw_close at index 2)
|
||||
let n = cached.bar_count;
|
||||
let mut bars = Vec::with_capacity(n);
|
||||
for i in 0..n {
|
||||
let close = cached.targets[i][2]; // raw_close
|
||||
let next_close = cached.targets[i][3]; // raw_next_close
|
||||
bars.push(ml::features::extraction::OHLCVBar {
|
||||
timestamp: chrono::Utc::now(), // placeholder — walk-forward uses index not timestamp
|
||||
open: close,
|
||||
high: close,
|
||||
low: close,
|
||||
close,
|
||||
volume: 0.0,
|
||||
});
|
||||
}
|
||||
info!(" Loaded {} bars + features from fxcache in {:.1}s",
|
||||
n, data_load_start.elapsed().as_secs_f64());
|
||||
(bars, cached.features)
|
||||
} else {
|
||||
// Fall back to DBN loading
|
||||
info!(" Loading OHLCV bars from DBN files...");
|
||||
let bars = load_all_bars(&args.data_dir, &args.symbol)?;
|
||||
if bars.is_empty() {
|
||||
anyhow::bail!("No bars loaded from {}", args.data_dir.display());
|
||||
}
|
||||
info!(" Loaded {} bars ({} to {})",
|
||||
bars.len(),
|
||||
bars.first().map(|b| b.timestamp.to_string()).unwrap_or_default(),
|
||||
bars.last().map(|b| b.timestamp.to_string()).unwrap_or_default(),
|
||||
);
|
||||
|
||||
// 2. Extract features
|
||||
info!(" Extracting {}-dimensional features...", args.feature_dim);
|
||||
let all_features = extract_ml_features(&bars)
|
||||
.context("Feature extraction failed")?;
|
||||
info!(" Extracted {} feature vectors (warmup period consumed {} bars)",
|
||||
all_features.len(),
|
||||
bars.len().saturating_sub(all_features.len()),
|
||||
);
|
||||
// Trim bars to align with features (skip warmup)
|
||||
let warmup_offset = bars.len().saturating_sub(all_features.len());
|
||||
let bars = bars[warmup_offset..].to_vec();
|
||||
(bars, all_features)
|
||||
};
|
||||
|
||||
// Since features skip the warmup period, we need bars aligned to features.
|
||||
// Features start at bar index warmup_offset (typically 50).
|
||||
|
||||
Reference in New Issue
Block a user