fix: filter DBN files by --symbol in precompute (was mixing ES+NQ+ZN+6E)

collect_dbn_files_filtered() checks for symbol subdirectory first,
falls back to filename matching. Precompute now loads only the
target symbol's data instead of all 4 instruments mixed together.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-04-03 18:20:42 +02:00
parent 9f7c14978f
commit b7840fc978
3 changed files with 46 additions and 5 deletions

View File

@@ -96,7 +96,8 @@ use clap::Parser;
use tracing::info;
use ml::trainers::dqn::{
collect_dbn_files_recursive, extract_features_from_bars, extract_ohlcv_bars_from_dbn,
collect_dbn_files_filtered, collect_dbn_files_recursive, extract_features_from_bars,
extract_ohlcv_bars_from_dbn,
};
use ml::features::extraction::OHLCVBar;
@@ -210,10 +211,24 @@ async fn main() -> Result<()> {
let t0 = Instant::now();
// ── Step 1: Load OHLCV bars from DBN files ──────────────────────────────
info!("Loading OHLCV bars from {}...", data_dir.display());
let mut dbn_files = collect_dbn_files_recursive(&data_dir);
info!("Loading OHLCV bars from {} (symbol={})...", data_dir.display(), opts.symbol);
let mut dbn_files = collect_dbn_files_filtered(&data_dir, Some(&opts.symbol));
if dbn_files.is_empty() {
anyhow::bail!(
"No DBN files found for symbol '{}' in {}. Available: {:?}",
opts.symbol,
data_dir.display(),
std::fs::read_dir(&data_dir)
.ok()
.map(|d| d.filter_map(|e| e.ok())
.filter(|e| e.path().is_dir())
.map(|e| e.file_name().to_string_lossy().into_owned())
.collect::<Vec<_>>())
.unwrap_or_default()
);
}
dbn_files.sort();
info!("Found {} DBN files", dbn_files.len());
info!("Found {} DBN files for symbol '{}'", dbn_files.len(), opts.symbol);
let mut all_bars: Vec<OHLCVBar> = Vec::new();
for file in &dbn_files {

View File

@@ -48,6 +48,32 @@ pub fn collect_dbn_files_recursive(dir: &Path) -> Vec<std::path::PathBuf> {
files
}
/// Collect DBN files, filtering to a specific symbol subdirectory.
/// If `symbol` is `Some("ES.FUT")`, only loads files from `dir/ES.FUT/`.
/// If the symbol subdirectory doesn't exist, falls back to filename matching.
pub fn collect_dbn_files_filtered(dir: &Path, symbol: Option<&str>) -> Vec<std::path::PathBuf> {
match symbol {
Some(sym) => {
let symbol_dir = dir.join(sym);
if symbol_dir.is_dir() {
collect_dbn_files_recursive(&symbol_dir)
} else {
// No subdirectory — filter files by name containing symbol
collect_dbn_files_recursive(dir)
.into_iter()
.filter(|p| {
p.file_name()
.and_then(|n| n.to_str())
.map(|n| n.contains(sym))
.unwrap_or(false)
})
.collect()
}
}
None => collect_dbn_files_recursive(dir),
}
}
/// Extract OHLCV bars from a single DBN file — no DQNTrainer required.
pub fn extract_ohlcv_bars_from_dbn(file_path: &Path) -> Result<Vec<OHLCVBar>> {
if is_zstd_file(file_path)? {

View File

@@ -38,7 +38,7 @@ pub use config::{DQNAgentType, DQNHyperparameters};
pub use early_stopping::EarlyStopping;
pub use lr_scheduler::{LRDecayType, LRScheduler};
pub use statistics::{FeatureStatistics, QValueStats};
pub use data_loading::{collect_dbn_files_recursive, extract_ohlcv_bars_from_dbn};
pub use data_loading::{collect_dbn_files_filtered, collect_dbn_files_recursive, extract_ohlcv_bars_from_dbn};
pub use features::extract_features_from_bars;
pub use trainer::DQNTrainer;