fix(train): emit per-fold norm_stats.json for evaluate_baseline

evaluate_baseline looks for <output_dir>/norm_stats_fold{N}.json per fold
(matching the convention written by train_baseline_supervised.rs:812).
train_baseline_rl never produced this file — the fxcache has a single
<hex_key>.norm_stats.json written by precompute_features, which RL
training uses implicitly for pre-normalized features but never copies to
the per-fold paths evaluate wants. Result (L40S train-7r9zf):

  Error: NormStats not found at /workspace/output/norm_stats_fold0.json
  - cannot evaluate without training-set statistics (computing from test
    data would introduce lookahead bias). Run training first to generate
    this file.
  WARN: Evaluation failed, continuing

Evaluate fails gracefully but no report is produced.

Fix: new helper `fxcache::norm_stats_path_for_key(data_dir, override, cache_key)`
returns the canonical <hex_key>.norm_stats.json path that sits next to
the .fxcache file. train_baseline_rl resolves this once after the fxcache
load (falling back to None if the key is zero — i.e., DBN fallback path
with no precomputed stats) and copies it into the output dir as
norm_stats_fold{N}.json for every fold processed.

The RL-side fxcache norm stats are fold-independent (one z-score per
cache key, applied to all walk-forward windows), so the per-fold copies
are intentional duplicates of the same file — this matches evaluate's
per-fold lookup semantics without diverging from supervised convention.

Closes task #32.
This commit is contained in:
jgrusewski
2026-04-21 18:09:18 +02:00
parent 679881fa53
commit f988ca384c
2 changed files with 51 additions and 0 deletions

View File

@@ -625,6 +625,24 @@ fn run_training(args: &Args) -> Result<Vec<RlTrainingResult>> {
std::fs::create_dir_all(&args.output_dir)
.with_context(|| format!("Failed to create output dir: {}", args.output_dir.display()))?;
// Resolve the shared norm_stats.json sibling of the fxcache. precompute_features
// writes one norm_stats per cache key — RL training uses the same stats for
// every fold (unlike supervised which recomputes per-fold). evaluate_baseline
// expects per-fold files, so we copy the shared file into output_dir once per
// fold below. Falls back silently if the cache_key is zero (DBN-fallback load)
// or the stats file doesn't exist — evaluate will warn rather than fail the
// workflow.
let shared_norm_stats_src = if fxcache.cache_key == [0u8; 32] {
None
} else {
ml::fxcache::norm_stats_path_for_key(
&args.data_dir,
cache_dir_override.as_deref(),
&fxcache.cache_key,
)
.filter(|p| p.exists())
};
// 3. Create tokio runtime ONCE, create DQN trainer ONCE, upload fxcache to GPU ONCE
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
@@ -721,6 +739,20 @@ fn run_training(args: &Args) -> Result<Vec<RlTrainingResult>> {
range.val_end - range.val_start,
);
// Persist per-fold norm_stats copy for evaluate_baseline. Source is the
// single `.norm_stats.json` emitted by precompute_features; destination
// path matches the supervised training convention so evaluate finds it
// at `<output_dir>/norm_stats_fold{N}.json`.
if let Some(src) = &shared_norm_stats_src {
let dst = args.output_dir.join(format!("norm_stats_fold{}.json", range.fold));
if let Err(e) = std::fs::copy(src, &dst) {
warn!(
" norm_stats_fold{} copy failed ({} -> {}): {}",
range.fold, src.display(), dst.display(), e
);
}
}
// Slice features/targets for this fold (zero-copy from fxcache arrays)
let train_feat = &fxcache.features[range.train_start..range.train_end];
let val_feat = &fxcache.features[range.val_start..range.val_end];

View File

@@ -526,6 +526,25 @@ pub fn resolve_cache_dir(data_dir: &Path, override_dir: Option<&Path>) -> Option
None
}
/// Returns the path to the `{hex_key}.norm_stats.json` file that sits
/// alongside the `.fxcache` for the given cache key.
///
/// `precompute_features` writes this file at the same time as the fxcache
/// (see `precompute_features.rs:607`). Supervised training writes a per-fold
/// copy into the output dir for evaluation; RL training should use this
/// helper to produce the same per-fold copies (`norm_stats_fold{N}.json`)
/// so `evaluate_baseline` can find them. Returns `None` if the cache
/// directory can't be resolved.
pub fn norm_stats_path_for_key(
data_dir: &Path,
cache_dir_override: Option<&Path>,
cache_key: &[u8; 32],
) -> Option<PathBuf> {
let cache_dir = resolve_cache_dir(data_dir, cache_dir_override)?;
let hex_key = hex::encode(cache_key);
Some(cache_dir.join(format!("{hex_key}.norm_stats.json")))
}
/// Discover and load an fxcache file. Single source of truth for all callers.
///
/// Cache dir priority: `cache_dir_override` > `FOXHUNT_FEATURE_CACHE_DIR` env > walk-up sibling.