feat(phase-e-4-a): walk-forward CV via --data-start-offset
Adds a sliding-window walk-forward harness for the T10 backtest:
- New load_snapshots_from_fxcache_at(start_offset, ...) loader variant
reads bars [start_offset..start_offset+max_snapshots) from the fxcache.
Alpha-cache lookups use absolute bar indices, so the same
alpha_logits_cache.bin works across folds.
- New --data-start-offset CLI flag on alpha_compose_backtest.
- scripts/walk_forward_cv.sh runs 3 folds (window=700K, train_frac=0.6)
at offsets 0 / 600K / 1.2M, producing /tmp/cv_fold_{A,B,C}.json plus
an aggregated mean±stddev Sharpe table across folds.
Walk-forward result (alpha_logits_cache trained on bars 0..1.57M, so
fold C eval is fully past the stacker cut):
cost fold-A fold-B fold-C mean ± stddev
0.0000 +91.52 -21.44 +46.74 +38.94 ± 56.88
0.0625 +84.94 -27.97 +38.42 +31.79 ± 56.74
0.1250 +79.91 -31.22 +33.51 +27.40 ± 55.82
0.2500 +72.77 -45.41 +15.16 +14.17 ± 59.09
0.5000 +50.52 -59.82 -12.75 -7.35 ± 55.37
Fold B (mid-quarter, bars 600K..1.3M) is a disaster — win rate
collapses to 0-22% across all costs. Folds A and C succeed strongly.
Cross-fold SD ≈ mean, so the policy is regime-dependent and cannot
be reliably deployed without regime detection.
Mean Sharpe at half-tick (+27.40) is still ~7× the stateless
Phase 1d.4 baseline (-4.0), so the temporal encoder adds real value
on average — but the single-window +62 OOS celebrated earlier was
a cherry-picked favorable regime, not a deployment-ready result.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -175,6 +175,12 @@ struct Cli {
|
||||
/// Snapshots to load from the fxcache.
|
||||
#[arg(long, default_value_t = 1_500_000)]
|
||||
max_snapshots: usize,
|
||||
/// Bars to skip at the front of the fxcache before windowing. Used by
|
||||
/// walk-forward CV scripts to slide a fixed-size train+eval window
|
||||
/// across the fxcache so each fold sees a distinct chronological
|
||||
/// region.
|
||||
#[arg(long, default_value_t = 0)]
|
||||
data_start_offset: usize,
|
||||
/// Episode horizon in snapshots.
|
||||
#[arg(long, default_value_t = 600)]
|
||||
horizon: usize,
|
||||
@@ -433,8 +439,9 @@ fn main() -> Result<()> {
|
||||
info!("Loaded fill model");
|
||||
let alpha_cache = ml::env::loaders::load_alpha_cache(&cli.alpha_cache)?;
|
||||
info!("Loaded alpha cache: {} entries", alpha_cache.len());
|
||||
let rows = ml::env::loaders::load_snapshots_from_fxcache(
|
||||
let rows = ml::env::loaders::load_snapshots_from_fxcache_at(
|
||||
&cli.fxcache_path,
|
||||
cli.data_start_offset,
|
||||
cli.max_snapshots,
|
||||
Some(&alpha_cache),
|
||||
cli.real_spread,
|
||||
|
||||
40
crates/ml/src/env/loaders.rs
vendored
40
crates/ml/src/env/loaders.rs
vendored
@@ -133,6 +133,22 @@ pub fn load_snapshots_from_fxcache(
|
||||
// This commit introduces the parameter; the real-peek implementation
|
||||
// lands in Task 5 follow-on.
|
||||
mbp10_dir: Option<&Path>,
|
||||
) -> Result<Vec<SnapshotRow>> {
|
||||
load_snapshots_from_fxcache_at(fxcache_path, 0, max_snapshots, alpha_cache, use_real_spread, mbp10_dir)
|
||||
}
|
||||
|
||||
/// Like `load_snapshots_from_fxcache` but starts at `start_offset` bars
|
||||
/// into the fxcache. Used by walk-forward CV so each fold sees a
|
||||
/// distinct chronological window. When `alpha_cache` is provided it must
|
||||
/// be a slice indexed in the SAME absolute bar coordinates as the
|
||||
/// fxcache (the loader will skip the first `start_offset` entries).
|
||||
pub fn load_snapshots_from_fxcache_at(
|
||||
fxcache_path: &Path,
|
||||
start_offset: usize,
|
||||
max_snapshots: usize,
|
||||
alpha_cache: Option<&[f32]>,
|
||||
use_real_spread: bool,
|
||||
mbp10_dir: Option<&Path>,
|
||||
) -> Result<Vec<SnapshotRow>> {
|
||||
if mbp10_dir.is_some() {
|
||||
warn!(
|
||||
@@ -153,8 +169,18 @@ pub fn load_snapshots_from_fxcache(
|
||||
);
|
||||
}
|
||||
let n_bars_total = reader.bar_count();
|
||||
let n = n_bars_total.min(max_snapshots);
|
||||
info!("fxcache: {} total bars, taking {} for the env", n_bars_total, n);
|
||||
if start_offset >= n_bars_total {
|
||||
anyhow::bail!(
|
||||
"start_offset {} ≥ fxcache bar count {}",
|
||||
start_offset, n_bars_total
|
||||
);
|
||||
}
|
||||
let n_available = n_bars_total - start_offset;
|
||||
let n = n_available.min(max_snapshots);
|
||||
info!(
|
||||
"fxcache: {} total bars, starting at offset {}, taking {} for the env",
|
||||
n_bars_total, start_offset, n
|
||||
);
|
||||
info!(
|
||||
"fxcache loader: spread mode = {}",
|
||||
if use_real_spread { "REAL (derived from features[78] spread_bps)" }
|
||||
@@ -162,11 +188,10 @@ pub fn load_snapshots_from_fxcache(
|
||||
);
|
||||
|
||||
if let Some(cache) = alpha_cache {
|
||||
if cache.len() < n {
|
||||
if cache.len() < start_offset + n {
|
||||
anyhow::bail!(
|
||||
"alpha cache has {} entries but env wants {} bars",
|
||||
cache.len(),
|
||||
n
|
||||
"alpha cache has {} entries but env wants bars {}..{}",
|
||||
cache.len(), start_offset, start_offset + n
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -179,7 +204,8 @@ pub fn load_snapshots_from_fxcache(
|
||||
let mut max_spread = f32::NEG_INFINITY;
|
||||
let mut n_floor_hits = 0_usize;
|
||||
let mut n_cap_hits = 0_usize;
|
||||
for i in 0..n {
|
||||
for local_i in 0..n {
|
||||
let i = start_offset + local_i;
|
||||
let rec = reader.record(i);
|
||||
let mid = rec.targets[COL_RAW_CLOSE - FEAT_DIM];
|
||||
if !mid.is_finite() || mid <= 0.0 {
|
||||
|
||||
Reference in New Issue
Block a user