diff --git a/crates/ml/examples/alpha_compose_backtest.rs b/crates/ml/examples/alpha_compose_backtest.rs index 005d36347..fe0a9314b 100644 --- a/crates/ml/examples/alpha_compose_backtest.rs +++ b/crates/ml/examples/alpha_compose_backtest.rs @@ -175,6 +175,12 @@ struct Cli { /// Snapshots to load from the fxcache. #[arg(long, default_value_t = 1_500_000)] max_snapshots: usize, + /// Bars to skip at the front of the fxcache before windowing. Used by + /// walk-forward CV scripts to slide a fixed-size train+eval window + /// across the fxcache so each fold sees a distinct chronological + /// region. + #[arg(long, default_value_t = 0)] + data_start_offset: usize, /// Episode horizon in snapshots. #[arg(long, default_value_t = 600)] horizon: usize, @@ -433,8 +439,9 @@ fn main() -> Result<()> { info!("Loaded fill model"); let alpha_cache = ml::env::loaders::load_alpha_cache(&cli.alpha_cache)?; info!("Loaded alpha cache: {} entries", alpha_cache.len()); - let rows = ml::env::loaders::load_snapshots_from_fxcache( + let rows = ml::env::loaders::load_snapshots_from_fxcache_at( &cli.fxcache_path, + cli.data_start_offset, cli.max_snapshots, Some(&alpha_cache), cli.real_spread, diff --git a/crates/ml/src/env/loaders.rs b/crates/ml/src/env/loaders.rs index 99f96f90f..5f6b022b1 100644 --- a/crates/ml/src/env/loaders.rs +++ b/crates/ml/src/env/loaders.rs @@ -133,6 +133,22 @@ pub fn load_snapshots_from_fxcache( // This commit introduces the parameter; the real-peek implementation // lands in Task 5 follow-on. mbp10_dir: Option<&Path>, +) -> Result> { + load_snapshots_from_fxcache_at(fxcache_path, 0, max_snapshots, alpha_cache, use_real_spread, mbp10_dir) +} + +/// Like `load_snapshots_from_fxcache` but starts at `start_offset` bars +/// into the fxcache. Used by walk-forward CV so each fold sees a +/// distinct chronological window. When `alpha_cache` is provided it must +/// be a slice indexed in the SAME absolute bar coordinates as the +/// fxcache (the loader will skip the first `start_offset` entries). +pub fn load_snapshots_from_fxcache_at( + fxcache_path: &Path, + start_offset: usize, + max_snapshots: usize, + alpha_cache: Option<&[f32]>, + use_real_spread: bool, + mbp10_dir: Option<&Path>, ) -> Result> { if mbp10_dir.is_some() { warn!( @@ -153,8 +169,18 @@ pub fn load_snapshots_from_fxcache( ); } let n_bars_total = reader.bar_count(); - let n = n_bars_total.min(max_snapshots); - info!("fxcache: {} total bars, taking {} for the env", n_bars_total, n); + if start_offset >= n_bars_total { + anyhow::bail!( + "start_offset {} ≥ fxcache bar count {}", + start_offset, n_bars_total + ); + } + let n_available = n_bars_total - start_offset; + let n = n_available.min(max_snapshots); + info!( + "fxcache: {} total bars, starting at offset {}, taking {} for the env", + n_bars_total, start_offset, n + ); info!( "fxcache loader: spread mode = {}", if use_real_spread { "REAL (derived from features[78] spread_bps)" } @@ -162,11 +188,10 @@ pub fn load_snapshots_from_fxcache( ); if let Some(cache) = alpha_cache { - if cache.len() < n { + if cache.len() < start_offset + n { anyhow::bail!( - "alpha cache has {} entries but env wants {} bars", - cache.len(), - n + "alpha cache has {} entries but env wants bars {}..{}", + cache.len(), start_offset, start_offset + n ); } } @@ -179,7 +204,8 @@ pub fn load_snapshots_from_fxcache( let mut max_spread = f32::NEG_INFINITY; let mut n_floor_hits = 0_usize; let mut n_cap_hits = 0_usize; - for i in 0..n { + for local_i in 0..n { + let i = start_offset + local_i; let rec = reader.record(i); let mid = rec.targets[COL_RAW_CLOSE - FEAT_DIM]; if !mid.is_finite() || mid <= 0.0 { diff --git a/scripts/walk_forward_cv.sh b/scripts/walk_forward_cv.sh new file mode 100755 index 000000000..bc0c59d22 --- /dev/null +++ b/scripts/walk_forward_cv.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Walk-forward CV for the Phase E.4.A.T10 compose backtest. +# +# Slides a fixed-size window across the fxcache. Each fold trains DQN +# from scratch on the front of the window and evaluates on the back. +# Fold C is the cleanest stacker-OOS test (eval bars 1.62M..1.9M lie +# entirely past the stacker training cut at 1.57M). +# +# Outputs per-fold JSON to /tmp/cv_fold_*.json. Run from repo root. + +set -euo pipefail + +FXCACHE="${FXCACHE:-/home/jgrusewski/Work/foxhunt/test_data/feature-cache/9297017b6db6795f75e57be8aefb03e45e6427513f42c08e22521e58fa025d5e.fxcache}" +ALPHA="${ALPHA:-config/ml/alpha_logits_cache.bin}" +FILL="${FILL:-config/ml/alpha_fill_coeffs.json}" +BIN="${BIN:-./target/release/examples/alpha_compose_backtest}" + +WINDOW=700000 +TRAIN_FRAC=0.6 # 420K train, 280K eval per fold + +declare -a FOLDS=( + "A:0" + "B:600000" + "C:1200000" +) + +for fold_spec in "${FOLDS[@]}"; do + name="${fold_spec%%:*}" + offset="${fold_spec##*:}" + out="/tmp/cv_fold_${name}.json" + echo "====================================================================" + echo "Fold ${name}: offset=${offset} window=${WINDOW} train_frac=${TRAIN_FRAC}" + echo "====================================================================" + SQLX_OFFLINE=true RUST_LOG=info "$BIN" \ + --fxcache-path "$FXCACHE" \ + --alpha-cache "$ALPHA" \ + --fill-coeffs "$FILL" \ + --data-start-offset "$offset" \ + --max-snapshots "$WINDOW" \ + --train-frac "$TRAIN_FRAC" \ + --c51 --temporal --window-k 16 --isv-continual \ + --out-path "$out" +done + +echo +echo "====================================================================" +echo "Walk-forward summary (best Sharpe_ann per cost, per fold)" +echo "====================================================================" +python3 - <<'PY' +import json, statistics +folds = ["A", "B", "C"] +rows = [] +for f in folds: + with open(f"/tmp/cv_fold_{f}.json") as fh: + j = json.load(fh) + by_cost = {} + for b in j["bins"]: + c = b["cost"] + if c not in by_cost or b["sharpe_annualised"] > by_cost[c][1]: + by_cost[c] = (b["threshold"], b["sharpe_annualised"], b["win_rate"], b["avg_n_trades"]) + rows.append((f, by_cost)) + +costs = sorted(rows[0][1].keys()) +print(f"{'cost':>8} " + " ".join(f"fold-{f}" for f, _ in rows)) +for c in costs: + cells = [] + for _, by_cost in rows: + tau, sa, wr, tp = by_cost[c] + cells.append(f"τ={tau:.2f} S={sa:+6.2f} ({wr*100:.0f}%/{tp:.0f})") + print(f"{c:>8.4f} " + " ".join(cells)) + +print() +print("Per-cost mean Sharpe_ann ± stddev across folds:") +for c in costs: + sas = [by_cost[c][1] for _, by_cost in rows] + m = statistics.mean(sas) + sd = statistics.stdev(sas) if len(sas) > 1 else 0.0 + print(f" cost={c:.4f} mean Sharpe_ann = {m:+7.2f} ± {sd:5.2f}") +PY