gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(phase-e-4-a): walk-forward CV via --data-start-offset

Browse Source 
Adds a sliding-window walk-forward harness for the T10 backtest:

- New load_snapshots_from_fxcache_at(start_offset, ...) loader variant
  reads bars [start_offset..start_offset+max_snapshots) from the fxcache.
  Alpha-cache lookups use absolute bar indices, so the same
  alpha_logits_cache.bin works across folds.
- New --data-start-offset CLI flag on alpha_compose_backtest.
- scripts/walk_forward_cv.sh runs 3 folds (window=700K, train_frac=0.6)
  at offsets 0 / 600K / 1.2M, producing /tmp/cv_fold_{A,B,C}.json plus
  an aggregated mean±stddev Sharpe table across folds.

Walk-forward result (alpha_logits_cache trained on bars 0..1.57M, so
fold C eval is fully past the stacker cut):

  cost     fold-A  fold-B  fold-C   mean ± stddev
  0.0000   +91.52  -21.44  +46.74   +38.94 ± 56.88
  0.0625   +84.94  -27.97  +38.42   +31.79 ± 56.74
  0.1250   +79.91  -31.22  +33.51   +27.40 ± 55.82
  0.2500   +72.77  -45.41  +15.16   +14.17 ± 59.09
  0.5000   +50.52  -59.82  -12.75    -7.35 ± 55.37

Fold B (mid-quarter, bars 600K..1.3M) is a disaster — win rate
collapses to 0-22% across all costs. Folds A and C succeed strongly.
Cross-fold SD ≈ mean, so the policy is regime-dependent and cannot
be reliably deployed without regime detection.

Mean Sharpe at half-tick (+27.40) is still ~7× the stateless
Phase 1d.4 baseline (-4.0), so the temporal encoder adds real value
on average — but the single-window +62 OOS celebrated earlier was
a cherry-picked favorable regime, not a deployment-ready result.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-05-16 00:02:24 +02:00
parent 3dc022b843
commit 3aef276255
3 changed files with 120 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
crates/ml/examples/alpha_compose_backtest.rs
View File

@@ -175,6 +175,12 @@ struct Cli {
/// Snapshots to load from the fxcache.
#[arg(long, default_value_t = 1_500_000)]
max_snapshots: usize,
/// Bars to skip at the front of the fxcache before windowing. Used by
/// walk-forward CV scripts to slide a fixed-size train+eval window
/// across the fxcache so each fold sees a distinct chronological
/// region.
#[arg(long, default_value_t = 0)]
data_start_offset: usize,
/// Episode horizon in snapshots.
#[arg(long, default_value_t = 600)]
horizon: usize,
@@ -433,8 +439,9 @@ fn main() -> Result<()> {
info!("Loaded fill model");
let alpha_cache = ml::env::loaders::load_alpha_cache(&cli.alpha_cache)?;
info!("Loaded alpha cache: {} entries", alpha_cache.len());
let rows = ml::env::loaders::load_snapshots_from_fxcache(
let rows = ml::env::loaders::load_snapshots_from_fxcache_at(
&cli.fxcache_path,
cli.data_start_offset,
cli.max_snapshots,
Some(&alpha_cache),
cli.real_spread,

40
crates/ml/src/env/loaders.rs vendored
View File

@@ -133,6 +133,22 @@ pub fn load_snapshots_from_fxcache(
// This commit introduces the parameter; the real-peek implementation
// lands in Task 5 follow-on.
mbp10_dir: Option<&Path>,
) -> Result<Vec<SnapshotRow>> {
load_snapshots_from_fxcache_at(fxcache_path, 0, max_snapshots, alpha_cache, use_real_spread, mbp10_dir)
}
/// Like `load_snapshots_from_fxcache` but starts at `start_offset` bars
/// into the fxcache. Used by walk-forward CV so each fold sees a
/// distinct chronological window. When `alpha_cache` is provided it must
/// be a slice indexed in the SAME absolute bar coordinates as the
/// fxcache (the loader will skip the first `start_offset` entries).
pub fn load_snapshots_from_fxcache_at(
fxcache_path: &Path,
start_offset: usize,
max_snapshots: usize,
alpha_cache: Option<&[f32]>,
use_real_spread: bool,
mbp10_dir: Option<&Path>,
) -> Result<Vec<SnapshotRow>> {
if mbp10_dir.is_some() {
warn!(
@@ -153,8 +169,18 @@ pub fn load_snapshots_from_fxcache(
);
}
let n_bars_total = reader.bar_count();
let n = n_bars_total.min(max_snapshots);
info!("fxcache: {} total bars, taking {} for the env", n_bars_total, n);
if start_offset >= n_bars_total {
anyhow::bail!(
"start_offset {} ≥ fxcache bar count {}",
start_offset, n_bars_total
);
}
let n_available = n_bars_total - start_offset;
let n = n_available.min(max_snapshots);
info!(
"fxcache: {} total bars, starting at offset {}, taking {} for the env",
n_bars_total, start_offset, n
);
info!(
"fxcache loader: spread mode = {}",
if use_real_spread { "REAL (derived from features[78] spread_bps)" }
@@ -162,11 +188,10 @@ pub fn load_snapshots_from_fxcache(
);
if let Some(cache) = alpha_cache {
if cache.len() < n {
if cache.len() < start_offset + n {
anyhow::bail!(
"alpha cache has {} entries but env wants {} bars",
cache.len(),
n
"alpha cache has {} entries but env wants bars {}..{}",
cache.len(), start_offset, start_offset + n
);
}
}
@@ -179,7 +204,8 @@ pub fn load_snapshots_from_fxcache(
let mut max_spread = f32::NEG_INFINITY;
let mut n_floor_hits = 0_usize;
let mut n_cap_hits = 0_usize;
for i in 0..n {
for local_i in 0..n {
let i = start_offset + local_i;
let rec = reader.record(i);
let mid = rec.targets[COL_RAW_CLOSE - FEAT_DIM];
if !mid.is_finite() || mid <= 0.0 {

79
scripts/walk_forward_cv.sh Executable file
View File

@@ -0,0 +1,79 @@
#!/usr/bin/env bash
# Walk-forward CV for the Phase E.4.A.T10 compose backtest.
#
# Slides a fixed-size window across the fxcache. Each fold trains DQN
# from scratch on the front of the window and evaluates on the back.
# Fold C is the cleanest stacker-OOS test (eval bars 1.62M..1.9M lie
# entirely past the stacker training cut at 1.57M).
#
# Outputs per-fold JSON to /tmp/cv_fold_*.json. Run from repo root.
set -euo pipefail
FXCACHE="${FXCACHE:-/home/jgrusewski/Work/foxhunt/test_data/feature-cache/9297017b6db6795f75e57be8aefb03e45e6427513f42c08e22521e58fa025d5e.fxcache}"
ALPHA="${ALPHA:-config/ml/alpha_logits_cache.bin}"
FILL="${FILL:-config/ml/alpha_fill_coeffs.json}"
BIN="${BIN:-./target/release/examples/alpha_compose_backtest}"
WINDOW=700000
TRAIN_FRAC=0.6 # 420K train, 280K eval per fold
declare -a FOLDS=(
"A:0"
"B:600000"
"C:1200000"
)
for fold_spec in "${FOLDS[@]}"; do
name="${fold_spec%%:*}"
offset="${fold_spec##*:}"
out="/tmp/cv_fold_${name}.json"
echo "===================================================================="
echo "Fold ${name}: offset=${offset} window=${WINDOW} train_frac=${TRAIN_FRAC}"
echo "===================================================================="
SQLX_OFFLINE=true RUST_LOG=info "$BIN" \
--fxcache-path "$FXCACHE" \
--alpha-cache "$ALPHA" \
--fill-coeffs "$FILL" \
--data-start-offset "$offset" \
--max-snapshots "$WINDOW" \
--train-frac "$TRAIN_FRAC" \
--c51 --temporal --window-k 16 --isv-continual \
--out-path "$out"
done
echo
echo "===================================================================="
echo "Walk-forward summary (best Sharpe_ann per cost, per fold)"
echo "===================================================================="
python3 - <<'PY'
import json, statistics
folds = ["A", "B", "C"]
rows = []
for f in folds:
with open(f"/tmp/cv_fold_{f}.json") as fh:
j = json.load(fh)
by_cost = {}
for b in j["bins"]:
c = b["cost"]
if c not in by_cost or b["sharpe_annualised"] > by_cost[c][1]:
by_cost[c] = (b["threshold"], b["sharpe_annualised"], b["win_rate"], b["avg_n_trades"])
rows.append((f, by_cost))
costs = sorted(rows[0][1].keys())
print(f"{'cost':>8} " + " ".join(f"fold-{f}" for f, _ in rows))
for c in costs:
cells = []
for _, by_cost in rows:
tau, sa, wr, tp = by_cost[c]
cells.append(f"τ={tau:.2f} S={sa:+6.2f} ({wr*100:.0f}%/{tp:.0f})")
print(f"{c:>8.4f} " + " ".join(cells))
print()
print("Per-cost mean Sharpe_ann ± stddev across folds:")
for c in costs:
sas = [by_cost[c][1] for _, by_cost in rows]
m = statistics.mean(sas)
sd = statistics.stdev(sas) if len(sas) > 1 else 0.0
print(f" cost={c:.4f} mean Sharpe_ann = {m:+7.2f} ± {sd:5.2f}")
PY