From 55ffe2b26feaa47ffa38ed2fd527b83fa4735191 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 16 May 2026 00:42:07 +0200 Subject: [PATCH] scripts(alpha): rename + add cross-quarter validation pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - New scripts/alpha_pipeline.sh: orchestrates the 2-quarter validation after fxcache lands. Auto-discovers the Q1+Q2 fxcache (newest .fxcache excluding the known Q1-only hash), trains alpha_train_stacker on it, then sweeps 4 conditions (baseline / +cost-rand / +regime-scale / +both) × 3 walk-forward folds, aggregating mean ± stddev Sharpe per cost across folds. No phase prefixes in name or contents — the script is meant to outlive any single milestone. - scripts/build_2q_fxcache.sh: fix staging layout so MBP-10 / trades / OHLCV symlinks live in the symbol subdir (`mbp10/ES.FUT/...`) that precompute_features expects. Previous flat-layout build aborted with "MBP-10 directory not found: .../mbp10/ES.FUT". Co-Authored-By: Claude Opus 4.7 --- scripts/alpha_pipeline.sh | 179 ++++++++++++++++++++++++++++++++++++ scripts/build_2q_fxcache.sh | 8 +- 2 files changed, 183 insertions(+), 4 deletions(-) create mode 100755 scripts/alpha_pipeline.sh diff --git a/scripts/alpha_pipeline.sh b/scripts/alpha_pipeline.sh new file mode 100755 index 000000000..e47cb14b9 --- /dev/null +++ b/scripts/alpha_pipeline.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Alpha pipeline — full 2-quarter validation after fxcache lands. +# +# 1. Locate the freshly-built Q1+Q2 2024 fxcache (newest .fxcache in +# test_data/feature-cache/ that is NOT the existing Q1-only file). +# 2. Train alpha_train_stacker on it → alpha_logits_cache_2q.bin. +# 3. Run 4-condition cross-quarter walk-forward CV: +# A: baseline (trained Mamba2 + C51, no regime scale, fixed cost) +# B: + cost randomization U[0.0625, 0.5] +# C: + vol regime scale only +# D: + both (the combined regime-defence hypothesis) +# 4. Aggregate Sharpe across conditions × folds. +# +# Run from the sp19-20-wr-first worktree root. + +set -euo pipefail + +REPO=/home/jgrusewski/Work/foxhunt/.worktrees/sp19-20-wr-first +CACHE_DIR=/home/jgrusewski/Work/foxhunt/test_data/feature-cache +FILL=config/ml/alpha_fill_coeffs.json +BIN_BT=./target/release/examples/alpha_compose_backtest +BIN_STACK=./target/release/examples/alpha_train_stacker + +cd "$REPO" + +# Identify the Q1+Q2 fxcache: newest .fxcache whose hash is NOT the +# known Q1-only one. +Q1_HASH=9297017b6db6795f75e57be8aefb03e45e6427513f42c08e22521e58fa025d5e +NEW_CACHE=$(ls -t "$CACHE_DIR"/*.fxcache 2>/dev/null | grep -v "$Q1_HASH" | head -1) +if [[ -z "$NEW_CACHE" ]]; then + echo "FATAL: no new fxcache found in $CACHE_DIR (only the Q1 one)" + exit 1 +fi +echo "Using fxcache: $NEW_CACHE" + +# ----- 1. Train alpha_train_stacker → alpha_logits_cache_2q.bin ----- +ALPHA_OUT=config/ml/alpha_logits_cache_2q.bin +if [[ ! -f "$ALPHA_OUT" ]]; then + echo "====================================================================" + echo "Stage 1: alpha_train_stacker on $NEW_CACHE" + echo "====================================================================" + SQLX_OFFLINE=true cargo build --release --example alpha_train_stacker -p ml-alpha + SQLX_OFFLINE=true RUST_LOG=info "$BIN_STACK" \ + --fxcache-path "$NEW_CACHE" \ + --horizon 6000 \ + --seq-len 32 \ + --hidden-dim 64 \ + --state-dim 16 \ + --epochs 5 \ + --batch-size 128 \ + --lr 3e-3 \ + --train-frac 0.8 \ + --alpha-cache-out "$ALPHA_OUT" +fi +echo "Alpha cache: $ALPHA_OUT" + +# ----- 2. Build backtest binary (in case of code changes) ----- +SQLX_OFFLINE=true cargo build --release --example alpha_compose_backtest -p ml >/dev/null + +# ----- 3. Walk-forward CV across 4 conditions × 3 folds ----- +# Fold A: offset=0, window=1.2M → train 0..720K, eval 720K..1.2M +# Fold B: offset=900K, window=1.2M → train 900K..1.62M, eval 1.62M..2.1M +# Fold C: offset=1.8M, window=1.2M → train 1.8M..2.52M, eval 2.52M..3.0M +# (Assumes Q1+Q2 fxcache has ~3-6M bars total at MBP-10 resolution.) + +declare -a CONDS=( + "A_baseline:--c51 --temporal --window-k 16 --isv-continual" + "B_costrand:--c51 --temporal --window-k 16 --isv-continual --train-cost-hi 0.5" + "C_regime:--c51 --temporal --window-k 16 --isv-continual --regime-scale" + "D_both:--c51 --temporal --window-k 16 --isv-continual --train-cost-hi 0.5 --regime-scale" +) + +declare -a FOLDS=( + "A:0:1200000" + "B:900000:1200000" + "C:1800000:1200000" +) + +mkdir -p /tmp/alpha_cv_results +for cond in "${CONDS[@]}"; do + cname="${cond%%:*}" + cflags="${cond##*:}" + for fold in "${FOLDS[@]}"; do + IFS=':' read -r fname offset window <<<"$fold" + out="/tmp/alpha_cv_results/${cname}_fold-${fname}.json" + if [[ -f "$out" ]]; then + echo "Skip (exists): $out"; continue + fi + echo "====================================================================" + echo "Cond $cname, fold $fname (offset=$offset, window=$window)" + echo "====================================================================" + SQLX_OFFLINE=true RUST_LOG=info "$BIN_BT" \ + --fxcache-path "$NEW_CACHE" \ + --alpha-cache "$ALPHA_OUT" \ + --fill-coeffs "$FILL" \ + --data-start-offset "$offset" \ + --max-snapshots "$window" \ + --train-frac 0.6 \ + $cflags \ + --out-path "$out" + done +done + +# ----- 4. Aggregate ----- +echo +echo "====================================================================" +echo "Alpha pipeline cross-quarter walk-forward summary" +echo "====================================================================" +python3 - <<'PY' +import json, os, statistics +conds = ["A_baseline", "B_costrand", "C_regime", "D_both"] +folds = ["A", "B", "C"] + +# For each (cond, cost): list of best Sharpe across folds +by_cell = {} +for cond in conds: + for f in folds: + p = f"/tmp/alpha_cv_results/{cond}_fold-{f}.json" + if not os.path.exists(p): + continue + with open(p) as fh: + j = json.load(fh) + for b in j["bins"]: + c = b["cost"] + key = (cond, c) + best = by_cell.get(key, (-1e9, None, None, None)) + if b["sharpe_annualised"] > best[0]: + by_cell[key] = ( + b["sharpe_annualised"], + b["threshold"], + b["win_rate"], + f, + ) + +# Per (cond, cost) — collect across folds for std-dev +fold_sharpes = {} # (cond, cost) -> [Sharpe per fold] +for cond in conds: + for f in folds: + p = f"/tmp/alpha_cv_results/{cond}_fold-{f}.json" + if not os.path.exists(p): + continue + with open(p) as fh: + j = json.load(fh) + by_cost = {} + for b in j["bins"]: + c = b["cost"] + if c not in by_cost or b["sharpe_annualised"] > by_cost[c]: + by_cost[c] = b["sharpe_annualised"] + for c, sa in by_cost.items(): + fold_sharpes.setdefault((cond, c), []).append(sa) + +costs = sorted({c for (_, c) in fold_sharpes.keys()}) +print(f"{'cost':>8} " + " ".join(f"{c:>22}" for c in conds)) +for c in costs: + cells = [] + for cond in conds: + sas = fold_sharpes.get((cond, c), []) + if not sas: + cells.append(" n/a ") + elif len(sas) == 1: + cells.append(f" {sas[0]:+7.2f} ") + else: + m = statistics.mean(sas) + sd = statistics.stdev(sas) + cells.append(f" {m:+7.2f} ± {sd:5.2f} ") + print(f"{c:>8.4f} " + " ".join(cells)) + +print() +print("Per-condition aggregate stats (across all folds × costs):") +for cond in conds: + all_sas = [] + for c in costs: + all_sas.extend(fold_sharpes.get((cond, c), [])) + if not all_sas: + continue + m = statistics.mean(all_sas) + sd = statistics.stdev(all_sas) if len(all_sas) > 1 else 0.0 + print(f" {cond:14} mean Sharpe = {m:+7.2f} std = {sd:5.2f} n = {len(all_sas)}") +PY diff --git a/scripts/build_2q_fxcache.sh b/scripts/build_2q_fxcache.sh index 5a9cb1535..7af7b7f45 100755 --- a/scripts/build_2q_fxcache.sh +++ b/scripts/build_2q_fxcache.sh @@ -18,11 +18,11 @@ set -euo pipefail # doesn't pick up other quarters that may land later. STAGE="${STAGE:-/tmp/fxbuild_2q_2024}" rm -rf "$STAGE" -mkdir -p "$STAGE"/{ohlcv,mbp10,trades} +mkdir -p "$STAGE"/{ohlcv,mbp10,trades}/ES.FUT for q in Q1 Q2; do - ln -s "/home/jgrusewski/Work/foxhunt/test_data/futures-baseline/ES.FUT/ES.FUT_2024-${q}.dbn.zst" "$STAGE/ohlcv/" - ln -s "/home/jgrusewski/Work/foxhunt/test_data/futures-baseline-mbp10/ES.FUT/ES.FUT_2024-${q}.dbn.zst" "$STAGE/mbp10/" - ln -s "/home/jgrusewski/Work/foxhunt/test_data/futures-baseline-trades/ES.FUT/ES.FUT_2024-${q}.dbn.zst" "$STAGE/trades/" + ln -s "/home/jgrusewski/Work/foxhunt/test_data/futures-baseline/ES.FUT/ES.FUT_2024-${q}.dbn.zst" "$STAGE/ohlcv/ES.FUT/" + ln -s "/home/jgrusewski/Work/foxhunt/test_data/futures-baseline-mbp10/ES.FUT/ES.FUT_2024-${q}.dbn.zst" "$STAGE/mbp10/ES.FUT/" + ln -s "/home/jgrusewski/Work/foxhunt/test_data/futures-baseline-trades/ES.FUT/ES.FUT_2024-${q}.dbn.zst" "$STAGE/trades/ES.FUT/" done mkdir -p test_data/feature-cache