scripts(alpha): rename + add cross-quarter validation pipeline
- New scripts/alpha_pipeline.sh: orchestrates the 2-quarter validation after fxcache lands. Auto-discovers the Q1+Q2 fxcache (newest .fxcache excluding the known Q1-only hash), trains alpha_train_stacker on it, then sweeps 4 conditions (baseline / +cost-rand / +regime-scale / +both) × 3 walk-forward folds, aggregating mean ± stddev Sharpe per cost across folds. No phase prefixes in name or contents — the script is meant to outlive any single milestone. - scripts/build_2q_fxcache.sh: fix staging layout so MBP-10 / trades / OHLCV symlinks live in the symbol subdir (`mbp10/ES.FUT/...`) that precompute_features expects. Previous flat-layout build aborted with "MBP-10 directory not found: .../mbp10/ES.FUT". Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
179
scripts/alpha_pipeline.sh
Executable file
179
scripts/alpha_pipeline.sh
Executable file
@@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env bash
|
||||
# Alpha pipeline — full 2-quarter validation after fxcache lands.
|
||||
#
|
||||
# 1. Locate the freshly-built Q1+Q2 2024 fxcache (newest .fxcache in
|
||||
# test_data/feature-cache/ that is NOT the existing Q1-only file).
|
||||
# 2. Train alpha_train_stacker on it → alpha_logits_cache_2q.bin.
|
||||
# 3. Run 4-condition cross-quarter walk-forward CV:
|
||||
# A: baseline (trained Mamba2 + C51, no regime scale, fixed cost)
|
||||
# B: + cost randomization U[0.0625, 0.5]
|
||||
# C: + vol regime scale only
|
||||
# D: + both (the combined regime-defence hypothesis)
|
||||
# 4. Aggregate Sharpe across conditions × folds.
|
||||
#
|
||||
# Run from the sp19-20-wr-first worktree root.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO=/home/jgrusewski/Work/foxhunt/.worktrees/sp19-20-wr-first
|
||||
CACHE_DIR=/home/jgrusewski/Work/foxhunt/test_data/feature-cache
|
||||
FILL=config/ml/alpha_fill_coeffs.json
|
||||
BIN_BT=./target/release/examples/alpha_compose_backtest
|
||||
BIN_STACK=./target/release/examples/alpha_train_stacker
|
||||
|
||||
cd "$REPO"
|
||||
|
||||
# Identify the Q1+Q2 fxcache: newest .fxcache whose hash is NOT the
|
||||
# known Q1-only one.
|
||||
Q1_HASH=9297017b6db6795f75e57be8aefb03e45e6427513f42c08e22521e58fa025d5e
|
||||
NEW_CACHE=$(ls -t "$CACHE_DIR"/*.fxcache 2>/dev/null | grep -v "$Q1_HASH" | head -1)
|
||||
if [[ -z "$NEW_CACHE" ]]; then
|
||||
echo "FATAL: no new fxcache found in $CACHE_DIR (only the Q1 one)"
|
||||
exit 1
|
||||
fi
|
||||
echo "Using fxcache: $NEW_CACHE"
|
||||
|
||||
# ----- 1. Train alpha_train_stacker → alpha_logits_cache_2q.bin -----
|
||||
ALPHA_OUT=config/ml/alpha_logits_cache_2q.bin
|
||||
if [[ ! -f "$ALPHA_OUT" ]]; then
|
||||
echo "===================================================================="
|
||||
echo "Stage 1: alpha_train_stacker on $NEW_CACHE"
|
||||
echo "===================================================================="
|
||||
SQLX_OFFLINE=true cargo build --release --example alpha_train_stacker -p ml-alpha
|
||||
SQLX_OFFLINE=true RUST_LOG=info "$BIN_STACK" \
|
||||
--fxcache-path "$NEW_CACHE" \
|
||||
--horizon 6000 \
|
||||
--seq-len 32 \
|
||||
--hidden-dim 64 \
|
||||
--state-dim 16 \
|
||||
--epochs 5 \
|
||||
--batch-size 128 \
|
||||
--lr 3e-3 \
|
||||
--train-frac 0.8 \
|
||||
--alpha-cache-out "$ALPHA_OUT"
|
||||
fi
|
||||
echo "Alpha cache: $ALPHA_OUT"
|
||||
|
||||
# ----- 2. Build backtest binary (in case of code changes) -----
|
||||
SQLX_OFFLINE=true cargo build --release --example alpha_compose_backtest -p ml >/dev/null
|
||||
|
||||
# ----- 3. Walk-forward CV across 4 conditions × 3 folds -----
|
||||
# Fold A: offset=0, window=1.2M → train 0..720K, eval 720K..1.2M
|
||||
# Fold B: offset=900K, window=1.2M → train 900K..1.62M, eval 1.62M..2.1M
|
||||
# Fold C: offset=1.8M, window=1.2M → train 1.8M..2.52M, eval 2.52M..3.0M
|
||||
# (Assumes Q1+Q2 fxcache has ~3-6M bars total at MBP-10 resolution.)
|
||||
|
||||
declare -a CONDS=(
|
||||
"A_baseline:--c51 --temporal --window-k 16 --isv-continual"
|
||||
"B_costrand:--c51 --temporal --window-k 16 --isv-continual --train-cost-hi 0.5"
|
||||
"C_regime:--c51 --temporal --window-k 16 --isv-continual --regime-scale"
|
||||
"D_both:--c51 --temporal --window-k 16 --isv-continual --train-cost-hi 0.5 --regime-scale"
|
||||
)
|
||||
|
||||
declare -a FOLDS=(
|
||||
"A:0:1200000"
|
||||
"B:900000:1200000"
|
||||
"C:1800000:1200000"
|
||||
)
|
||||
|
||||
mkdir -p /tmp/alpha_cv_results
|
||||
for cond in "${CONDS[@]}"; do
|
||||
cname="${cond%%:*}"
|
||||
cflags="${cond##*:}"
|
||||
for fold in "${FOLDS[@]}"; do
|
||||
IFS=':' read -r fname offset window <<<"$fold"
|
||||
out="/tmp/alpha_cv_results/${cname}_fold-${fname}.json"
|
||||
if [[ -f "$out" ]]; then
|
||||
echo "Skip (exists): $out"; continue
|
||||
fi
|
||||
echo "===================================================================="
|
||||
echo "Cond $cname, fold $fname (offset=$offset, window=$window)"
|
||||
echo "===================================================================="
|
||||
SQLX_OFFLINE=true RUST_LOG=info "$BIN_BT" \
|
||||
--fxcache-path "$NEW_CACHE" \
|
||||
--alpha-cache "$ALPHA_OUT" \
|
||||
--fill-coeffs "$FILL" \
|
||||
--data-start-offset "$offset" \
|
||||
--max-snapshots "$window" \
|
||||
--train-frac 0.6 \
|
||||
$cflags \
|
||||
--out-path "$out"
|
||||
done
|
||||
done
|
||||
|
||||
# ----- 4. Aggregate -----
|
||||
echo
|
||||
echo "===================================================================="
|
||||
echo "Alpha pipeline cross-quarter walk-forward summary"
|
||||
echo "===================================================================="
|
||||
python3 - <<'PY'
|
||||
import json, os, statistics
|
||||
conds = ["A_baseline", "B_costrand", "C_regime", "D_both"]
|
||||
folds = ["A", "B", "C"]
|
||||
|
||||
# For each (cond, cost): list of best Sharpe across folds
|
||||
by_cell = {}
|
||||
for cond in conds:
|
||||
for f in folds:
|
||||
p = f"/tmp/alpha_cv_results/{cond}_fold-{f}.json"
|
||||
if not os.path.exists(p):
|
||||
continue
|
||||
with open(p) as fh:
|
||||
j = json.load(fh)
|
||||
for b in j["bins"]:
|
||||
c = b["cost"]
|
||||
key = (cond, c)
|
||||
best = by_cell.get(key, (-1e9, None, None, None))
|
||||
if b["sharpe_annualised"] > best[0]:
|
||||
by_cell[key] = (
|
||||
b["sharpe_annualised"],
|
||||
b["threshold"],
|
||||
b["win_rate"],
|
||||
f,
|
||||
)
|
||||
|
||||
# Per (cond, cost) — collect across folds for std-dev
|
||||
fold_sharpes = {} # (cond, cost) -> [Sharpe per fold]
|
||||
for cond in conds:
|
||||
for f in folds:
|
||||
p = f"/tmp/alpha_cv_results/{cond}_fold-{f}.json"
|
||||
if not os.path.exists(p):
|
||||
continue
|
||||
with open(p) as fh:
|
||||
j = json.load(fh)
|
||||
by_cost = {}
|
||||
for b in j["bins"]:
|
||||
c = b["cost"]
|
||||
if c not in by_cost or b["sharpe_annualised"] > by_cost[c]:
|
||||
by_cost[c] = b["sharpe_annualised"]
|
||||
for c, sa in by_cost.items():
|
||||
fold_sharpes.setdefault((cond, c), []).append(sa)
|
||||
|
||||
costs = sorted({c for (_, c) in fold_sharpes.keys()})
|
||||
print(f"{'cost':>8} " + " ".join(f"{c:>22}" for c in conds))
|
||||
for c in costs:
|
||||
cells = []
|
||||
for cond in conds:
|
||||
sas = fold_sharpes.get((cond, c), [])
|
||||
if not sas:
|
||||
cells.append(" n/a ")
|
||||
elif len(sas) == 1:
|
||||
cells.append(f" {sas[0]:+7.2f} ")
|
||||
else:
|
||||
m = statistics.mean(sas)
|
||||
sd = statistics.stdev(sas)
|
||||
cells.append(f" {m:+7.2f} ± {sd:5.2f} ")
|
||||
print(f"{c:>8.4f} " + " ".join(cells))
|
||||
|
||||
print()
|
||||
print("Per-condition aggregate stats (across all folds × costs):")
|
||||
for cond in conds:
|
||||
all_sas = []
|
||||
for c in costs:
|
||||
all_sas.extend(fold_sharpes.get((cond, c), []))
|
||||
if not all_sas:
|
||||
continue
|
||||
m = statistics.mean(all_sas)
|
||||
sd = statistics.stdev(all_sas) if len(all_sas) > 1 else 0.0
|
||||
print(f" {cond:14} mean Sharpe = {m:+7.2f} std = {sd:5.2f} n = {len(all_sas)}")
|
||||
PY
|
||||
@@ -18,11 +18,11 @@ set -euo pipefail
|
||||
# doesn't pick up other quarters that may land later.
|
||||
STAGE="${STAGE:-/tmp/fxbuild_2q_2024}"
|
||||
rm -rf "$STAGE"
|
||||
mkdir -p "$STAGE"/{ohlcv,mbp10,trades}
|
||||
mkdir -p "$STAGE"/{ohlcv,mbp10,trades}/ES.FUT
|
||||
for q in Q1 Q2; do
|
||||
ln -s "/home/jgrusewski/Work/foxhunt/test_data/futures-baseline/ES.FUT/ES.FUT_2024-${q}.dbn.zst" "$STAGE/ohlcv/"
|
||||
ln -s "/home/jgrusewski/Work/foxhunt/test_data/futures-baseline-mbp10/ES.FUT/ES.FUT_2024-${q}.dbn.zst" "$STAGE/mbp10/"
|
||||
ln -s "/home/jgrusewski/Work/foxhunt/test_data/futures-baseline-trades/ES.FUT/ES.FUT_2024-${q}.dbn.zst" "$STAGE/trades/"
|
||||
ln -s "/home/jgrusewski/Work/foxhunt/test_data/futures-baseline/ES.FUT/ES.FUT_2024-${q}.dbn.zst" "$STAGE/ohlcv/ES.FUT/"
|
||||
ln -s "/home/jgrusewski/Work/foxhunt/test_data/futures-baseline-mbp10/ES.FUT/ES.FUT_2024-${q}.dbn.zst" "$STAGE/mbp10/ES.FUT/"
|
||||
ln -s "/home/jgrusewski/Work/foxhunt/test_data/futures-baseline-trades/ES.FUT/ES.FUT_2024-${q}.dbn.zst" "$STAGE/trades/ES.FUT/"
|
||||
done
|
||||
|
||||
mkdir -p test_data/feature-cache
|
||||
|
||||
Reference in New Issue
Block a user