From f4bd1e2432fbb5a6a3ba41b20f8a4e2452e04966 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 21 Jun 2026 10:40:34 +0200 Subject: [PATCH] =?UTF-8?q?research(surfer):=20crypto=20edge-falsification?= =?UTF-8?q?=20probes=20=E2=80=94=20intraday/cascade=20dead,=20stablecoin?= =?UTF-8?q?=20peg-reversion=20validated?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - crypto_mft_xsec / mft_*: intraday/MFT XS price edge FALSIFIED (fee-trap) - crypto_cascade_reversion: liquidation-cascade reversion FALSIFIED (continuation) - crypto_trend_sizing: TS-trend return-engine reconfirmed (Sharpe ~1.25) - crypto_stablecoin_dislocation/harden/intraday: short-rich peg-reversion VALIDATED (bounded, uncorrelated); long-cheap = death-spiral trap Co-Authored-By: Claude Opus 4.8 --- scripts/surfer/crypto_cascade_reversion.py | 116 +++++++++++++ scripts/surfer/crypto_mft_xsec.py | 129 ++++++++++++++ .../surfer/crypto_stablecoin_dislocation.py | 119 +++++++++++++ scripts/surfer/crypto_stablecoin_harden.py | 145 ++++++++++++++++ scripts/surfer/crypto_stablecoin_intraday.py | 120 +++++++++++++ scripts/surfer/crypto_trend_sizing.py | 120 +++++++++++++ scripts/surfer/mft_4h_harden.py | 163 ++++++++++++++++++ scripts/surfer/mft_harden_v2.py | 114 ++++++++++++ scripts/surfer/mft_horizon_sweep.py | 154 +++++++++++++++++ 9 files changed, 1180 insertions(+) create mode 100644 scripts/surfer/crypto_cascade_reversion.py create mode 100644 scripts/surfer/crypto_mft_xsec.py create mode 100644 scripts/surfer/crypto_stablecoin_dislocation.py create mode 100644 scripts/surfer/crypto_stablecoin_harden.py create mode 100644 scripts/surfer/crypto_stablecoin_intraday.py create mode 100644 scripts/surfer/crypto_trend_sizing.py create mode 100644 scripts/surfer/mft_4h_harden.py create mode 100644 scripts/surfer/mft_harden_v2.py create mode 100644 scripts/surfer/mft_horizon_sweep.py diff --git a/scripts/surfer/crypto_cascade_reversion.py b/scripts/surfer/crypto_cascade_reversion.py new file mode 100644 index 000000000..1425101d3 --- /dev/null +++ b/scripts/surfer/crypto_cascade_reversion.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Cascade reversion — do crypto coins BOUNCE after an extreme forced-flow down-move? + +Liquidation cascades are forced deleveraging that overshoots; the harvestable consequence is the +mean-reversion bounce. Binance killed the historical liquidation feed (live websocket only), so we +test the PROXY already on disk: extreme negative trailing-H return = a cascade. Pre-registered: + + formation: trailing-H log return per coin (cross-sectional) + signal: LONG the bottom-q fraction (biggest losers) — long-only on crashed names + hold: next H hours, NON-OVERLAPPING rebalance + cost: 5 bp / leg on actual weight turnover + two builds: raw_long (equal-weight losers, has crypto beta) + hedged (long losers − short equal-weight UNIVERSE basket; beta-neutral, short leg + is the diversified basket NOT a single coin → dodges the short-melt-up ruin + that killed generic XS reversal) + KILL: net annualized SR < 0.5 OR not positive in most years (chrono) → cascade reversion closed. + +LIMITATION: npz panel has close only (no volume) → cannot condition on the volume spike that +confirms a true liquidation cascade; the extreme-return tail is the proxy. Flagged, not hidden. +""" +import math +import os + +import numpy as np + +OUT = "data/surfer/crypto1h" +COST_BP = 5.0 +SYMS = ["BTCUSDT", "ETHUSDT", "SOLUSDT", "XRPUSDT", "BNBUSDT", "DOGEUSDT", "ADAUSDT", "AVAXUSDT", + "LINKUSDT", "LTCUSDT", "DOTUSDT", "TRXUSDT", "BCHUSDT", "ETCUSDT", "FILUSDT", "ATOMUSDT"] +HOUR_MS = 3_600_000 +HOLDS = [1, 2, 4, 8, 12, 24] # formation == hold, hours +Q = 0.20 # bottom quintile = biggest losers + + +def build_panel(): + series = {} + for s in SYMS: + cache = f"{OUT}/{s}.npz" + if not os.path.exists(cache): + continue + d = np.load(cache) + if len(d["ts"]) > 24 * 60: + series[s] = (d["ts"], d["close"]) + allts = sorted(set().union(*[set((ts // HOUR_MS).tolist()) for ts, _ in series.values()])) + tindex = {t: i for i, t in enumerate(allts)} + syms = sorted(series) + P = np.full((len(allts), len(syms)), np.nan) + for j, s in enumerate(syms): + ts, c = series[s] + for t, px in zip(ts // HOUR_MS, c): + P[tindex[int(t)], j] = px + yrs = np.array([1970 + int(t) // int(365.25 * 24) for t in allts]) + return P, syms, yrs + + +def ann(rets, per_year_factor): + rets = np.asarray(rets) + if len(rets) < 20 or rets.std() == 0: + return float("nan"), float("nan") + sr = rets.mean() / rets.std() + t = rets.mean() / (rets.std() / math.sqrt(len(rets))) + return sr * per_year_factor, t + + +def run(): + P, syms, yrs = build_panel() + T, N = P.shape + logP = np.log(P) + print(f"\n===== CRYPTO CASCADE REVERSION (long the crashed names, net {COST_BP}bp/leg) =====") + print(f"panel: {T} hours x {N} coins years {int(yrs.min())}-{int(yrs.max())} q={Q} (bottom quintile)") + print("LONG bottom-q losers; hedged = long losers - short equal-wt universe basket\n") + hdr = f"{'hold':>5} {'periods':>8} {'build':>8} {'net_SR':>8} {'t(net)':>7} {'hit%':>6} {'per-year net-SR (chrono)':>30}" + print(hdr); print("-" * len(hdr)) + + for H in HOLDS: + pf = math.sqrt(24 * 365 / H) + idx = np.arange(H, T - H, H) # need t-H for formation, t+H for fwd + raw, hed, yr_raw, yr_hed = [], [], [], [] + w_raw_prev = np.zeros(N); w_hed_prev = np.zeros(N) + for t in idx: + past = logP[t] - logP[t - H] + fwd = logP[t + H] - logP[t] + ok = np.isfinite(past) & np.isfinite(fwd) + n_ok = int(ok.sum()) + if n_ok < 6: + continue + okidx = np.where(ok)[0] + k = max(1, int(round(Q * n_ok))) + losers = okidx[np.argsort(past[okidx])[:k]] # most-negative trailing return + # raw long + w_raw = np.zeros(N); w_raw[losers] = 1.0 / k + # hedged: long losers - short equal-weight universe basket + w_hed = np.zeros(N); w_hed[losers] += 1.0 / k; w_hed[okidx] -= 1.0 / n_ok + r_raw = float(np.nansum(w_raw * fwd)) - np.abs(w_raw - w_raw_prev).sum() * COST_BP / 1e4 + r_hed = float(np.nansum(w_hed * fwd)) - np.abs(w_hed - w_hed_prev).sum() * COST_BP / 1e4 + raw.append(r_raw); hed.append(r_hed) + yr_raw.append(int(yrs[t])); yr_hed.append(int(yrs[t])) + w_raw_prev = w_raw; w_hed_prev = w_hed + if len(raw) < 20: + print(f"{H:>5} (too few periods)"); continue + for name, series, yrl in (("raw_long", raw, yr_raw), ("hedged", hed, yr_hed)): + sr, t = ann(series, pf) + hit = 100.0 * np.mean(np.asarray(series) > 0) + py = {} + for r, y in zip(series, yrl): + py.setdefault(y, []).append(r) + pystr = " ".join(f"{y}:{(np.mean(v)/(np.std(v)+1e-12)):+.2f}" + for y, v in sorted(py.items()) if len(v) >= 10) + tag = f"{H}h" if name == "raw_long" else "" + print(f"{tag:>5} {len(series):>8} {name:>8} {sr:>+8.2f} {t:>+7.2f} {hit:>6.1f} {pystr}") + print("-" * len(hdr)) + print("PASS: hedged net SR > 0.5, t>=2, positive in most years. Else cascade reversion closed.") + + +if __name__ == "__main__": + run() diff --git a/scripts/surfer/crypto_mft_xsec.py b/scripts/surfer/crypto_mft_xsec.py new file mode 100644 index 000000000..3fbc893c3 --- /dev/null +++ b/scripts/surfer/crypto_mft_xsec.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""MFT crypto pulse — cross-sectional momentum at HOUR holds, net of turnover cost, hardened-lite. + +The proven crypto edge is DAILY cross-sectional momentum (Sharpe ~1.4). micro_gate showed MINUTE-horizon +(per-coin) is dead. This tests the gap: does cross-sectional momentum work at HOUR holds (1h..48h)? If a +pulse survives 5bp/leg turnover cost with stable per-year sign, it justifies a full multi-year hardened +build; if not, intraday/MFT is closed and the edge is purely daily. + +Hourly Binance klines (free, timestamped) for liquid perps → aligned panel. Pre-registered: lookback=hold=H, +weights = dollar-neutral cross-sectional z-score of trailing-H return (gross 1), rebalance every H hours +(NON-OVERLAPPING), forward-H return, cost = turnover · 5bp. Report gross/net annualized Sharpe + t + per-year. +""" +import json +import math +import os +import time +import urllib.request + +import numpy as np + +OUT = "data/surfer/crypto1h" +COST_BP = 5.0 +HOLDS = [1, 2, 4, 8, 12, 24, 48] +SYMS = ["BTCUSDT", "ETHUSDT", "SOLUSDT", "XRPUSDT", "BNBUSDT", "DOGEUSDT", "ADAUSDT", "AVAXUSDT", + "LINKUSDT", "LTCUSDT", "DOTUSDT", "TRXUSDT", "BCHUSDT", "ETCUSDT", "FILUSDT", "ATOMUSDT"] +HOUR_MS = 3_600_000 + + +def _get(u): + return json.load(urllib.request.urlopen(urllib.request.Request(u, headers={"User-Agent": "curl/8"}), timeout=30)) + + +def fetch(sym): + cache = f"{OUT}/{sym}.npz" + if os.path.exists(cache): + d = np.load(cache); return d["ts"], d["close"] + end = int(time.time() * 1000); start = end - 5 * 365 * 86_400_000 # up to ~5y + rows, cur = [], start + for _ in range(400): + k = _get(f"https://fapi.binance.com/fapi/v1/klines?symbol={sym}&interval=1h&startTime={cur}&limit=1500") + if not k: + break + rows += k + cur = k[-1][0] + 1 + if len(k) < 1500 or cur >= end: + break + time.sleep(0.05) + if not rows: + return np.array([]), np.array([]) + d = {int(r[0]): float(r[4]) for r in rows} + ts = np.array(sorted(d)); close = np.array([d[t] for t in ts]) + os.makedirs(OUT, exist_ok=True) + np.savez(cache, ts=ts, close=close) + return ts, close + + +def build_panel(): + series = {} + for s in SYMS: + ts, c = fetch(s) + if len(ts) > 24 * 60: # need >~2 months + series[s] = (ts, c) + # align on the union of hourly timestamps (floored to the hour) + allts = sorted(set().union(*[set((ts // HOUR_MS).tolist()) for ts, _ in series.values()])) + tindex = {t: i for i, t in enumerate(allts)} + syms = sorted(series) + P = np.full((len(allts), len(syms)), np.nan) + for j, s in enumerate(syms): + ts, c = series[s] + for t, px in zip(ts // HOUR_MS, c): + P[tindex[int(t)], j] = px + yrs = np.array([1970 + int(t) // (365.25 * 24) for t in allts]) + return P, syms, yrs + + +def ann_sharpe(x): + x = x[np.isfinite(x)] + if len(x) < 20 or x.std() == 0: + return float("nan"), float("nan"), 0 + return float(x.mean() / x.std()), x.mean() / (x.std() / math.sqrt(len(x))), len(x) + + +def run(): + P, syms, yrs = build_panel() + T, N = P.shape + logP = np.log(P) + print(f"\n========== MFT CRYPTO CROSS-SECTIONAL MOMENTUM (hour holds, net {COST_BP}bp/turnover) ==========") + print(f"panel: {T} hours × {N} coins ({syms[0]}..{syms[-1]}) years {int(yrs.min())}-{int(yrs.max())}") + print("dollar-neutral xs z-score momentum, lookback=hold=H, non-overlapping rebalance, cost=turnover·5bp\n") + print(f"{'hold':>6} {'periods':>8} {'gross_SR':>9} {'net_SR':>8} {'t(net)':>7} {'pos%':>6} {'per-year net-SR (chrono)':>26}") + print("-" * 92) + for H in HOLDS: + idx = np.arange(0, T - H, H) # non-overlapping rebalance points + rets, yr_of = [], [] + w_prev = np.zeros(N) + per_year = {} + for t in idx: + past = logP[t] - logP[t - H] if t - H >= 0 else np.full(N, np.nan) + fwd = logP[t + H] - logP[t] + ok = np.isfinite(past) & np.isfinite(fwd) + if ok.sum() < 6: + continue + z = np.zeros(N) + pv = past[ok]; zz = (pv - pv.mean()) / (pv.std() + 1e-12) + z[ok] = zz + g = np.abs(z).sum() + w = z / g if g > 0 else z # dollar-neutral, gross 1 + turn = np.abs(w - w_prev).sum() + r = float(np.nansum(w * fwd)) - turn * COST_BP / 1e4 + rets.append(r); yr_of.append(int(yrs[t])) + per_year.setdefault(int(yrs[t]), []).append(r) + w_prev = w + rets = np.array(rets) + if len(rets) < 20: + print(f"{H:>6} (too few periods)"); continue + gross_sr, _, _ = ann_sharpe(rets + 0) # gross approximated below + # recompute gross (no cost) quickly + per_yr_str = " ".join(f"{y}:{(np.mean(v)/ (np.std(v)+1e-12)):+.2f}" for y, v in sorted(per_year.items()) if len(v) >= 10) + per_period = math.sqrt(24 * 365 / H) # annualization factor for H-hour periods + nsr, nt, n = ann_sharpe(rets) + # gross series + # (recompute gross by adding back mean turnover cost is approximate; instead report net only + per-year) + print(f"{H:>6}h {len(rets):>8} {'':>9} {nsr*per_period:>+8.2f} {nt:>+7.2f} {100*np.mean(rets>0):>6.1f} {per_yr_str}") + print("-" * 92) + print("PASS: net annualized SR > 0 with t≥2 AND positive in most years. Else MFT-crypto closed → daily only.") + + +if __name__ == "__main__": + run() diff --git a/scripts/surfer/crypto_stablecoin_dislocation.py b/scripts/surfer/crypto_stablecoin_dislocation.py new file mode 100644 index 000000000..9c7aa3fa9 --- /dev/null +++ b/scripts/surfer/crypto_stablecoin_dislocation.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Stablecoin peg-reversion existence test — the cleanest crypto-FX dislocation. + +Stablecoins are supposed to be worth $1; mint/redeem arbitrage (forced flow) pulls a deviation back +to parity. This tests whether that reversion is harvestable NET OF COST on free Binance spot data, and +— critically — how bad the DEATH-SPIRAL tail is (some "depegs" never revert: UST->0, BUSD wind-down). + +Pre-registered: pair STABLE/USDT, deviation d=close-1. Reversion bet when |d|>thresh: position = +-sign(d) (rich->short, cheap->long), hold H days, pnl_net = -sign(d)*(close[t+H]/close[t]-1) - cost_rt. +Report per-pair + pooled: trade count, mean net pnl (bp), hit%, WORST trade (bp, the death-spiral tail), +and a daily-strategy Sharpe. Cost levels bracket Binance stablecoin fees {0,2,10}bp round-trip. +KILL: pooled net mean<=0 OR edge entirely from one terminal name OR worst-trade tail dwarfs mean edge. + +LIMITATION: overlapping forward windows (first-look existence test, not a hardened backtest); daily +granularity (intraday depeg spikes under-sampled). Flagged, not hidden. +""" +import json +import math +import os +import time +import urllib.request + +import numpy as np + +OUT = "data/surfer/stablecoin" +PAIRS = ["USDCUSDT", "FDUSDUSDT", "TUSDUSDT", "DAIUSDT", "BUSDUSDT", "USDPUSDT"] +DAY_MS = 86_400_000 +THRESHOLDS_BP = [10, 25, 50] +HOLDS = [1, 2, 5] +COSTS_BP = [0.0, 2.0, 10.0] + + +def _get(u): + return json.load(urllib.request.urlopen(urllib.request.Request(u, headers={"User-Agent": "curl/8"}), timeout=30)) + + +def fetch(sym): + cache = f"{OUT}/{sym}.npz" + if os.path.exists(cache): + d = np.load(cache); return d["ts"], d["close"] + start = 1_546_300_800_000 # 2019-01-01 + end = int(time.time() * 1000) + rows, cur = [], start + for _ in range(400): + k = _get(f"https://api.binance.com/api/v3/klines?symbol={sym}&interval=1d&startTime={cur}&limit=1000") + if not k: + break + rows += k + cur = k[-1][0] + DAY_MS + if len(k) < 1000 or cur >= end: + break + time.sleep(0.05) + if not rows: + return np.array([]), np.array([]) + d = {int(r[0]): float(r[4]) for r in rows} + ts = np.array(sorted(d)); close = np.array([d[t] for t in ts]) + os.makedirs(OUT, exist_ok=True) + np.savez(cache, ts=ts, close=close) + return ts, close + + +def reversion(close, thresh_bp, H, cost_bp): + """Event reversion bet: enter when |d|>thresh, pnl=-sign(d)*(fwd_ret)-cost_rt. Returns net-pnl array (bp).""" + d = close - 1.0 + th = thresh_bp / 1e4 + pnls = [] + for t in range(len(close) - H): + if abs(d[t]) > th and close[t] > 0: + fwd = close[t + H] / close[t] - 1.0 + pnl = -np.sign(d[t]) * fwd - cost_bp / 1e4 + pnls.append(pnl * 1e4) # in bp + return np.array(pnls) + + +def run(): + series = {} + for s in PAIRS: + ts, c = fetch(s) + if len(ts) > 60: + series[s] = c + print("\n========== STABLECOIN PEG-REVERSION (crypto-FX dislocation, free Binance spot daily) ==========") + print(f"pairs: {', '.join(f'{s}({len(c)}d)' for s, c in series.items())}\n") + + # 1) dislocation frequency per pair + print("--- dislocation magnitude (|close-1|) per pair ---") + print(f"{'pair':>9} {'days':>5} {'med_bp':>7} {'p95_bp':>7} {'max_bp':>8} {'>10bp%':>7} {'>50bp%':>7}") + for s, c in series.items(): + d = np.abs(c - 1.0) * 1e4 + print(f"{s:>9} {len(c):>5} {np.median(d):>7.1f} {np.percentile(d,95):>7.1f} {d.max():>8.0f} " + f"{100*np.mean(d>10):>6.1f}% {100*np.mean(d>50):>6.1f}%") + + # 2) reversion edge, pooled across pairs, by (thresh, H, cost) + print("\n--- reversion edge (pooled all pairs); pnl in bp/trade, net of round-trip cost ---") + print(f"{'thr_bp':>6} {'H':>2} {'cost':>5} {'trades':>7} {'mean_bp':>8} {'hit%':>6} {'worst_bp':>9} {'daily_SR':>9}") + for th in THRESHOLDS_BP: + for H in HOLDS: + for cost in COSTS_BP: + allp = np.concatenate([reversion(c, th, H, cost) for c in series.values()]) if series else np.array([]) + if len(allp) < 10: + continue + sr = (allp.mean() / allp.std() * math.sqrt(365 / H)) if allp.std() > 0 else float("nan") + print(f"{th:>6} {H:>2} {cost:>4.0f}b {len(allp):>7} {allp.mean():>+8.1f} " + f"{100*np.mean(allp>0):>5.1f}% {allp.min():>+9.0f} {sr:>+9.2f}") + print(" " + "-" * 60) + + # 3) per-pair edge at a fixed mid setting (thr=25bp, H=2, cost=2bp) — is it one terminal name? + print("\n--- per-pair edge @ thr=25bp H=2 cost=2bp (is the edge concentrated/terminal?) ---") + print(f"{'pair':>9} {'trades':>7} {'mean_bp':>8} {'hit%':>6} {'worst_bp':>9}") + for s, c in series.items(): + p = reversion(c, 25, 2, 2.0) + if len(p) >= 5: + print(f"{s:>9} {len(p):>7} {p.mean():>+8.1f} {100*np.mean(p>0):>5.1f}% {p.min():>+9.0f}") + else: + print(f"{s:>9} {len(p):>7} (too few events)") + print("\nKILL if: pooled mean<=0, OR edge is one terminal name, OR |worst_bp| dwarfs mean (death-spiral tail).") + + +if __name__ == "__main__": + run() diff --git a/scripts/surfer/crypto_stablecoin_harden.py b/scripts/surfer/crypto_stablecoin_harden.py new file mode 100644 index 000000000..edcbaf7de --- /dev/null +++ b/scripts/surfer/crypto_stablecoin_harden.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Stablecoin peg-reversion — HARDENED. Make-or-break test of the crypto-FX dislocation edge. + +The spike (crypto_stablecoin_dislocation.py) found a real reversion edge (SR 3-4) but was blind to: +(1) survivorship — all sampled stables reverted; the true tail is buying a cheap stable that goes to + ZERO (UST -75%->delist). (2) overlap inflation. (3) stress slippage. (4) long/short asymmetry. + +This hardens all four: + * UST included (the real death-spiral). FRAX excluded (garbage prints). USTC excluded (post-death). + * NON-OVERLAPPING event walk with explicit exit (revert / max_hold / stop-loss). + * deviation-SCALED slippage: cost_rt = base_bp + slip_k*|deviation| (big depeg = thin book). + * LONG-cheap (unbounded -100% tail) vs SHORT-rich (bounded) decomposed separately. + * collapse filters compared: raw / stop-loss / skip-falling-knife(+stop). + * per-year sign + BTC-stress correlation proxy (does it lose when crypto crashes?). +KILL: long-side net<=0 after UST+filters, OR edge only pre-cost, OR loses concentrated in BTC crashes. +""" +import math +import os + +import numpy as np + +OUT = "data/surfer/stablecoin" +HEALTHY = ["USDCUSDT", "FDUSDUSDT", "TUSDUSDT", "BUSDUSDT", "USDPUSDT", "SUSDUSDT", "DAIUSDT"] +TERMINAL = ["USTUSDT"] +DAY_MS = 86_400_000 +THRESH_BP = 50 +EXIT_BP = 10 +MAX_HOLD = 5 +BASE_BP = 2.0 +SLIP_K = 0.15 # slippage = 15% of the entry deviation (stress thinness) +STOP = 0.03 # 3% stop-loss + + +def fetch(sym): + import json, time, urllib.request + cache = f"{OUT}/{sym}.npz" + if os.path.exists(cache): + d = np.load(cache); return d["ts"], d["close"] + g = lambda u: json.load(urllib.request.urlopen(urllib.request.Request(u, headers={"User-Agent": "curl/8"}), timeout=30)) + start, end = 1_546_300_800_000, int(time.time() * 1000) + rows, cur = [], start + for _ in range(400): + k = g(f"https://api.binance.com/api/v3/klines?symbol={sym}&interval=1d&startTime={cur}&limit=1000") + if not k: + break + rows += k; cur = k[-1][0] + DAY_MS + if len(k) < 1000 or cur >= end: + break + time.sleep(0.05) + if not rows: + return np.array([]), np.array([]) + d = {int(r[0]): float(r[4]) for r in rows} + ts = np.array(sorted(d)); close = np.array([d[t] for t in ts]) + os.makedirs(OUT, exist_ok=True); np.savez(cache, ts=ts, close=close) + return ts, close + + +def walk(ts, close, mode): + """Non-overlapping reversion trades. mode: 'raw'|'stop'|'skip'. Returns list of (dir, year, net_bp, entry_dev_bp).""" + n = len(close); d = close - 1.0; th = THRESH_BP / 1e4; ex = EXIT_BP / 1e4 + trades = []; i = 1 + while i < n - 1: + if abs(d[i]) <= th: + i += 1; continue + direction = -1.0 if d[i] > 0 else 1.0 # rich->short(-1), cheap->long(+1) + # skip-falling-knife: only buy a cheap stable if it is NOT still dropping (bounce confirmation) + if mode == "skip" and direction > 0 and close[i] < close[i - 1]: + i += 1; continue + entry = close[i]; j = i + 1; stop = STOP if mode in ("stop", "skip") else 9.9 + while j < n and (j - i) <= MAX_HOLD: + pnl = direction * (close[j] / entry - 1.0) + if pnl <= -stop or abs(close[j] - 1.0) < ex: + break + j += 1 + j = min(j, n - 1) + gross = direction * (close[j] / entry - 1.0) + slip = (BASE_BP + SLIP_K * abs(d[i]) * 1e4) / 1e4 + yr = 1970 + int(ts[i]) // (365 * DAY_MS) + trades.append((direction, yr, (gross - slip) * 1e4, abs(d[i]) * 1e4)) + i = j + 1 + return trades + + +def stats(pnls): + p = np.array(pnls) + if len(p) < 3: + return f"n={len(p)} (too few)" + sr = p.mean() / p.std() * math.sqrt(365 / ((MAX_HOLD + 1) / 2)) if p.std() > 0 else float("nan") + return f"n={len(p):>4} mean={p.mean():>+7.1f}bp hit={100*np.mean(p>0):>4.1f}% worst={p.min():>+7.0f}bp SR~{sr:>+5.2f}" + + +def run(): + data = {} + for s in HEALTHY + TERMINAL: + ts, c = fetch(s) + if len(ts) > 30: + data[s] = (ts, c) + print("\n===== STABLECOIN PEG-REVERSION — HARDENED (non-overlap, UST-in, slip-scaled, long/short split) =====") + print(f"pairs: {', '.join(data)} thr={THRESH_BP}bp hold<={MAX_HOLD}d exit<{EXIT_BP}bp slip={BASE_BP}+{SLIP_K}*|dev| stop={STOP*100:.0f}%\n") + + for mode in ("raw", "stop", "skip"): + allt = [] + for s in data: + allt += [(s, *t) for t in walk(*data[s], mode)] + longs = [t[3] for t in allt if t[1] > 0] + shorts = [t[3] for t in allt if t[1] < 0] + ust = [t[3] for t in allt if t[0] == "USTUSDT"] + print(f"[{mode:>4}] ALL : {stats([t[3] for t in allt])}") + print(f" LONG(cheap, -100% tail): {stats(longs)}") + print(f" SHORT(rich, bounded) : {stats(shorts)}") + print(f" UST trades only : {stats(ust)}") + print() + + # per-year (SHORT-only, skip mode = the candidate deployable core) + print("--- per-year SHORT-rich net mean bp (the bounded-risk core), skip mode ---") + by_yr = {} + for s in data: + for t in walk(*data[s], "skip"): + if t[0] < 0: + by_yr.setdefault(t[1], []).append(t[2]) + print(" ".join(f"{y}:{np.mean(v):+.0f}({len(v)})" for y, v in sorted(by_yr.items()) if v)) + + # BTC-stress proxy: does the LONG side lose during BTC crashes? + bts, btc = fetch("BTCUSDT") + if len(btc) > 100: + bret = {int(bts[k]) // DAY_MS: (btc[k] / btc[k - 1] - 1.0) for k in range(1, len(btc))} + crash, calm = [], [] + for s in data: + ts, c = data[s] + for dirn, yr, net, dev in walk(ts, c, "skip"): + pass + # simpler: tag each long trade entry day with BTC 1d return + for s in data: + ts, c = data[s]; d = c - 1.0; th = THRESH_BP / 1e4 + for i in range(1, len(c) - 1): + if abs(d[i]) > th and d[i] < 0: # long-cheap entries + br = bret.get(int(ts[i]) // DAY_MS, 0.0) + (crash if br < -0.05 else calm).append(1) + print(f"\nLONG-cheap entries on BTC-crash days (>-5%): {len(crash)} vs calm: {len(calm)} " + f"({100*len(crash)/max(1,len(crash)+len(calm)):.0f}% in crashes = tail-correlation flag)") + print("\nREAD: deployable core = SHORT-rich (bounded). LONG-cheap viable only if filters tame the UST tail.") + + +if __name__ == "__main__": + run() diff --git a/scripts/surfer/crypto_stablecoin_intraday.py b/scripts/surfer/crypto_stablecoin_intraday.py new file mode 100644 index 000000000..effb3a013 --- /dev/null +++ b/scripts/surfer/crypto_stablecoin_intraday.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Stablecoin peg-reversion — INTRADAY EXECUTION validation (the make-or-break-on-fills test). + +Daily test found short-rich reversion ~+25bp/SR4. This checks it survives REALISTIC intraday execution: + * 1h bars, REALISTIC fills: enter at NEXT bar's OPEN (reaction lag — you can't transact at the peak), + exit at the reverting bar's close; pay round-trip spread (calibrated from live book: ~1bp/side). + * deviation HALF-LIFE: of rich events at hour t, what % still rich at t+1/+2/+4/+8 (do you have time?). + * short-rich (validated, bounded) focus; long-cheap with skip-falling-knife for contrast. +Universe = live-tradeable liquid stables only (USDC/FDUSD/TUSD/USDP; DAI/BUSD books are empty). +KILL: edge gone after next-open fills + spread, OR deviations vanish within 1h (no reaction window). +""" +import math +import os + +import numpy as np + +OUT = "data/surfer/stable1h" +PAIRS = ["USDCUSDT", "FDUSDUSDT", "TUSDUSDT", "USDPUSDT"] +HOUR_MS = 3_600_000 +THRESHES = [25, 50] +EXIT_BP = 10 +MAX_HOLD_H = 48 +SPREAD_BP = 1.0 # per side, from live book; round-trip = 2*SPREAD_BP + + +def fetch(sym): + import json, time, urllib.request + cache = f"{OUT}/{sym}.npz" + if os.path.exists(cache): + d = np.load(cache); return d["ts"], d["open"], d["close"] + g = lambda u: json.load(urllib.request.urlopen(urllib.request.Request(u, headers={"User-Agent": "curl/8"}), timeout=30)) + start, end = int(time.time() * 1000) - 3 * 365 * 86_400_000, int(time.time() * 1000) + rows, cur = [], start + for _ in range(800): + k = g(f"https://api.binance.com/api/v3/klines?symbol={sym}&interval=1h&startTime={cur}&limit=1000") + if not k: + break + rows += k; cur = k[-1][0] + HOUR_MS + if len(k) < 1000 or cur >= end: + break + time.sleep(0.04) + if not rows: + return np.array([]), np.array([]), np.array([]) + o = {int(r[0]): (float(r[1]), float(r[4])) for r in rows} + ts = np.array(sorted(o)); op = np.array([o[t][0] for t in ts]); cl = np.array([o[t][1] for t in ts]) + os.makedirs(OUT, exist_ok=True); np.savez(cache, ts=ts, open=op, close=cl) + return ts, op, cl + + +def walk(op, cl, thresh, side, skip=False): + """Non-overlapping. side='short'(rich) or 'long'(cheap). enter next-bar OPEN, exit revert close, pay spread.""" + n = len(cl); d = cl - 1.0; th = thresh / 1e4; ex = EXIT_BP / 1e4; rt = 2 * SPREAD_BP / 1e4 + out = []; i = 1 + while i < n - 2: + rich = d[i] > th; cheap = d[i] < -th + take = (side == "short" and rich) or (side == "long" and cheap) + if not take: + i += 1; continue + if skip and side == "long" and cl[i] < cl[i - 1]: # don't catch a still-falling knife + i += 1; continue + direction = -1.0 if rich else 1.0 + entry = op[i + 1] # realistic: fill at next bar open + j = i + 2 + while j < n and (j - (i + 1)) <= MAX_HOLD_H: + if abs(cl[j] - 1.0) < ex: + break + j += 1 + j = min(j, n - 1) + gross = direction * (cl[j] / entry - 1.0) + out.append((gross - rt) * 1e4) + i = j + 1 + return np.array(out) + + +def half_life(cl, thresh): + d = cl - 1.0; th = thresh / 1e4 + ev = np.where(np.abs(d) > th)[0]; ev = ev[ev < len(cl) - 9] + if len(ev) == 0: + return None + return {h: 100 * np.mean(np.abs(d[ev + h]) > th) for h in (1, 2, 4, 8)} + + +def st(p): + if len(p) < 3: + return f"n={len(p)} (few)" + sr = p.mean() / p.std() * math.sqrt(365 * 24 / (MAX_HOLD_H / 2)) if p.std() > 0 else float("nan") + return f"n={len(p):>4} mean={p.mean():>+7.1f}bp hit={100*np.mean(p>0):>4.1f}% worst={p.min():>+7.0f}bp SR~{sr:>+5.2f}" + + +def run(): + data = {} + for s in PAIRS: + ts, op, cl = fetch(s) + if len(ts) > 500: + data[s] = (op, cl, ts) + print("\n===== STABLECOIN INTRADAY EXECUTION (1h, next-open fills, spread 2bp rt) =====") + print(f"pairs: {', '.join(f'{s}({len(v[1])}h)' for s,v in data.items())}\n") + + print("--- deviation HALF-LIFE: % of events still beyond threshold after H hours (reaction window) ---") + for s, (op, cl, ts) in data.items(): + for th in THRESHES: + hl = half_life(cl, th) + if hl: + print(f" {s:>9} thr{th}: t+1={hl[1]:.0f}% t+2={hl[2]:.0f}% t+4={hl[4]:.0f}% t+8={hl[8]:.0f}%") + print() + + for th in THRESHES: + sh = np.concatenate([walk(d[0], d[1], th, "short") for d in data.values()]) if data else np.array([]) + lo = np.concatenate([walk(d[0], d[1], th, "long", skip=True) for d in data.values()]) if data else np.array([]) + print(f"thr={th}bp SHORT-rich : {st(sh)}") + print(f" LONG-cheap*: {st(lo)} (*skip-falling-knife)") + # per-pair short + for s, (op, cl, ts) in data.items(): + print(f" {s:>9} short: {st(walk(op, cl, th, 'short'))}") + print() + print("READ: edge survives if SHORT-rich stays +ve net of next-open+spread AND half-life>~2h (time to act).") + + +if __name__ == "__main__": + run() diff --git a/scripts/surfer/crypto_trend_sizing.py b/scripts/surfer/crypto_trend_sizing.py new file mode 100644 index 000000000..2fa3aee78 --- /dev/null +++ b/scripts/surfer/crypto_trend_sizing.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Crypto TS-trend sizing — reconfirm the validated return engine + find the deployable CAGR. + +The TSMOM crypto sleeve is VALIDATED (memory: Sharpe ~1.23, CAGR 75.7% @ 61% vol, corr~0 to funding). +The 76% CAGR is just 61%-vol sizing, not a free lunch. This re-runs the SAME pre-registered config +(LONG/FLAT, lookbacks 20/60/120, on crypto_pit daily close, liquidity floor on qvol) and sweeps a +vol-target overlay to answer the #2 question: at a SANE vol target, what is the deployable CAGR and +does the Sharpe hold (it should — vol-target is a rescale, but per-period sizing adds timing). + +NOT a search: the 20/60/120 long/flat config is replicated verbatim from the validated harness. +""" +import glob +import math + +import numpy as np + +PIT = "data/surfer/crypto_pit" +LOOKBACKS = [20, 60, 120] +COST_BP = 5.0 +ADV_FLOOR = 5e6 # $5M trailing quote-volume liquidity floor +VOL_TARGETS = [0.10, 0.15, 0.20] +LEV_CAP = 3.0 +PPY = 365 # crypto trades every day + + +def build(): + days = set() + raw = {} + for f in sorted(glob.glob(f"{PIT}/*.npz")): + d = np.load(f) + if len(d["day"]) < 130: + continue + sym = f.split("/")[-1][:-4] + raw[sym] = {int(dd): (c, q) for dd, c, q in zip(d["day"], d["close"], d["qvol"])} + days.update(int(x) for x in d["day"]) + days = sorted(days) + di = {d: i for i, d in enumerate(days)} + syms = sorted(raw) + C = np.full((len(days), len(syms)), np.nan) + V = np.full((len(days), len(syms)), np.nan) + for j, s in enumerate(syms): + for dd, (c, q) in raw[s].items(): + C[di[dd], j] = c; V[di[dd], j] = q + return np.array(days), syms, C, V + + +def maxdd(equity): + peak = np.maximum.accumulate(equity) + return float((equity / peak - 1).min()) + + +def run(): + days, syms, C, V = build() + T, N = C.shape + logC = np.log(C) + ret = np.full((T, N), np.nan) + ret[1:] = C[1:] / C[:-1] - 1.0 + # long/flat signal = mean over lookbacks of 1[trailing-L log return > 0] + sig = np.zeros((T, N)) + cnt = np.zeros((T, N)) + for L in LOOKBACKS: + tr = np.full((T, N), np.nan) + tr[L:] = logC[L:] - logC[:-L] + on = np.where(np.isfinite(tr), (tr > 0).astype(float), np.nan) + m = np.isfinite(on) + sig[m] += on[m]; cnt[m] += 1 + sig = np.where(cnt > 0, sig / np.maximum(cnt, 1), 0.0) # 0..1 long/flat conviction + + print(f"\n===== CRYPTO TS-TREND SIZING (long/flat {LOOKBACKS}, ${ADV_FLOOR/1e6:.0f}M ADV floor, {COST_BP}bp/leg) =====") + print(f"panel: {T} days x {N} coins (survivorship-free incl. dead)\n") + + # base book (gross 1, daily rebalance, equal-weight by conviction among liquid+on coins) + w_prev = np.zeros(N) + book = np.zeros(T) + for t in range(1, T): + eligible = np.isfinite(ret[t]) & np.isfinite(V[t - 1]) & (V[t - 1] > ADV_FLOOR) + s = sig[t - 1] * eligible + g = s.sum() + w = s / g if g > 0 else np.zeros(N) + turn = np.abs(w - w_prev).sum() + book[t] = float(np.nansum(w * ret[t])) - turn * COST_BP / 1e4 + w_prev = w + + valid = np.arange(T) > max(LOOKBACKS) + base = book[valid] + base_sr = base.mean() / base.std() * math.sqrt(PPY) if base.std() > 0 else float("nan") + base_vol = base.std() * math.sqrt(PPY) + base_cagr = float(np.prod(1 + base) ** (PPY / len(base)) - 1) + print(f"BASE (un-vol-targeted): Sharpe {base_sr:+.2f} vol {base_vol*100:.0f}% CAGR {base_cagr*100:+.1f}% maxDD {maxdd(np.cumprod(1+base))*100:.1f}%") + print(f" (memory reference: Sharpe ~1.23, CAGR ~76% @ ~61% vol — replicating)\n") + + print(f"{'vol_tgt':>8} {'Sharpe':>7} {'CAGR':>7} {'real_vol':>9} {'maxDD':>7} {'avg_lev':>8} {'per-year SR':>30}") + print("-" * 86) + # vol-target overlay on the base book (trailing 30d realized, lagged, leverage-capped) + rv = np.full(T, np.nan) + for t in range(31, T): + w = book[t - 30:t] + rv[t] = w.std() * math.sqrt(PPY) + yrs = 1970 + days // 365 + for vt in VOL_TARGETS: + lev = np.where(np.isfinite(rv) & (rv > 0), np.clip(vt / rv, 0, LEV_CAP), 0.0) + tgt = book * np.concatenate([[0], lev[:-1]]) # lag leverage by 1 day + s = tgt[valid] + sr = s.mean() / s.std() * math.sqrt(PPY) if s.std() > 0 else float("nan") + vol = s.std() * math.sqrt(PPY) + cagr = float(np.prod(1 + s) ** (PPY / len(s)) - 1) + dd = maxdd(np.cumprod(1 + s)) + avglev = float(np.mean(lev[valid])) + yv = {} + for r, y in zip(s, yrs[valid]): + yv.setdefault(int(y), []).append(r) + ystr = " ".join(f"{y}:{(np.mean(v)/(np.std(v)+1e-12)*math.sqrt(PPY)):+.1f}" + for y, v in sorted(yv.items()) if len(v) >= 30) + print(f"{vt*100:>6.0f}% {sr:>+7.2f} {cagr*100:>+6.1f}% {vol*100:>8.0f}% {dd*100:>+6.1f}% {avglev:>8.2f} {ystr}") + print("-" * 86) + print("Deployable read: pick the vol target whose maxDD you can stomach; Sharpe should ~hold across targets.") + + +if __name__ == "__main__": + run() diff --git a/scripts/surfer/mft_4h_harden.py b/scripts/surfer/mft_4h_harden.py new file mode 100644 index 000000000..4d7b90520 --- /dev/null +++ b/scripts/surfer/mft_4h_harden.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Harden the 4h MFT trend edge — kill the inflation sources before trusting it. + +The mft_horizon_sweep found a 4h selective trend edge (OOS net +5.9t, t+7.4). But that t is +OVERSTATED: overlapping 4h holds across 5-min entries are not independent. This re-tests honestly: + + 1. ONE TRADE PER SESSION — the single highest-conviction morning setup, ~4h hold, flat-by-close. + Non-overlapping ⇒ trades are ~independent across sessions ⇒ honest t-stat. Also the realistic strat. + 2. IS-DERIVED conviction cutoff — set on the in-sample sessions, applied to OOS (no pooled-threshold leak). + 3. REAL RTH session — ET 09:30–16:00 (not UTC-midnight); entry window = first 2.5h so 4h fits before close. + 4. PER-YEAR breakdown — is the edge in every year, or one regime? + 5. Cost robustness (1 and 2 ticks) + multiple-testing context. + +Pre-registered: dir=sign(ret over H bars), conviction=|t-stat|=|ret_H|/(vol·√H), vol=trailing 12-bar. +Entry = highest-conviction valid bar in 09:30-12:00, exit = +H bars (same session), cost 1 (and 2) ticks. +Trade the session only if its best conviction ≥ IS cutoff (selectivity sweep). H=48 (4h) primary; 36/60 robustness. +""" +import glob +import math + +import numpy as np +import pandas as pd + +TICK = 0.25 +BAR_NS = 5 * 60 * 10**9 +HOLDS = [36, 48, 60] # 3h, 4h (primary), 5h +ENTRY_CUTOFF_MIN = 12 * 60 # latest entry 12:00 ET so a 4h hold fits before 16:00 +RTH_LO, RTH_HI = 9 * 60 + 30, 16 * 60 + + +def load_es_5min(): + import databento as db + ts_all, c_all = [], [] + for p in sorted(glob.glob("data/surfer/es1m/*.dbn")): + try: + arr = db.DBNStore.from_file(p).to_ndarray() + except Exception: + continue + if len(arr) == 0 or "close" not in arr.dtype.names: + continue + ts = arr["ts_event"].astype(np.int64) + c = arr["close"].astype(np.float64) / 1e9 + m = c > 0 + ts_all.append(ts[m]); c_all.append(c[m]) + ts = np.concatenate(ts_all); c = np.concatenate(c_all) + _u, ui = np.unique(ts, return_index=True); ts, c = ts[ui], c[ui] + o = np.argsort(ts, kind="stable"); ts, c = ts[o], c[o] + b5 = ts // BAR_NS + _, first = np.unique(b5, return_index=True) + last = np.append(first[1:] - 1, len(b5) - 1) + return ts[last], c[last] + + +def rth_sessions(ts5, close5): + """Restrict to ET RTH 09:30-16:00, return (close, session_id, minute_of_day_ET, year) RTH-only, sorted.""" + idx = pd.to_datetime(ts5, utc=True).tz_convert("America/New_York") + mins = (idx.hour * 60 + idx.minute).to_numpy() + rth = (mins >= RTH_LO) & (mins < RTH_HI) + et_date = idx.strftime("%Y-%m-%d").to_numpy() + year = idx.year.to_numpy() + close = close5[rth]; sess_str = et_date[rth]; mod = mins[rth]; yr = year[rth] + sid = pd.factorize(sess_str)[0] # contiguous session ids (data already time-sorted) + return close, sid, mod, yr + + +def per_session_trades(close, sid, mod, yr, H, cost): + """For each RTH session pick the single highest-conviction entry in 09:30-12:00, hold H bars to a + same-session exit. Returns arrays over sessions that HAVE a valid candidate: + (best_conv, net_ticks, year, sess_order).""" + lc = np.log(close) + n = len(lc) + # trailing per-bar vol (12-bar) and H-bar trend, only valid within the same session + r1 = np.zeros(n); r1[1:] = lc[1:] - lc[:-1] + same1 = np.zeros(n, bool); same1[1:] = sid[1:] == sid[:-1] + r1 = np.where(same1, r1, 0.0) + # rolling 12-bar std via cumsum + w = 12 + cs = np.concatenate([[0.0], np.cumsum(r1)]); cs2 = np.concatenate([[0.0], np.cumsum(r1 * r1)]) + vol = np.full(n, np.nan) + s = cs[w:] - cs[:-w]; s2 = cs2[w:] - cs2[:-w] + vol[w - 1:] = np.sqrt(np.maximum(s2 / w - (s / w) ** 2, 0.0)) + # trailing H-bar trend over the RTH-contiguous series (overnight gap collapsed — conservative: + # the overnight move, where much drift lives, is excluded from the signal). No same-session + # requirement on the lookback (a 4h trailing trend can't fit in the morning entry window otherwise). + retH = np.full(n, np.nan); retH[H:] = lc[H:] - lc[:-H] + tstat = retH / np.maximum(vol * math.sqrt(H), 1e-9) + # forward H-bar move (ticks), exit MUST be same session (flat-by-close) + fwd = np.full(n, np.nan); fwd[:-H] = (close[H:] - close[:-H]) / TICK + sameH_fwd = np.zeros(n, bool); sameH_fwd[:-H] = sid[:-H] == sid[H:] + entry_ok = (mod <= ENTRY_CUTOFF_MIN) & sameH_fwd & np.isfinite(tstat) & np.isfinite(fwd) & (vol > 0) + direction = np.sign(retH) + net = direction * fwd - cost + conv = np.abs(tstat) + + out_conv, out_net, out_yr, out_ord = [], [], [], [] + # group by session, pick max-conviction valid entry + order = np.argsort(sid, kind="stable") + sid_s = sid[order] + bounds = np.append(np.append(0, np.nonzero(np.diff(sid_s))[0] + 1), len(sid_s)) + for b in range(len(bounds) - 1): + rows = order[bounds[b]:bounds[b + 1]] + rows = rows[entry_ok[rows] & (direction[rows] != 0)] + if len(rows) == 0: + continue + best = rows[np.argmax(conv[rows])] + out_conv.append(conv[best]); out_net.append(net[best]) + out_yr.append(yr[best]); out_ord.append(sid[best]) + return (np.array(out_conv), np.array(out_net), np.array(out_yr), np.array(out_ord)) + + +def stats(net): + n = len(net) + if n < 20: + return n, float("nan"), float("nan"), float("nan"), float("nan") + m = float(net.mean()); sd = float(net.std()) + 1e-12 + t = m / (sd / math.sqrt(n)) + hit = 100.0 * float((net > 0).mean()) + ann_sharpe = (m / sd) * math.sqrt(252.0) # ~1 trade/session ⇒ ~252 trades/yr + return n, m, hit, t, ann_sharpe + + +def run(): + ts5, close5 = load_es_5min() + close, sid, mod, yr = rth_sessions(ts5, close5) + n_sess = int(sid.max()) + 1 + print("\n================ 4h MFT HARDENING (RTH, 1 trade/session, IS-cutoff, per-year) ================") + print(f"RTH 5-min bars={len(close)} sessions={n_sess} years={int(yr.min())}-{int(yr.max())}") + print("honest non-overlapping t-stat (1 trade/session); multiple-testing bar ≈ |t|>2.3 for ~10 configs\n") + + for cost in (1.0, 2.0): + print(f"================= COST = {cost:.0f} tick round-trip =================") + for H in HOLDS: + conv, net, tyr, sord = per_session_trades(close, sid, mod, yr, H, cost) + if len(conv) < 50: + print(f" H={H}: too few sessions with a candidate ({len(conv)})"); continue + split = np.quantile(sord, 0.70) # first 70% of sessions = IS + is_m = sord < split; oos_m = ~is_m + hh = H * 5 / 60.0 + print(f" --- H={H} ({hh:.0f}h hold) --- (IS sessions {int(is_m.sum())} / OOS {int(oos_m.sum())})") + print(f" {'select':>8} {'cutoff':>7} {'n_OOS':>6} {'net_t':>7} {'hit%':>6} {'t':>6} {'annSR':>6} {'trd/yr':>7}") + for sel in (1.00, 0.50, 0.25, 0.10): # trade top-sel% of IS-conviction days + cutoff = np.quantile(conv[is_m], 1.0 - sel) if sel < 1.0 else -np.inf + take = oos_m & (conv >= cutoff) + n, m, hit, t, sr = stats(net[take]) + oos_years = (sord[oos_m].max() - split) / n_sess * (int(yr.max()) - int(yr.min()) + 1) + trd_yr = n / max(oos_years, 1e-9) + cuts = f"{cutoff:.2f}" if np.isfinite(cutoff) else "all" + print(f" {sel:>8.2f} {cuts:>7} {n:>6} {m:>+7.2f} {hit:>6.1f} {t:>+6.2f} {sr:>+6.2f} {trd_yr:>7.1f}") + # per-year at the pre-registered selectivity = top 25% of IS-conviction days + cutoff = np.quantile(conv[is_m], 0.75) + print(f" per-year (sel=0.25, cost={cost:.0f}, OOS+IS shown; IS years are not OOS):") + for y in range(int(yr.min()), int(yr.max()) + 1): + ym = (tyr == y) & (conv >= cutoff) + if int(ym.sum()) < 10: + continue + n, m, hit, t, sr = stats(net[ym]) + flag = "OOS" if (np.median(sord[tyr == y]) >= split) else "is " + print(f" {y} [{flag}] n={n:>4} net={m:>+6.2f}t hit={hit:>5.1f}% t={t:>+5.2f}") + print() + + +if __name__ == "__main__": + run() diff --git a/scripts/surfer/mft_harden_v2.py b/scripts/surfer/mft_harden_v2.py new file mode 100644 index 000000000..fab0f14e1 --- /dev/null +++ b/scripts/surfer/mft_harden_v2.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Apples-to-apples hardening of the sweep's OWN 4h edge — same 24h-continuous signal, but kill the +statistical inflation: ONE non-overlapping trade per UTC-day, IS-derived conviction cutoff, per-year. + +The sweep (mft_horizon_sweep) used a 24h-continuous 5-min grid, day = UTC-day, hold = 48 bars = 4h of +clock time, top-decile conviction over HEAVILY OVERLAPPING entries → t+7.4 (inflated by overlap). Here: +same grid + signal, but max-conviction single entry per UTC-day (non-overlapping ⇒ honest t), threshold +fit on IS, reported per-year. cost 1 and 2 ticks. PASS: OOS net>0 & hit>50% & honest t≥2, stable per-year. +""" +import glob +import math + +import numpy as np + +TICK = 0.25 +BAR_NS = 5 * 60 * 10**9 +DAY_NS = 86_400 * 10**9 +HOLDS = [48] # 4h (the sweep's standout); same grid as the sweep + + +def load_es_5min(): + import databento as db + ts_all, c_all = [], [] + for p in sorted(glob.glob("data/surfer/es1m/*.dbn")): + try: + arr = db.DBNStore.from_file(p).to_ndarray() + except Exception: + continue + if len(arr) == 0 or "close" not in arr.dtype.names: + continue + ts = arr["ts_event"].astype(np.int64); c = arr["close"].astype(np.float64) / 1e9 + m = c > 0; ts_all.append(ts[m]); c_all.append(c[m]) + ts = np.concatenate(ts_all); c = np.concatenate(c_all) + _u, ui = np.unique(ts, return_index=True); ts, c = ts[ui], c[ui] + o = np.argsort(ts, kind="stable"); ts, c = ts[o], c[o] + b5 = ts // BAR_NS + _, first = np.unique(b5, return_index=True) + last = np.append(first[1:] - 1, len(b5) - 1) + return ts[last], c[last] + + +def per_day_trades(close, day, yr, H, cost): + lc = np.log(close); n = len(lc) + r1 = np.zeros(n); r1[1:] = lc[1:] - lc[:-1] + same1 = np.zeros(n, bool); same1[1:] = day[1:] == day[:-1] + r1 = np.where(same1, r1, 0.0) + w = 12 + cs = np.concatenate([[0.0], np.cumsum(r1)]); cs2 = np.concatenate([[0.0], np.cumsum(r1 * r1)]) + vol = np.full(n, np.nan); s = cs[w:] - cs[:-w]; s2 = cs2[w:] - cs2[:-w] + vol[w - 1:] = np.sqrt(np.maximum(s2 / w - (s / w) ** 2, 0.0)) + retH = np.full(n, np.nan); retH[H:] = lc[H:] - lc[:-H] + sameH_back = np.zeros(n, bool); sameH_back[H:] = day[H:] == day[:-H] # like the sweep: sameday(H) + tstat = retH / np.maximum(vol * math.sqrt(H), 1e-9) + fwd = np.full(n, np.nan); fwd[:-H] = (close[H:] - close[:-H]) / TICK + sameH_fwd = np.zeros(n, bool); sameH_fwd[:-H] = day[:-H] == day[H:] + ok = sameH_fwd & sameH_back & np.isfinite(tstat) & np.isfinite(fwd) & (vol > 0) + direction = np.sign(retH); net = direction * fwd - cost; conv = np.abs(tstat) + + oc, on, oy, od = [], [], [], [] + order = np.argsort(day, kind="stable"); ds = day[order] + bounds = np.append(np.append(0, np.nonzero(np.diff(ds))[0] + 1), len(ds)) + for b in range(len(bounds) - 1): + rows = order[bounds[b]:bounds[b + 1]] + rows = rows[ok[rows] & (direction[rows] != 0)] + if len(rows) == 0: + continue + best = rows[np.argmax(conv[rows])] + oc.append(conv[best]); on.append(net[best]); oy.append(yr[best]); od.append(day[best]) + return np.array(oc), np.array(on), np.array(oy), np.array(od) + + +def stats(net): + n = len(net) + if n < 20: + return n, float("nan"), float("nan"), float("nan"), float("nan") + m = float(net.mean()); sd = float(net.std()) + 1e-12 + return n, m, 100.0 * float((net > 0).mean()), m / (sd / math.sqrt(n)), (m / sd) * math.sqrt(252.0) + + +def run(): + ts5, close5 = load_es_5min() + day = (ts5 // DAY_NS).astype(np.int64) + yr = (1970 + (ts5 // (365.25 * DAY_NS))).astype(int) # approx calendar year from epoch days + print("\n============ APPLES-TO-APPLES 4h HARDENING (24h grid, 1 trade/UTC-day, IS-cutoff) ============") + print(f"5-min bars={len(close5)} days={int(day.max()-day.min())} multiple-testing bar ≈ |t|>2.3\n") + for cost in (1.0, 2.0): + for H in HOLDS: + conv, net, tyr, dd = per_day_trades(close5, day, yr, H, cost) + if len(conv) < 50: + print(f"cost={cost:.0f} H={H}: too few ({len(conv)})"); continue + split = np.quantile(dd, 0.70); is_m = dd < split; oos_m = ~is_m + print(f"=== cost={cost:.0f} tick, H={H} (4h), 1 trade/day (IS days {int(is_m.sum())} / OOS {int(oos_m.sum())}) ===") + print(f" {'select':>7} {'cutoff':>7} {'n_OOS':>6} {'net_t':>7} {'hit%':>6} {'t':>6} {'annSR':>6} {'trd/yr':>7}") + for sel in (1.00, 0.50, 0.25, 0.10): + cut = np.quantile(conv[is_m], 1.0 - sel) if sel < 1.0 else -np.inf + take = oos_m & (conv >= cut) + n, m, hit, t, sr = stats(net[take]) + oosyrs = max((int(tyr[oos_m].max()) - int(tyr[oos_m].min()) + 1), 1) + cs = f"{cut:.2f}" if np.isfinite(cut) else "all" + print(f" {sel:>7.2f} {cs:>7} {n:>6} {m:>+7.2f} {hit:>6.1f} {t:>+6.2f} {sr:>+6.2f} {n/oosyrs:>7.1f}") + cut = np.quantile(conv[is_m], 0.75) + print(f" per-year (sel=0.25; IS years are not OOS):") + for y in range(int(tyr.min()), int(tyr.max()) + 1): + ym = (tyr == y) & (conv >= cut) + if int(ym.sum()) < 10: + continue + n, m, hit, t, sr = stats(net[ym]) + flag = "OOS" if np.median(dd[tyr == y]) >= split else "is " + print(f" {y} [{flag}] n={n:>4} net={m:>+6.2f}t hit={hit:>5.1f}% t={t:>+5.2f}") + print() + + +if __name__ == "__main__": + run() diff --git a/scripts/surfer/mft_horizon_sweep.py b/scripts/surfer/mft_horizon_sweep.py new file mode 100644 index 000000000..c339498f9 --- /dev/null +++ b/scripts/surfer/mft_horizon_sweep.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""MFT horizon sweep — does a SLOWER (lower-turnover) intraday trend edge clear costs? + +Prong A (mft_quality_test) showed: at 15-min holds the net ≈ -1 tick = the cost itself (gross ≈ 0). +This sweeps the hold horizon UP (15m → full session) on clean ES OHLCV-1m. As the hold grows the +forward move (in ticks) grows ~√time while the 1-tick cost stays fixed, so cost/gross shrinks — this +isolates the question the user posed: keep costs low by trading SLOWER. If gross edge grows with +horizon (trend persistence), there's an MFT sweet spot; if gross stays ~0 everywhere, the price signal +is dead regardless of speed. + +Pre-registered (NOT tuned): trend-follow, lookback = hold = H bars (5-min each); direction = sign(ret_H); +conviction = |t-stat| = |ret_H| / (per-bar vol · √H); entry close[t], exit close[t+H], intraday only +(same-day, flat-by-close); cost = 1.0 tick round-trip (crossing). Quality curve by conviction decile, +IS(first 70%) / OOS(last 30%). PASS bar: OOS top-decile net > 0 ticks & hit > 50% & |t| ≥ 2, at a +horizon with low trades/day (genuinely MFT). +""" +import glob +import math + +import numpy as np +import torch + +DEV = "cuda" if torch.cuda.is_available() else None +TICK = 0.25 +COST_TICKS = 1.0 +BAR_NS = 5 * 60 * 10**9 +DAY_NS = 86_400 * 10**9 +HOLDS = [3, 12, 24, 48, 78] # 15m, 1h, 2h, 4h, ~full session (5-min bars) + + +def load_es_5min(): + import databento as db + es1m = sorted(glob.glob("data/surfer/es1m/*.dbn")) + ts_all, c_all = [], [] + if es1m: + for p in es1m: + try: + arr = db.DBNStore.from_file(p).to_ndarray() + except Exception: + continue + if len(arr) == 0 or "close" not in arr.dtype.names: + continue + ts = arr["ts_event"].astype(np.int64) + c = arr["close"].astype(np.float64) / 1e9 + m = c > 0 + ts_all.append(ts[m]); c_all.append(c[m]) + else: + for p in sorted(glob.glob("test_data/futures-baseline/ES.FUT/*.dbn.zst")): + try: + df = db.DBNStore.from_file(p).to_df().reset_index() + except Exception: + continue + if df.empty or "close" not in df.columns: + continue + df = df[df["close"] > 0] + if df.empty: + continue + dom = df["instrument_id"].value_counts().idxmax() + df = df[df["instrument_id"] == dom].sort_values("ts_event") + ts_all.append(df["ts_event"].astype("int64").to_numpy()) + c_all.append(df["close"].to_numpy(np.float64)) + ts = np.concatenate(ts_all); c = np.concatenate(c_all) + _u, _ui = np.unique(ts, return_index=True) + ts, c = ts[_ui], c[_ui] + o = np.argsort(ts, kind="stable"); ts, c = ts[o], c[o] + b5 = ts // BAR_NS + _, first = np.unique(b5, return_index=True) + last = np.append(first[1:] - 1, len(b5) - 1) + return ts[last], c[last] + + +def rolling_std(x, w): + z = torch.zeros(1, dtype=x.dtype, device=x.device) + cs = torch.cat([z, torch.cumsum(x, 0)]) + cs2 = torch.cat([z, torch.cumsum(x * x, 0)]) + s = cs[w:] - cs[:-w] + s2 = cs2[w:] - cs2[:-w] + mean = s / w + var = (s2 / w - mean * mean).clamp_min(0.0) + out = torch.full_like(x, float("nan")) + out[w - 1:] = var.sqrt() + return out + + +def run(): + ts5, close5 = load_es_5min() + day = ts5 // DAY_NS + n_days = int(day.max() - day.min()) + lc = torch.tensor(np.log(close5), device=DEV, dtype=torch.float64) + px = torch.tensor(close5, device=DEV, dtype=torch.float64) + dy = torch.tensor(day, device=DEV) + T = len(lc) + t = torch.arange(T, device=DEV) + split = int(0.7 * T) + + def sameday(lag): + m = torch.zeros(T, dtype=torch.bool, device=DEV) + m[lag:] = dy[lag:] == dy[:-lag] + return m + + r1 = torch.zeros_like(lc); r1[1:] = lc[1:] - lc[:-1] + r1 = torch.where(sameday(1), r1, torch.zeros_like(r1)) + vol1 = rolling_std(r1, 12) # per-bar 5-min vol (fixed 1h window, pre-registered) + + print("\n================ MFT HORIZON SWEEP (GPU, intraday trend, flat-by-close) ================") + print(f"device={DEV} 5-min bars={T} span days≈{n_days} cost={COST_TICKS} tick round-trip") + print("trend-follow: dir=sign(ret_H), conv=|ret_H|/(vol·√H). PASS: OOS top-decile net>0 & hit>50% & |t|≥2\n") + print(f"{'hold':>6} {'top-q':>6} {'n_OOS':>7} {'gross_t':>8} {'net_t':>8} {'cost/gr':>8} {'hit%':>6} {'t-stat':>7} {'trd/day':>8}") + print("-" * 78) + + for H in HOLDS: + retH = torch.full_like(lc, float("nan")); retH[H:] = lc[H:] - lc[:-H] + tstat = retH / (vol1 * math.sqrt(H)).clamp_min(1e-9) + fwd_ticks = torch.full_like(px, float("nan")) + fwd_ticks[:-H] = (px[H:] - px[:-H]) / TICK + fwd_valid = torch.zeros(T, dtype=torch.bool, device=DEV) + fwd_valid[:-H] = dy[H:] == dy[:-H] # exit same day (flat-by-close) + valid = fwd_valid & sameday(H) & torch.isfinite(tstat) & torch.isfinite(vol1) & (vol1 > 0) + direction = torch.sign(retH) + net = direction * fwd_ticks - COST_TICKS + gross = direction * fwd_ticks + cand = valid & (direction != 0) + idx = torch.nonzero(cand, as_tuple=True)[0] + if len(idx) < 100: + print(f"{H:>6} (too few candidates: {len(idx)})"); continue + conv = tstat.abs()[idx]; tt = t[idx] + order = torch.argsort(conv, descending=True) + hold_hours = H * 5 / 60.0 + for q in [0.05, 0.10, 1.00]: + k = max(int(q * len(idx)), 1) + sel = idx[order[:k]] + oos = sel[tt[order[:k]] >= split] + no = int(len(oos)) + if no < 20: + print(f"{H:>6} {q:>6.2f} (OOS n<20)"); continue + g = gross[oos]; nt = net[oos] + gm = float(g.mean()); nm = float(nt.mean()) + tval = nm / (float(nt.std()) / math.sqrt(no) + 1e-12) + hit = 100.0 * float((nt > 0).float().mean()) + cost_gr = (COST_TICKS / gm * 100.0) if gm > 0 else float("nan") + trd_day = no / (0.3 * n_days) # OOS is last 30% of days + tag = f"{H}({hold_hours:.1f}h)" if q == 0.05 else "" + print(f"{tag:>6} {q:>6.2f} {no:>7} {gm:>+8.3f} {nm:>+8.3f} {cost_gr:>7.0f}% {hit:>6.1f} {tval:>+7.2f} {trd_day:>8.2f}") + print("-" * 78) + + +def main(): + if DEV is None: + raise SystemExit("CUDA not available") + run() + + +if __name__ == "__main__": + main()