From 027d73a504bebcd7b49edfb919bbe44840f2850e Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sun, 7 Jun 2026 18:33:28 +0200 Subject: [PATCH] research(crypto): cross-venue funding arb survives net-of-cost with hysteresis Backtested the cross-venue funding arb on historical funding (Binance/OKX/Hyperliquid). Gross +21%/yr, spreads persist (capture 0.71), but NAIVE daily rebalance is cost-killed (net Sharpe -4.5, negative every month). HYSTERESIS (hold winners until spread decays, entry>10bp/exit>5bp) flips net to +10-14%/yr market-neutral (Sharpe +11-15, but inflated by ~1% vol + idealized fills; realistic ~3-6 / return ~10-14%). Turnover is the swing factor. First edge of the whole search to survive the net-of-cost horde -- market-neutral, persistent, no spot leg (solves hedgeability), operational not predictive. Caveats: 94d/one period, idealized fills, counterparty. Next: switch live harness to hysteresis, deepen history to full year, micro-live. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/surfer/fetch_xvenue_hist.py | 90 +++++++++++++++++++++ scripts/surfer/fetch_xvenue_hist2.py | 113 +++++++++++++++++++++++++++ scripts/surfer/xvenue_sim.py | 67 ++++++++++++++++ scripts/surfer/xvenue_sim2.py | 84 ++++++++++++++++++++ scripts/surfer/xvenue_sim3.py | 79 +++++++++++++++++++ 5 files changed, 433 insertions(+) create mode 100644 scripts/surfer/fetch_xvenue_hist.py create mode 100644 scripts/surfer/fetch_xvenue_hist2.py create mode 100644 scripts/surfer/xvenue_sim.py create mode 100644 scripts/surfer/xvenue_sim2.py create mode 100644 scripts/surfer/xvenue_sim3.py diff --git a/scripts/surfer/fetch_xvenue_hist.py b/scripts/surfer/fetch_xvenue_hist.py new file mode 100644 index 000000000..f433b51de --- /dev/null +++ b/scripts/surfer/fetch_xvenue_hist.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Fetch historical funding (Binance + Bybit) for liquid coins on both, to BACKTEST the cross-venue +funding arb instead of waiting for live paper-forward. Aggregates 8h settlements to daily per venue. +Cached. Saves data/surfer/xvenue/panel.json = {coin: {date: [binance_daily, bybit_daily]}}. +""" +import datetime +import json +import os +import time +import urllib.request + +OUT = "data/surfer/xvenue" +NCOINS = 60 +DAYS = 330 + + +def get(u, post=None): + data = json.dumps(post).encode() if post else None + h = {"User-Agent": "Mozilla/5.0", "Accept": "application/json"} + for a in range(4): + try: + return json.loads(urllib.request.urlopen(urllib.request.Request(u, data=data, headers=h), timeout=25).read()) + except Exception: + if a == 3: + return None + time.sleep(3 * (a + 1)) + return None + + +def base(s): + return s[:-4] if s.endswith("USDT") else s + + +def day_of(ms): + return datetime.datetime.utcfromtimestamp(int(ms) / 1000).strftime("%Y-%m-%d") + + +def binance_hist(coin): + r = get(f"https://fapi.binance.com/fapi/v1/fundingRate?symbol={coin}USDT&limit=1000") + d = {} + for x in (r or []): + d.setdefault(day_of(x["fundingTime"]), 0.0) + d[day_of(x["fundingTime"])] += float(x["fundingRate"]) + return d + + +def bybit_hist(coin): + out, end = {}, None + for _ in range(6): + u = f"https://api.bybit.com/v5/market/funding/history?category=linear&symbol={coin}USDT&limit=200" + if end: + u += f"&endTime={end}" + r = get(u) + lst = ((r or {}).get("result") or {}).get("list") or [] + if not lst: + break + for x in lst: + t = x["fundingRateTimestamp"] + out.setdefault(day_of(t), 0.0) + out[day_of(t)] += float(x["fundingRate"]) + end = int(lst[-1]["fundingRateTimestamp"]) - 1 + time.sleep(0.3) + return out + + +def main(): + os.makedirs(OUT, exist_ok=True) + binv = {base(x["symbol"]): float(x["quoteVolume"]) for x in get("https://fapi.binance.com/fapi/v1/ticker/24hr") if x["symbol"].endswith("USDT")} + byr = get("https://api.bybit.com/v5/market/tickers?category=linear")["result"]["list"] + bybv = {base(x["symbol"]): float(x.get("turnover24h", 0)) for x in byr if x["symbol"].endswith("USDT")} + coins = sorted([c for c in binv if c in bybv and binv[c] > 10e6 and bybv[c] > 10e6], key=lambda c: -binv[c])[:NCOINS] + print(f"universe: {len(coins)} coins on both venues >$10M/day") + panel = {} + for i, c in enumerate(coins): + cf = f"{OUT}/{c}.json" + if os.path.exists(cf): + panel[c] = json.load(open(cf)); continue + bn = binance_hist(c); by = bybit_hist(c) + days = sorted(set(bn) & set(by)) + panel[c] = {d: [bn[d], by[d]] for d in days} + json.dump(panel[c], open(cf, "w")) + if i % 10 == 0: + print(f" {i+1}/{len(coins)} {c}: {len(panel[c])} common days") + time.sleep(0.4) + json.dump(panel, open(f"{OUT}/panel.json", "w")) + print(f"DONE: {len(panel)} coins -> {OUT}/panel.json") + + +if __name__ == "__main__": + main() diff --git a/scripts/surfer/fetch_xvenue_hist2.py b/scripts/surfer/fetch_xvenue_hist2.py new file mode 100644 index 000000000..2b0585267 --- /dev/null +++ b/scripts/surfer/fetch_xvenue_hist2.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Deep multi-venue funding history (Binance + OKX + Hyperliquid, ~1yr) for a confident cross-venue +backtest. Binance & HL are deep (~330d); OKX ~3mo. Aggregate to daily per venue. Cache per coin. +Saves data/surfer/xvenue/panel2.json = {coin: {date: {bn, okx, hl}}}. +""" +import datetime +import json +import os +import time +import urllib.request + +OUT = "data/surfer/xvenue2" +NCOINS = 35 +DAYS = 330 + + +def get(u, post=None): + data = json.dumps(post).encode() if post else None + h = {"User-Agent": "Mozilla/5.0", "Accept": "application/json"} + if post: + h["Content-Type"] = "application/json" + for a in range(4): + try: + return json.loads(urllib.request.urlopen(urllib.request.Request(u, data=data, headers=h), timeout=25).read()) + except Exception: + if a == 3: + return None + time.sleep(3 * (a + 1)) + return None + + +def base(s): + return s[:-4] if s.endswith("USDT") else s + + +def day_of(ms): + return datetime.datetime.utcfromtimestamp(int(ms) / 1000).strftime("%Y-%m-%d") + + +def binance_hist(coin): + r = get(f"https://fapi.binance.com/fapi/v1/fundingRate?symbol={coin}USDT&limit=1000") + d = {} + for x in (r or []): + k = day_of(x["fundingTime"]); d[k] = d.get(k, 0.0) + float(x["fundingRate"]) + return d + + +def okx_hist(coin): + out, after = {}, None + for _ in range(6): + u = f"https://www.okx.com/api/v5/public/funding-rate-history?instId={coin}-USDT-SWAP&limit=100" + if after: + u += f"&after={after}" + r = get(u); data = (r or {}).get("data") or [] + if not data: + break + for x in data: + k = day_of(x["fundingTime"]); out[k] = out.get(k, 0.0) + float(x["fundingRate"]) + after = data[-1]["fundingTime"]; time.sleep(0.2) + return out + + +def hl_hist(coin, now_ms): + out, start = {}, now_ms - DAYS * 86400000 + for _ in range(30): + r = get("https://api.hyperliquid.xyz/info", post={"type": "fundingHistory", "coin": coin, "startTime": start}) + if not r: + break + for x in r: + k = day_of(x["time"]); out[k] = out.get(k, 0.0) + float(x["fundingRate"]) + last = int(r[-1]["time"]) + if last <= start or len(r) < 2: + break + start = last + 1 + return out + + +def main(): + os.makedirs(OUT, exist_ok=True) + now_ms = int(time.time() * 1000) + binv = {base(x["symbol"]): float(x["quoteVolume"]) for x in get("https://fapi.binance.com/fapi/v1/ticker/24hr") if x["symbol"].endswith("USDT")} + hlmeta = get("https://api.hyperliquid.xyz/info", post={"type": "metaAndAssetCtxs"}) + hlcoins = {u["name"] for u in hlmeta[0]["universe"]} + coins = sorted([c for c in binv if c in hlcoins and binv[c] > 10e6], key=lambda c: -binv[c])[:NCOINS] + print(f"universe: {len(coins)} coins (Binance>$10M & on Hyperliquid)") + panel = {} + for i, c in enumerate(coins): + cf = f"{OUT}/{c}.json" + if os.path.exists(cf): + panel[c] = json.load(open(cf)); continue + bn = binance_hist(c); okx = okx_hist(c); hl = hl_hist(c, now_ms) + days = sorted(set(bn) | set(okx) | set(hl)) + rec = {} + for d in days: + v = {} + if d in bn: + v["bn"] = bn[d] + if d in okx: + v["okx"] = okx[d] + if d in hl: + v["hl"] = hl[d] + if len(v) >= 2: + rec[d] = v + panel[c] = rec + json.dump(rec, open(cf, "w")) + print(f" {i+1}/{len(coins)} {c}: {len(rec)} days w/ >=2 venues (bn{len(bn)} okx{len(okx)} hl{len(hl)})") + time.sleep(0.3) + json.dump(panel, open(f"{OUT}/panel2.json", "w")) + print(f"DONE: {len(panel)} coins -> {OUT}/panel2.json") + + +if __name__ == "__main__": + main() diff --git a/scripts/surfer/xvenue_sim.py b/scripts/surfer/xvenue_sim.py new file mode 100644 index 000000000..ef9964cb5 --- /dev/null +++ b/scripts/surfer/xvenue_sim.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Backtest the cross-venue funding arb on historical funding (Binance vs Bybit). + +The persistence test, on history: each day pick the top-K coins by |binance-bybit funding spread|, +position to collect it (short higher-funding venue, long lower), and book the REALIZED next-day +funding difference (not the snapshot). If spreads persist -> positive; if they mean-revert before +you collect -> ~0 net. Net of round-trip cost on turnover. Reports gross/net, Sharpe, by top-K. +""" +import json +import math +import os +import sys + +import numpy as np + +PANEL = "data/surfer/xvenue/panel.json" +COST_RT = 0.0010 +HURDLE = 0.0005 + + +def main(): + panel = json.load(open(PANEL)) + dates = sorted(set().union(*[set(v) for v in panel.values()])) + di = {d: i for i, d in enumerate(dates)} + coins = list(panel) + T, N = len(dates), len(coins) + bn = np.full((T, N), np.nan); by = np.full((T, N), np.nan) + for j, c in enumerate(coins): + for d, (b, y) in panel[c].items(): + bn[di[d], j] = b; by[di[d], j] = y + spread = bn - by # signed: + means binance funding higher + print(f"cross-venue backtest: {N} coins, {T} days ({dates[0]}..{dates[-1]})") + + def sim(K, cost): + rets = [] + prev = set() + for t in range(T - 1): + s_t = spread[t]; s_n = spread[t + 1] + ok = np.isfinite(s_t) & np.isfinite(s_n) & (np.abs(s_t) > HURDLE) + idx = np.where(ok)[0] + if len(idx) == 0: + rets.append(0.0); continue + top = idx[np.argsort(-np.abs(s_t[idx]))[:K]] + p = np.sign(s_t[top]) + realized = float(np.mean(p * s_n[top])) # collect next-day actual difference + cur = set(coins[j] for j in top) + turn = len(cur ^ prev) / max(len(cur), 1) + rets.append(realized - turn * (cost / 2)); prev = cur + r = np.array(rets) + ann = r.mean() * 365; vol = r.std() * math.sqrt(365) + eq = np.cumprod(1 + r); dd = float((eq / np.maximum.accumulate(eq) - 1).min()) + return ann, vol, (ann / vol if vol > 0 else float("nan")), dd, eq[-1] - 1 + + print(f"\n{'topK':>5} {'annNET%':>8} {'vol%':>6} {'Sharpe':>7} {'maxDD%':>7} {'totalNET%':>9}") + for K in [5, 10, 20]: + a, v, sh, dd, tot = sim(K, COST_RT) + g = sim(K, 0.0)[0] + print(f"{K:>5} {100*a:>+8.1f} {100*v:>6.1f} {sh:>+7.2f} {100*dd:>+7.1f} {100*tot:>+9.1f} (gross ann {100*g:+.0f}%)") + # persistence diagnostic: sign(spread_t) == sign(spread_t+1) fraction + fin = np.isfinite(spread[:-1]) & np.isfinite(spread[1:]) & (np.abs(spread[:-1]) > HURDLE) + persist = np.mean(np.sign(spread[:-1][fin]) == np.sign(spread[1:][fin])) + print(f"\n spread-sign persistence (1 day): {100*persist:.0f}% (>>50% = spreads persist = real; ~50% = noise/revert)") + print(" VERDICT: net Sharpe>1 + persistence>>50% = real capturable edge; net~0/persist~50% = mean-reverts before you collect.") + + +if __name__ == "__main__": + main() diff --git a/scripts/surfer/xvenue_sim2.py b/scripts/surfer/xvenue_sim2.py new file mode 100644 index 000000000..16d053838 --- /dev/null +++ b/scripts/surfer/xvenue_sim2.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""N-venue cross-venue funding backtest (Binance/OKX/Hyperliquid, deep history). + +Per coin-day: spread = max-min daily funding across available venues (short max, long min). +Pick top-K by |spread|, book the REALIZED next-day max-min difference for the held pair, net of +turnover cost. Reports Sharpe/return by top-K, persistence, and PER-MONTH Sharpe (regime check). +""" +import json +import math +import os +import sys + +import numpy as np + +PANEL = "data/surfer/xvenue2/panel2.json" +COST_RT = 0.0010 +HURDLE = 0.0005 + + +def main(): + panel = json.load(open(PANEL)) + dates = sorted(set().union(*[set(v) for v in panel.values()])) + di = {d: i for i, d in enumerate(dates)} + coins = list(panel) + T, N = len(dates), len(coins) + # per coin-day: best (max) and worst (min) venue funding, and which venues + hi = np.full((T, N), np.nan); lo = np.full((T, N), np.nan) + for j, c in enumerate(coins): + for d, v in panel[c].items(): + vals = list(v.values()) + if len(vals) >= 2: + hi[di[d], j] = max(vals); lo[di[d], j] = min(vals) + spread = hi - lo # always >=0 (max-min) + print(f"N-venue backtest: {N} coins, {T} days ({dates[0]}..{dates[-1]})") + + def sim(K, cost): + rets, prev = [], set() + for t in range(T - 1): + s_t, s_n = spread[t], spread[t + 1] + ok = np.isfinite(s_t) & np.isfinite(s_n) & (s_t > HURDLE) + idx = np.where(ok)[0] + if len(idx) == 0: + rets.append(0.0); continue + top = idx[np.argsort(-s_t[idx])[:K]] + realized = float(np.mean(s_n[top])) # next-day max-min still captured if pair persists + cur = set(coins[j] for j in top) + turn = len(cur ^ prev) / max(len(cur), 1) + rets.append(realized - turn * (cost / 2)); prev = cur + r = np.array(rets) + ann = r.mean() * 365; vol = r.std() * math.sqrt(365) + eq = np.cumprod(1 + r); dd = float((eq / np.maximum.accumulate(eq) - 1).min()) + return ann, vol, (ann / vol if vol > 0 else float("nan")), dd, r + + print(f"\n{'topK':>5} {'annNET%':>8} {'Sharpe':>7} {'maxDD%':>7} {'gross%':>7}") + R = {} + for K in [5, 10, 20]: + a, v, sh, dd, r = sim(K, COST_RT); g = sim(K, 0.0)[0]; R[K] = r + print(f"{K:>5} {100*a:>+8.1f} {sh:>+7.2f} {100*dd:>+7.1f} {100*g:>+7.0f}") + + # persistence: top-spread coin still positive-spread direction next day. spread is max-min(>=0); + # the real persistence q = does the SAME venue stay highest? approximate via spread autocorr sign>0 always, + # so report realized/snapshot ratio = how much of today's spread you actually collect next day. + capt = [] + for t in range(T - 1): + ok = np.isfinite(spread[t]) & np.isfinite(spread[t + 1]) & (spread[t] > HURDLE) + idx = np.where(ok)[0] + if len(idx): + top = idx[np.argsort(-spread[t][idx])[:10]] + capt.append(np.mean(spread[t + 1][top]) / max(np.mean(spread[t][top]), 1e-9)) + print(f"\n capture ratio (next-day spread / today's spread, top-10): {np.mean(capt):.2f} (1.0=fully persists, ~0=collapses)") + + # per-month Sharpe (regime check) on top-10 + r10 = R[10] + mo = [dates[t][:7] for t in range(T - 1)] + print(" per-month Sharpe (top-10, net):") + for m in sorted(set(mo)): + seg = r10[np.array(mo) == m] + if len(seg) > 8 and seg.std() > 0: + print(f" {m}: {seg.mean()/seg.std()*math.sqrt(365):+.1f} (n={len(seg)})") + print("\n VERDICT: net Sharpe>1 across MONTHS + capture ratio>0.5 = robust deployable edge.") + + +if __name__ == "__main__": + main() diff --git a/scripts/surfer/xvenue_sim3.py b/scripts/surfer/xvenue_sim3.py new file mode 100644 index 000000000..378d81ce5 --- /dev/null +++ b/scripts/surfer/xvenue_sim3.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Low-turnover rescue test: gross cross-venue spread is +21% with 0.71 persistence, but naive daily +rebalancing is cost-killed. Test if HYSTERESIS (hold a coin until its spread decays below an exit +floor) + lower maker-fee cost flips net positive. If yes -> real but needs patient execution; if no +-> cost-walled.""" +import json +import math + +import numpy as np + +panel = json.load(open("data/surfer/xvenue2/panel2.json")) +dates = sorted(set().union(*[set(v) for v in panel.values()])) +di = {d: i for i, d in enumerate(dates)} +coins = list(panel) +T, N = len(dates), len(coins) +spread = np.full((T, N), np.nan) +for j, c in enumerate(coins): + for d, v in panel[c].items(): + vals = list(v.values()) + if len(vals) >= 2: + spread[di[d], j] = max(vals) - min(vals) + + +def sim(K, entry, exit_, cost): + held = {} # col -> True + rets = [] + for t in range(T - 1): + s_t, s_n = spread[t], spread[t + 1] + # drop held coins whose spread decayed below exit (or data gone) + held = {j: 1 for j in held if np.isfinite(s_t[j]) and s_t[j] > exit_} + # fill up to K from fresh entries above entry hurdle + if len(held) < K: + cand = [j for j in np.where(np.isfinite(s_t) & (s_t > entry))[0] if j not in held] + cand.sort(key=lambda j: -s_t[j]) + for j in cand[:K - len(held)]: + held[j] = 1 + if not held: + rets.append(0.0); continue + w = 1.0 / len(held) + realized = sum(w * s_n[j] for j in held if np.isfinite(s_n[j])) + # turnover: this sim only changes the set on entry/exit crossings (low churn) + rets.append(realized) + sim._prev = set(held) + return np.array(rets) + + +def turnover_cost(K, entry, exit_, cost): + """Re-run tracking set changes to charge cost properly.""" + held, rets, prev = {}, [], set() + for t in range(T - 1): + s_t, s_n = spread[t], spread[t + 1] + held = {j: 1 for j in held if np.isfinite(s_t[j]) and s_t[j] > exit_} + if len(held) < K: + cand = [j for j in np.where(np.isfinite(s_t) & (s_t > entry))[0] if j not in held] + cand.sort(key=lambda j: -s_t[j]) + for j in cand[:K - len(held)]: + held[j] = 1 + cur = set(held) + turn = len(cur ^ prev) / max(len(cur), 1) if cur else 0 + realized = (sum(s_n[j] for j in held if np.isfinite(s_n[j])) / len(held)) if held else 0.0 + rets.append(realized - turn * (cost / 2)); prev = cur + return np.array(rets) + + +def stats(r): + ann = r.mean() * 365; vol = r.std() * math.sqrt(365) + return ann, (ann / vol if vol > 0 else float("nan")) + + +print(f"low-turnover rescue: {N} coins, {T} days") +print(f"{'config':>34} {'annNET%':>8} {'Sharpe':>7}") +for K in [10, 20]: + for entry, exit_ in [(0.0005, 0.0003), (0.0010, 0.0005), (0.0020, 0.0010)]: + for cost, cl in [(0.0006, "maker6bp"), (0.0010, "taker10bp")]: + r = turnover_cost(K, entry, exit_, cost) + a, sh = stats(r) + print(f" K={K:>2} entry{entry*1e4:.0f}/exit{exit_*1e4:.0f}bp {cl:>9} {100*a:>+8.1f} {sh:>+7.2f}") +print("\nVERDICT: any config net Sharpe>1 = edge survives with patient/low-turnover execution.") +print("If all still negative = cost-walled even with hysteresis -> not deployable.")