diff --git a/scripts/surfer/fetch_crypto_pit.py b/scripts/surfer/fetch_crypto_pit.py new file mode 100644 index 000000000..a634b21e9 --- /dev/null +++ b/scripts/surfer/fetch_crypto_pit.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Point-in-time crypto fetch: broad live universe + KNOWN-DEAD perps, WITH volume. + +For a survivorship-free test: delisted Binance perps still return klines up to delisting, +so including them lets a date-by-date "top-N by trailing dollar-volume" universe contain +dead coins while they traded and drop them when they die. Saves day, close, qvol, funding +to data/surfer/crypto_pit/ (gitignored). +""" +import json +import os +import time +import urllib.request + +import numpy as np + +OUT = "data/surfer/crypto_pit" +DAY_MS = 86_400_000 +TOP_N = 120 +# famous delisted / dead Binance USDT perps (the survivorship risk); klines available post-delist +DEAD = ["LUNAUSDT", "ANCUSDT", "SRMUSDT", "HNTUSDT", "MATICUSDT", "FTTUSDT", "RAYUSDT", + "WAVESUSDT", "BNXUSDT", "SCUSDT", "OCEANUSDT", "AGIXUSDT", "CVCUSDT", "TLMUSDT", + "DENTUSDT", "KEYUSDT", "CTKUSDT", "TOMOUSDT", "AKROUSDT", "BLZUSDT", "COMBOUSDT", + "FTMUSDT", "DGBUSDT", "REEFUSDT", "SLPUSDT", "IDEXUSDT", "LINAUSDT", "NEBLUSDT", + "RADUSDT", "BTSUSDT", "STMXUSDT", "MDTUSDT", "AMBUSDT", "GTCUSDT", "DARUSDT"] + + +def get(url): + try: + return json.load(urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "curl/8"}), timeout=30)) + except Exception: + return None + + +def universe(n): + info = get("https://fapi.binance.com/fapi/v1/exchangeInfo") + perps = {s["symbol"] for s in info["symbols"] + if s.get("contractType") == "PERPETUAL" and s.get("quoteAsset") == "USDT" and s.get("status") == "TRADING"} + tick = get("https://fapi.binance.com/fapi/v1/ticker/24hr") + vol = {t["symbol"]: float(t["quoteVolume"]) for t in tick if t["symbol"] in perps} + return sorted(vol, key=lambda s: -vol[s])[:n] + + +def klines(sym): + out, end = [], None + for _ in range(20): + u = f"https://fapi.binance.com/fapi/v1/klines?symbol={sym}&interval=1d&limit=1500" + if end: + u += f"&endTime={end}" + k = get(u) + if not k: + break + out = k + out + end = k[0][0] - 1 + if len(k) < 1500: + break + time.sleep(0.15) + d = {int(r[0]): (float(r[4]), float(r[7])) for r in out} # close, quote-volume + days = np.array(sorted(d)) + cl = np.array([d[t][0] for t in days]); qv = np.array([d[t][1] for t in days]) + return days // DAY_MS, cl, qv + + +def funding(sym, start_ms): + out, st = [], start_ms + for _ in range(80): + f = get(f"https://fapi.binance.com/fapi/v1/fundingRate?symbol={sym}&startTime={st}&limit=1000") + if not f: + break + out += f + st = f[-1]["fundingTime"] + 1 + if len(f) < 1000: + break + time.sleep(0.12) + daily = {} + for r in out: + daily.setdefault(int(r["fundingTime"]) // DAY_MS, 0.0) + daily[int(r["fundingTime"]) // DAY_MS] += float(r["fundingRate"]) + return daily + + +def main(): + os.makedirs(OUT, exist_ok=True) + syms = sorted(set(universe(TOP_N)) | set(DEAD)) + ok = 0 + for sym in syms: + outp = f"{OUT}/{sym}.npz" + if os.path.exists(outp): + ok += 1; continue + kd, cl, qv = klines(sym) + if len(kd) < 200: + print(f" {sym}: short/none ({len(kd)}d), skip"); continue + fmap = funding(sym, int(kd.min()) * DAY_MS) + fund = np.array([fmap.get(int(d), 0.0) for d in kd]) + np.savez(outp, day=kd, close=cl, qvol=qv, funding=fund) + alive = "DEAD" if qv[-30:].mean() == 0 else "live" + print(f" {sym}: {len(kd)}d ({kd.min()}..{kd.max()}) {alive}") + ok += 1 + time.sleep(0.2) + print(f"DONE: {ok}/{len(syms)} -> {OUT}/") + + +if __name__ == "__main__": + main() diff --git a/scripts/surfer/pit_sweep.py b/scripts/surfer/pit_sweep.py new file mode 100644 index 000000000..3fffeebee --- /dev/null +++ b/scripts/surfer/pit_sweep.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Point-in-time survivorship test for crypto cross-sectional momentum. + +Universe is rebuilt EACH DAY: among coins with active trailing-30d dollar-volume (>0), +take the top-K by that volume. Dead coins (LUNA, SRM, ...) are IN the cross-section while +they trade and DROP OUT (after carrying their crash) when volume dies. No lookahead: +volume & momentum use only past data; weights apply to next-day return. If momentum's +edge survives here — especially 2022 with LUNA included — it is real beyond the bootstrap proxy. +""" +import glob +import math +import os +import sys + +import numpy as np + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from signal_sweep import xs_weights, pnl_w, validate, sharpe_t # noqa: E402 +import torch # noqa: E402 + +DEV = "cuda" if torch.cuda.is_available() else "cpu" + + +def load(): + syms, data = [], {} + for p in sorted(glob.glob("data/surfer/crypto_pit/*.npz")): + d = np.load(p); s = p.split("/")[-1][:-4] + data[s] = (d["day"].astype(np.int64), d["close"].astype(float), d["qvol"].astype(float), d["funding"].astype(float)) + syms.append(s) + syms = sorted(syms) + days = np.array(sorted(set().union(*[set(data[s][0].tolist()) for s in syms]))) + di = {int(v): i for i, v in enumerate(days.tolist())} + T, N = len(days), len(syms) + close = np.full((T, N), np.nan); qv = np.full((T, N), np.nan); fund = np.full((T, N), np.nan) + for j, s in enumerate(syms): + dd, cc, vv, ff = data[s] + for k in range(len(dd)): + r = di[int(dd[k])]; close[r, j] = cc[k]; qv[r, j] = vv[k]; fund[r, j] = ff[k] + return syms, days, close, qv, fund + + +def trailing(lc, L): + out = np.full_like(lc, np.nan); out[L:] = lc[L:] - lc[:-L]; return out + + +def main(): + syms, days, close, qv, fund = load() + T, N = close.shape + lc = np.log(close) + R = np.zeros((T, N)); R[1:] = lc[1:] - lc[:-1]; R = np.where(np.isfinite(R), R, 0.0) + Reff = R - np.where(np.isfinite(fund), fund, 0.0) + qv = np.nan_to_num(qv) + dv30 = np.full((T, N), 0.0) # trailing 30d mean dollar-volume + for t in range(30, T): + dv30[t] = qv[t - 30:t].mean(axis=0) + dead = sum(1 for j in range(N) if qv[-30:, j].mean() == 0) + year = (1970 + days / 365.25).astype(int) + + print(f"\n===== POINT-IN-TIME MOMENTUM — {N} coins ({dead} dead/delisted), {T}d =====") + for TOPK in [30, 50]: + # daily universe = top-K by trailing dollar-vol among actively-trading coins + univ = np.zeros((T, N), bool) + for t in range(T): + elig = np.where((dv30[t] > 0) & np.isfinite(close[t]))[0] + if len(elig) == 0: + continue + k = min(TOPK, len(elig)) + top = elig[np.argsort(-dv30[t, elig])[:k]] + univ[t, top] = True + print(f"\n--- TOPK={TOPK} (avg coins/day in-universe = {univ.sum(1).mean():.0f}) ---") + print(f"{'lookback':>9} {'full':>6} {'IS':>6} {'OOS':>6} {'CPCVmed':>8} {'DSR':>5} | per-year (2020..2026)") + for L in [10, 20, 30]: + sig = trailing(lc, L).copy() + sig[~univ] = np.nan # restrict signal to PIT universe + w = xs_weights(sig) + pnl = pnl_w(w, Reff, cost_bp=10) + v = validate(pnl, days, 6) + py = [] + for y in range(2020, 2027): + m = year[1:] == y + py.append(sharpe_t(torch.tensor(pnl[m], device=DEV, dtype=torch.float64)) if m.sum() > 30 else float("nan")) + pys = " ".join(f"{p:>+5.2f}" if not math.isnan(p) else " n/a" for p in py) + print(f"{L:>9} {v['full']:>+6.2f} {v['is_']:>+6.2f} {v['oos']:>+6.2f} {v['med']:>+8.2f} {v['dsr']:>5.2f} | {pys}") + print("\nVERDICT: if mom_20-30 stays positive (esp. 2022 with LUNA in-universe) => survivorship-robust REAL edge.") + + +if __name__ == "__main__": + main()