feat(surfer): point-in-time survivorship test PASSED — momentum is real
Built survivorship-free universe: 135 perps incl 28 known-dead (LUNA/SRM/MATIC...), universe rebuilt daily as top-K by trailing dollar-volume (dead coins in while trading, drop out after crash), no lookahead. mom_20 TOPK=30: full +0.76, IS+0.85/OOS+0.56, CPCV-med +0.73, DSR 0.86, POSITIVE EVERY YEAR 2020-2026 incl 2022 +0.72. Counterintuitive: survivorship was HIDING the edge (survivor-only 2022 -0.40 -> PIT +0.72) because momentum shorts the dying coins and profits from crashes. Decisive survivorship confirmation. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
103
scripts/surfer/fetch_crypto_pit.py
Normal file
103
scripts/surfer/fetch_crypto_pit.py
Normal file
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Point-in-time crypto fetch: broad live universe + KNOWN-DEAD perps, WITH volume.
|
||||
|
||||
For a survivorship-free test: delisted Binance perps still return klines up to delisting,
|
||||
so including them lets a date-by-date "top-N by trailing dollar-volume" universe contain
|
||||
dead coins while they traded and drop them when they die. Saves day, close, qvol, funding
|
||||
to data/surfer/crypto_pit/ (gitignored).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
import numpy as np
|
||||
|
||||
OUT = "data/surfer/crypto_pit"
|
||||
DAY_MS = 86_400_000
|
||||
TOP_N = 120
|
||||
# famous delisted / dead Binance USDT perps (the survivorship risk); klines available post-delist
|
||||
DEAD = ["LUNAUSDT", "ANCUSDT", "SRMUSDT", "HNTUSDT", "MATICUSDT", "FTTUSDT", "RAYUSDT",
|
||||
"WAVESUSDT", "BNXUSDT", "SCUSDT", "OCEANUSDT", "AGIXUSDT", "CVCUSDT", "TLMUSDT",
|
||||
"DENTUSDT", "KEYUSDT", "CTKUSDT", "TOMOUSDT", "AKROUSDT", "BLZUSDT", "COMBOUSDT",
|
||||
"FTMUSDT", "DGBUSDT", "REEFUSDT", "SLPUSDT", "IDEXUSDT", "LINAUSDT", "NEBLUSDT",
|
||||
"RADUSDT", "BTSUSDT", "STMXUSDT", "MDTUSDT", "AMBUSDT", "GTCUSDT", "DARUSDT"]
|
||||
|
||||
|
||||
def get(url):
|
||||
try:
|
||||
return json.load(urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "curl/8"}), timeout=30))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def universe(n):
|
||||
info = get("https://fapi.binance.com/fapi/v1/exchangeInfo")
|
||||
perps = {s["symbol"] for s in info["symbols"]
|
||||
if s.get("contractType") == "PERPETUAL" and s.get("quoteAsset") == "USDT" and s.get("status") == "TRADING"}
|
||||
tick = get("https://fapi.binance.com/fapi/v1/ticker/24hr")
|
||||
vol = {t["symbol"]: float(t["quoteVolume"]) for t in tick if t["symbol"] in perps}
|
||||
return sorted(vol, key=lambda s: -vol[s])[:n]
|
||||
|
||||
|
||||
def klines(sym):
|
||||
out, end = [], None
|
||||
for _ in range(20):
|
||||
u = f"https://fapi.binance.com/fapi/v1/klines?symbol={sym}&interval=1d&limit=1500"
|
||||
if end:
|
||||
u += f"&endTime={end}"
|
||||
k = get(u)
|
||||
if not k:
|
||||
break
|
||||
out = k + out
|
||||
end = k[0][0] - 1
|
||||
if len(k) < 1500:
|
||||
break
|
||||
time.sleep(0.15)
|
||||
d = {int(r[0]): (float(r[4]), float(r[7])) for r in out} # close, quote-volume
|
||||
days = np.array(sorted(d))
|
||||
cl = np.array([d[t][0] for t in days]); qv = np.array([d[t][1] for t in days])
|
||||
return days // DAY_MS, cl, qv
|
||||
|
||||
|
||||
def funding(sym, start_ms):
|
||||
out, st = [], start_ms
|
||||
for _ in range(80):
|
||||
f = get(f"https://fapi.binance.com/fapi/v1/fundingRate?symbol={sym}&startTime={st}&limit=1000")
|
||||
if not f:
|
||||
break
|
||||
out += f
|
||||
st = f[-1]["fundingTime"] + 1
|
||||
if len(f) < 1000:
|
||||
break
|
||||
time.sleep(0.12)
|
||||
daily = {}
|
||||
for r in out:
|
||||
daily.setdefault(int(r["fundingTime"]) // DAY_MS, 0.0)
|
||||
daily[int(r["fundingTime"]) // DAY_MS] += float(r["fundingRate"])
|
||||
return daily
|
||||
|
||||
|
||||
def main():
|
||||
os.makedirs(OUT, exist_ok=True)
|
||||
syms = sorted(set(universe(TOP_N)) | set(DEAD))
|
||||
ok = 0
|
||||
for sym in syms:
|
||||
outp = f"{OUT}/{sym}.npz"
|
||||
if os.path.exists(outp):
|
||||
ok += 1; continue
|
||||
kd, cl, qv = klines(sym)
|
||||
if len(kd) < 200:
|
||||
print(f" {sym}: short/none ({len(kd)}d), skip"); continue
|
||||
fmap = funding(sym, int(kd.min()) * DAY_MS)
|
||||
fund = np.array([fmap.get(int(d), 0.0) for d in kd])
|
||||
np.savez(outp, day=kd, close=cl, qvol=qv, funding=fund)
|
||||
alive = "DEAD" if qv[-30:].mean() == 0 else "live"
|
||||
print(f" {sym}: {len(kd)}d ({kd.min()}..{kd.max()}) {alive}")
|
||||
ok += 1
|
||||
time.sleep(0.2)
|
||||
print(f"DONE: {ok}/{len(syms)} -> {OUT}/")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user