gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

research(surfer): crypto edge-falsification probes — intraday/cascade dead, stablecoin peg-reversion validated

Browse Source 
- crypto_mft_xsec / mft_*: intraday/MFT XS price edge FALSIFIED (fee-trap)
- crypto_cascade_reversion: liquidation-cascade reversion FALSIFIED (continuation)
- crypto_trend_sizing: TS-trend return-engine reconfirmed (Sharpe ~1.25)
- crypto_stablecoin_dislocation/harden/intraday: short-rich peg-reversion VALIDATED (bounded, uncorrelated); long-cheap = death-spiral trap

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-06-21 10:40:34 +02:00
parent 6878aac950
commit f4bd1e2432
9 changed files with 1180 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
scripts/surfer/crypto_cascade_reversion.py Normal file
View File

@@ -0,0 +1,116 @@
#!/usr/bin/env python3
"""Cascade reversion — do crypto coins BOUNCE after an extreme forced-flow down-move?
Liquidation cascades are forced deleveraging that overshoots; the harvestable consequence is the
mean-reversion bounce. Binance killed the historical liquidation feed (live websocket only), so we
test the PROXY already on disk: extreme negative trailing-H return = a cascade. Pre-registered:
formation: trailing-H log return per coin (cross-sectional)
signal: LONG the bottom-q fraction (biggest losers) — long-only on crashed names
hold: next H hours, NON-OVERLAPPING rebalance
cost: 5 bp / leg on actual weight turnover
two builds: raw_long (equal-weight losers, has crypto beta)
hedged (long losers − short equal-weight UNIVERSE basket; beta-neutral, short leg
is the diversified basket NOT a single coin → dodges the short-melt-up ruin
that killed generic XS reversal)
KILL: net annualized SR < 0.5 OR not positive in most years (chrono) → cascade reversion closed.
LIMITATION: npz panel has close only (no volume) → cannot condition on the volume spike that
confirms a true liquidation cascade; the extreme-return tail is the proxy. Flagged, not hidden.
"""
import math
import os
import numpy as np
OUT = "data/surfer/crypto1h"
COST_BP = 5.0
SYMS = ["BTCUSDT", "ETHUSDT", "SOLUSDT", "XRPUSDT", "BNBUSDT", "DOGEUSDT", "ADAUSDT", "AVAXUSDT",
"LINKUSDT", "LTCUSDT", "DOTUSDT", "TRXUSDT", "BCHUSDT", "ETCUSDT", "FILUSDT", "ATOMUSDT"]
HOUR_MS = 3_600_000
HOLDS = [1, 2, 4, 8, 12, 24] # formation == hold, hours
Q = 0.20 # bottom quintile = biggest losers
def build_panel():
series = {}
for s in SYMS:
cache = f"{OUT}/{s}.npz"
if not os.path.exists(cache):
continue
d = np.load(cache)
if len(d["ts"]) > 24 * 60:
series[s] = (d["ts"], d["close"])
allts = sorted(set().union(*[set((ts // HOUR_MS).tolist()) for ts, _ in series.values()]))
tindex = {t: i for i, t in enumerate(allts)}
syms = sorted(series)
P = np.full((len(allts), len(syms)), np.nan)
for j, s in enumerate(syms):
ts, c = series[s]
for t, px in zip(ts // HOUR_MS, c):
P[tindex[int(t)], j] = px
yrs = np.array([1970 + int(t) // int(365.25 * 24) for t in allts])
return P, syms, yrs
def ann(rets, per_year_factor):
rets = np.asarray(rets)
if len(rets) < 20 or rets.std() == 0:
return float("nan"), float("nan")
sr = rets.mean() / rets.std()
t = rets.mean() / (rets.std() / math.sqrt(len(rets)))
return sr * per_year_factor, t
def run():
P, syms, yrs = build_panel()
T, N = P.shape
logP = np.log(P)
print(f"\n===== CRYPTO CASCADE REVERSION (long the crashed names, net {COST_BP}bp/leg) =====")
print(f"panel: {T} hours x {N} coins years {int(yrs.min())}-{int(yrs.max())} q={Q} (bottom quintile)")
print("LONG bottom-q losers; hedged = long losers - short equal-wt universe basket\n")
hdr = f"{'hold':>5} {'periods':>8} {'build':>8} {'net_SR':>8} {'t(net)':>7} {'hit%':>6} {'per-year net-SR (chrono)':>30}"
print(hdr); print("-" * len(hdr))
for H in HOLDS:
pf = math.sqrt(24 * 365 / H)
idx = np.arange(H, T - H, H) # need t-H for formation, t+H for fwd
raw, hed, yr_raw, yr_hed = [], [], [], []
w_raw_prev = np.zeros(N); w_hed_prev = np.zeros(N)
for t in idx:
past = logP[t] - logP[t - H]
fwd = logP[t + H] - logP[t]
ok = np.isfinite(past) & np.isfinite(fwd)
n_ok = int(ok.sum())
if n_ok < 6:
continue
okidx = np.where(ok)[0]
k = max(1, int(round(Q * n_ok)))
losers = okidx[np.argsort(past[okidx])[:k]] # most-negative trailing return
# raw long
w_raw = np.zeros(N); w_raw[losers] = 1.0 / k
# hedged: long losers - short equal-weight universe basket
w_hed = np.zeros(N); w_hed[losers] += 1.0 / k; w_hed[okidx] -= 1.0 / n_ok
r_raw = float(np.nansum(w_raw * fwd)) - np.abs(w_raw - w_raw_prev).sum() * COST_BP / 1e4
r_hed = float(np.nansum(w_hed * fwd)) - np.abs(w_hed - w_hed_prev).sum() * COST_BP / 1e4
raw.append(r_raw); hed.append(r_hed)
yr_raw.append(int(yrs[t])); yr_hed.append(int(yrs[t]))
w_raw_prev = w_raw; w_hed_prev = w_hed
if len(raw) < 20:
print(f"{H:>5} (too few periods)"); continue
for name, series, yrl in (("raw_long", raw, yr_raw), ("hedged", hed, yr_hed)):
sr, t = ann(series, pf)
hit = 100.0 * np.mean(np.asarray(series) > 0)
py = {}
for r, y in zip(series, yrl):
py.setdefault(y, []).append(r)
pystr = " ".join(f"{y}:{(np.mean(v)/(np.std(v)+1e-12)):+.2f}"
for y, v in sorted(py.items()) if len(v) >= 10)
tag = f"{H}h" if name == "raw_long" else ""
print(f"{tag:>5} {len(series):>8} {name:>8} {sr:>+8.2f} {t:>+7.2f} {hit:>6.1f} {pystr}")
print("-" * len(hdr))
print("PASS: hedged net SR > 0.5, t>=2, positive in most years. Else cascade reversion closed.")
if __name__ == "__main__":
run()

129
scripts/surfer/crypto_mft_xsec.py Normal file
View File

@@ -0,0 +1,129 @@
#!/usr/bin/env python3
"""MFT crypto pulse — cross-sectional momentum at HOUR holds, net of turnover cost, hardened-lite.
The proven crypto edge is DAILY cross-sectional momentum (Sharpe ~1.4). micro_gate showed MINUTE-horizon
(per-coin) is dead. This tests the gap: does cross-sectional momentum work at HOUR holds (1h..48h)? If a
pulse survives 5bp/leg turnover cost with stable per-year sign, it justifies a full multi-year hardened
build; if not, intraday/MFT is closed and the edge is purely daily.
Hourly Binance klines (free, timestamped) for liquid perps → aligned panel. Pre-registered: lookback=hold=H,
weights = dollar-neutral cross-sectional z-score of trailing-H return (gross 1), rebalance every H hours
(NON-OVERLAPPING), forward-H return, cost = turnover · 5bp. Report gross/net annualized Sharpe + t + per-year.
"""
import json
import math
import os
import time
import urllib.request
import numpy as np
OUT = "data/surfer/crypto1h"
COST_BP = 5.0
HOLDS = [1, 2, 4, 8, 12, 24, 48]
SYMS = ["BTCUSDT", "ETHUSDT", "SOLUSDT", "XRPUSDT", "BNBUSDT", "DOGEUSDT", "ADAUSDT", "AVAXUSDT",
"LINKUSDT", "LTCUSDT", "DOTUSDT", "TRXUSDT", "BCHUSDT", "ETCUSDT", "FILUSDT", "ATOMUSDT"]
HOUR_MS = 3_600_000
def _get(u):
return json.load(urllib.request.urlopen(urllib.request.Request(u, headers={"User-Agent": "curl/8"}), timeout=30))
def fetch(sym):
cache = f"{OUT}/{sym}.npz"
if os.path.exists(cache):
d = np.load(cache); return d["ts"], d["close"]
end = int(time.time() * 1000); start = end - 5 * 365 * 86_400_000 # up to ~5y
rows, cur = [], start
for _ in range(400):
k = _get(f"https://fapi.binance.com/fapi/v1/klines?symbol={sym}&interval=1h&startTime={cur}&limit=1500")
if not k:
break
rows += k
cur = k[-1][0] + 1
if len(k) < 1500 or cur >= end:
break
time.sleep(0.05)
if not rows:
return np.array([]), np.array([])
d = {int(r[0]): float(r[4]) for r in rows}
ts = np.array(sorted(d)); close = np.array([d[t] for t in ts])
os.makedirs(OUT, exist_ok=True)
np.savez(cache, ts=ts, close=close)
return ts, close
def build_panel():
series = {}
for s in SYMS:
ts, c = fetch(s)
if len(ts) > 24 * 60: # need >~2 months
series[s] = (ts, c)
# align on the union of hourly timestamps (floored to the hour)
allts = sorted(set().union(*[set((ts // HOUR_MS).tolist()) for ts, _ in series.values()]))
tindex = {t: i for i, t in enumerate(allts)}
syms = sorted(series)
P = np.full((len(allts), len(syms)), np.nan)
for j, s in enumerate(syms):
ts, c = series[s]
for t, px in zip(ts // HOUR_MS, c):
P[tindex[int(t)], j] = px
yrs = np.array([1970 + int(t) // (365.25 * 24) for t in allts])
return P, syms, yrs
def ann_sharpe(x):
x = x[np.isfinite(x)]
if len(x) < 20 or x.std() == 0:
return float("nan"), float("nan"), 0
return float(x.mean() / x.std()), x.mean() / (x.std() / math.sqrt(len(x))), len(x)
def run():
P, syms, yrs = build_panel()
T, N = P.shape
logP = np.log(P)
print(f"\n========== MFT CRYPTO CROSS-SECTIONAL MOMENTUM (hour holds, net {COST_BP}bp/turnover) ==========")
print(f"panel: {T} hours × {N} coins ({syms[0]}..{syms[-1]}) years {int(yrs.min())}-{int(yrs.max())}")
print("dollar-neutral xs z-score momentum, lookback=hold=H, non-overlapping rebalance, cost=turnover·5bp\n")
print(f"{'hold':>6} {'periods':>8} {'gross_SR':>9} {'net_SR':>8} {'t(net)':>7} {'pos%':>6} {'per-year net-SR (chrono)':>26}")
print("-" * 92)
for H in HOLDS:
idx = np.arange(0, T - H, H) # non-overlapping rebalance points
rets, yr_of = [], []
w_prev = np.zeros(N)
per_year = {}
for t in idx:
past = logP[t] - logP[t - H] if t - H >= 0 else np.full(N, np.nan)
fwd = logP[t + H] - logP[t]
ok = np.isfinite(past) & np.isfinite(fwd)
if ok.sum() < 6:
continue
z = np.zeros(N)
pv = past[ok]; zz = (pv - pv.mean()) / (pv.std() + 1e-12)
z[ok] = zz
g = np.abs(z).sum()
w = z / g if g > 0 else z # dollar-neutral, gross 1
turn = np.abs(w - w_prev).sum()
r = float(np.nansum(w * fwd)) - turn * COST_BP / 1e4
rets.append(r); yr_of.append(int(yrs[t]))
per_year.setdefault(int(yrs[t]), []).append(r)
w_prev = w
rets = np.array(rets)
if len(rets) < 20:
print(f"{H:>6} (too few periods)"); continue
gross_sr, _, _ = ann_sharpe(rets + 0) # gross approximated below
# recompute gross (no cost) quickly
per_yr_str = " ".join(f"{y}:{(np.mean(v)/ (np.std(v)+1e-12)):+.2f}" for y, v in sorted(per_year.items()) if len(v) >= 10)
per_period = math.sqrt(24 * 365 / H) # annualization factor for H-hour periods
nsr, nt, n = ann_sharpe(rets)
# gross series
# (recompute gross by adding back mean turnover cost is approximate; instead report net only + per-year)
print(f"{H:>6}h {len(rets):>8} {'':>9} {nsr*per_period:>+8.2f} {nt:>+7.2f} {100*np.mean(rets>0):>6.1f} {per_yr_str}")
print("-" * 92)
print("PASS: net annualized SR > 0 with t≥2 AND positive in most years. Else MFT-crypto closed → daily only.")
if __name__ == "__main__":
run()

119
scripts/surfer/crypto_stablecoin_dislocation.py Normal file
View File

@@ -0,0 +1,119 @@
#!/usr/bin/env python3
"""Stablecoin peg-reversion existence test — the cleanest crypto-FX dislocation.
Stablecoins are supposed to be worth $1; mint/redeem arbitrage (forced flow) pulls a deviation back
to parity. This tests whether that reversion is harvestable NET OF COST on free Binance spot data, and
— critically — how bad the DEATH-SPIRAL tail is (some "depegs" never revert: UST->0, BUSD wind-down).
Pre-registered: pair STABLE/USDT, deviation d=close-1. Reversion bet when |d|>thresh: position =
-sign(d) (rich->short, cheap->long), hold H days, pnl_net = -sign(d)*(close[t+H]/close[t]-1) - cost_rt.
Report per-pair + pooled: trade count, mean net pnl (bp), hit%, WORST trade (bp, the death-spiral tail),
and a daily-strategy Sharpe. Cost levels bracket Binance stablecoin fees {0,2,10}bp round-trip.
KILL: pooled net mean<=0 OR edge entirely from one terminal name OR worst-trade tail dwarfs mean edge.
LIMITATION: overlapping forward windows (first-look existence test, not a hardened backtest); daily
granularity (intraday depeg spikes under-sampled). Flagged, not hidden.
"""
import json
import math
import os
import time
import urllib.request
import numpy as np
OUT = "data/surfer/stablecoin"
PAIRS = ["USDCUSDT", "FDUSDUSDT", "TUSDUSDT", "DAIUSDT", "BUSDUSDT", "USDPUSDT"]
DAY_MS = 86_400_000
THRESHOLDS_BP = [10, 25, 50]
HOLDS = [1, 2, 5]
COSTS_BP = [0.0, 2.0, 10.0]
def _get(u):
return json.load(urllib.request.urlopen(urllib.request.Request(u, headers={"User-Agent": "curl/8"}), timeout=30))
def fetch(sym):
cache = f"{OUT}/{sym}.npz"
if os.path.exists(cache):
d = np.load(cache); return d["ts"], d["close"]
start = 1_546_300_800_000 # 2019-01-01
end = int(time.time() * 1000)
rows, cur = [], start
for _ in range(400):
k = _get(f"https://api.binance.com/api/v3/klines?symbol={sym}&interval=1d&startTime={cur}&limit=1000")
if not k:
break
rows += k
cur = k[-1][0] + DAY_MS
if len(k) < 1000 or cur >= end:
break
time.sleep(0.05)
if not rows:
return np.array([]), np.array([])
d = {int(r[0]): float(r[4]) for r in rows}
ts = np.array(sorted(d)); close = np.array([d[t] for t in ts])
os.makedirs(OUT, exist_ok=True)
np.savez(cache, ts=ts, close=close)
return ts, close
def reversion(close, thresh_bp, H, cost_bp):
"""Event reversion bet: enter when |d|>thresh, pnl=-sign(d)*(fwd_ret)-cost_rt. Returns net-pnl array (bp)."""
d = close - 1.0
th = thresh_bp / 1e4
pnls = []
for t in range(len(close) - H):
if abs(d[t]) > th and close[t] > 0:
fwd = close[t + H] / close[t] - 1.0
pnl = -np.sign(d[t]) * fwd - cost_bp / 1e4
pnls.append(pnl * 1e4) # in bp
return np.array(pnls)
def run():
series = {}
for s in PAIRS:
ts, c = fetch(s)
if len(ts) > 60:
series[s] = c
print("\n========== STABLECOIN PEG-REVERSION (crypto-FX dislocation, free Binance spot daily) ==========")
print(f"pairs: {', '.join(f'{s}({len(c)}d)' for s, c in series.items())}\n")
# 1) dislocation frequency per pair
print("--- dislocation magnitude (|close-1|) per pair ---")
print(f"{'pair':>9} {'days':>5} {'med_bp':>7} {'p95_bp':>7} {'max_bp':>8} {'>10bp%':>7} {'>50bp%':>7}")
for s, c in series.items():
d = np.abs(c - 1.0) * 1e4
print(f"{s:>9} {len(c):>5} {np.median(d):>7.1f} {np.percentile(d,95):>7.1f} {d.max():>8.0f} "
f"{100*np.mean(d>10):>6.1f}% {100*np.mean(d>50):>6.1f}%")
# 2) reversion edge, pooled across pairs, by (thresh, H, cost)
print("\n--- reversion edge (pooled all pairs); pnl in bp/trade, net of round-trip cost ---")
print(f"{'thr_bp':>6} {'H':>2} {'cost':>5} {'trades':>7} {'mean_bp':>8} {'hit%':>6} {'worst_bp':>9} {'daily_SR':>9}")
for th in THRESHOLDS_BP:
for H in HOLDS:
for cost in COSTS_BP:
allp = np.concatenate([reversion(c, th, H, cost) for c in series.values()]) if series else np.array([])
if len(allp) < 10:
continue
sr = (allp.mean() / allp.std() * math.sqrt(365 / H)) if allp.std() > 0 else float("nan")
print(f"{th:>6} {H:>2} {cost:>4.0f}b {len(allp):>7} {allp.mean():>+8.1f} "
f"{100*np.mean(allp>0):>5.1f}% {allp.min():>+9.0f} {sr:>+9.2f}")
print(" " + "-" * 60)
# 3) per-pair edge at a fixed mid setting (thr=25bp, H=2, cost=2bp) — is it one terminal name?
print("\n--- per-pair edge @ thr=25bp H=2 cost=2bp (is the edge concentrated/terminal?) ---")
print(f"{'pair':>9} {'trades':>7} {'mean_bp':>8} {'hit%':>6} {'worst_bp':>9}")
for s, c in series.items():
p = reversion(c, 25, 2, 2.0)
if len(p) >= 5:
print(f"{s:>9} {len(p):>7} {p.mean():>+8.1f} {100*np.mean(p>0):>5.1f}% {p.min():>+9.0f}")
else:
print(f"{s:>9} {len(p):>7} (too few events)")
print("\nKILL if: pooled mean<=0, OR edge is one terminal name, OR |worst_bp| dwarfs mean (death-spiral tail).")
if __name__ == "__main__":
run()

145
scripts/surfer/crypto_stablecoin_harden.py Normal file
View File

@@ -0,0 +1,145 @@
#!/usr/bin/env python3
"""Stablecoin peg-reversion — HARDENED. Make-or-break test of the crypto-FX dislocation edge.
The spike (crypto_stablecoin_dislocation.py) found a real reversion edge (SR 3-4) but was blind to:
(1) survivorship — all sampled stables reverted; the true tail is buying a cheap stable that goes to
ZERO (UST -75%->delist). (2) overlap inflation. (3) stress slippage. (4) long/short asymmetry.
This hardens all four:
* UST included (the real death-spiral). FRAX excluded (garbage prints). USTC excluded (post-death).
* NON-OVERLAPPING event walk with explicit exit (revert / max_hold / stop-loss).
* deviation-SCALED slippage: cost_rt = base_bp + slip_k*|deviation| (big depeg = thin book).
* LONG-cheap (unbounded -100% tail) vs SHORT-rich (bounded) decomposed separately.
* collapse filters compared: raw / stop-loss / skip-falling-knife(+stop).
* per-year sign + BTC-stress correlation proxy (does it lose when crypto crashes?).
KILL: long-side net<=0 after UST+filters, OR edge only pre-cost, OR loses concentrated in BTC crashes.
"""
import math
import os
import numpy as np
OUT = "data/surfer/stablecoin"
HEALTHY = ["USDCUSDT", "FDUSDUSDT", "TUSDUSDT", "BUSDUSDT", "USDPUSDT", "SUSDUSDT", "DAIUSDT"]
TERMINAL = ["USTUSDT"]
DAY_MS = 86_400_000
THRESH_BP = 50
EXIT_BP = 10
MAX_HOLD = 5
BASE_BP = 2.0
SLIP_K = 0.15 # slippage = 15% of the entry deviation (stress thinness)
STOP = 0.03 # 3% stop-loss
def fetch(sym):
import json, time, urllib.request
cache = f"{OUT}/{sym}.npz"
if os.path.exists(cache):
d = np.load(cache); return d["ts"], d["close"]
g = lambda u: json.load(urllib.request.urlopen(urllib.request.Request(u, headers={"User-Agent": "curl/8"}), timeout=30))
start, end = 1_546_300_800_000, int(time.time() * 1000)
rows, cur = [], start
for _ in range(400):
k = g(f"https://api.binance.com/api/v3/klines?symbol={sym}&interval=1d&startTime={cur}&limit=1000")
if not k:
break
rows += k; cur = k[-1][0] + DAY_MS
if len(k) < 1000 or cur >= end:
break
time.sleep(0.05)
if not rows:
return np.array([]), np.array([])
d = {int(r[0]): float(r[4]) for r in rows}
ts = np.array(sorted(d)); close = np.array([d[t] for t in ts])
os.makedirs(OUT, exist_ok=True); np.savez(cache, ts=ts, close=close)
return ts, close
def walk(ts, close, mode):
"""Non-overlapping reversion trades. mode: 'raw'|'stop'|'skip'. Returns list of (dir, year, net_bp, entry_dev_bp)."""
n = len(close); d = close - 1.0; th = THRESH_BP / 1e4; ex = EXIT_BP / 1e4
trades = []; i = 1
while i < n - 1:
if abs(d[i]) <= th:
i += 1; continue
direction = -1.0 if d[i] > 0 else 1.0 # rich->short(-1), cheap->long(+1)
# skip-falling-knife: only buy a cheap stable if it is NOT still dropping (bounce confirmation)
if mode == "skip" and direction > 0 and close[i] < close[i - 1]:
i += 1; continue
entry = close[i]; j = i + 1; stop = STOP if mode in ("stop", "skip") else 9.9
while j < n and (j - i) <= MAX_HOLD:
pnl = direction * (close[j] / entry - 1.0)
if pnl <= -stop or abs(close[j] - 1.0) < ex:
break
j += 1
j = min(j, n - 1)
gross = direction * (close[j] / entry - 1.0)
slip = (BASE_BP + SLIP_K * abs(d[i]) * 1e4) / 1e4
yr = 1970 + int(ts[i]) // (365 * DAY_MS)
trades.append((direction, yr, (gross - slip) * 1e4, abs(d[i]) * 1e4))
i = j + 1
return trades
def stats(pnls):
p = np.array(pnls)
if len(p) < 3:
return f"n={len(p)} (too few)"
sr = p.mean() / p.std() * math.sqrt(365 / ((MAX_HOLD + 1) / 2)) if p.std() > 0 else float("nan")
return f"n={len(p):>4} mean={p.mean():>+7.1f}bp hit={100*np.mean(p>0):>4.1f}% worst={p.min():>+7.0f}bp SR~{sr:>+5.2f}"
def run():
data = {}
for s in HEALTHY + TERMINAL:
ts, c = fetch(s)
if len(ts) > 30:
data[s] = (ts, c)
print("\n===== STABLECOIN PEG-REVERSION — HARDENED (non-overlap, UST-in, slip-scaled, long/short split) =====")
print(f"pairs: {', '.join(data)} thr={THRESH_BP}bp hold<={MAX_HOLD}d exit<{EXIT_BP}bp slip={BASE_BP}+{SLIP_K}*|dev| stop={STOP*100:.0f}%\n")
for mode in ("raw", "stop", "skip"):
allt = []
for s in data:
allt += [(s, *t) for t in walk(*data[s], mode)]
longs = [t[3] for t in allt if t[1] > 0]
shorts = [t[3] for t in allt if t[1] < 0]
ust = [t[3] for t in allt if t[0] == "USTUSDT"]
print(f"[{mode:>4}] ALL : {stats([t[3] for t in allt])}")
print(f" LONG(cheap, -100% tail): {stats(longs)}")
print(f" SHORT(rich, bounded) : {stats(shorts)}")
print(f" UST trades only : {stats(ust)}")
print()
# per-year (SHORT-only, skip mode = the candidate deployable core)
print("--- per-year SHORT-rich net mean bp (the bounded-risk core), skip mode ---")
by_yr = {}
for s in data:
for t in walk(*data[s], "skip"):
if t[0] < 0:
by_yr.setdefault(t[1], []).append(t[2])
print(" ".join(f"{y}:{np.mean(v):+.0f}({len(v)})" for y, v in sorted(by_yr.items()) if v))
# BTC-stress proxy: does the LONG side lose during BTC crashes?
bts, btc = fetch("BTCUSDT")
if len(btc) > 100:
bret = {int(bts[k]) // DAY_MS: (btc[k] / btc[k - 1] - 1.0) for k in range(1, len(btc))}
crash, calm = [], []
for s in data:
ts, c = data[s]
for dirn, yr, net, dev in walk(ts, c, "skip"):
pass
# simpler: tag each long trade entry day with BTC 1d return
for s in data:
ts, c = data[s]; d = c - 1.0; th = THRESH_BP / 1e4
for i in range(1, len(c) - 1):
if abs(d[i]) > th and d[i] < 0: # long-cheap entries
br = bret.get(int(ts[i]) // DAY_MS, 0.0)
(crash if br < -0.05 else calm).append(1)
print(f"\nLONG-cheap entries on BTC-crash days (>-5%): {len(crash)} vs calm: {len(calm)} "
f"({100*len(crash)/max(1,len(crash)+len(calm)):.0f}% in crashes = tail-correlation flag)")
print("\nREAD: deployable core = SHORT-rich (bounded). LONG-cheap viable only if filters tame the UST tail.")
if __name__ == "__main__":
run()

120
scripts/surfer/crypto_stablecoin_intraday.py Normal file
View File

@@ -0,0 +1,120 @@
#!/usr/bin/env python3
"""Stablecoin peg-reversion — INTRADAY EXECUTION validation (the make-or-break-on-fills test).
Daily test found short-rich reversion ~+25bp/SR4. This checks it survives REALISTIC intraday execution:
* 1h bars, REALISTIC fills: enter at NEXT bar's OPEN (reaction lag — you can't transact at the peak),
exit at the reverting bar's close; pay round-trip spread (calibrated from live book: ~1bp/side).
* deviation HALF-LIFE: of rich events at hour t, what % still rich at t+1/+2/+4/+8 (do you have time?).
* short-rich (validated, bounded) focus; long-cheap with skip-falling-knife for contrast.
Universe = live-tradeable liquid stables only (USDC/FDUSD/TUSD/USDP; DAI/BUSD books are empty).
KILL: edge gone after next-open fills + spread, OR deviations vanish within 1h (no reaction window).
"""
import math
import os
import numpy as np
OUT = "data/surfer/stable1h"
PAIRS = ["USDCUSDT", "FDUSDUSDT", "TUSDUSDT", "USDPUSDT"]
HOUR_MS = 3_600_000
THRESHES = [25, 50]
EXIT_BP = 10
MAX_HOLD_H = 48
SPREAD_BP = 1.0 # per side, from live book; round-trip = 2*SPREAD_BP
def fetch(sym):
import json, time, urllib.request
cache = f"{OUT}/{sym}.npz"
if os.path.exists(cache):
d = np.load(cache); return d["ts"], d["open"], d["close"]
g = lambda u: json.load(urllib.request.urlopen(urllib.request.Request(u, headers={"User-Agent": "curl/8"}), timeout=30))
start, end = int(time.time() * 1000) - 3 * 365 * 86_400_000, int(time.time() * 1000)
rows, cur = [], start
for _ in range(800):
k = g(f"https://api.binance.com/api/v3/klines?symbol={sym}&interval=1h&startTime={cur}&limit=1000")
if not k:
break
rows += k; cur = k[-1][0] + HOUR_MS
if len(k) < 1000 or cur >= end:
break
time.sleep(0.04)
if not rows:
return np.array([]), np.array([]), np.array([])
o = {int(r[0]): (float(r[1]), float(r[4])) for r in rows}
ts = np.array(sorted(o)); op = np.array([o[t][0] for t in ts]); cl = np.array([o[t][1] for t in ts])
os.makedirs(OUT, exist_ok=True); np.savez(cache, ts=ts, open=op, close=cl)
return ts, op, cl
def walk(op, cl, thresh, side, skip=False):
"""Non-overlapping. side='short'(rich) or 'long'(cheap). enter next-bar OPEN, exit revert close, pay spread."""
n = len(cl); d = cl - 1.0; th = thresh / 1e4; ex = EXIT_BP / 1e4; rt = 2 * SPREAD_BP / 1e4
out = []; i = 1
while i < n - 2:
rich = d[i] > th; cheap = d[i] < -th
take = (side == "short" and rich) or (side == "long" and cheap)
if not take:
i += 1; continue
if skip and side == "long" and cl[i] < cl[i - 1]: # don't catch a still-falling knife
i += 1; continue
direction = -1.0 if rich else 1.0
entry = op[i + 1] # realistic: fill at next bar open
j = i + 2
while j < n and (j - (i + 1)) <= MAX_HOLD_H:
if abs(cl[j] - 1.0) < ex:
break
j += 1
j = min(j, n - 1)
gross = direction * (cl[j] / entry - 1.0)
out.append((gross - rt) * 1e4)
i = j + 1
return np.array(out)
def half_life(cl, thresh):
d = cl - 1.0; th = thresh / 1e4
ev = np.where(np.abs(d) > th)[0]; ev = ev[ev < len(cl) - 9]
if len(ev) == 0:
return None
return {h: 100 * np.mean(np.abs(d[ev + h]) > th) for h in (1, 2, 4, 8)}
def st(p):
if len(p) < 3:
return f"n={len(p)} (few)"
sr = p.mean() / p.std() * math.sqrt(365 * 24 / (MAX_HOLD_H / 2)) if p.std() > 0 else float("nan")
return f"n={len(p):>4} mean={p.mean():>+7.1f}bp hit={100*np.mean(p>0):>4.1f}% worst={p.min():>+7.0f}bp SR~{sr:>+5.2f}"
def run():
data = {}
for s in PAIRS:
ts, op, cl = fetch(s)
if len(ts) > 500:
data[s] = (op, cl, ts)
print("\n===== STABLECOIN INTRADAY EXECUTION (1h, next-open fills, spread 2bp rt) =====")
print(f"pairs: {', '.join(f'{s}({len(v[1])}h)' for s,v in data.items())}\n")
print("--- deviation HALF-LIFE: % of events still beyond threshold after H hours (reaction window) ---")
for s, (op, cl, ts) in data.items():
for th in THRESHES:
hl = half_life(cl, th)
if hl:
print(f" {s:>9} thr{th}: t+1={hl[1]:.0f}% t+2={hl[2]:.0f}% t+4={hl[4]:.0f}% t+8={hl[8]:.0f}%")
print()
for th in THRESHES:
sh = np.concatenate([walk(d[0], d[1], th, "short") for d in data.values()]) if data else np.array([])
lo = np.concatenate([walk(d[0], d[1], th, "long", skip=True) for d in data.values()]) if data else np.array([])
print(f"thr={th}bp SHORT-rich : {st(sh)}")
print(f" LONG-cheap*: {st(lo)} (*skip-falling-knife)")
# per-pair short
for s, (op, cl, ts) in data.items():
print(f" {s:>9} short: {st(walk(op, cl, th, 'short'))}")
print()
print("READ: edge survives if SHORT-rich stays +ve net of next-open+spread AND half-life>~2h (time to act).")
if __name__ == "__main__":
run()

120
scripts/surfer/crypto_trend_sizing.py Normal file
View File

@@ -0,0 +1,120 @@
#!/usr/bin/env python3
"""Crypto TS-trend sizing — reconfirm the validated return engine + find the deployable CAGR.
The TSMOM crypto sleeve is VALIDATED (memory: Sharpe ~1.23, CAGR 75.7% @ 61% vol, corr~0 to funding).
The 76% CAGR is just 61%-vol sizing, not a free lunch. This re-runs the SAME pre-registered config
(LONG/FLAT, lookbacks 20/60/120, on crypto_pit daily close, liquidity floor on qvol) and sweeps a
vol-target overlay to answer the #2 question: at a SANE vol target, what is the deployable CAGR and
does the Sharpe hold (it should — vol-target is a rescale, but per-period sizing adds timing).
NOT a search: the 20/60/120 long/flat config is replicated verbatim from the validated harness.
"""
import glob
import math
import numpy as np
PIT = "data/surfer/crypto_pit"
LOOKBACKS = [20, 60, 120]
COST_BP = 5.0
ADV_FLOOR = 5e6 # $5M trailing quote-volume liquidity floor
VOL_TARGETS = [0.10, 0.15, 0.20]
LEV_CAP = 3.0
PPY = 365 # crypto trades every day
def build():
days = set()
raw = {}
for f in sorted(glob.glob(f"{PIT}/*.npz")):
d = np.load(f)
if len(d["day"]) < 130:
continue
sym = f.split("/")[-1][:-4]
raw[sym] = {int(dd): (c, q) for dd, c, q in zip(d["day"], d["close"], d["qvol"])}
days.update(int(x) for x in d["day"])
days = sorted(days)
di = {d: i for i, d in enumerate(days)}
syms = sorted(raw)
C = np.full((len(days), len(syms)), np.nan)
V = np.full((len(days), len(syms)), np.nan)
for j, s in enumerate(syms):
for dd, (c, q) in raw[s].items():
C[di[dd], j] = c; V[di[dd], j] = q
return np.array(days), syms, C, V
def maxdd(equity):
peak = np.maximum.accumulate(equity)
return float((equity / peak - 1).min())
def run():
days, syms, C, V = build()
T, N = C.shape
logC = np.log(C)
ret = np.full((T, N), np.nan)
ret[1:] = C[1:] / C[:-1] - 1.0
# long/flat signal = mean over lookbacks of 1[trailing-L log return > 0]
sig = np.zeros((T, N))
cnt = np.zeros((T, N))
for L in LOOKBACKS:
tr = np.full((T, N), np.nan)
tr[L:] = logC[L:] - logC[:-L]
on = np.where(np.isfinite(tr), (tr > 0).astype(float), np.nan)
m = np.isfinite(on)
sig[m] += on[m]; cnt[m] += 1
sig = np.where(cnt > 0, sig / np.maximum(cnt, 1), 0.0) # 0..1 long/flat conviction
print(f"\n===== CRYPTO TS-TREND SIZING (long/flat {LOOKBACKS}, ${ADV_FLOOR/1e6:.0f}M ADV floor, {COST_BP}bp/leg) =====")
print(f"panel: {T} days x {N} coins (survivorship-free incl. dead)\n")
# base book (gross 1, daily rebalance, equal-weight by conviction among liquid+on coins)
w_prev = np.zeros(N)
book = np.zeros(T)
for t in range(1, T):
eligible = np.isfinite(ret[t]) & np.isfinite(V[t - 1]) & (V[t - 1] > ADV_FLOOR)
s = sig[t - 1] * eligible
g = s.sum()
w = s / g if g > 0 else np.zeros(N)
turn = np.abs(w - w_prev).sum()
book[t] = float(np.nansum(w * ret[t])) - turn * COST_BP / 1e4
w_prev = w
valid = np.arange(T) > max(LOOKBACKS)
base = book[valid]
base_sr = base.mean() / base.std() * math.sqrt(PPY) if base.std() > 0 else float("nan")
base_vol = base.std() * math.sqrt(PPY)
base_cagr = float(np.prod(1 + base) ** (PPY / len(base)) - 1)
print(f"BASE (un-vol-targeted): Sharpe {base_sr:+.2f} vol {base_vol*100:.0f}% CAGR {base_cagr*100:+.1f}% maxDD {maxdd(np.cumprod(1+base))*100:.1f}%")
print(f" (memory reference: Sharpe ~1.23, CAGR ~76% @ ~61% vol — replicating)\n")
print(f"{'vol_tgt':>8} {'Sharpe':>7} {'CAGR':>7} {'real_vol':>9} {'maxDD':>7} {'avg_lev':>8} {'per-year SR':>30}")
print("-" * 86)
# vol-target overlay on the base book (trailing 30d realized, lagged, leverage-capped)
rv = np.full(T, np.nan)
for t in range(31, T):
w = book[t - 30:t]
rv[t] = w.std() * math.sqrt(PPY)
yrs = 1970 + days // 365
for vt in VOL_TARGETS:
lev = np.where(np.isfinite(rv) & (rv > 0), np.clip(vt / rv, 0, LEV_CAP), 0.0)
tgt = book * np.concatenate([[0], lev[:-1]]) # lag leverage by 1 day
s = tgt[valid]
sr = s.mean() / s.std() * math.sqrt(PPY) if s.std() > 0 else float("nan")
vol = s.std() * math.sqrt(PPY)
cagr = float(np.prod(1 + s) ** (PPY / len(s)) - 1)
dd = maxdd(np.cumprod(1 + s))
avglev = float(np.mean(lev[valid]))
yv = {}
for r, y in zip(s, yrs[valid]):
yv.setdefault(int(y), []).append(r)
ystr = " ".join(f"{y}:{(np.mean(v)/(np.std(v)+1e-12)*math.sqrt(PPY)):+.1f}"
for y, v in sorted(yv.items()) if len(v) >= 30)
print(f"{vt*100:>6.0f}% {sr:>+7.2f} {cagr*100:>+6.1f}% {vol*100:>8.0f}% {dd*100:>+6.1f}% {avglev:>8.2f} {ystr}")
print("-" * 86)
print("Deployable read: pick the vol target whose maxDD you can stomach; Sharpe should ~hold across targets.")
if __name__ == "__main__":
run()

163
scripts/surfer/mft_4h_harden.py Normal file
View File

@@ -0,0 +1,163 @@
#!/usr/bin/env python3
"""Harden the 4h MFT trend edge — kill the inflation sources before trusting it.
The mft_horizon_sweep found a 4h selective trend edge (OOS net +5.9t, t+7.4). But that t is
OVERSTATED: overlapping 4h holds across 5-min entries are not independent. This re-tests honestly:
1. ONE TRADE PER SESSION — the single highest-conviction morning setup, ~4h hold, flat-by-close.
Non-overlapping ⇒ trades are ~independent across sessions ⇒ honest t-stat. Also the realistic strat.
2. IS-DERIVED conviction cutoff — set on the in-sample sessions, applied to OOS (no pooled-threshold leak).
3. REAL RTH session — ET 09:30–16:00 (not UTC-midnight); entry window = first 2.5h so 4h fits before close.
4. PER-YEAR breakdown — is the edge in every year, or one regime?
5. Cost robustness (1 and 2 ticks) + multiple-testing context.
Pre-registered: dir=sign(ret over H bars), conviction=|t-stat|=|ret_H|/(vol·√H), vol=trailing 12-bar.
Entry = highest-conviction valid bar in 09:30-12:00, exit = +H bars (same session), cost 1 (and 2) ticks.
Trade the session only if its best conviction ≥ IS cutoff (selectivity sweep). H=48 (4h) primary; 36/60 robustness.
"""
import glob
import math
import numpy as np
import pandas as pd
TICK = 0.25
BAR_NS = 5 * 60 * 10**9
HOLDS = [36, 48, 60] # 3h, 4h (primary), 5h
ENTRY_CUTOFF_MIN = 12 * 60 # latest entry 12:00 ET so a 4h hold fits before 16:00
RTH_LO, RTH_HI = 9 * 60 + 30, 16 * 60
def load_es_5min():
import databento as db
ts_all, c_all = [], []
for p in sorted(glob.glob("data/surfer/es1m/*.dbn")):
try:
arr = db.DBNStore.from_file(p).to_ndarray()
except Exception:
continue
if len(arr) == 0 or "close" not in arr.dtype.names:
continue
ts = arr["ts_event"].astype(np.int64)
c = arr["close"].astype(np.float64) / 1e9
m = c > 0
ts_all.append(ts[m]); c_all.append(c[m])
ts = np.concatenate(ts_all); c = np.concatenate(c_all)
_u, ui = np.unique(ts, return_index=True); ts, c = ts[ui], c[ui]
o = np.argsort(ts, kind="stable"); ts, c = ts[o], c[o]
b5 = ts // BAR_NS
_, first = np.unique(b5, return_index=True)
last = np.append(first[1:] - 1, len(b5) - 1)
return ts[last], c[last]
def rth_sessions(ts5, close5):
"""Restrict to ET RTH 09:30-16:00, return (close, session_id, minute_of_day_ET, year) RTH-only, sorted."""
idx = pd.to_datetime(ts5, utc=True).tz_convert("America/New_York")
mins = (idx.hour * 60 + idx.minute).to_numpy()
rth = (mins >= RTH_LO) & (mins < RTH_HI)
et_date = idx.strftime("%Y-%m-%d").to_numpy()
year = idx.year.to_numpy()
close = close5[rth]; sess_str = et_date[rth]; mod = mins[rth]; yr = year[rth]
sid = pd.factorize(sess_str)[0] # contiguous session ids (data already time-sorted)
return close, sid, mod, yr
def per_session_trades(close, sid, mod, yr, H, cost):
"""For each RTH session pick the single highest-conviction entry in 09:30-12:00, hold H bars to a
same-session exit. Returns arrays over sessions that HAVE a valid candidate:
(best_conv, net_ticks, year, sess_order)."""
lc = np.log(close)
n = len(lc)
# trailing per-bar vol (12-bar) and H-bar trend, only valid within the same session
r1 = np.zeros(n); r1[1:] = lc[1:] - lc[:-1]
same1 = np.zeros(n, bool); same1[1:] = sid[1:] == sid[:-1]
r1 = np.where(same1, r1, 0.0)
# rolling 12-bar std via cumsum
w = 12
cs = np.concatenate([[0.0], np.cumsum(r1)]); cs2 = np.concatenate([[0.0], np.cumsum(r1 * r1)])
vol = np.full(n, np.nan)
s = cs[w:] - cs[:-w]; s2 = cs2[w:] - cs2[:-w]
vol[w - 1:] = np.sqrt(np.maximum(s2 / w - (s / w) ** 2, 0.0))
# trailing H-bar trend over the RTH-contiguous series (overnight gap collapsed — conservative:
# the overnight move, where much drift lives, is excluded from the signal). No same-session
# requirement on the lookback (a 4h trailing trend can't fit in the morning entry window otherwise).
retH = np.full(n, np.nan); retH[H:] = lc[H:] - lc[:-H]
tstat = retH / np.maximum(vol * math.sqrt(H), 1e-9)
# forward H-bar move (ticks), exit MUST be same session (flat-by-close)
fwd = np.full(n, np.nan); fwd[:-H] = (close[H:] - close[:-H]) / TICK
sameH_fwd = np.zeros(n, bool); sameH_fwd[:-H] = sid[:-H] == sid[H:]
entry_ok = (mod <= ENTRY_CUTOFF_MIN) & sameH_fwd & np.isfinite(tstat) & np.isfinite(fwd) & (vol > 0)
direction = np.sign(retH)
net = direction * fwd - cost
conv = np.abs(tstat)
out_conv, out_net, out_yr, out_ord = [], [], [], []
# group by session, pick max-conviction valid entry
order = np.argsort(sid, kind="stable")
sid_s = sid[order]
bounds = np.append(np.append(0, np.nonzero(np.diff(sid_s))[0] + 1), len(sid_s))
for b in range(len(bounds) - 1):
rows = order[bounds[b]:bounds[b + 1]]
rows = rows[entry_ok[rows] & (direction[rows] != 0)]
if len(rows) == 0:
continue
best = rows[np.argmax(conv[rows])]
out_conv.append(conv[best]); out_net.append(net[best])
out_yr.append(yr[best]); out_ord.append(sid[best])
return (np.array(out_conv), np.array(out_net), np.array(out_yr), np.array(out_ord))
def stats(net):
n = len(net)
if n < 20:
return n, float("nan"), float("nan"), float("nan"), float("nan")
m = float(net.mean()); sd = float(net.std()) + 1e-12
t = m / (sd / math.sqrt(n))
hit = 100.0 * float((net > 0).mean())
ann_sharpe = (m / sd) * math.sqrt(252.0) # ~1 trade/session ⇒ ~252 trades/yr
return n, m, hit, t, ann_sharpe
def run():
ts5, close5 = load_es_5min()
close, sid, mod, yr = rth_sessions(ts5, close5)
n_sess = int(sid.max()) + 1
print("\n================ 4h MFT HARDENING (RTH, 1 trade/session, IS-cutoff, per-year) ================")
print(f"RTH 5-min bars={len(close)} sessions={n_sess} years={int(yr.min())}-{int(yr.max())}")
print("honest non-overlapping t-stat (1 trade/session); multiple-testing bar ≈ |t|>2.3 for ~10 configs\n")
for cost in (1.0, 2.0):
print(f"================= COST = {cost:.0f} tick round-trip =================")
for H in HOLDS:
conv, net, tyr, sord = per_session_trades(close, sid, mod, yr, H, cost)
if len(conv) < 50:
print(f" H={H}: too few sessions with a candidate ({len(conv)})"); continue
split = np.quantile(sord, 0.70) # first 70% of sessions = IS
is_m = sord < split; oos_m = ~is_m
hh = H * 5 / 60.0
print(f" --- H={H} ({hh:.0f}h hold) --- (IS sessions {int(is_m.sum())} / OOS {int(oos_m.sum())})")
print(f" {'select':>8} {'cutoff':>7} {'n_OOS':>6} {'net_t':>7} {'hit%':>6} {'t':>6} {'annSR':>6} {'trd/yr':>7}")
for sel in (1.00, 0.50, 0.25, 0.10): # trade top-sel% of IS-conviction days
cutoff = np.quantile(conv[is_m], 1.0 - sel) if sel < 1.0 else -np.inf
take = oos_m & (conv >= cutoff)
n, m, hit, t, sr = stats(net[take])
oos_years = (sord[oos_m].max() - split) / n_sess * (int(yr.max()) - int(yr.min()) + 1)
trd_yr = n / max(oos_years, 1e-9)
cuts = f"{cutoff:.2f}" if np.isfinite(cutoff) else "all"
print(f" {sel:>8.2f} {cuts:>7} {n:>6} {m:>+7.2f} {hit:>6.1f} {t:>+6.2f} {sr:>+6.2f} {trd_yr:>7.1f}")
# per-year at the pre-registered selectivity = top 25% of IS-conviction days
cutoff = np.quantile(conv[is_m], 0.75)
print(f" per-year (sel=0.25, cost={cost:.0f}, OOS+IS shown; IS years are not OOS):")
for y in range(int(yr.min()), int(yr.max()) + 1):
ym = (tyr == y) & (conv >= cutoff)
if int(ym.sum()) < 10:
continue
n, m, hit, t, sr = stats(net[ym])
flag = "OOS" if (np.median(sord[tyr == y]) >= split) else "is "
print(f" {y} [{flag}] n={n:>4} net={m:>+6.2f}t hit={hit:>5.1f}% t={t:>+5.2f}")
print()
if __name__ == "__main__":
run()

114
scripts/surfer/mft_harden_v2.py Normal file
View File

@@ -0,0 +1,114 @@
#!/usr/bin/env python3
"""Apples-to-apples hardening of the sweep's OWN 4h edge — same 24h-continuous signal, but kill the
statistical inflation: ONE non-overlapping trade per UTC-day, IS-derived conviction cutoff, per-year.
The sweep (mft_horizon_sweep) used a 24h-continuous 5-min grid, day = UTC-day, hold = 48 bars = 4h of
clock time, top-decile conviction over HEAVILY OVERLAPPING entries → t+7.4 (inflated by overlap). Here:
same grid + signal, but max-conviction single entry per UTC-day (non-overlapping ⇒ honest t), threshold
fit on IS, reported per-year. cost 1 and 2 ticks. PASS: OOS net>0 & hit>50% & honest t≥2, stable per-year.
"""
import glob
import math
import numpy as np
TICK = 0.25
BAR_NS = 5 * 60 * 10**9
DAY_NS = 86_400 * 10**9
HOLDS = [48] # 4h (the sweep's standout); same grid as the sweep
def load_es_5min():
import databento as db
ts_all, c_all = [], []
for p in sorted(glob.glob("data/surfer/es1m/*.dbn")):
try:
arr = db.DBNStore.from_file(p).to_ndarray()
except Exception:
continue
if len(arr) == 0 or "close" not in arr.dtype.names:
continue
ts = arr["ts_event"].astype(np.int64); c = arr["close"].astype(np.float64) / 1e9
m = c > 0; ts_all.append(ts[m]); c_all.append(c[m])
ts = np.concatenate(ts_all); c = np.concatenate(c_all)
_u, ui = np.unique(ts, return_index=True); ts, c = ts[ui], c[ui]
o = np.argsort(ts, kind="stable"); ts, c = ts[o], c[o]
b5 = ts // BAR_NS
_, first = np.unique(b5, return_index=True)
last = np.append(first[1:] - 1, len(b5) - 1)
return ts[last], c[last]
def per_day_trades(close, day, yr, H, cost):
lc = np.log(close); n = len(lc)
r1 = np.zeros(n); r1[1:] = lc[1:] - lc[:-1]
same1 = np.zeros(n, bool); same1[1:] = day[1:] == day[:-1]
r1 = np.where(same1, r1, 0.0)
w = 12
cs = np.concatenate([[0.0], np.cumsum(r1)]); cs2 = np.concatenate([[0.0], np.cumsum(r1 * r1)])
vol = np.full(n, np.nan); s = cs[w:] - cs[:-w]; s2 = cs2[w:] - cs2[:-w]
vol[w - 1:] = np.sqrt(np.maximum(s2 / w - (s / w) ** 2, 0.0))
retH = np.full(n, np.nan); retH[H:] = lc[H:] - lc[:-H]
sameH_back = np.zeros(n, bool); sameH_back[H:] = day[H:] == day[:-H] # like the sweep: sameday(H)
tstat = retH / np.maximum(vol * math.sqrt(H), 1e-9)
fwd = np.full(n, np.nan); fwd[:-H] = (close[H:] - close[:-H]) / TICK
sameH_fwd = np.zeros(n, bool); sameH_fwd[:-H] = day[:-H] == day[H:]
ok = sameH_fwd & sameH_back & np.isfinite(tstat) & np.isfinite(fwd) & (vol > 0)
direction = np.sign(retH); net = direction * fwd - cost; conv = np.abs(tstat)
oc, on, oy, od = [], [], [], []
order = np.argsort(day, kind="stable"); ds = day[order]
bounds = np.append(np.append(0, np.nonzero(np.diff(ds))[0] + 1), len(ds))
for b in range(len(bounds) - 1):
rows = order[bounds[b]:bounds[b + 1]]
rows = rows[ok[rows] & (direction[rows] != 0)]
if len(rows) == 0:
continue
best = rows[np.argmax(conv[rows])]
oc.append(conv[best]); on.append(net[best]); oy.append(yr[best]); od.append(day[best])
return np.array(oc), np.array(on), np.array(oy), np.array(od)
def stats(net):
n = len(net)
if n < 20:
return n, float("nan"), float("nan"), float("nan"), float("nan")
m = float(net.mean()); sd = float(net.std()) + 1e-12
return n, m, 100.0 * float((net > 0).mean()), m / (sd / math.sqrt(n)), (m / sd) * math.sqrt(252.0)
def run():
ts5, close5 = load_es_5min()
day = (ts5 // DAY_NS).astype(np.int64)
yr = (1970 + (ts5 // (365.25 * DAY_NS))).astype(int) # approx calendar year from epoch days
print("\n============ APPLES-TO-APPLES 4h HARDENING (24h grid, 1 trade/UTC-day, IS-cutoff) ============")
print(f"5-min bars={len(close5)} days={int(day.max()-day.min())} multiple-testing bar ≈ |t|>2.3\n")
for cost in (1.0, 2.0):
for H in HOLDS:
conv, net, tyr, dd = per_day_trades(close5, day, yr, H, cost)
if len(conv) < 50:
print(f"cost={cost:.0f} H={H}: too few ({len(conv)})"); continue
split = np.quantile(dd, 0.70); is_m = dd < split; oos_m = ~is_m
print(f"=== cost={cost:.0f} tick, H={H} (4h), 1 trade/day (IS days {int(is_m.sum())} / OOS {int(oos_m.sum())}) ===")
print(f" {'select':>7} {'cutoff':>7} {'n_OOS':>6} {'net_t':>7} {'hit%':>6} {'t':>6} {'annSR':>6} {'trd/yr':>7}")
for sel in (1.00, 0.50, 0.25, 0.10):
cut = np.quantile(conv[is_m], 1.0 - sel) if sel < 1.0 else -np.inf
take = oos_m & (conv >= cut)
n, m, hit, t, sr = stats(net[take])
oosyrs = max((int(tyr[oos_m].max()) - int(tyr[oos_m].min()) + 1), 1)
cs = f"{cut:.2f}" if np.isfinite(cut) else "all"
print(f" {sel:>7.2f} {cs:>7} {n:>6} {m:>+7.2f} {hit:>6.1f} {t:>+6.2f} {sr:>+6.2f} {n/oosyrs:>7.1f}")
cut = np.quantile(conv[is_m], 0.75)
print(f" per-year (sel=0.25; IS years are not OOS):")
for y in range(int(tyr.min()), int(tyr.max()) + 1):
ym = (tyr == y) & (conv >= cut)
if int(ym.sum()) < 10:
continue
n, m, hit, t, sr = stats(net[ym])
flag = "OOS" if np.median(dd[tyr == y]) >= split else "is "
print(f" {y} [{flag}] n={n:>4} net={m:>+6.2f}t hit={hit:>5.1f}% t={t:>+5.2f}")
print()
if __name__ == "__main__":
run()

154
scripts/surfer/mft_horizon_sweep.py Normal file
View File

@@ -0,0 +1,154 @@
#!/usr/bin/env python3
"""MFT horizon sweep — does a SLOWER (lower-turnover) intraday trend edge clear costs?
Prong A (mft_quality_test) showed: at 15-min holds the net ≈ -1 tick = the cost itself (gross ≈ 0).
This sweeps the hold horizon UP (15m → full session) on clean ES OHLCV-1m. As the hold grows the
forward move (in ticks) grows ~√time while the 1-tick cost stays fixed, so cost/gross shrinks — this
isolates the question the user posed: keep costs low by trading SLOWER. If gross edge grows with
horizon (trend persistence), there's an MFT sweet spot; if gross stays ~0 everywhere, the price signal
is dead regardless of speed.
Pre-registered (NOT tuned): trend-follow, lookback = hold = H bars (5-min each); direction = sign(ret_H);
conviction = |t-stat| = |ret_H| / (per-bar vol · √H); entry close[t], exit close[t+H], intraday only
(same-day, flat-by-close); cost = 1.0 tick round-trip (crossing). Quality curve by conviction decile,
IS(first 70%) / OOS(last 30%). PASS bar: OOS top-decile net > 0 ticks & hit > 50% & |t| ≥ 2, at a
horizon with low trades/day (genuinely MFT).
"""
import glob
import math
import numpy as np
import torch
DEV = "cuda" if torch.cuda.is_available() else None
TICK = 0.25
COST_TICKS = 1.0
BAR_NS = 5 * 60 * 10**9
DAY_NS = 86_400 * 10**9
HOLDS = [3, 12, 24, 48, 78] # 15m, 1h, 2h, 4h, ~full session (5-min bars)
def load_es_5min():
import databento as db
es1m = sorted(glob.glob("data/surfer/es1m/*.dbn"))
ts_all, c_all = [], []
if es1m:
for p in es1m:
try:
arr = db.DBNStore.from_file(p).to_ndarray()
except Exception:
continue
if len(arr) == 0 or "close" not in arr.dtype.names:
continue
ts = arr["ts_event"].astype(np.int64)
c = arr["close"].astype(np.float64) / 1e9
m = c > 0
ts_all.append(ts[m]); c_all.append(c[m])
else:
for p in sorted(glob.glob("test_data/futures-baseline/ES.FUT/*.dbn.zst")):
try:
df = db.DBNStore.from_file(p).to_df().reset_index()
except Exception:
continue
if df.empty or "close" not in df.columns:
continue
df = df[df["close"] > 0]
if df.empty:
continue
dom = df["instrument_id"].value_counts().idxmax()
df = df[df["instrument_id"] == dom].sort_values("ts_event")
ts_all.append(df["ts_event"].astype("int64").to_numpy())
c_all.append(df["close"].to_numpy(np.float64))
ts = np.concatenate(ts_all); c = np.concatenate(c_all)
_u, _ui = np.unique(ts, return_index=True)
ts, c = ts[_ui], c[_ui]
o = np.argsort(ts, kind="stable"); ts, c = ts[o], c[o]
b5 = ts // BAR_NS
_, first = np.unique(b5, return_index=True)
last = np.append(first[1:] - 1, len(b5) - 1)
return ts[last], c[last]
def rolling_std(x, w):
z = torch.zeros(1, dtype=x.dtype, device=x.device)
cs = torch.cat([z, torch.cumsum(x, 0)])
cs2 = torch.cat([z, torch.cumsum(x * x, 0)])
s = cs[w:] - cs[:-w]
s2 = cs2[w:] - cs2[:-w]
mean = s / w
var = (s2 / w - mean * mean).clamp_min(0.0)
out = torch.full_like(x, float("nan"))
out[w - 1:] = var.sqrt()
return out
def run():
ts5, close5 = load_es_5min()
day = ts5 // DAY_NS
n_days = int(day.max() - day.min())
lc = torch.tensor(np.log(close5), device=DEV, dtype=torch.float64)
px = torch.tensor(close5, device=DEV, dtype=torch.float64)
dy = torch.tensor(day, device=DEV)
T = len(lc)
t = torch.arange(T, device=DEV)
split = int(0.7 * T)
def sameday(lag):
m = torch.zeros(T, dtype=torch.bool, device=DEV)
m[lag:] = dy[lag:] == dy[:-lag]
return m
r1 = torch.zeros_like(lc); r1[1:] = lc[1:] - lc[:-1]
r1 = torch.where(sameday(1), r1, torch.zeros_like(r1))
vol1 = rolling_std(r1, 12) # per-bar 5-min vol (fixed 1h window, pre-registered)
print("\n================ MFT HORIZON SWEEP (GPU, intraday trend, flat-by-close) ================")
print(f"device={DEV} 5-min bars={T} span days≈{n_days} cost={COST_TICKS} tick round-trip")
print("trend-follow: dir=sign(ret_H), conv=|ret_H|/(vol·√H). PASS: OOS top-decile net>0 & hit>50% & |t|≥2\n")
print(f"{'hold':>6} {'top-q':>6} {'n_OOS':>7} {'gross_t':>8} {'net_t':>8} {'cost/gr':>8} {'hit%':>6} {'t-stat':>7} {'trd/day':>8}")
print("-" * 78)
for H in HOLDS:
retH = torch.full_like(lc, float("nan")); retH[H:] = lc[H:] - lc[:-H]
tstat = retH / (vol1 * math.sqrt(H)).clamp_min(1e-9)
fwd_ticks = torch.full_like(px, float("nan"))
fwd_ticks[:-H] = (px[H:] - px[:-H]) / TICK
fwd_valid = torch.zeros(T, dtype=torch.bool, device=DEV)
fwd_valid[:-H] = dy[H:] == dy[:-H] # exit same day (flat-by-close)
valid = fwd_valid & sameday(H) & torch.isfinite(tstat) & torch.isfinite(vol1) & (vol1 > 0)
direction = torch.sign(retH)
net = direction * fwd_ticks - COST_TICKS
gross = direction * fwd_ticks
cand = valid & (direction != 0)
idx = torch.nonzero(cand, as_tuple=True)[0]
if len(idx) < 100:
print(f"{H:>6} (too few candidates: {len(idx)})"); continue
conv = tstat.abs()[idx]; tt = t[idx]
order = torch.argsort(conv, descending=True)
hold_hours = H * 5 / 60.0
for q in [0.05, 0.10, 1.00]:
k = max(int(q * len(idx)), 1)
sel = idx[order[:k]]
oos = sel[tt[order[:k]] >= split]
no = int(len(oos))
if no < 20:
print(f"{H:>6} {q:>6.2f} (OOS n<20)"); continue
g = gross[oos]; nt = net[oos]
gm = float(g.mean()); nm = float(nt.mean())
tval = nm / (float(nt.std()) / math.sqrt(no) + 1e-12)
hit = 100.0 * float((nt > 0).float().mean())
cost_gr = (COST_TICKS / gm * 100.0) if gm > 0 else float("nan")
trd_day = no / (0.3 * n_days) # OOS is last 30% of days
tag = f"{H}({hold_hours:.1f}h)" if q == 0.05 else ""
print(f"{tag:>6} {q:>6.2f} {no:>7} {gm:>+8.3f} {nm:>+8.3f} {cost_gr:>7.0f}% {hit:>6.1f} {tval:>+7.2f} {trd_day:>8.2f}")
print("-" * 78)
def main():
if DEV is None:
raise SystemExit("CUDA not available")
run()
if __name__ == "__main__":
main()