From 1cf38232e80daa0e89a1c08e2fb628468ed5b752 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Sat, 6 Jun 2026 12:15:37 +0200 Subject: [PATCH] =?UTF-8?q?fix(surfer):=20full-16y=20MFT=20settle=20?= =?UTF-8?q?=E2=80=94=20regime-adaptive=20surfer=20significantly=20negative?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pulled 16y ES ohlcv-1m continuous (year-chunked, $20 credits, no 504) + light to_ndarray loader (to_df OOM'd at 20GB). On 1.1M 5-min bars the 1.3y +0.50/t=0.67 top-5% hint collapsed: regime-adaptive top-5% = -0.52 ticks/trade t=-2.51 (significantly negative); ALL cells/signals/convictions significantly negative. Decisive: no capturable intraday directional edge for crossing/non-colocated setup. Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/surfer/fetch_es_1m.py | 81 ++++++++++++++++++++++++++++++ scripts/surfer/mft_quality_test.py | 46 +++++++++++------ 2 files changed, 113 insertions(+), 14 deletions(-) create mode 100644 scripts/surfer/fetch_es_1m.py diff --git a/scripts/surfer/fetch_es_1m.py b/scripts/surfer/fetch_es_1m.py new file mode 100644 index 000000000..0f352cfa9 --- /dev/null +++ b/scripts/surfer/fetch_es_1m.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Fetch full-history ES front-month OHLCV-1m (continuous .c.0), year-chunked, BUDGET-CAPPED. + +get_cost-gated (aborts over cap, no download); year chunks with quarter fallback on 504. +Saves per-chunk DBN to data/surfer/es1m/ (gitignored). Key from env, never printed. +""" +import os +import sys +import time + +import databento as db + +CAP_USD = 25.00 +DS = "GLBX.MDP3" +SCHEMA = "ohlcv-1m" +SYM = "ES.c.0" +STYPE = "continuous" +OUT_DIR = "data/surfer/es1m" + + +def fetch(client, start, end, out): + data = client.timeseries.get_range(dataset=DS, symbols=[SYM], schema=SCHEMA, + start=start, end=end, stype_in=STYPE) + data.to_file(out) + return sum(1 for _ in data) + + +def main(): + key = os.environ.get("DATABENTO_API_KEY") + if not key: + print("DATABENTO_API_KEY not set"); return 2 + client = db.Historical(key) + cost = client.metadata.get_cost(dataset=DS, symbols=[SYM], schema=SCHEMA, + start="2010-06-06", end="2026-06-05", stype_in=STYPE) + print(f"aggregate get_cost=${cost:.4f} cap=${CAP_USD:.2f}") + if cost > CAP_USD: + print("ABORT: over cap — nothing downloaded."); return 1 + os.makedirs(OUT_DIR, exist_ok=True) + total = 0 + for year in range(2010, 2027): + y0, y1 = f"{year}-01-01", f"{year+1}-01-01" + if year == 2010: + y0 = "2010-06-06" + if year == 2026: + y1 = "2026-06-05" + out = f"{OUT_DIR}/ES_{year}.dbn" + if os.path.exists(out): + print(f" {year}: exists, skip"); continue + ok = False + for attempt in range(1, 3): + try: + n = fetch(client, y0, y1, out); total += n + print(f" {year}: {n:,} recs -> {out}") + ok = True; break + except Exception as e: + print(f" {year}: attempt {attempt} {type(e).__name__}; retry") + time.sleep(3) + if not ok: # fall back to quarter chunks + print(f" {year}: year failed → quarter fallback") + for q, (m0, m1) in enumerate([("01-01", "04-01"), ("04-01", "07-01"), + ("07-01", "10-01"), ("10-01", "12-31")], 1): + qs, qe = f"{year}-{m0}", f"{year}-{m1}" + if year == 2010 and q == 1: + qs = "2010-06-06" + if year == 2026 and q >= 3: + continue + qout = f"{OUT_DIR}/ES_{year}_q{q}.dbn" + if os.path.exists(qout): + continue + try: + n = fetch(client, qs, qe, qout); total += n + print(f" {year} q{q}: {n:,} -> {qout}") + except Exception as e: + print(f" {year} q{q}: FAILED {type(e).__name__}") + time.sleep(1) + print(f"DONE: {total:,} records into {OUT_DIR}/") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/surfer/mft_quality_test.py b/scripts/surfer/mft_quality_test.py index 00edd6e9c..1eaab7eb1 100644 --- a/scripts/surfer/mft_quality_test.py +++ b/scripts/surfer/mft_quality_test.py @@ -35,22 +35,40 @@ DAY_NS = 86_400 * 10**9 def load_es_5min(): import databento as db + # Prefer the full-history continuous front-month pull (data/surfer/es1m/), else fall back + # to the local ~2y parent OHLCV-1m (futures-baseline/ES, per-quarter front-month pick). + es1m = sorted(glob.glob("data/surfer/es1m/*.dbn")) ts_all, c_all = [], [] - for p in sorted(glob.glob("test_data/futures-baseline/ES.FUT/*.dbn.zst")): - try: - df = db.DBNStore.from_file(p).to_df().reset_index() - except Exception: - continue - if df.empty or "close" not in df.columns: - continue - df = df[df["close"] > 0] - if df.empty: - continue - dom = df["instrument_id"].value_counts().idxmax() # front month for the quarter - df = df[df["instrument_id"] == dom].sort_values("ts_event") - ts_all.append(df["ts_event"].astype("int64").to_numpy()) - c_all.append(df["close"].to_numpy(np.float64)) + if es1m: + for p in es1m: # to_ndarray = light (to_df OOMs at 20GB) + try: + arr = db.DBNStore.from_file(p).to_ndarray() + except Exception: + continue + if len(arr) == 0 or "close" not in arr.dtype.names: + continue + ts = arr["ts_event"].astype(np.int64) + c = arr["close"].astype(np.float64) / 1e9 # raw 1e9 fixed-point → price + m = c > 0 + ts_all.append(ts[m]); c_all.append(c[m]) + else: + for p in sorted(glob.glob("test_data/futures-baseline/ES.FUT/*.dbn.zst")): + try: + df = db.DBNStore.from_file(p).to_df().reset_index() + except Exception: + continue + if df.empty or "close" not in df.columns: + continue + df = df[df["close"] > 0] + if df.empty: + continue + dom = df["instrument_id"].value_counts().idxmax() # front month for the quarter + df = df[df["instrument_id"] == dom].sort_values("ts_event") + ts_all.append(df["ts_event"].astype("int64").to_numpy()) + c_all.append(df["close"].to_numpy(np.float64)) ts = np.concatenate(ts_all); c = np.concatenate(c_all) + _u, _ui = np.unique(ts, return_index=True) # dedup any overlapping ts at chunk seams + ts, c = ts[_ui], c[_ui] o = np.argsort(ts, kind="stable"); ts, c = ts[o], c[o] b5 = ts // BAR_NS # 5-min bin id _, first = np.unique(b5, return_index=True)