fix(surfer): full-16y MFT settle — regime-adaptive surfer significantly negative
Pulled 16y ES ohlcv-1m continuous (year-chunked, $20 credits, no 504) + light to_ndarray loader (to_df OOM'd at 20GB). On 1.1M 5-min bars the 1.3y +0.50/t=0.67 top-5% hint collapsed: regime-adaptive top-5% = -0.52 ticks/trade t=-2.51 (significantly negative); ALL cells/signals/convictions significantly negative. Decisive: no capturable intraday directional edge for crossing/non-colocated setup. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
81
scripts/surfer/fetch_es_1m.py
Normal file
81
scripts/surfer/fetch_es_1m.py
Normal file
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fetch full-history ES front-month OHLCV-1m (continuous .c.0), year-chunked, BUDGET-CAPPED.
|
||||
|
||||
get_cost-gated (aborts over cap, no download); year chunks with quarter fallback on 504.
|
||||
Saves per-chunk DBN to data/surfer/es1m/ (gitignored). Key from env, never printed.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import databento as db
|
||||
|
||||
CAP_USD = 25.00
|
||||
DS = "GLBX.MDP3"
|
||||
SCHEMA = "ohlcv-1m"
|
||||
SYM = "ES.c.0"
|
||||
STYPE = "continuous"
|
||||
OUT_DIR = "data/surfer/es1m"
|
||||
|
||||
|
||||
def fetch(client, start, end, out):
|
||||
data = client.timeseries.get_range(dataset=DS, symbols=[SYM], schema=SCHEMA,
|
||||
start=start, end=end, stype_in=STYPE)
|
||||
data.to_file(out)
|
||||
return sum(1 for _ in data)
|
||||
|
||||
|
||||
def main():
|
||||
key = os.environ.get("DATABENTO_API_KEY")
|
||||
if not key:
|
||||
print("DATABENTO_API_KEY not set"); return 2
|
||||
client = db.Historical(key)
|
||||
cost = client.metadata.get_cost(dataset=DS, symbols=[SYM], schema=SCHEMA,
|
||||
start="2010-06-06", end="2026-06-05", stype_in=STYPE)
|
||||
print(f"aggregate get_cost=${cost:.4f} cap=${CAP_USD:.2f}")
|
||||
if cost > CAP_USD:
|
||||
print("ABORT: over cap — nothing downloaded."); return 1
|
||||
os.makedirs(OUT_DIR, exist_ok=True)
|
||||
total = 0
|
||||
for year in range(2010, 2027):
|
||||
y0, y1 = f"{year}-01-01", f"{year+1}-01-01"
|
||||
if year == 2010:
|
||||
y0 = "2010-06-06"
|
||||
if year == 2026:
|
||||
y1 = "2026-06-05"
|
||||
out = f"{OUT_DIR}/ES_{year}.dbn"
|
||||
if os.path.exists(out):
|
||||
print(f" {year}: exists, skip"); continue
|
||||
ok = False
|
||||
for attempt in range(1, 3):
|
||||
try:
|
||||
n = fetch(client, y0, y1, out); total += n
|
||||
print(f" {year}: {n:,} recs -> {out}")
|
||||
ok = True; break
|
||||
except Exception as e:
|
||||
print(f" {year}: attempt {attempt} {type(e).__name__}; retry")
|
||||
time.sleep(3)
|
||||
if not ok: # fall back to quarter chunks
|
||||
print(f" {year}: year failed → quarter fallback")
|
||||
for q, (m0, m1) in enumerate([("01-01", "04-01"), ("04-01", "07-01"),
|
||||
("07-01", "10-01"), ("10-01", "12-31")], 1):
|
||||
qs, qe = f"{year}-{m0}", f"{year}-{m1}"
|
||||
if year == 2010 and q == 1:
|
||||
qs = "2010-06-06"
|
||||
if year == 2026 and q >= 3:
|
||||
continue
|
||||
qout = f"{OUT_DIR}/ES_{year}_q{q}.dbn"
|
||||
if os.path.exists(qout):
|
||||
continue
|
||||
try:
|
||||
n = fetch(client, qs, qe, qout); total += n
|
||||
print(f" {year} q{q}: {n:,} -> {qout}")
|
||||
except Exception as e:
|
||||
print(f" {year} q{q}: FAILED {type(e).__name__}")
|
||||
time.sleep(1)
|
||||
print(f"DONE: {total:,} records into {OUT_DIR}/")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -35,22 +35,40 @@ DAY_NS = 86_400 * 10**9
|
||||
|
||||
def load_es_5min():
|
||||
import databento as db
|
||||
# Prefer the full-history continuous front-month pull (data/surfer/es1m/), else fall back
|
||||
# to the local ~2y parent OHLCV-1m (futures-baseline/ES, per-quarter front-month pick).
|
||||
es1m = sorted(glob.glob("data/surfer/es1m/*.dbn"))
|
||||
ts_all, c_all = [], []
|
||||
for p in sorted(glob.glob("test_data/futures-baseline/ES.FUT/*.dbn.zst")):
|
||||
try:
|
||||
df = db.DBNStore.from_file(p).to_df().reset_index()
|
||||
except Exception:
|
||||
continue
|
||||
if df.empty or "close" not in df.columns:
|
||||
continue
|
||||
df = df[df["close"] > 0]
|
||||
if df.empty:
|
||||
continue
|
||||
dom = df["instrument_id"].value_counts().idxmax() # front month for the quarter
|
||||
df = df[df["instrument_id"] == dom].sort_values("ts_event")
|
||||
ts_all.append(df["ts_event"].astype("int64").to_numpy())
|
||||
c_all.append(df["close"].to_numpy(np.float64))
|
||||
if es1m:
|
||||
for p in es1m: # to_ndarray = light (to_df OOMs at 20GB)
|
||||
try:
|
||||
arr = db.DBNStore.from_file(p).to_ndarray()
|
||||
except Exception:
|
||||
continue
|
||||
if len(arr) == 0 or "close" not in arr.dtype.names:
|
||||
continue
|
||||
ts = arr["ts_event"].astype(np.int64)
|
||||
c = arr["close"].astype(np.float64) / 1e9 # raw 1e9 fixed-point → price
|
||||
m = c > 0
|
||||
ts_all.append(ts[m]); c_all.append(c[m])
|
||||
else:
|
||||
for p in sorted(glob.glob("test_data/futures-baseline/ES.FUT/*.dbn.zst")):
|
||||
try:
|
||||
df = db.DBNStore.from_file(p).to_df().reset_index()
|
||||
except Exception:
|
||||
continue
|
||||
if df.empty or "close" not in df.columns:
|
||||
continue
|
||||
df = df[df["close"] > 0]
|
||||
if df.empty:
|
||||
continue
|
||||
dom = df["instrument_id"].value_counts().idxmax() # front month for the quarter
|
||||
df = df[df["instrument_id"] == dom].sort_values("ts_event")
|
||||
ts_all.append(df["ts_event"].astype("int64").to_numpy())
|
||||
c_all.append(df["close"].to_numpy(np.float64))
|
||||
ts = np.concatenate(ts_all); c = np.concatenate(c_all)
|
||||
_u, _ui = np.unique(ts, return_index=True) # dedup any overlapping ts at chunk seams
|
||||
ts, c = ts[_ui], c[_ui]
|
||||
o = np.argsort(ts, kind="stable"); ts, c = ts[o], c[o]
|
||||
b5 = ts // BAR_NS # 5-min bin id
|
||||
_, first = np.unique(b5, return_index=True)
|
||||
|
||||
Reference in New Issue
Block a user