#!/usr/bin/env python3
"""Fetch daily klines + funding history for major USDT perps (Binance, free, no key).

Caches per-symbol npz to data/surfer/crypto/ (gitignored): day, open, close, funding_daily
(sum of the 8h funding rates that day). Curated long-history majors → reduces (not eliminates)
survivorship; v1 caveat documented. Crypto is 24/7 → no roll / no overnight gap.
"""
import json
import os
import time
import urllib.request

import numpy as np

OUT = "data/surfer/crypto"
DAY_MS = 86_400_000
TOP_N = 80  # programmatic universe: top-N USDT perps by 24h quote-volume (removes hand-selection bias)


def get(url):
    req = urllib.request.Request(url, headers={"User-Agent": "curl/8"})
    return json.load(urllib.request.urlopen(req, timeout=30))


def universe(n):
    info = get("https://fapi.binance.com/fapi/v1/exchangeInfo")
    perps = {s["symbol"] for s in info["symbols"]
             if s.get("contractType") == "PERPETUAL" and s.get("quoteAsset") == "USDT"
             and s.get("status") == "TRADING"}
    tick = get("https://fapi.binance.com/fapi/v1/ticker/24hr")
    vol = {t["symbol"]: float(t["quoteVolume"]) for t in tick if t["symbol"] in perps}
    return sorted(vol, key=lambda s: -vol[s])[:n]


SYMS = universe(TOP_N)


def klines(sym):
    out = []
    end = None
    for _ in range(20):
        u = f"https://fapi.binance.com/fapi/v1/klines?symbol={sym}&interval=1d&limit=1500"
        if end:
            u += f"&endTime={end}"
        k = get(u)
        if not k:
            break
        out = k + out
        end = k[0][0] - 1
        if len(k) < 1500:
            break
        time.sleep(0.15)
    # dedup by openTime
    d = {int(r[0]): (float(r[1]), float(r[4])) for r in out}
    days = np.array(sorted(d))
    op = np.array([d[t][0] for t in days]); cl = np.array([d[t][1] for t in days])
    return days // DAY_MS, op, cl


def funding(sym, start_ms):
    out = []
    st = start_ms
    for _ in range(80):                                          # forward pagination (startTime works; endTime didn't)
        u = f"https://fapi.binance.com/fapi/v1/fundingRate?symbol={sym}&startTime={st}&limit=1000"
        f = get(u)
        if not f:
            break
        out += f
        st = f[-1]["fundingTime"] + 1
        if len(f) < 1000:
            break
        time.sleep(0.12)
    daily = {}
    for r in out:
        daily.setdefault(int(r["fundingTime"]) // DAY_MS, 0.0)
        daily[int(r["fundingTime"]) // DAY_MS] += float(r["fundingRate"])
    return daily


def main():
    os.makedirs(OUT, exist_ok=True)
    ok = 0
    for sym in SYMS:
        outp = f"{OUT}/{sym}.npz"
        if os.path.exists(outp):
            print(f"  {sym}: cached"); ok += 1; continue
        try:
            kd, op, cl = klines(sym)
            if len(kd) < 400:
                print(f"  {sym}: too short ({len(kd)}d), skip"); continue
            fmap = funding(sym, int(kd.min()) * DAY_MS)
            fund = np.array([fmap.get(int(d), 0.0) for d in kd])
            np.savez(outp, day=kd, open=op, close=cl, funding=fund)
            print(f"  {sym}: {len(kd)}d  ({kd.min()}..{kd.max()})  fundcov={np.mean(fund!=0):.2f}")
            ok += 1
            time.sleep(0.2)
        except Exception as e:
            print(f"  {sym}: FAIL {type(e).__name__} {str(e)[:80]}")
    print(f"DONE: {ok}/{len(SYMS)} symbols -> {OUT}/")


if __name__ == "__main__":
    main()