#!/usr/bin/env python3
"""Fetch historical funding (Binance + Bybit) for liquid coins on both, to BACKTEST the cross-venue
funding arb instead of waiting for live paper-forward. Aggregates 8h settlements to daily per venue.
Cached. Saves data/surfer/xvenue/panel.json = {coin: {date: [binance_daily, bybit_daily]}}.
"""
import datetime
import json
import os
import time
import urllib.request

OUT = "data/surfer/xvenue"
NCOINS = 60
DAYS = 330


def get(u, post=None):
    data = json.dumps(post).encode() if post else None
    h = {"User-Agent": "Mozilla/5.0", "Accept": "application/json"}
    for a in range(4):
        try:
            return json.loads(urllib.request.urlopen(urllib.request.Request(u, data=data, headers=h), timeout=25).read())
        except Exception:
            if a == 3:
                return None
            time.sleep(3 * (a + 1))
    return None


def base(s):
    return s[:-4] if s.endswith("USDT") else s


def day_of(ms):
    return datetime.datetime.utcfromtimestamp(int(ms) / 1000).strftime("%Y-%m-%d")


def binance_hist(coin):
    r = get(f"https://fapi.binance.com/fapi/v1/fundingRate?symbol={coin}USDT&limit=1000")
    d = {}
    for x in (r or []):
        d.setdefault(day_of(x["fundingTime"]), 0.0)
        d[day_of(x["fundingTime"])] += float(x["fundingRate"])
    return d


def bybit_hist(coin):
    out, end = {}, None
    for _ in range(6):
        u = f"https://api.bybit.com/v5/market/funding/history?category=linear&symbol={coin}USDT&limit=200"
        if end:
            u += f"&endTime={end}"
        r = get(u)
        lst = ((r or {}).get("result") or {}).get("list") or []
        if not lst:
            break
        for x in lst:
            t = x["fundingRateTimestamp"]
            out.setdefault(day_of(t), 0.0)
            out[day_of(t)] += float(x["fundingRate"])
        end = int(lst[-1]["fundingRateTimestamp"]) - 1
        time.sleep(0.3)
    return out


def main():
    os.makedirs(OUT, exist_ok=True)
    binv = {base(x["symbol"]): float(x["quoteVolume"]) for x in get("https://fapi.binance.com/fapi/v1/ticker/24hr") if x["symbol"].endswith("USDT")}
    byr = get("https://api.bybit.com/v5/market/tickers?category=linear")["result"]["list"]
    bybv = {base(x["symbol"]): float(x.get("turnover24h", 0)) for x in byr if x["symbol"].endswith("USDT")}
    coins = sorted([c for c in binv if c in bybv and binv[c] > 10e6 and bybv[c] > 10e6], key=lambda c: -binv[c])[:NCOINS]
    print(f"universe: {len(coins)} coins on both venues >$10M/day")
    panel = {}
    for i, c in enumerate(coins):
        cf = f"{OUT}/{c}.json"
        if os.path.exists(cf):
            panel[c] = json.load(open(cf)); continue
        bn = binance_hist(c); by = bybit_hist(c)
        days = sorted(set(bn) & set(by))
        panel[c] = {d: [bn[d], by[d]] for d in days}
        json.dump(panel[c], open(cf, "w"))
        if i % 10 == 0:
            print(f"  {i+1}/{len(coins)} {c}: {len(panel[c])} common days")
        time.sleep(0.4)
    json.dump(panel, open(f"{OUT}/panel.json", "w"))
    print(f"DONE: {len(panel)} coins -> {OUT}/panel.json")


if __name__ == "__main__":
    main()