#!/usr/bin/env python3
"""Apples-to-apples hardening of the sweep's OWN 4h edge — same 24h-continuous signal, but kill the
statistical inflation: ONE non-overlapping trade per UTC-day, IS-derived conviction cutoff, per-year.

The sweep (mft_horizon_sweep) used a 24h-continuous 5-min grid, day = UTC-day, hold = 48 bars = 4h of
clock time, top-decile conviction over HEAVILY OVERLAPPING entries → t+7.4 (inflated by overlap). Here:
same grid + signal, but max-conviction single entry per UTC-day (non-overlapping ⇒ honest t), threshold
fit on IS, reported per-year. cost 1 and 2 ticks. PASS: OOS net>0 & hit>50% & honest t≥2, stable per-year.
"""
import glob
import math

import numpy as np

TICK = 0.25
BAR_NS = 5 * 60 * 10**9
DAY_NS = 86_400 * 10**9
HOLDS = [48]                       # 4h (the sweep's standout); same grid as the sweep


def load_es_5min():
    import databento as db
    ts_all, c_all = [], []
    for p in sorted(glob.glob("data/surfer/es1m/*.dbn")):
        try:
            arr = db.DBNStore.from_file(p).to_ndarray()
        except Exception:
            continue
        if len(arr) == 0 or "close" not in arr.dtype.names:
            continue
        ts = arr["ts_event"].astype(np.int64); c = arr["close"].astype(np.float64) / 1e9
        m = c > 0; ts_all.append(ts[m]); c_all.append(c[m])
    ts = np.concatenate(ts_all); c = np.concatenate(c_all)
    _u, ui = np.unique(ts, return_index=True); ts, c = ts[ui], c[ui]
    o = np.argsort(ts, kind="stable"); ts, c = ts[o], c[o]
    b5 = ts // BAR_NS
    _, first = np.unique(b5, return_index=True)
    last = np.append(first[1:] - 1, len(b5) - 1)
    return ts[last], c[last]


def per_day_trades(close, day, yr, H, cost):
    lc = np.log(close); n = len(lc)
    r1 = np.zeros(n); r1[1:] = lc[1:] - lc[:-1]
    same1 = np.zeros(n, bool); same1[1:] = day[1:] == day[:-1]
    r1 = np.where(same1, r1, 0.0)
    w = 12
    cs = np.concatenate([[0.0], np.cumsum(r1)]); cs2 = np.concatenate([[0.0], np.cumsum(r1 * r1)])
    vol = np.full(n, np.nan); s = cs[w:] - cs[:-w]; s2 = cs2[w:] - cs2[:-w]
    vol[w - 1:] = np.sqrt(np.maximum(s2 / w - (s / w) ** 2, 0.0))
    retH = np.full(n, np.nan); retH[H:] = lc[H:] - lc[:-H]
    sameH_back = np.zeros(n, bool); sameH_back[H:] = day[H:] == day[:-H]      # like the sweep: sameday(H)
    tstat = retH / np.maximum(vol * math.sqrt(H), 1e-9)
    fwd = np.full(n, np.nan); fwd[:-H] = (close[H:] - close[:-H]) / TICK
    sameH_fwd = np.zeros(n, bool); sameH_fwd[:-H] = day[:-H] == day[H:]
    ok = sameH_fwd & sameH_back & np.isfinite(tstat) & np.isfinite(fwd) & (vol > 0)
    direction = np.sign(retH); net = direction * fwd - cost; conv = np.abs(tstat)

    oc, on, oy, od = [], [], [], []
    order = np.argsort(day, kind="stable"); ds = day[order]
    bounds = np.append(np.append(0, np.nonzero(np.diff(ds))[0] + 1), len(ds))
    for b in range(len(bounds) - 1):
        rows = order[bounds[b]:bounds[b + 1]]
        rows = rows[ok[rows] & (direction[rows] != 0)]
        if len(rows) == 0:
            continue
        best = rows[np.argmax(conv[rows])]
        oc.append(conv[best]); on.append(net[best]); oy.append(yr[best]); od.append(day[best])
    return np.array(oc), np.array(on), np.array(oy), np.array(od)


def stats(net):
    n = len(net)
    if n < 20:
        return n, float("nan"), float("nan"), float("nan"), float("nan")
    m = float(net.mean()); sd = float(net.std()) + 1e-12
    return n, m, 100.0 * float((net > 0).mean()), m / (sd / math.sqrt(n)), (m / sd) * math.sqrt(252.0)


def run():
    ts5, close5 = load_es_5min()
    day = (ts5 // DAY_NS).astype(np.int64)
    yr = (1970 + (ts5 // (365.25 * DAY_NS))).astype(int)        # approx calendar year from epoch days
    print("\n============ APPLES-TO-APPLES 4h HARDENING (24h grid, 1 trade/UTC-day, IS-cutoff) ============")
    print(f"5-min bars={len(close5)}  days={int(day.max()-day.min())}  multiple-testing bar ≈ |t|>2.3\n")
    for cost in (1.0, 2.0):
        for H in HOLDS:
            conv, net, tyr, dd = per_day_trades(close5, day, yr, H, cost)
            if len(conv) < 50:
                print(f"cost={cost:.0f} H={H}: too few ({len(conv)})"); continue
            split = np.quantile(dd, 0.70); is_m = dd < split; oos_m = ~is_m
            print(f"=== cost={cost:.0f} tick, H={H} (4h), 1 trade/day  (IS days {int(is_m.sum())} / OOS {int(oos_m.sum())}) ===")
            print(f"    {'select':>7} {'cutoff':>7} {'n_OOS':>6} {'net_t':>7} {'hit%':>6} {'t':>6} {'annSR':>6} {'trd/yr':>7}")
            for sel in (1.00, 0.50, 0.25, 0.10):
                cut = np.quantile(conv[is_m], 1.0 - sel) if sel < 1.0 else -np.inf
                take = oos_m & (conv >= cut)
                n, m, hit, t, sr = stats(net[take])
                oosyrs = max((int(tyr[oos_m].max()) - int(tyr[oos_m].min()) + 1), 1)
                cs = f"{cut:.2f}" if np.isfinite(cut) else "all"
                print(f"    {sel:>7.2f} {cs:>7} {n:>6} {m:>+7.2f} {hit:>6.1f} {t:>+6.2f} {sr:>+6.2f} {n/oosyrs:>7.1f}")
            cut = np.quantile(conv[is_m], 0.75)
            print(f"    per-year (sel=0.25; IS years are not OOS):")
            for y in range(int(tyr.min()), int(tyr.max()) + 1):
                ym = (tyr == y) & (conv >= cut)
                if int(ym.sum()) < 10:
                    continue
                n, m, hit, t, sr = stats(net[ym])
                flag = "OOS" if np.median(dd[tyr == y]) >= split else "is "
                print(f"      {y} [{flag}] n={n:>4} net={m:>+6.2f}t hit={hit:>5.1f}% t={t:>+5.2f}")
            print()


if __name__ == "__main__":
    run()