"""OOS-disciplined overlay A/B harness for the Bybit 4-edge book — READ-ONLY, memory-bounded.

The DEPLOYED live overlay is legacy-venue-tuned (full-Kelly 30% vol-target + ISV-adaptive vol budget). On the
Bybit book that overlay HURTS: naive equal-weight Sharpe ~1.81 collapses to ~0.98 because the overlay
over-levers (~2.79×) a book whose realized vol it was never calibrated to. Our own memory (A3g/A3h) warns
that in-sample overlay TUNING over-estimates real Sharpe by ~0.6 and is an overfit trap.

So this harness is OOS-DISCIPLINED, not a sweep-to-best:
  1. Assemble the 4-sleeve `returns_by_sleeve` for the Bybit book (`sleeve_returns_from_store`).
  2. Run a SMALL, PRINCIPLED config grid (CONFIG_NAMES) — naive eq-wt, inverse-vol, calibrated vol-target
     (15%/20% to the book's OWN vol, cap ~1.5-2×), +killswitch, +marginal-gate, +ISV-adaptive, and the
     current live_overlay reference.
  3. Score EACH config on a TRAIN split (first `train_frac`) AND a held-out TEST split.
  4. The TRAIN-chosen winner is adopted ONLY if it beats naive eq-wt OUT-OF-SAMPLE (TEST). A config that
     wins TRAIN but not TEST is FLAGGED overfit and NOT recommended; the recommendation falls back to the
     best OOS config that beats naive, or to "stay naive eq-wt" if none does. The train->test Sharpe haircut
     is surfaced (memory expects ~0.6).

REUSE — the overlay is NOT reimplemented. Every config is driven by ONE causal forward loop
(`_overlay_curve`) that composes the EXISTING `paper_book` overlay primitives:
  * `isv_adaptive_weights` (the live ISV weighting / A3h) — `weighting="isv"`;
  * `book_allocator._inverse_vol_weights` (the live inverse-vol allocator) — `weighting="inverse_vol"`;
  * static equal weight — `weighting="equal"` (the naive baseline);
  * `vol_target_leverage` (the live fractional-Kelly vol-target) with the config's `target_vol`/`kelly`/
    `max_leverage`, or unit leverage when `vol_target=False`;
  * `adaptive_target_vol` (the live ISV-adaptive vol budget / A3h) when `isv_vol_budget=True`;
  * `DrawdownKillSwitch` (the live kill switch) when `killswitch=True`;
  * `marginal_gate_active` (the live leave-one-out marginal-Sharpe gate) when `marginal_gate=True`.
The `live_overlay` config delegates to the EXACT live `simulate_book`/`IncrementalRiskOverlay` path
(controller="killswitch", full-Kelly, 30% base + ISV budget) so the "what we have now" reference is the real
deployed overlay, not a hand-rolled lookalike. A golden test pins live_overlay == simulate_book on the full
sample. The causal convention (weights/leverage for day d from returns STRICTLY BEFORE d) is identical to
`paper_book_returns` / `paper_sim`.

READ-ONLY: no PaperRepo, no store.write_* — the live paper_* tables are untouched (WriteTripwire test proves
it). MEMORY-BOUNDED: a `universe` restricts the panel reads to the liquid subset via the same
`_UniverseRestrictedStore` seam `bybit_book_eval` uses (FullPanel tripwire test proves it).
"""
from __future__ import annotations

import math
import statistics as st
from dataclasses import dataclass
from typing import Any, Protocol

import numpy as np

from fxhnt.application.bybit_book_eval import (
    _DEFAULT_BYBIT_SLEEVES,
    _equal_weight_metrics,
    sleeve_returns_from_store,
)
from fxhnt.application.paper_book import (
    DrawdownKillSwitch,
    adaptive_target_vol,
    isv_adaptive_weights,
    marginal_gate_active,
    vol_target_leverage,
)
from fxhnt.application.paper_sim import simulate_book

_PERIODS_PER_YEAR = 365


class _FeatureStore(Protocol):
    def crypto_close_panel(self, *, suffix: str = "USDT") -> dict[str, dict[int, float]]: ...
    def crypto_funding_panel(self, *, suffix: str = "USDT") -> dict[str, dict[int, float]]: ...
    def crypto_spot_panel(self, *, suffix: str = "USDT") -> dict[str, dict[int, float]]: ...
    def read_panel(self, symbols: list[str], feature: str) -> dict[str, dict[int, float]]: ...


@dataclass(frozen=True)
class OverlayConfig:
    """ONE overlay configuration = a set of knobs over the REUSED overlay primitives.

    weighting:        "equal" (naive) | "inverse_vol" (book_allocator) | "isv" (the live A3h weighting).
    vol_target:       apply `vol_target_leverage`? False → unit leverage (risk-weight only, no leverage).
    target_vol:       annualized vol budget for `vol_target_leverage` (the calibrated configs use 0.15/0.20).
    kelly:            fractional-Kelly knob for `vol_target_leverage`.
    max_leverage:     leverage cap for `vol_target_leverage` (the robust over-levering fix uses ~1.5-2×).
    isv_vol_budget:   scale target_vol by trailing book Sharpe via `adaptive_target_vol` (the A3h budget)?
    killswitch:       apply the live `DrawdownKillSwitch` (0 leverage while killed)?
    marginal_gate:    apply the live `marginal_gate_active` leave-one-out edge gate to the active set?
    live:             delegate to the EXACT live `simulate_book` path (the deployed reference); ignores the
                      knobs above except for documentation.
    """
    weighting: str = "equal"
    vol_target: bool = False
    target_vol: float = 0.20
    kelly: float = 1.0
    max_leverage: float = 2.0
    isv_vol_budget: bool = False
    killswitch: bool = False
    marginal_gate: bool = False
    live: bool = False


# The SMALL, PRINCIPLED grid (~8 configs, not a giant sweep). Order = the report order.
#   naive_eqwt           — the ~1.81 baseline (no overlay).
#   inverse_vol          — risk-weight the sleeves, NO leverage (pure de-risking).
#   voltarget_15 / _20   — calibrated vol-target to the book's OWN vol at 15%/20%, cap 2× (the ROBUST fix
#                          for the over-levering: fits LEVERAGE to VOL, not returns).
#   voltarget_20_kill    — + drawdown killswitch.
#   voltarget_20_gate    — + marginal-gate.
#   isv_adaptive         — the A3h-style ISV-adaptive vol budget (most overfit-prone; included to be tested).
#   live_overlay         — the deployed legacy-venue-tuned overlay (expected ~0.98).
_GRID: dict[str, OverlayConfig] = {
    "naive_eqwt": OverlayConfig(weighting="equal", vol_target=False),
    "inverse_vol": OverlayConfig(weighting="inverse_vol", vol_target=False),
    "voltarget_15": OverlayConfig(weighting="inverse_vol", vol_target=True, target_vol=0.15,
                                  kelly=1.0, max_leverage=2.0),
    "voltarget_20": OverlayConfig(weighting="inverse_vol", vol_target=True, target_vol=0.20,
                                  kelly=1.0, max_leverage=2.0),
    "voltarget_20_kill": OverlayConfig(weighting="inverse_vol", vol_target=True, target_vol=0.20,
                                       kelly=1.0, max_leverage=2.0, killswitch=True),
    "voltarget_20_gate": OverlayConfig(weighting="inverse_vol", vol_target=True, target_vol=0.20,
                                       kelly=1.0, max_leverage=2.0, marginal_gate=True),
    "isv_adaptive": OverlayConfig(weighting="isv", vol_target=True, target_vol=0.30, kelly=1.0,
                                  max_leverage=3.0, isv_vol_budget=True),
    "live_overlay": OverlayConfig(live=True),
}

CONFIG_NAMES: tuple[str, ...] = tuple(_GRID)


# --- weighting (reused primitives) -------------------------------------------------------------

def _weights_for(weighting: str, before: dict[str, dict[int, float]], active: list[str], *,
                 vol_lookback: int = 60) -> dict[str, float]:
    """Per-date sleeve weights from the strictly-prior returns `before`, by mode — each REUSES a live
    weighting primitive (no reimplementation). `active` = sleeves with prior history (cold-start → equal)."""
    if not active:
        return {}
    if weighting == "isv":
        return isv_adaptive_weights(before, active)
    if weighting == "inverse_vol":
        from fxhnt.application.book_allocator import _inverse_vol_weights
        window: dict[str, np.ndarray] = {}
        for s in active:
            vals = [before[s][d] for d in sorted(before.get(s, {}))][-vol_lookback:]
            window[s] = np.asarray(vals, dtype=float)
        # max_sleeve_weight 1.0 = no cap (the harness's own concentration control lives elsewhere); the
        # book_allocator inverse-vol path is reused verbatim.
        return _inverse_vol_weights(window, 1.0)
    # equal weight (the naive baseline) over the active set.
    n = len(active)
    return {s: 1.0 / n for s in active}


def _overlay_curve(returns_by_sleeve: dict[str, dict[int, float]], sleeves: list[str],
                   cfg: OverlayConfig, *, cost_bps: float, vol_lookback: int = 60) -> dict[int, float]:
    """One config's CAUSAL daily NET return series `{epoch_day: ret}` over the union date axis.

    Reuses the live overlay primitives in a single forward loop (mirrors `paper_book_returns` /
    `paper_sim.overlay_pass`): for each day d the weights + leverage + kill state are computed from returns
    STRICTLY BEFORE d (no look-ahead). bret(d) = (0 if killed else lev)·Σ w·r_d − turnover·cost. Pure.

    `cfg.live` delegates to `simulate_book` (the exact deployed overlay) so the reference config IS the live
    code path. Otherwise the knobs select weighting / vol-target / ISV budget / killswitch / marginal-gate —
    each a reused `paper_book` function."""
    selected = [s for s in sleeves if returns_by_sleeve.get(s)]
    if not selected:
        return {}

    if cfg.live:
        res = simulate_book({s: returns_by_sleeve[s] for s in selected}, capital=100_000.0,
                            sleeves=selected, controller="killswitch", kelly_fraction=1.0,
                            cost_bps=cost_bps)
        # SimResult carries the equity curve; recover the per-day net return for a uniform Sharpe basis.
        return _curve_to_returns(res.dates, res.equity, 100_000.0)

    all_days = sorted({d for s in selected for d in returns_by_sleeve[s]})
    before: dict[str, dict[int, float]] = {s: {} for s in selected}
    out: dict[int, float] = {}
    out_vals: list[float] = []
    ks = DrawdownKillSwitch() if cfg.killswitch else None
    prev_eff: dict[str, float] = {}

    for d in all_days:
        active = [s for s in selected if before[s]]
        if cfg.marginal_gate and active:
            active = marginal_gate_active(before, active, vol_lookback=vol_lookback)
        weights = _weights_for(cfg.weighting, before, active, vol_lookback=vol_lookback)

        if cfg.vol_target and weights:
            tv = cfg.target_vol
            if cfg.isv_vol_budget:
                tv = adaptive_target_vol(out_vals, base=cfg.target_vol)
            lev = vol_target_leverage(weights, before, target_vol=tv, kelly_fraction=cfg.kelly,
                                      vol_lookback=vol_lookback, max_leverage=cfg.max_leverage)
        else:
            lev = 1.0

        killed = ks.killed if ks is not None else False
        eff_lev = 0.0 if killed else lev
        gross = eff_lev * sum(weights.get(s, 0.0) * returns_by_sleeve[s].get(d, 0.0) for s in weights)
        # turnover cost on the EFFECTIVE (leveraged) weights vs the prior day (mirrors apply_knobs).
        eff_w = {s: weights.get(s, 0.0) * eff_lev for s in weights}
        if cost_bps:
            turnover = sum(abs(eff_w.get(s, 0.0) - prev_eff.get(s, 0.0))
                           for s in set(eff_w) | set(prev_eff))
            gross -= turnover * cost_bps / 1e4
        prev_eff = eff_w

        if math.isfinite(gross):
            out[d] = gross
            out_vals.append(gross)

        if ks is not None:
            # The killswitch observes the UN-throttled (unit-leverage, killed-aware off) book return so it
            # tracks the same regime signal the live overlay does — the gross weighted return for the day.
            unthrottled = sum(weights.get(s, 0.0) * returns_by_sleeve[s].get(d, 0.0) for s in weights)
            ks.observe(unthrottled)

        for s in selected:
            if d in returns_by_sleeve[s]:
                before[s][d] = returns_by_sleeve[s][d]
    return out


def _curve_to_returns(dates: list[int], equity: list[float], capital: float) -> dict[int, float]:
    """Recover the per-day net return series from a `simulate_book` equity curve (eq[i]/eq[i-1] − 1, with
    the first day relative to `capital`). Lets the live config share the same Sharpe basis as the others."""
    out: dict[int, float] = {}
    prev = capital
    for d, e in zip(dates, equity, strict=True):
        if prev != 0.0:
            out[int(d)] = e / prev - 1.0
        prev = e
    return out


# --- metrics + split ---------------------------------------------------------------------------

def _metrics(series: dict[int, float]) -> dict[str, Any]:
    """Sharpe (mean/pstdev × √365) / CAGR (over the date span) / maxDD / total-return / days of a `{day: ret}`
    series. <2 obs → zeros. maxDD <= 0."""
    days = sorted(series)
    rets = [series[d] for d in days]
    if len(rets) < 2:
        return {"sharpe": 0.0, "cagr": 0.0, "max_dd": 0.0, "total_return": 0.0, "days": len(rets)}
    sd = st.pstdev(rets)
    sharpe = (st.mean(rets) / sd * math.sqrt(_PERIODS_PER_YEAR)) if sd > 0 else 0.0
    equity = 1.0
    peak = 1.0
    max_dd = 0.0
    for r in rets:
        equity *= (1.0 + r)
        peak = max(peak, equity)
        if peak > 0:
            max_dd = min(max_dd, equity / peak - 1.0)
    span = days[-1] - days[0]
    years = span / _PERIODS_PER_YEAR if span > 0 else 0.0
    cagr = (equity ** (1.0 / years) - 1.0) if years > 0 and equity > 0 else 0.0
    return {"sharpe": sharpe, "cagr": cagr, "max_dd": max_dd, "total_return": equity - 1.0,
            "days": len(rets)}


def _split_days(returns_by_sleeve: dict[str, dict[int, float]], train_frac: float) -> tuple[int, int]:
    """(train_hi, test_lo) epoch-days from the union date axis: TRAIN = first `train_frac`, TEST = the rest."""
    days = sorted({d for r in returns_by_sleeve.values() for d in r})
    if len(days) < 4:
        return days[-1] if days else 0, (days[-1] + 1) if days else 1
    idx = max(1, min(len(days) - 1, int(len(days) * train_frac)))
    return days[idx - 1], days[idx]


def _window(series: dict[int, float], *, day_hi: int | None = None,
            day_lo: int | None = None) -> dict[int, float]:
    return {d: r for d, r in series.items()
            if (day_hi is None or d <= day_hi) and (day_lo is None or d >= day_lo)}


def _naive_metrics(returns_by_sleeve: dict[str, dict[int, float]], active: list[str], *,
                   cost_bps: float, day_hi: int | None = None,
                   day_lo: int | None = None) -> dict[str, Any]:
    """Naive equal-weight metrics over a date window, via the EXACT `_equal_weight_metrics`
    `risk_managed_book_from_store` uses — so the harness's `naive_eqwt` is bit-identical to the deployed
    book's naive baseline (the consistency guarantee). Windows the per-sleeve returns first."""
    windowed = {s: _window(returns_by_sleeve[s], day_hi=day_hi, day_lo=day_lo) for s in active}
    present = [s for s in active if windowed.get(s)]
    m = _equal_weight_metrics(windowed, present, capital=100_000.0, cost_bps=cost_bps)
    return {"sharpe": m["sharpe"], "cagr": m["cagr"], "max_dd": m["max_dd"],
            "total_return": m["total_return"], "days": m["days"]}


# --- the harness -------------------------------------------------------------------------------

def overlay_ab(returns_by_sleeve_or_store: Any, *, universe: set[str] | None = None,
               cost_bps: float = 5.5, train_frac: float = 0.6,
               sleeves: list[str] | None = None,
               unlock_events: list[dict[str, Any]] | None = None,
               stablecoin_spot_panel: dict[str, dict[int, float]] | None = None,
               ) -> dict[str, Any]:
    """OOS-disciplined overlay A/B over the Bybit 4-edge book. READ-ONLY, memory-bounded.

    `returns_by_sleeve_or_store` may be a `{sleeve: {epoch_day: ret}}` dict (the pure path) OR a feature
    store (then the 4-sleeve returns are assembled via `sleeve_returns_from_store`, universe-restricted).

    For EACH config in the grid: compute TRAIN Sharpe, TEST (OOS) Sharpe, and FULL-sample metrics, plus CAGR
    / maxDD per split. Then:
      * `train_winner`  = the config with the best TRAIN Sharpe;
      * `best_test`     = the config with the best TEST Sharpe;
      * VERDICT: a re-tuned overlay is adopted ONLY if the TRAIN winner beats naive_eqwt on TEST. If it does
        not, it is FLAGGED overfit and the recommendation falls back to the best config that beats naive OOS,
        or to "naive_eqwt" if none does. The train->test Sharpe haircut is reported.

    Returns `{available, sleeves, table {config: {train, test, full, config}}, train_winner, best_test,
    verdict {recommendation, train_winner_overfit, train_to_test_haircut, naive_test_sharpe, ...}, cost_bps,
    train_frac, days}`. Writes NOTHING."""
    returns_by_sleeve = _assemble_returns(
        returns_by_sleeve_or_store, universe=universe, cost_bps=cost_bps, sleeves=sleeves,
        unlock_events=unlock_events, stablecoin_spot_panel=stablecoin_spot_panel)
    active = [s for s in (sleeves or _DEFAULT_BYBIT_SLEEVES) if returns_by_sleeve.get(s)]
    # the pure path may pass arbitrary sleeve names; keep any with data.
    for s in returns_by_sleeve:
        if returns_by_sleeve[s] and s not in active:
            active.append(s)

    n_days = len({d for r in returns_by_sleeve.values() for d in r})
    if not active or n_days < 4:
        return {"available": False, "sleeves": active, "days": n_days,
                "reason": "need >= 2 sleeves-with-data and >= 4 union days for a train/test split "
                          "(run the Phase-A/B Bybit ingests first)"}

    train_hi, test_lo = _split_days(returns_by_sleeve, train_frac)

    table: dict[str, dict[str, Any]] = {}
    for name, cfg in _GRID.items():
        if name == "naive_eqwt":
            # Consistency: the naive baseline's metrics come from the SAME `_equal_weight_metrics` the
            # established `risk_managed_book_from_store` reports, computed per split window — so the harness's
            # naive number is bit-identical to the deployed book's naive number (a golden-consistency test).
            table[name] = {
                "config": cfg,
                "train": _naive_metrics(returns_by_sleeve, active, cost_bps=cost_bps, day_hi=train_hi),
                "test": _naive_metrics(returns_by_sleeve, active, cost_bps=cost_bps, day_lo=test_lo),
                "full": _naive_metrics(returns_by_sleeve, active, cost_bps=cost_bps),
            }
            continue
        full = _overlay_curve(returns_by_sleeve, active, cfg, cost_bps=cost_bps)
        table[name] = {
            "config": cfg,
            "train": _metrics(_window(full, day_hi=train_hi)),
            "test": _metrics(_window(full, day_lo=test_lo)),
            "full": _metrics(full),
        }

    verdict = _verdict(table)
    train_winner = verdict["train_winner"]
    best_test = max(table, key=lambda n: table[n]["test"]["sharpe"])

    return {"available": True, "sleeves": active, "days": n_days,
            "table": table, "train_winner": train_winner, "best_test": best_test,
            "verdict": verdict, "cost_bps": cost_bps, "train_frac": train_frac,
            "split_train_hi": train_hi, "split_test_lo": test_lo}


def _verdict(table: dict[str, dict[str, Any]]) -> dict[str, Any]:
    """The OOS-discipline verdict. `train_winner` = best TRAIN Sharpe. It is ADOPTED only if it beats
    naive_eqwt on TEST; else it is flagged overfit and the recommendation is the best config that beats naive
    OOS (or naive_eqwt if none does). The train->test haircut is the train winner's Sharpe drop."""
    naive_test = table["naive_eqwt"]["test"]["sharpe"]
    train_winner = max(table, key=lambda n: table[n]["train"]["sharpe"])
    tw_test = table[train_winner]["test"]["sharpe"]
    tw_train = table[train_winner]["train"]["sharpe"]
    haircut = tw_train - tw_test

    train_winner_beats_naive_oos = tw_test > naive_test + 1e-9
    # candidates that beat naive OOS, ranked by TEST Sharpe (the OOS-robust set).
    oos_winners = sorted(
        (n for n in table if n != "naive_eqwt" and table[n]["test"]["sharpe"] > naive_test + 1e-9),
        key=lambda n: table[n]["test"]["sharpe"], reverse=True)

    if train_winner_beats_naive_oos:
        recommendation = train_winner
        overfit = False
        note = (f"adopt '{train_winner}': it won TRAIN (Sharpe {tw_train:.2f}) AND beats naive eq-wt "
                f"OUT-OF-SAMPLE (TEST {tw_test:.2f} > naive {naive_test:.2f}).")
    else:
        overfit = train_winner != "naive_eqwt"
        if oos_winners:
            recommendation = oos_winners[0]
            rec_test = table[recommendation]["test"]["sharpe"]
            note = (f"train winner '{train_winner}' is OVERFIT (TEST {tw_test:.2f} <= naive {naive_test:.2f}); "
                    f"adopt the OOS-robust '{recommendation}' instead (TEST {rec_test:.2f} > naive).")
        else:
            recommendation = "naive_eqwt"
            note = (f"NO overlay config beats naive eq-wt OUT-OF-SAMPLE (train winner '{train_winner}' "
                    f"TEST {tw_test:.2f} <= naive {naive_test:.2f}) — STAY NAIVE EQ-WT.")

    return {"recommendation": recommendation, "train_winner": train_winner,
            "train_winner_overfit": overfit, "train_to_test_haircut": haircut,
            "naive_test_sharpe": naive_test, "oos_winners": oos_winners, "note": note}


def _assemble_returns(returns_by_sleeve_or_store: Any, *, universe: set[str] | None, cost_bps: float,
                      sleeves: list[str] | None, unlock_events: list[dict[str, Any]] | None,
                      stablecoin_spot_panel: dict[str, dict[int, float]] | None,
                      ) -> dict[str, dict[int, float]]:
    """Either pass through a prebuilt `{sleeve: {day: ret}}` dict, or assemble the 4-sleeve returns from a
    feature store via `sleeve_returns_from_store` (universe-restricted READS → memory-bounded). A dict is
    detected by its values being dicts; anything else is treated as a store."""
    obj = returns_by_sleeve_or_store
    if isinstance(obj, dict) and (not obj or all(isinstance(v, dict) for v in obj.values())):
        return {s: dict(v) for s, v in obj.items()}
    candidate = list(sleeves) if sleeves is not None else list(_DEFAULT_BYBIT_SLEEVES)
    out: dict[str, dict[int, float]] = {}
    for sleeve in candidate:
        series = sleeve_returns_from_store(
            obj, sleeve, universe=universe, cost_bps=cost_bps,
            unlock_events=unlock_events, stablecoin_spot_panel=stablecoin_spot_panel)
        if len(series) >= 2:
            out[sleeve] = series
    return out
