The Bybit liquid book was costed off a daily-high/low Corwin-Schultz spread proxy that over-states the true quoted spread ~104x (measured 1.4bp real vs 143.6bp CS), so the cockpit showed a 0.22-Sharpe cost-model artifact instead of the book's true ~1.5-1.8 Sharpe. - Part 1: `bybit_spread_refresh` samples the live L1 top-of-book several times (injectable sleep, no bare time.sleep) and stores the MEDIAN per coin in a new `bybit_real_spread` table (idempotent upsert + reader on PaperRepo). New CLI `fxhnt bybit-spread-refresh [--samples N]`. - Part 2: `_bybit_measured_net` + the precompute take an optional `real_spread_by_coin` override (reuses the maker-recost `recost_net_series` plumbing at f=0 = exact measured formula); coins without a stored real spread fall back to CS. Threaded for bybit_4edge ONLY -- binance_combined stays on CS (its broad/illiquid mirage finding stands). - Part 3: the cockpit caption honestly states "real L1 quoted spread (forward-realistic -- current liquidity; 2021-22 spreads were wider, so early- history cost is understated)"; the data-cost-mode="measured" marker is kept. - Part 4: the daily compare-measured-precompute CronJob now runs `bybit-spread-refresh && compare-measured-precompute`; added 443 egress for api.bybit.com. GROSS returns unchanged (cost-only fix). Tests: median-of-samples, missing/zero skip, idempotent upsert, reader round-trip; override lowers cost-drag + lifts Sharpe; partial-override CS fallback; precompute threads real spread for bybit not binance; caption renders with marker intact. Full suite: 1687 passed. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
130 lines
5.5 KiB
Python
130 lines
5.5 KiB
Python
"""Robust real per-coin L1 spread refresh + storage (Part 1 of the real-spread cost-model fix).
|
||
|
||
The Bybit liquid book was costed off a daily-high/low Corwin–Schultz spread proxy that over-states the true
|
||
quoted spread ~104× (1.4bp real vs 143.6bp CS). Part 1 SAMPLES the live L1 top-of-book several times and
|
||
stores the MEDIAN per coin (robust against one weird moment) in `bybit_real_spread`. All tests are NO-network:
|
||
a fixtured multi-sample fetcher + an in-memory SQLite repo; the inter-sample `sleep` is injected (no bare
|
||
time.sleep).
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import datetime as dt
|
||
|
||
from fxhnt.adapters.data.bybit_true_spread import BybitTrueSpread
|
||
from fxhnt.adapters.persistence.paper_repo import PaperRepo
|
||
from fxhnt.application.bybit_spread_refresh import (
|
||
read_real_spread_by_coin,
|
||
refresh_real_spreads,
|
||
sample_median_spreads,
|
||
)
|
||
|
||
_AT = dt.datetime(2026, 6, 26, tzinfo=dt.UTC)
|
||
|
||
|
||
def _ticker(symbol: str, bid, ask, last="100"): # noqa: ANN001, ANN202
|
||
return {"symbol": symbol, "bid1Price": bid, "ask1Price": ask, "lastPrice": last}
|
||
|
||
|
||
def _payload(rows): # noqa: ANN001, ANN202
|
||
return {"retCode": 0, "result": {"list": rows}}
|
||
|
||
|
||
class _MultiSampleSource:
|
||
"""A BybitTrueSpread stand-in serving a SEQUENCE of per-coin spread snapshots (one per sample call)."""
|
||
|
||
def __init__(self, snapshots: list[dict[str, float]]) -> None:
|
||
self._snaps = snapshots
|
||
self.calls = 0
|
||
|
||
def real_spread_bps(self, *, universe=None): # noqa: ANN001, ANN202
|
||
snap = self._snaps[min(self.calls, len(self._snaps) - 1)]
|
||
self.calls += 1
|
||
if universe is not None:
|
||
snap = {c: v for c, v in snap.items() if c in universe}
|
||
return dict(snap)
|
||
|
||
|
||
# --- median-of-samples -----------------------------------------------------------------------------
|
||
|
||
def test_sample_median_is_per_coin_median_across_samples() -> None:
|
||
"""The stored spread is the MEDIAN of each coin's samples — robust against one wide outlier tick."""
|
||
src = _MultiSampleSource([
|
||
{"BTCUSDT": 1.0, "ETHUSDT": 2.0},
|
||
{"BTCUSDT": 99.0, "ETHUSDT": 2.5}, # BTC outlier moment
|
||
{"BTCUSDT": 1.2, "ETHUSDT": 1.5},
|
||
])
|
||
slept: list[float] = []
|
||
out = sample_median_spreads(src, samples=3, interval_s=2.0, sleep=slept.append)
|
||
assert out == {"BTCUSDT": 1.2, "ETHUSDT": 2.0} # median(1,99,1.2)=1.2 ; median(2,2.5,1.5)=2.0
|
||
assert src.calls == 3
|
||
assert slept == [2.0, 2.0] # sleeps BETWEEN samples only (n-1), injected — no real wait
|
||
|
||
|
||
def test_sample_handles_coin_present_in_only_some_samples() -> None:
|
||
"""A coin quoted in only some samples uses the median of the samples it DOES have (not penalized)."""
|
||
src = _MultiSampleSource([
|
||
{"BTCUSDT": 1.0, "NEWUSDT": 5.0},
|
||
{"BTCUSDT": 3.0}, # NEWUSDT absent this sample
|
||
{"BTCUSDT": 2.0, "NEWUSDT": 7.0},
|
||
])
|
||
out = sample_median_spreads(src, samples=3, sleep=lambda _s: None)
|
||
assert out["BTCUSDT"] == 2.0 # median(1,3,2)
|
||
assert out["NEWUSDT"] == 6.0 # median(5,7)
|
||
|
||
|
||
def test_sample_skips_missing_and_zero_quotes_via_fetcher() -> None:
|
||
"""Missing / zero / crossed quotes are skipped (BybitTrueSpread drops them) → not stored as garbage cost."""
|
||
payloads = [
|
||
_payload([_ticker("BTCUSDT", "99999.5", "100000.5"),
|
||
_ticker("DEADUSDT", "0", "0"), # zero quote → skip
|
||
_ticker("CROSSUSDT", "101", "100")]), # crossed → skip
|
||
_payload([_ticker("BTCUSDT", "99998.5", "100000.5"),
|
||
{"symbol": "NOQUOTE"}]), # missing fields → skip
|
||
]
|
||
state = {"i": 0}
|
||
|
||
def fetch(_url): # noqa: ANN001, ANN202
|
||
p = payloads[min(state["i"], len(payloads) - 1)]
|
||
state["i"] += 1
|
||
return p
|
||
|
||
src = BybitTrueSpread(fetch=fetch, sleep=lambda _s: None)
|
||
out = sample_median_spreads(src, samples=2, sleep=lambda _s: None)
|
||
assert set(out) == {"BTCUSDT"} # only the valid quote survives both samples
|
||
assert out["BTCUSDT"] > 0.0
|
||
|
||
|
||
# --- persist + read round-trip + idempotency -------------------------------------------------------
|
||
|
||
def test_refresh_persists_and_reader_round_trips() -> None:
|
||
repo = PaperRepo("sqlite://")
|
||
repo.migrate()
|
||
src = _MultiSampleSource([{"BTCUSDT": 1.0, "ETHUSDT": 2.0}])
|
||
medians = refresh_real_spreads(repo, src, samples=1, sleep=lambda _s: None, at=_AT)
|
||
assert medians == {"BTCUSDT": 1.0, "ETHUSDT": 2.0}
|
||
assert read_real_spread_by_coin(repo) == {"BTCUSDT": 1.0, "ETHUSDT": 2.0}
|
||
|
||
|
||
def test_refresh_is_idempotent_upsert() -> None:
|
||
"""A second refresh REPLACES each coin's row (no dup rows, latest value wins)."""
|
||
repo = PaperRepo("sqlite://")
|
||
repo.migrate()
|
||
refresh_real_spreads(repo, _MultiSampleSource([{"BTCUSDT": 5.0}]), samples=1, sleep=lambda _s: None, at=_AT)
|
||
refresh_real_spreads(repo, _MultiSampleSource([{"BTCUSDT": 1.5, "ETHUSDT": 2.0}]), samples=1,
|
||
sleep=lambda _s: None, at=_AT)
|
||
assert read_real_spread_by_coin(repo) == {"BTCUSDT": 1.5, "ETHUSDT": 2.0}
|
||
|
||
|
||
def test_upsert_skips_non_finite_and_negative() -> None:
|
||
"""Defensive: a NaN / negative spread is never stored as a garbage cost."""
|
||
repo = PaperRepo("sqlite://")
|
||
repo.migrate()
|
||
repo.upsert_real_spreads({"AUSDT": 3.0, "BUSDT": float("nan"), "CUSDT": -2.0, "DUSDT": float("inf")}, at=_AT)
|
||
assert read_real_spread_by_coin(repo) == {"AUSDT": 3.0}
|
||
|
||
|
||
def test_reader_empty_when_no_refresh() -> None:
|
||
repo = PaperRepo("sqlite://")
|
||
repo.migrate()
|
||
assert read_real_spread_by_coin(repo) == {}
|