From a2e8516f77b7873289577d5c19c8025e6ac8dca3 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 24 Jun 2026 13:05:37 +0200 Subject: [PATCH] =?UTF-8?q?fix(bybit):=20stablecoin=20detector=20requires?= =?UTF-8?q?=20staying=20near=20peg=20(dispersion),=20not=20just=20median?= =?UTF-8?q?=20=E2=80=94=20excludes=20volatile=20coins=20averaging=20~$1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 --- .../warehouse/timescale_feature_store.py | 31 ++++++++ .../application/bybit_stablecoin_eval.py | 73 ++++++++++++------ .../integration/test_bybit_stablecoin_eval.py | 76 ++++++++++++++++--- 3 files changed, 147 insertions(+), 33 deletions(-) diff --git a/src/fxhnt/adapters/warehouse/timescale_feature_store.py b/src/fxhnt/adapters/warehouse/timescale_feature_store.py index 03f9b68..5d90c12 100644 --- a/src/fxhnt/adapters/warehouse/timescale_feature_store.py +++ b/src/fxhnt/adapters/warehouse/timescale_feature_store.py @@ -353,6 +353,37 @@ class TimescaleFeatureStore: import statistics as _st return {s: _st.median(v) for s, v in per_symbol.items() if v} + def spot_close_peg_stats( + self, *, suffix: str = "USDT", band: float = 0.05 + ) -> dict[str, tuple[float, float, int]]: + """{symbol: (median, frac_within_band, n)} for symbols ending in `suffix` — the LIGHT pass that lets + a caller auto-detect symbols that STAY near $1 (true stablecoins), not merely AVERAGE ~$1. + + `frac_within_band` is the fraction of observations with `spot_close ∈ [1-band, 1+band]` (default + ±5% of $1). A volatile coin whose median ≈ $1 but which wanders (SUSHI/THETA/TWT/VIRTUAL) has a LOW + frac; a real peg (USDC/USDE) has frac ≈ 1.0. `n` is the observation count (history length). + + Reads only `(symbol, value)` (no ts) for `spot_close` and streams the per-symbol median + in-band + fraction in Python (portable across PG/SQLite; SQL median isn't standard). Memory-bounded by total + spot rows × one float, not the panel-dict-per-day blow-up. READ-ONLY (one SELECT).""" + with self._engine.connect() as c: + rows = c.execute(text( + f"SELECT symbol, value FROM {self._table} WHERE feature = 'spot_close' AND symbol LIKE :pat" + ), {"pat": f"%{suffix}"}).fetchall() + per_symbol: dict[str, list[float]] = {} + for symbol, value in rows: + if value is not None: + per_symbol.setdefault(str(symbol), []).append(float(value)) + import statistics as _st + lo, hi = 1.0 - band, 1.0 + band + out: dict[str, tuple[float, float, int]] = {} + for s, v in per_symbol.items(): + if not v: + continue + in_band = sum(1 for x in v if lo <= x <= hi) + out[s] = (_st.median(v), in_band / len(v), len(v)) + return out + def trading_days_epoch(self) -> list[int]: """Sorted distinct adjclose trading days as epoch-DAYS (ts // 86400).""" with self._engine.connect() as c: diff --git a/src/fxhnt/application/bybit_stablecoin_eval.py b/src/fxhnt/application/bybit_stablecoin_eval.py index b092e26..210405c 100644 --- a/src/fxhnt/application/bybit_stablecoin_eval.py +++ b/src/fxhnt/application/bybit_stablecoin_eval.py @@ -8,17 +8,20 @@ stablecoins, using the `spot_close` data ALREADY in `bybit_features` — BEFORE USDC-margined perps (Phase 2)? HOW THE EDGE LOGIC IS REUSED (nothing reinvented): - * `bybit_stable_spot_panel(store)` auto-detects which symbols in the warehouse are near-$1 stablecoins - (median `spot_close` ∈ [0.95, 1.05]) and returns a `{sym: {epoch_day: close}}` panel — the EXACT shape - `StableReversionRunner` consumes. + * `bybit_stable_spot_panel(store)` auto-detects which symbols in the warehouse are TRUE near-$1 + stablecoins — ones that STAY near peg (tiny dispersion), not merely AVERAGE ~$1: median `spot_close` ∈ + [0.98, 1.02] AND >=90% of daily closes within ±5% of $1 AND >=60 days of history — and returns a + `{sym: {epoch_day: close}}` panel, the EXACT shape `StableReversionRunner` consumes. (A median-only + filter wrongly admitted volatile coins averaging ~$1 like SUSHI/THETA/TWT/VIRTUAL → bogus −90% DD.) * `stablecoin_reversion_metrics_from_store` feeds that panel straight into `StableReversionRunner(panel)` — the live sleeve's own runner, UNCHANGED (same short-rich logic, same threshold, same turnover cost) — and wraps its (dates, returns) via the shared `_curve_metrics` (the carry/edges template). -MEMORY-BOUNDED auto-detect: a near-$1 symbol can't be known a-priori, but the full `spot_close` table is -~hundreds of coins. We do a LIGHT pass — `spot_close_medians(store)` reads only `(symbol, value)` (no ts), -streams a per-symbol median, and selects the near-$1 names — then read ONLY those full series via -`read_panel(near_one_syms, "spot_close")`. The whole multi-day panel is never materialised for non-stables. +MEMORY-BOUNDED auto-detect: a stablecoin symbol can't be known a-priori, but the full `spot_close` table is +~hundreds of coins. We do a LIGHT pass — `spot_close_peg_stats(store)` reads only `(symbol, value)` (no ts), +streams per-symbol `(median, frac_within_5pct, n)`, and selects the names that STAY near $1 — then read ONLY +those full series via `read_panel(stable_syms, "spot_close")`. The whole multi-day panel is never +materialised for non-stables. READ-ONLY: there is no PaperRepo and no `store.write_*` / `paper_*` write is ever issued. Both the median scan and the panel read are SELECTs; `StableReversionRunner` is pure. A write-tripwire store in the tests @@ -37,36 +40,56 @@ from typing import Any, Protocol from fxhnt.application.bybit_carry_revalidate import _curve_metrics -# A symbol whose MEDIAN spot_close falls in this band is treated as a near-$1 stablecoin. Wide enough to -# admit USDC/USDE/TUSD (and a transient depeg), tight enough to EXCLUDE gold (PAXG/XAUT ~3000), a dead peg -# (USTC ~0.02), and any normal coin. -_PEG_LO = 0.95 -_PEG_HI = 1.05 +# A stablecoin is defined by STAYING near $1 (tiny dispersion), NOT merely by AVERAGING ~$1. A loose +# median-only filter admits VOLATILE coins that happen to average ~$1 (SUSHI/THETA/TWT/VIRTUAL all wander +# $0.5–$5 but median ≈ $1) — treating those as pegged ("short when rich") produced a bogus −90% DD. So a +# symbol qualifies ONLY if it is BOTH centered at peg AND stays there: +# 1. median(spot_close) ∈ [STABLE_MEDIAN_LO, STABLE_MEDIAN_HI] (centered at the $1 peg), AND +# 2. the FRACTION of days with spot_close ∈ [1±STABLE_PEG_BAND] is >= STABLE_MIN_FRAC_IN_BAND +# (i.e. >=90% of observations within ±5% of $1 — it STAYS near peg), AND +# 3. n >= STABLE_MIN_HISTORY days (a brand-new coin with a few near-$1 prints isn't admitted). +# This keeps USDC/USDE/BUSD/RLUSD/USD1 (near $1 ~always; brief depegs like USDE's intraday event are +# invisible at the daily close anyway) while EXCLUDING SUSHI/THETA/TWT/VIRTUAL, gold (PAXG/XAUT ~3000), +# and dead pegs (USTC ~0.02). +STABLE_MEDIAN_LO = 0.98 +STABLE_MEDIAN_HI = 1.02 +STABLE_PEG_BAND = 0.05 # half-width of the ±$1 "stays near peg" band +STABLE_MIN_FRAC_IN_BAND = 0.90 # >=90% of observations must lie within ±5% of $1 +STABLE_MIN_HISTORY = 60 # require at least this many daily observations class _FeatureStore(Protocol): """Just the reads this evaluator needs (TimescaleFeatureStore satisfies it). READ-ONLY.""" def read_panel(self, symbols: list[str], feature: str) -> dict[str, dict[int, float]]: ... - def spot_close_medians(self, *, suffix: str = "USDT") -> dict[str, float]: ... + def spot_close_peg_stats( + self, *, suffix: str = "USDT", band: float = STABLE_PEG_BAND + ) -> dict[str, tuple[float, float, int]]: ... -def _near_one(median: float) -> bool: - return _PEG_LO <= median <= _PEG_HI +def _is_stablecoin(median: float, frac_in_band: float, n: int) -> bool: + """A symbol qualifies as a stablecoin only if it is centered at peg AND stays there AND has history.""" + return ( + STABLE_MEDIAN_LO <= median <= STABLE_MEDIAN_HI + and frac_in_band >= STABLE_MIN_FRAC_IN_BAND + and n >= STABLE_MIN_HISTORY + ) def bybit_stable_spot_panel(store: _FeatureStore, *, suffix: str = "USDT") -> dict[str, dict[int, float]]: """Auto-detect Bybit's near-$1 stablecoin spot pairs and return the panel `StableReversionRunner` wants. - Step 1 (LIGHT, memory-bounded): `store.spot_close_medians()` streams a per-symbol median of `spot_close` - WITHOUT materialising the multi-day panel; we keep the symbols whose median ∈ [0.95, 1.05] (USDC/USDE/ - TUSD), dropping gold (~3000), dead pegs (~0.02), and normal coins. - Step 2: read ONLY those near-$1 symbols' full series via `read_panel(near, "spot_close")` — so the heavy - panel materialises for a HANDFUL of stablecoins, never the whole table. + Step 1 (LIGHT, memory-bounded): `store.spot_close_peg_stats()` streams per-symbol `(median, + frac_within_5pct, n)` WITHOUT materialising the multi-day panel; we keep ONLY symbols that BOTH center at + peg (median ∈ [0.98, 1.02]) AND stay there (>=90% of observations within ±5% of $1) AND have >=60 days of + history. This excludes VOLATILE coins that merely average ~$1 (SUSHI/THETA/TWT/VIRTUAL), gold (~3000), + dead pegs (~0.02), and brand-new coins — keeping real pegs (USDC/USDE/BUSD/RLUSD/USD1). + Step 2: read ONLY those qualifying symbols' full series via `read_panel(near, "spot_close")` — so the + heavy panel materialises for a HANDFUL of stablecoins, never the whole table. Returns `{symbol: {epoch_day: spot_close}}` (empty {} dropped). READ-ONLY (two SELECTs).""" - medians = store.spot_close_medians(suffix=suffix) - near = sorted(s for s, med in medians.items() if _near_one(med)) + stats = store.spot_close_peg_stats(suffix=suffix, band=STABLE_PEG_BAND) + near = sorted(s for s, (med, frac, n) in stats.items() if _is_stablecoin(med, frac, n)) if not near: return {} panel = store.read_panel(near, "spot_close") @@ -89,8 +112,10 @@ def stablecoin_reversion_metrics_from_store( panel = bybit_stable_spot_panel(store, suffix=suffix) pairs = sorted(panel) if not panel: - return _na("no near-$1 stablecoin spot pair in bybit_features " - f"(median spot_close in [{_PEG_LO}, {_PEG_HI}]); ingest spot_close first", pairs) + return _na("no near-$1 stablecoin spot pair in bybit_features (median spot_close in " + f"[{STABLE_MEDIAN_LO}, {STABLE_MEDIAN_HI}] AND >={STABLE_MIN_FRAC_IN_BAND:.0%} of days " + f"within +-{STABLE_PEG_BAND:.0%} of $1 over >={STABLE_MIN_HISTORY} days); " + "ingest spot_close first", pairs) res = StableReversionRunner(panel, cost_bps=cost_bps).run() if res.returns.size == 0: diff --git a/tests/integration/test_bybit_stablecoin_eval.py b/tests/integration/test_bybit_stablecoin_eval.py index 31548a1..b1b20e2 100644 --- a/tests/integration/test_bybit_stablecoin_eval.py +++ b/tests/integration/test_bybit_stablecoin_eval.py @@ -2,8 +2,9 @@ Seeds a synthetic in-memory `TimescaleFeatureStore("sqlite://", table="bybit_features")` with `spot_close` rows and asserts: - * `bybit_stable_spot_panel` auto-detects ONLY the near-$1 symbols (median spot_close ∈ [0.95, 1.05]) — - excluding a gold-like (~3000), a dead peg (~0.02), and a normal coin (~100); + * `bybit_stable_spot_panel` auto-detects ONLY symbols that STAY near $1 (median ∈ [0.98, 1.02] AND >=90% + of days within ±5% of $1 AND >=60 days history) — excluding a gold-like (~3000), a dead peg (~0.02), a + normal coin (~100), AND a VOLATILE coin that merely averages ~$1 (SUSHI-like: regression for the bug); * `stablecoin_reversion_metrics_from_store` runs the LIVE `StableReversionRunner` UNCHANGED on that panel — a rich→revert panel produces a positive (short-rich) curve with sane finite metrics; * the evaluator is READ-ONLY (write-tripwire store proxy); @@ -28,13 +29,19 @@ def _seed_spot(store: TimescaleFeatureStore, symbol: str, series: dict[int, floa store.write_features(symbol, [(d * _DAY, {"spot_close": px}) for d, px in series.items()]) -def _seed_mixed_universe(store: TimescaleFeatureStore, *, days: int = 40) -> None: - """A realistic mix: two near-$1 stablecoins (rich→revert), a gold-like (~3000), a dead peg (~0.02), and a - normal coin (~100). Only the two near-$1 names should be auto-detected as stablecoin spot pairs.""" - # USDC: persistently slightly rich (above peg) on even days, reverts to peg on odd days. +def _seed_mixed_universe(store: TimescaleFeatureStore, *, days: int = 80) -> None: + """A realistic mix: two TRUE near-$1 stablecoins (rich→revert, stay near peg), a SUSHI-like volatile coin + (median ~1 but WANDERS $0.5–$5), a gold-like (~3000), a dead peg (~0.02), and a normal coin (~100). Only + the two true stablecoins should be auto-detected — the SUSHI-like one is the regression case for the bug. + `days >= 60` so the true stablecoins clear the min-history requirement.""" + # USDC: persistently slightly rich (above peg) on even days, reverts to peg on odd days. Stays near $1. _seed_spot(store, "USDCUSDT", {d: (1.01 if d % 2 == 0 else 1.0) for d in range(days)}) - # USDE (Ethena): same rich→revert shape, slightly smaller deviation. + # USDE (Ethena): same rich→revert shape, slightly smaller deviation. Stays near $1. _seed_spot(store, "USDEUSDT", {d: (1.008 if d % 2 == 0 else 1.0) for d in range(days)}) + # SUSHI-like: values cycle 0.5, 0.8, 1.0, 2.5, 4.0 — median ~1 but WANDERS far from peg most days. + # Loose median-only filter wrongly admitted this (bogus −90% DD); the dispersion filter must EXCLUDE it. + _wander = [0.5, 0.8, 1.0, 2.5, 4.0] + _seed_spot(store, "SUSHIUSDT", {d: _wander[d % len(_wander)] for d in range(days)}) # Gold-proxy — median ~3000, must be EXCLUDED. _seed_spot(store, "PAXGUSDT", {d: 3000.0 + d for d in range(days)}) # Dead peg — median ~0.02, must be EXCLUDED. @@ -78,10 +85,61 @@ def test_panel_selects_only_near_one_stablecoins() -> None: assert all(isinstance(v, dict) and v for v in panel.values()) +def test_panel_excludes_volatile_coin_averaging_one() -> None: + """REGRESSION for the bug: a SUSHI-like coin whose MEDIAN is ~$1 but which WANDERS $0.5–$5 must NOT be + treated as a stablecoin — its median passes the old loose filter but it spends most days far from peg, so + the frac-in-band dispersion filter excludes it. (Admitting it caused a bogus −90% DD.)""" + store = TimescaleFeatureStore("sqlite://", table="bybit_features") + _seed_mixed_universe(store) + panel = bybit_stable_spot_panel(store) + store.close() + + assert "SUSHIUSDT" not in panel + assert set(panel) == {"USDCUSDT", "USDEUSDT"} + + +def test_panel_frac_in_band_boundary() -> None: + """A coin near-$1 for MOST days but spiking occasionally: frac_in_band just under 0.90 → excluded; + just over 0.90 → included (boundary around STABLE_MIN_FRAC_IN_BAND).""" + from fxhnt.application.bybit_stablecoin_eval import STABLE_MIN_FRAC_IN_BAND + + assert STABLE_MIN_FRAC_IN_BAND == 0.90 + + # 100 days: 11 spike days (off-peg) → 89% in-band → just UNDER 0.90 → excluded. + under = {d: (2.0 if d < 11 else 1.0) for d in range(100)} + # 100 days: 10 spike days → 90% in-band → exactly at threshold → included. + over = {d: (2.0 if d < 10 else 1.0) for d in range(100)} + + store = TimescaleFeatureStore("sqlite://", table="bybit_features") + _seed_spot(store, "UNDERUSDT", under) + _seed_spot(store, "OVERUSDT", over) + panel = bybit_stable_spot_panel(store) + store.close() + + assert "UNDERUSDT" not in panel # 0.89 < 0.90 + assert "OVERUSDT" in panel # 0.90 >= 0.90 + + +def test_panel_excludes_short_history_coin() -> None: + """Min-history: a brand-new coin with only ~5 near-$1 prints must be EXCLUDED (n < STABLE_MIN_HISTORY), + even though every print is exactly at peg.""" + from fxhnt.application.bybit_stablecoin_eval import STABLE_MIN_HISTORY + + store = TimescaleFeatureStore("sqlite://", table="bybit_features") + _seed_spot(store, "NEWUSDT", {d: 1.0 for d in range(5)}) + # A real stablecoin with ample history, for contrast. + _seed_spot(store, "USDCUSDT", {d: 1.0 for d in range(STABLE_MIN_HISTORY + 5)}) + panel = bybit_stable_spot_panel(store) + store.close() + + assert "NEWUSDT" not in panel + assert "USDCUSDT" in panel + + def test_panel_empty_when_no_near_one_symbol() -> None: store = TimescaleFeatureStore("sqlite://", table="bybit_features") - _seed_spot(store, "ETHUSDT", {d: 100.0 + d for d in range(40)}) - _seed_spot(store, "PAXGUSDT", {d: 3000.0 for d in range(40)}) + _seed_spot(store, "ETHUSDT", {d: 100.0 + d for d in range(80)}) + _seed_spot(store, "PAXGUSDT", {d: 3000.0 for d in range(80)}) panel = bybit_stable_spot_panel(store) store.close()