350 lines
16 KiB
Python
350 lines
16 KiB
Python
"""OOS / WALK-FORWARD validation harness for the Bybit POSITIONING edge — the DEPLOY GATE.
|
|
|
|
The in-sample adversarial verify (`verify_positioning_edge`) already passed (Sharpe ~3.1@5.5bp, positive
|
|
every year, top-5 share ~46%, uncorrelated with the 3 edges). This suite tests the OOS harness
|
|
(`walk_forward_positioning`) that must clear BEFORE the edge deploys. Because the signal is
|
|
near-parameterless (cross-sectional demean of buyRatio, contrarian sign), OOS validation is about:
|
|
|
|
1. CONSISTENCY across held-out time -> rolling non-overlapping windows, fraction positive;
|
|
2. the DIRECTION CHOICE survives a real train/test split (we picked "contrarian" AFTER seeing the
|
|
full sample, so re-pick it on TRAIN only and apply it to never-looked-at TEST);
|
|
3. ROBUSTNESS to construction choices -> drop-top-N contributors, liquidity sweep, signal variant;
|
|
4. STRICT CAUSALITY -> a look-ahead audit (buyRatio at d -> d->d+1 return only).
|
|
|
|
All fixtures are in-memory (sqlite), NO network, READ-ONLY (a write-tripwire never trips), memory-bounded.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
|
|
from fxhnt.adapters.warehouse.timescale_feature_store import TimescaleFeatureStore
|
|
from fxhnt.application.bybit_positioning_eval import (
|
|
look_ahead_audit,
|
|
walk_forward_positioning,
|
|
)
|
|
|
|
_DAY = 86_400
|
|
_START = 19_358 # 2023-01-01 (epoch day) — the live edge spans 2023-26
|
|
|
|
|
|
class _WriteTripwireStore:
|
|
"""Read-only proxy: forwards reads, but ANY write/persist call trips an assertion."""
|
|
|
|
_FORBIDDEN = frozenset({
|
|
"write_features", "write_features_bulk", "upsert_feature_rows", "upsert_membership",
|
|
"replace_positions", "upsert_nav", "upsert_sleeve_ret", "replace_shadow_positions",
|
|
"replace_trades", "_create_schema", "_bulk_upsert",
|
|
})
|
|
|
|
def __init__(self, inner: TimescaleFeatureStore) -> None:
|
|
object.__setattr__(self, "_inner", inner)
|
|
|
|
def __getattr__(self, name: str):
|
|
if name in _WriteTripwireStore._FORBIDDEN:
|
|
raise AssertionError(f"READ-ONLY violation: harness called write method {name!r}")
|
|
return getattr(self._inner, name)
|
|
|
|
|
|
# --- fixtures ----------------------------------------------------------------------------------
|
|
|
|
def _seed_broad_contrarian(store: TimescaleFeatureStore, *, days: int = 720,
|
|
coins: tuple[str, ...] = ("AAAUSDT", "BBBUSDT", "CCCUSDT", "DDDUSDT",
|
|
"EEEUSDT", "FFFUSDT", "GGGUSDT", "HHHUSDT")) -> None:
|
|
"""A consistently-positive, BROAD contrarian edge across many coins for the FULL span: whichever coin
|
|
is over-LONG this day (high buyRatio) falls next day (crowd wrong -> SHORT earns). Rotate the over-long
|
|
coin so no single coin dominates the attribution. Spans ~2 years for rolling windows + per-year."""
|
|
px = {c: 100.0 for c in coins}
|
|
for i in range(days):
|
|
d = _START + i
|
|
over_long = coins[i % len(coins)]
|
|
for c in coins:
|
|
r = 0.70 if c == over_long else 0.40
|
|
store.write_features(c, [(d * _DAY, {"long_ratio": r, "close": px[c]})])
|
|
for c in coins:
|
|
# over-long coin reverts DOWN next day; the rest drift UP slightly (contrarian pays broadly).
|
|
px[c] *= (1.0 - 0.006) if c == over_long else (1.0 + 0.001)
|
|
d = _START + days
|
|
for c in coins:
|
|
store.write_features(c, [(d * _DAY, {"long_ratio": 0.5, "close": px[c]})])
|
|
|
|
|
|
def _seed_regime_dependent(store: TimescaleFeatureStore, *, days: int = 720) -> None:
|
|
"""Contrarian pays ONLY in the FIRST ~1/7 of the span, then the relationship dies (price flat regardless
|
|
of positioning). A real edge is positive in most windows; this one is positive in ~1 window only, so the
|
|
rolling-window harness must report a LOW fraction-positive."""
|
|
coins = ("AAAUSDT", "BBBUSDT", "CCCUSDT", "DDDUSDT")
|
|
px = {c: 100.0 for c in coins}
|
|
live_until = days // 7
|
|
for i in range(days):
|
|
d = _START + i
|
|
over_long = coins[i % len(coins)]
|
|
for c in coins:
|
|
r = 0.70 if c == over_long else 0.40
|
|
store.write_features(c, [(d * _DAY, {"long_ratio": r, "close": px[c]})])
|
|
if i < live_until:
|
|
for c in coins:
|
|
px[c] *= (1.0 - 0.008) if c == over_long else (1.0 + 0.0026)
|
|
# else: prices unchanged — positioning carries no information (dead regime).
|
|
d = _START + days
|
|
for c in coins:
|
|
store.write_features(c, [(d * _DAY, {"long_ratio": 0.5, "close": px[c]})])
|
|
|
|
|
|
def _seed_train_only(store: TimescaleFeatureStore, *, days: int = 600) -> None:
|
|
"""Contrarian pays in TRAIN (first 60%) but the sign FLIPS in TEST (last 40%): the over-long coin RISES
|
|
in the test era (momentum, not reversion). Train picks 'contrarian'; applied to TEST it LOSES. The
|
|
train/holdout test must catch this overfit (TEST Sharpe < 0)."""
|
|
coins = ("AAAUSDT", "BBBUSDT", "CCCUSDT", "DDDUSDT")
|
|
px = {c: 100.0 for c in coins}
|
|
split = int(days * 0.6)
|
|
for i in range(days):
|
|
d = _START + i
|
|
over_long = coins[i % len(coins)]
|
|
for c in coins:
|
|
r = 0.70 if c == over_long else 0.40
|
|
store.write_features(c, [(d * _DAY, {"long_ratio": r, "close": px[c]})])
|
|
if i < split:
|
|
# TRAIN: contrarian pays (over-long reverts down).
|
|
for c in coins:
|
|
px[c] *= (1.0 - 0.006) if c == over_long else (1.0 + 0.002)
|
|
else:
|
|
# TEST: sign flips (over-long continues UP) -> contrarian loses there.
|
|
for c in coins:
|
|
px[c] *= (1.0 + 0.006) if c == over_long else (1.0 - 0.002)
|
|
d = _START + days
|
|
for c in coins:
|
|
store.write_features(c, [(d * _DAY, {"long_ratio": 0.5, "close": px[c]})])
|
|
|
|
|
|
def _seed_one_coin(store: TimescaleFeatureStore, *, days: int = 400) -> None:
|
|
"""The contrarian edge lives in ONE coin only: AAA reverts (tradeable), the rest are pure noise around
|
|
flat. Dropping the top contributor should KILL it (concentration-driven / fragile)."""
|
|
coins = ("AAAUSDT", "BBBUSDT", "CCCUSDT", "DDDUSDT")
|
|
px = {c: 100.0 for c in coins}
|
|
for i in range(days):
|
|
d = _START + i
|
|
# AAA oscillates over-long/over-short with a clean reversion; others sit at mid (no signal).
|
|
aaa_long = (i % 2 == 0)
|
|
store.write_features("AAAUSDT",
|
|
[(d * _DAY, {"long_ratio": 0.75 if aaa_long else 0.25, "close": px["AAAUSDT"]})])
|
|
for c in ("BBBUSDT", "CCCUSDT", "DDDUSDT"):
|
|
store.write_features(c, [(d * _DAY, {"long_ratio": 0.5, "close": px[c]})])
|
|
px["AAAUSDT"] *= (1.0 - 0.01) if aaa_long else (1.0 + 0.01) # clean reversion on AAA only
|
|
# the others wander with a tiny deterministic zig-zag that nets ~0 and carries no positioning signal
|
|
for c in ("BBBUSDT", "CCCUSDT", "DDDUSDT"):
|
|
px[c] *= (1.0 + 0.0005) if (i % 2 == 0) else (1.0 - 0.0005)
|
|
d = _START + days
|
|
for c in coins:
|
|
store.write_features(c, [(d * _DAY, {"long_ratio": 0.5, "close": px[c]})])
|
|
|
|
|
|
# --- 1. rolling non-overlapping windows --------------------------------------------------------
|
|
|
|
def test_rolling_windows_broad_edge_mostly_positive() -> None:
|
|
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
|
_seed_broad_contrarian(store)
|
|
rep = walk_forward_positioning(store, universe=None, cost_bps=5.5, window_days=180)
|
|
store.close()
|
|
rw = rep["rolling_windows"]
|
|
assert rw["n_windows"] >= 3
|
|
assert rw["fraction_positive"] > 0.7 # a real edge is positive in most windows
|
|
assert len(rw["windows"]) == rw["n_windows"]
|
|
for w in rw["windows"]:
|
|
assert {"start", "end", "sharpe", "total_return", "days"} <= set(w)
|
|
|
|
|
|
def test_rolling_windows_regime_dependent_low_fraction() -> None:
|
|
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
|
_seed_regime_dependent(store)
|
|
rep = walk_forward_positioning(store, universe=None, cost_bps=5.5, window_days=180)
|
|
store.close()
|
|
rw = rep["rolling_windows"]
|
|
assert rw["n_windows"] >= 3
|
|
# only the first window has the edge -> the harness must flag regime-dependence (well below 70%).
|
|
assert rw["fraction_positive"] <= 0.5
|
|
|
|
|
|
# --- 2. train / holdout DIRECTION test ---------------------------------------------------------
|
|
|
|
def test_train_holdout_direction_generalises_on_broad_edge() -> None:
|
|
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
|
_seed_broad_contrarian(store)
|
|
rep = walk_forward_positioning(store, universe=None, cost_bps=5.5)
|
|
store.close()
|
|
th = rep["train_holdout"]
|
|
assert th["train_direction"] == "contrarian" # train re-discovers the contrarian sign
|
|
assert th["train_sharpe"] > 0.0
|
|
assert th["test_sharpe"] > 0.0 # the train-chosen direction generalises OOS
|
|
|
|
|
|
def test_train_holdout_catches_overfit_when_sign_flips_in_test() -> None:
|
|
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
|
_seed_train_only(store)
|
|
rep = walk_forward_positioning(store, universe=None, cost_bps=5.5)
|
|
store.close()
|
|
th = rep["train_holdout"]
|
|
assert th["train_direction"] == "contrarian" # contrarian wins on TRAIN
|
|
assert th["train_sharpe"] > 0.0
|
|
assert th["test_sharpe"] < 0.0 # ...but loses OOS -> overfit caught
|
|
|
|
|
|
# --- 3a. drop-top-N robustness -----------------------------------------------------------------
|
|
|
|
def test_drop_top_n_broad_edge_survives() -> None:
|
|
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
|
_seed_broad_contrarian(store)
|
|
rep = walk_forward_positioning(store, universe=None, cost_bps=5.5)
|
|
store.close()
|
|
dt = rep["drop_top_n"]
|
|
assert dt["full_sharpe"] > 0.0
|
|
assert dt["drop5"]["sharpe"] > 0.0 # broad edge holds without its top-5 contributors
|
|
assert dt["drop5"]["n_dropped"] >= 1
|
|
# dropped names are identified on TRAIN (causal), not on the full/test sample.
|
|
assert isinstance(dt["drop5"]["dropped"], list)
|
|
|
|
|
|
def test_drop_top_n_one_coin_edge_collapses() -> None:
|
|
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
|
_seed_one_coin(store)
|
|
rep = walk_forward_positioning(store, universe=None, cost_bps=5.5)
|
|
store.close()
|
|
dt = rep["drop_top_n"]
|
|
assert dt["full_sharpe"] > 0.0
|
|
# remove the single load-bearing coin -> the edge collapses (Sharpe no longer clearly positive).
|
|
assert dt["drop5"]["sharpe"] < dt["full_sharpe"]
|
|
assert dt["drop5"]["sharpe"] < 0.5
|
|
|
|
|
|
# --- 3b. liquidity-threshold sweep -------------------------------------------------------------
|
|
|
|
def test_liquidity_sweep_returns_metric_per_threshold() -> None:
|
|
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
|
_seed_broad_contrarian(store)
|
|
# turnover panel so liquid_universe resolves (all coins comfortably liquid here).
|
|
for i in range(720):
|
|
d = _START + i
|
|
for c in ("AAAUSDT", "BBBUSDT", "CCCUSDT", "DDDUSDT",
|
|
"EEEUSDT", "FFFUSDT", "GGGUSDT", "HHHUSDT"):
|
|
store.write_features(c, [(d * _DAY, {"turnover": 50_000_000.0})])
|
|
rep = walk_forward_positioning(store, universe=None, cost_bps=5.5,
|
|
liquidity_grid=(5_000_000.0, 10_000_000.0, 25_000_000.0))
|
|
store.close()
|
|
ls = rep["liquidity_sweep"]
|
|
assert set(ls) == {5_000_000.0, 10_000_000.0, 25_000_000.0}
|
|
for _thr, m in ls.items():
|
|
assert "sharpe" in m and "n_coins" in m
|
|
assert math.isfinite(m["sharpe"])
|
|
assert m["n_coins"] >= 0
|
|
|
|
|
|
# --- 3c. signal-variant sweep ------------------------------------------------------------------
|
|
|
|
def test_signal_variant_sweep_reports_each_construction() -> None:
|
|
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
|
_seed_broad_contrarian(store)
|
|
rep = walk_forward_positioning(store, universe=None, cost_bps=5.5)
|
|
store.close()
|
|
sv = rep["signal_variants"]
|
|
assert set(sv) == {"demean", "zscore", "rank"}
|
|
for _name, m in sv.items():
|
|
assert "sharpe" in m and math.isfinite(m["sharpe"])
|
|
# all three constructions express the same contrarian reversion -> all positive on the broad fixture.
|
|
assert sv["demean"]["sharpe"] > 0.0
|
|
assert sv["zscore"]["sharpe"] > 0.0
|
|
assert sv["rank"]["sharpe"] > 0.0
|
|
|
|
|
|
# --- 4. look-ahead audit -----------------------------------------------------------------------
|
|
|
|
def test_look_ahead_audit_passes_on_causal_signal() -> None:
|
|
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
|
_seed_broad_contrarian(store, days=120)
|
|
rep = look_ahead_audit(store, universe=None)
|
|
store.close()
|
|
assert rep["causal"] is True
|
|
assert rep["leak_days"] == 0
|
|
|
|
|
|
def test_look_ahead_audit_catches_a_deliberately_leaked_mapping() -> None:
|
|
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
|
_seed_broad_contrarian(store, days=120)
|
|
rep = look_ahead_audit(store, universe=None, _leak=True) # use the SAME-day return (leak)
|
|
store.close()
|
|
assert rep["causal"] is False
|
|
assert rep["leak_days"] > 0
|
|
|
|
|
|
def test_walk_forward_includes_look_ahead_audit() -> None:
|
|
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
|
_seed_broad_contrarian(store)
|
|
rep = walk_forward_positioning(store, universe=None, cost_bps=5.5)
|
|
store.close()
|
|
assert rep["look_ahead"]["causal"] is True
|
|
|
|
|
|
# --- 5. PASS/FAIL verdict ----------------------------------------------------------------------
|
|
|
|
def test_verdict_pass_on_broad_generalising_edge() -> None:
|
|
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
|
_seed_broad_contrarian(store)
|
|
for i in range(720):
|
|
d = _START + i
|
|
for c in ("AAAUSDT", "BBBUSDT", "CCCUSDT", "DDDUSDT",
|
|
"EEEUSDT", "FFFUSDT", "GGGUSDT", "HHHUSDT"):
|
|
store.write_features(c, [(d * _DAY, {"turnover": 50_000_000.0})])
|
|
rep = walk_forward_positioning(store, universe=None, cost_bps=5.5, window_days=180,
|
|
liquidity_grid=(5_000_000.0, 10_000_000.0, 25_000_000.0))
|
|
store.close()
|
|
v = rep["verdict"]
|
|
assert v["deployable"] is True
|
|
assert v["checks"]["rolling_windows_positive"] is True
|
|
assert v["checks"]["oos_direction_positive"] is True
|
|
assert v["checks"]["survives_drop_top5"] is True
|
|
assert v["checks"]["liquidity_robust"] is True
|
|
|
|
|
|
def test_verdict_fail_on_overfit_edge() -> None:
|
|
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
|
_seed_train_only(store)
|
|
rep = walk_forward_positioning(store, universe=None, cost_bps=5.5)
|
|
store.close()
|
|
v = rep["verdict"]
|
|
assert v["deployable"] is False
|
|
assert v["checks"]["oos_direction_positive"] is False
|
|
|
|
|
|
# --- 6. read-only ------------------------------------------------------------------------------
|
|
|
|
def test_walk_forward_is_read_only() -> None:
|
|
inner = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
|
_seed_broad_contrarian(inner)
|
|
store = _WriteTripwireStore(inner)
|
|
rep = walk_forward_positioning(store, universe=None, cost_bps=5.5) # must not trip the tripwire
|
|
inner.close()
|
|
assert rep["rolling_windows"]["n_windows"] >= 1
|
|
|
|
|
|
# --- 7. CLI ------------------------------------------------------------------------------------
|
|
|
|
def test_cli_walk_forward_prints_verdict(monkeypatch) -> None:
|
|
from typer.testing import CliRunner
|
|
|
|
import fxhnt.cli as cli
|
|
|
|
store = TimescaleFeatureStore("sqlite://", table="bybit_features")
|
|
_seed_broad_contrarian(store)
|
|
|
|
monkeypatch.setattr(
|
|
"fxhnt.adapters.warehouse.timescale_feature_store.TimescaleFeatureStore",
|
|
lambda *_a, **_k: store, raising=False)
|
|
|
|
result = CliRunner().invoke(cli.app, ["bybit-positioning-eval", "--all", "--walk-forward"])
|
|
store.close()
|
|
assert result.exit_code == 0, result.output
|
|
out = result.output.lower()
|
|
assert "walk-forward" in out or "walk forward" in out
|
|
assert "rolling" in out
|
|
assert "oos" in out or "holdout" in out
|
|
assert "verdict" in out
|
|
assert "pass" in out or "fail" in out
|