258 lines
11 KiB
Python
258 lines
11 KiB
Python
import datetime as dt
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from fxhnt.adapters.warehouse.duckdb_feature_store import DuckDbFeatureStore
|
|
from fxhnt.application.equity_backtest_runner import (
|
|
BacktestRunResult,
|
|
EquityBacktestRunner,
|
|
evaluate_constructions,
|
|
select_backtest_universe,
|
|
)
|
|
from fxhnt.domain.equity_backtest import month_end_rebalance_dates
|
|
|
|
_SPD = 86_400
|
|
|
|
|
|
def _ts(iso: str) -> int:
|
|
d = dt.date.fromisoformat(iso)
|
|
return (d - dt.date(1970, 1, 1)).days * _SPD
|
|
|
|
|
|
def _build_warehouse(path: str) -> DuckDbFeatureStore:
|
|
"""3 names over ~2 years of daily bars. AAA strong uptrend, BBB mild,
|
|
CCC downtrend — gives a non-degenerate cross-section for long/ls/tilt."""
|
|
store = DuckDbFeatureStore(path)
|
|
start = dt.date(2022, 1, 3)
|
|
n = 520 # ~2 trading years
|
|
drifts = {"AAA": 0.0015, "BBB": 0.0003, "CCC": -0.0010}
|
|
items = []
|
|
members = []
|
|
for sym, dr in drifts.items():
|
|
rows = []
|
|
px = 100.0
|
|
for i in range(n):
|
|
day = start + dt.timedelta(days=i)
|
|
px *= (1.0 + dr)
|
|
ts = (day - dt.date(1970, 1, 1)).days * _SPD
|
|
rows.append((ts, {"adjclose": px, "close": px, "volume": 1_000_000.0}))
|
|
items.append((sym, rows))
|
|
members.append((sym, "NYSE", "2000-01-01", "2026-06-01"))
|
|
store.write_features_bulk(items)
|
|
store.upsert_membership(members)
|
|
return store
|
|
|
|
|
|
def test_runner_produces_three_construction_series(tmp_path):
|
|
store = _build_warehouse(str(tmp_path / "wh.duckdb"))
|
|
runner = EquityBacktestRunner(store, n=10, cost_bps_per_turnover=15.0,
|
|
borrow_annual=0.0, momentum_min_history=252)
|
|
result = runner.run()
|
|
store.close()
|
|
assert set(result.returns_by_construction) == {"long", "ls", "tilt"}
|
|
for series in result.returns_by_construction.values():
|
|
assert len(series) > 50
|
|
assert np.isfinite(series).all()
|
|
assert float(np.prod(1.0 + result.returns_by_construction["long"]) - 1.0) > 0.0
|
|
|
|
# rebalance_dates lists only ACTUALLY-TRADED periods (len(kept) >= 2),
|
|
# never the 252d-warmup months where no book could be formed.
|
|
month_ends = month_end_rebalance_dates(
|
|
sorted({
|
|
(dt.date(2022, 1, 3) + dt.timedelta(days=i)).isoformat()
|
|
for i in range(520)
|
|
})
|
|
)
|
|
assert len(result.rebalance_dates) >= 1
|
|
assert all(d in month_ends for d in result.rebalance_dates)
|
|
# Warmup (252 trading-day history requirement) means fewer traded periods
|
|
# than the total monthly rebalance count.
|
|
assert len(result.rebalance_dates) < len(month_ends)
|
|
# All 3 synthetic names qualify once warmed up, so the average over TRADED
|
|
# periods is ~3.0 (not ~1.67, which would average in the empty warmup months).
|
|
assert abs(result.n_names_avg - 3.0) < 1e-9
|
|
|
|
|
|
def test_runner_is_deterministic(tmp_path):
|
|
store = _build_warehouse(str(tmp_path / "wh.duckdb"))
|
|
runner = EquityBacktestRunner(store, n=10, cost_bps_per_turnover=15.0,
|
|
borrow_annual=0.0, momentum_min_history=252)
|
|
a = runner.run().returns_by_construction["ls"]
|
|
b = runner.run().returns_by_construction["ls"]
|
|
store.close()
|
|
assert np.array_equal(a, b)
|
|
|
|
|
|
def test_momentum_min_history_gates_inclusion(tmp_path):
|
|
# With an absurdly high min-history requirement, NO name ever has enough
|
|
# history, so every construction series is empty and n_names_avg is 0.
|
|
store = _build_warehouse(str(tmp_path / "wh.duckdb"))
|
|
runner = EquityBacktestRunner(store, n=10, cost_bps_per_turnover=15.0,
|
|
borrow_annual=0.0, momentum_min_history=100_000)
|
|
result = runner.run()
|
|
store.close()
|
|
assert result.n_names_avg == 0.0
|
|
for series in result.returns_by_construction.values():
|
|
assert len(series) == 0
|
|
|
|
|
|
def test_evaluate_constructions_emits_stats_and_verdict_per_construction(tmp_path):
|
|
store = _build_warehouse(str(tmp_path / "wh.duckdb"))
|
|
runner = EquityBacktestRunner(store, n=10, cost_bps_per_turnover=15.0,
|
|
borrow_annual=0.0, momentum_min_history=252)
|
|
result = runner.run()
|
|
store.close()
|
|
report = evaluate_constructions(result, oos_fraction=0.40, dsr_min=0.95,
|
|
oos_min_sharpe=0.0, max_is_oos_decay=0.50)
|
|
assert set(report["constructions"]) == {"long", "ls", "tilt"}
|
|
for c, block in report["constructions"].items():
|
|
assert {"stats", "verdict"} <= set(block)
|
|
assert {"sharpe", "cagr", "max_drawdown", "n_obs"} <= set(block["stats"])
|
|
assert {"passed", "dsr", "is_sharpe", "oos_sharpe", "n_trials", "reasons"} <= set(block["verdict"])
|
|
assert block["verdict"]["n_trials"] == 3
|
|
assert report["n_names_avg"] >= 0.0
|
|
assert isinstance(report["rebalance_dates"], list)
|
|
import json
|
|
json.dumps(report) # must not raise
|
|
|
|
|
|
def test_evaluate_constructions_short_series_emits_clean_insufficient_data_verdict():
|
|
# FIX #2: a construction with len < 3 must NOT call evaluate() (which would
|
|
# return dsr=NaN and break json.dumps). Instead emit a clean fallback verdict.
|
|
result = BacktestRunResult(
|
|
returns_by_construction={c: np.array([0.01, 0.02]) for c in ("long", "ls", "tilt")},
|
|
rebalance_dates=[],
|
|
n_names_avg=0.0,
|
|
)
|
|
report = evaluate_constructions(result)
|
|
assert set(report["constructions"]) == {"long", "ls", "tilt"}
|
|
for c, block in report["constructions"].items():
|
|
v = block["verdict"]
|
|
assert v["passed"] is False
|
|
assert v["reasons"] == ["insufficient data"]
|
|
assert v["dsr"] == 0.0
|
|
assert v["pvalue"] == 1.0
|
|
assert v["n_trials"] == 3
|
|
# fallback verdict must have the SAME keys as Verdict.model_dump()
|
|
assert set(v) == {"passed", "dsr", "is_sharpe", "oos_sharpe",
|
|
"n_trials", "reasons", "pvalue"}
|
|
import json
|
|
json.dumps(report) # must not raise (no NaN)
|
|
|
|
|
|
def _build_liquidity_warehouse(path: str) -> DuckDbFeatureStore:
|
|
"""5 symbols with deliberately differing history-length and dollar-volume so
|
|
the peak-yearly-liquidity pre-filter can be exercised:
|
|
|
|
- LIVELONG : long history, HIGH dollar-volume throughout (top liquidity)
|
|
- DELISTED : SHORT lifetime (well above min_history though) but VERY HIGH
|
|
dollar-volume while alive — must be INCLUDED (survivorship-safe;
|
|
peak-yearly ranking sees its liquid alive years)
|
|
- ILLIQUID : long history but tiny dollar-volume throughout — ranks low,
|
|
dropped by a small top_k
|
|
- RECENTIPO : short-but-sufficient history, moderate dollar-volume
|
|
- TOOSHORT : fewer than min_history bars — must be EXCLUDED
|
|
"""
|
|
store = DuckDbFeatureStore(path)
|
|
|
|
def write(sym: str, start_iso: str, n_bars: int, close: float, vol: float) -> None:
|
|
start = dt.date.fromisoformat(start_iso)
|
|
rows = []
|
|
for i in range(n_bars):
|
|
day = start + dt.timedelta(days=i)
|
|
ts = (day - dt.date(1970, 1, 1)).days * _SPD
|
|
rows.append((ts, {"adjclose": close, "close": close, "volume": vol}))
|
|
store.write_features(sym, rows)
|
|
store.upsert_membership([(sym, "NYSE", start_iso,
|
|
(start + dt.timedelta(days=n_bars)).isoformat())])
|
|
|
|
write("LIVELONG", "2010-01-01", 3000, close=100.0, vol=5_000_000.0) # dv = 5e8
|
|
write("DELISTED", "2012-01-01", 600, close=200.0, vol=8_000_000.0) # dv = 1.6e9 (highest)
|
|
write("ILLIQUID", "2010-01-01", 3000, close=10.0, vol=1_000.0) # dv = 1e4 (lowest)
|
|
write("RECENTIPO", "2023-01-01", 500, close=50.0, vol=2_000_000.0) # dv = 1e8
|
|
write("TOOSHORT", "2024-01-01", 50, close=100.0, vol=9_000_000.0) # < min_history
|
|
return store
|
|
|
|
|
|
def test_select_backtest_universe_filters_and_ranks(tmp_path):
|
|
store = _build_liquidity_warehouse(str(tmp_path / "liq.duckdb"))
|
|
liq = select_backtest_universe(store, min_history=300, top_k=2500)
|
|
store.close()
|
|
|
|
# dict[str, float] with positive values
|
|
assert isinstance(liq, dict)
|
|
assert all(isinstance(s, str) for s in liq)
|
|
assert all(isinstance(v, float) and v > 0.0 for v in liq.values())
|
|
|
|
# TOOSHORT (< min_history bars) is excluded
|
|
assert "TOOSHORT" not in liq
|
|
# DELISTED short-but-liquid name is INCLUDED (survivorship-safe)
|
|
assert "DELISTED" in liq
|
|
# long-history liquid name present
|
|
assert "LIVELONG" in liq
|
|
|
|
# ranking by peak-yearly dollar-volume: DELISTED (1.6e9) > LIVELONG (5e8) > ILLIQUID (1e4)
|
|
assert liq["DELISTED"] > liq["LIVELONG"] > liq["ILLIQUID"]
|
|
|
|
|
|
def test_select_backtest_universe_top_k_drops_illiquid_tail(tmp_path):
|
|
store = _build_liquidity_warehouse(str(tmp_path / "liq.duckdb"))
|
|
# top_k=2 keeps only the two most-liquid (DELISTED, LIVELONG); ILLIQUID dropped
|
|
liq = select_backtest_universe(store, min_history=300, top_k=2)
|
|
store.close()
|
|
assert len(liq) == 2
|
|
assert set(liq) == {"DELISTED", "LIVELONG"}
|
|
assert "ILLIQUID" not in liq
|
|
|
|
|
|
def test_runner_candidate_path_matches_membership_gating(tmp_path):
|
|
store = _build_warehouse(str(tmp_path / "wh.duckdb"))
|
|
runner = EquityBacktestRunner(
|
|
store, n=10, candidate_top_k=100, candidate_min_history=252,
|
|
)
|
|
result = runner.run()
|
|
store.close()
|
|
# bounded path works end-to-end: 3 construction series, finite, non-trivial
|
|
assert set(result.returns_by_construction) == {"long", "ls", "tilt"}
|
|
for series in result.returns_by_construction.values():
|
|
assert len(series) > 50
|
|
assert np.isfinite(series).all()
|
|
assert result.n_names_avg > 0.0
|
|
|
|
|
|
def test_evaluate_constructions_sr_variance_is_cross_construction_computed_once(monkeypatch):
|
|
# FIX #1: sr_variance must be the variance of the per-construction IS Sharpes
|
|
# ACROSS the constructions, computed ONCE and shared by every evaluate() call —
|
|
# NOT the per-construction (1/len(r)) value computed inside the loop.
|
|
import fxhnt.application.equity_backtest_runner as runner_mod
|
|
|
|
rng = np.random.default_rng(0)
|
|
series = {
|
|
"long": rng.normal(0.001, 0.01, 300),
|
|
"ls": rng.normal(0.0, 0.02, 300),
|
|
"tilt": rng.normal(0.0005, 0.015, 300),
|
|
}
|
|
result = BacktestRunResult(
|
|
returns_by_construction=series, rebalance_dates=[], n_names_avg=0.0,
|
|
)
|
|
|
|
seen_sr_variance: list[float] = []
|
|
real_evaluate = runner_mod.evaluate
|
|
|
|
def spy_evaluate(*args, **kwargs):
|
|
seen_sr_variance.append(kwargs["sr_variance"])
|
|
return real_evaluate(*args, **kwargs)
|
|
|
|
monkeypatch.setattr(runner_mod, "evaluate", spy_evaluate)
|
|
evaluate_constructions(result, oos_fraction=0.40)
|
|
|
|
# one evaluate() call per construction, all sharing the SAME sr_variance value
|
|
assert len(seen_sr_variance) == 3
|
|
assert len(set(seen_sr_variance)) == 1
|
|
# and that single value equals the cross-construction variance of IS Sharpes
|
|
is_sharpes = [
|
|
runner_mod.per_period_sharpe(r[: int(0.60 * len(r))]) for r in series.values()
|
|
]
|
|
assert seen_sr_variance[0] == pytest.approx(float(np.var(is_sharpes)))
|