fix(b3b): cross-construction sr_variance for DSR; guard short series (no NaN JSON); epoch_day public
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -13,9 +13,9 @@ import numpy as np
|
||||
|
||||
from fxhnt.adapters.warehouse.duckdb_feature_store import DuckDbFeatureStore
|
||||
from fxhnt.domain.backtest import compute_stats
|
||||
from fxhnt.domain.gauntlet.core import evaluate
|
||||
from fxhnt.domain.gauntlet.core import evaluate, per_period_sharpe
|
||||
from fxhnt.domain.equity_backtest import (
|
||||
_epoch_day,
|
||||
epoch_day,
|
||||
daily_returns_for_weights,
|
||||
month_end_rebalance_dates,
|
||||
universe_asof,
|
||||
@@ -79,7 +79,7 @@ class EquityBacktestRunner:
|
||||
traded_dates: list[str] = []
|
||||
|
||||
for k, rebal in enumerate(rebals):
|
||||
rebal_day = _epoch_day(rebal)
|
||||
rebal_day = epoch_day(rebal)
|
||||
lo = rebal_day - _DOLLAR_VOL_WINDOW_DAYS
|
||||
dollar_vol: dict[str, float] = {}
|
||||
for s in syms:
|
||||
@@ -111,7 +111,7 @@ class EquityBacktestRunner:
|
||||
|
||||
scores = composite_price(momentum_score(mom), lowvol_score(vols))
|
||||
|
||||
end_day = _epoch_day(rebals[k + 1]) if k + 1 < len(rebals) else all_days[-1]
|
||||
end_day = epoch_day(rebals[k + 1]) if k + 1 < len(rebals) else all_days[-1]
|
||||
hold_days = [d for d in all_days if rebal_day <= d <= end_day]
|
||||
|
||||
for c in _CONSTRUCTIONS:
|
||||
@@ -152,11 +152,29 @@ def evaluate_constructions(
|
||||
"n_trials": n_trials,
|
||||
},
|
||||
}
|
||||
# DSR's sr_variance is the variance of the per-construction IS Sharpe estimates ACROSS the
|
||||
# trials (the constructions) — the cross-trial Sharpe dispersion, computed ONCE and shared by
|
||||
# every evaluate() call. Mirrors fxhnt.application.discovery.search (per_period_sharpe over the
|
||||
# IS slice; np.var when ≥2 trials else 0.0). per_period_sharpe is NaN-safe (0.0 for len<3).
|
||||
is_sharpes = [
|
||||
per_period_sharpe(r[: int((1.0 - oos_fraction) * len(r))])
|
||||
for r in result.returns_by_construction.values()
|
||||
if len(r[: int((1.0 - oos_fraction) * len(r))]) >= 3
|
||||
]
|
||||
sr_variance = float(np.var(is_sharpes)) if len(is_sharpes) > 1 else 0.0
|
||||
|
||||
for c, r in result.returns_by_construction.items():
|
||||
stats = compute_stats(r)
|
||||
if len(r) < 3:
|
||||
out["constructions"][c] = {
|
||||
"stats": stats.model_dump(),
|
||||
"verdict": {"passed": False, "dsr": 0.0, "is_sharpe": 0.0,
|
||||
"oos_sharpe": 0.0, "n_trials": n_trials,
|
||||
"reasons": ["insufficient data"], "pvalue": 1.0},
|
||||
}
|
||||
continue
|
||||
split = int((1.0 - oos_fraction) * len(r))
|
||||
is_r, oos_r = r[:split], r[split:]
|
||||
sr_variance = (1.0 / len(r)) if len(r) > 1 else 0.0
|
||||
verdict = evaluate(
|
||||
is_r, oos_r, n_trials=n_trials, sr_variance=sr_variance,
|
||||
dsr_min=dsr_min, oos_min_sharpe=oos_min_sharpe,
|
||||
|
||||
@@ -9,7 +9,7 @@ from fxhnt.domain.strategies.equity_factor import book_return
|
||||
_SECONDS_PER_DAY = 86_400
|
||||
|
||||
|
||||
def _epoch_day(iso_date: str) -> int:
|
||||
def epoch_day(iso_date: str) -> int:
|
||||
"""ISO 'YYYY-MM-DD' -> epoch-day (matches DuckDbFeatureStore.read_panel keys)."""
|
||||
d = dt.date.fromisoformat(iso_date)
|
||||
return (d - dt.date(1970, 1, 1)).days
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import datetime as dt
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from fxhnt.adapters.warehouse.duckdb_feature_store import DuckDbFeatureStore
|
||||
from fxhnt.application.equity_backtest_runner import (
|
||||
BacktestRunResult,
|
||||
EquityBacktestRunner,
|
||||
evaluate_constructions,
|
||||
)
|
||||
@@ -112,3 +114,63 @@ def test_evaluate_constructions_emits_stats_and_verdict_per_construction(tmp_pat
|
||||
assert isinstance(report["rebalance_dates"], list)
|
||||
import json
|
||||
json.dumps(report) # must not raise
|
||||
|
||||
|
||||
def test_evaluate_constructions_short_series_emits_clean_insufficient_data_verdict():
|
||||
# FIX #2: a construction with len < 3 must NOT call evaluate() (which would
|
||||
# return dsr=NaN and break json.dumps). Instead emit a clean fallback verdict.
|
||||
result = BacktestRunResult(
|
||||
returns_by_construction={c: np.array([0.01, 0.02]) for c in ("long", "ls", "tilt")},
|
||||
rebalance_dates=[],
|
||||
n_names_avg=0.0,
|
||||
)
|
||||
report = evaluate_constructions(result)
|
||||
assert set(report["constructions"]) == {"long", "ls", "tilt"}
|
||||
for c, block in report["constructions"].items():
|
||||
v = block["verdict"]
|
||||
assert v["passed"] is False
|
||||
assert v["reasons"] == ["insufficient data"]
|
||||
assert v["dsr"] == 0.0
|
||||
assert v["pvalue"] == 1.0
|
||||
assert v["n_trials"] == 3
|
||||
# fallback verdict must have the SAME keys as Verdict.model_dump()
|
||||
assert set(v) == {"passed", "dsr", "is_sharpe", "oos_sharpe",
|
||||
"n_trials", "reasons", "pvalue"}
|
||||
import json
|
||||
json.dumps(report) # must not raise (no NaN)
|
||||
|
||||
|
||||
def test_evaluate_constructions_sr_variance_is_cross_construction_computed_once(monkeypatch):
|
||||
# FIX #1: sr_variance must be the variance of the per-construction IS Sharpes
|
||||
# ACROSS the constructions, computed ONCE and shared by every evaluate() call —
|
||||
# NOT the per-construction (1/len(r)) value computed inside the loop.
|
||||
import fxhnt.application.equity_backtest_runner as runner_mod
|
||||
|
||||
rng = np.random.default_rng(0)
|
||||
series = {
|
||||
"long": rng.normal(0.001, 0.01, 300),
|
||||
"ls": rng.normal(0.0, 0.02, 300),
|
||||
"tilt": rng.normal(0.0005, 0.015, 300),
|
||||
}
|
||||
result = BacktestRunResult(
|
||||
returns_by_construction=series, rebalance_dates=[], n_names_avg=0.0,
|
||||
)
|
||||
|
||||
seen_sr_variance: list[float] = []
|
||||
real_evaluate = runner_mod.evaluate
|
||||
|
||||
def spy_evaluate(*args, **kwargs):
|
||||
seen_sr_variance.append(kwargs["sr_variance"])
|
||||
return real_evaluate(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(runner_mod, "evaluate", spy_evaluate)
|
||||
evaluate_constructions(result, oos_fraction=0.40)
|
||||
|
||||
# one evaluate() call per construction, all sharing the SAME sr_variance value
|
||||
assert len(seen_sr_variance) == 3
|
||||
assert len(set(seen_sr_variance)) == 1
|
||||
# and that single value equals the cross-construction variance of IS Sharpes
|
||||
is_sharpes = [
|
||||
runner_mod.per_period_sharpe(r[: int(0.60 * len(r))]) for r in series.values()
|
||||
]
|
||||
assert seen_sr_variance[0] == pytest.approx(float(np.var(is_sharpes)))
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from fxhnt.domain.equity_backtest import (
|
||||
universe_asof,
|
||||
month_end_rebalance_dates,
|
||||
_epoch_day,
|
||||
epoch_day,
|
||||
)
|
||||
|
||||
MEMBERS = [
|
||||
@@ -38,8 +38,8 @@ def test_month_end_rebalance_dates_one_per_month():
|
||||
|
||||
|
||||
def test_epoch_day_roundtrip():
|
||||
assert _epoch_day("1970-01-01") == 0
|
||||
assert _epoch_day("1970-01-02") == 1
|
||||
assert epoch_day("1970-01-01") == 0
|
||||
assert epoch_day("1970-01-02") == 1
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
Reference in New Issue
Block a user