From c7602dbf66aa1934d051560e3445bc59be1ff456 Mon Sep 17 00:00:00 2001 From: jgrusewski Date: Wed, 17 Jun 2026 12:53:38 +0200 Subject: [PATCH] fix(b3b): cross-construction sr_variance for DSR; guard short series (no NaN JSON); epoch_day public Co-Authored-By: Claude Opus 4.8 (1M context) --- .../application/equity_backtest_runner.py | 28 +++++++-- src/fxhnt/domain/equity_backtest.py | 2 +- .../test_equity_backtest_runner.py | 62 +++++++++++++++++++ tests/unit/test_equity_backtest_domain.py | 6 +- 4 files changed, 89 insertions(+), 9 deletions(-) diff --git a/src/fxhnt/application/equity_backtest_runner.py b/src/fxhnt/application/equity_backtest_runner.py index f2aa194..9e06c3b 100644 --- a/src/fxhnt/application/equity_backtest_runner.py +++ b/src/fxhnt/application/equity_backtest_runner.py @@ -13,9 +13,9 @@ import numpy as np from fxhnt.adapters.warehouse.duckdb_feature_store import DuckDbFeatureStore from fxhnt.domain.backtest import compute_stats -from fxhnt.domain.gauntlet.core import evaluate +from fxhnt.domain.gauntlet.core import evaluate, per_period_sharpe from fxhnt.domain.equity_backtest import ( - _epoch_day, + epoch_day, daily_returns_for_weights, month_end_rebalance_dates, universe_asof, @@ -79,7 +79,7 @@ class EquityBacktestRunner: traded_dates: list[str] = [] for k, rebal in enumerate(rebals): - rebal_day = _epoch_day(rebal) + rebal_day = epoch_day(rebal) lo = rebal_day - _DOLLAR_VOL_WINDOW_DAYS dollar_vol: dict[str, float] = {} for s in syms: @@ -111,7 +111,7 @@ class EquityBacktestRunner: scores = composite_price(momentum_score(mom), lowvol_score(vols)) - end_day = _epoch_day(rebals[k + 1]) if k + 1 < len(rebals) else all_days[-1] + end_day = epoch_day(rebals[k + 1]) if k + 1 < len(rebals) else all_days[-1] hold_days = [d for d in all_days if rebal_day <= d <= end_day] for c in _CONSTRUCTIONS: @@ -152,11 +152,29 @@ def evaluate_constructions( "n_trials": n_trials, }, } + # DSR's sr_variance is the variance of the per-construction IS Sharpe estimates ACROSS the + # trials (the constructions) — the cross-trial Sharpe dispersion, computed ONCE and shared by + # every evaluate() call. Mirrors fxhnt.application.discovery.search (per_period_sharpe over the + # IS slice; np.var when ≥2 trials else 0.0). per_period_sharpe is NaN-safe (0.0 for len<3). + is_sharpes = [ + per_period_sharpe(r[: int((1.0 - oos_fraction) * len(r))]) + for r in result.returns_by_construction.values() + if len(r[: int((1.0 - oos_fraction) * len(r))]) >= 3 + ] + sr_variance = float(np.var(is_sharpes)) if len(is_sharpes) > 1 else 0.0 + for c, r in result.returns_by_construction.items(): stats = compute_stats(r) + if len(r) < 3: + out["constructions"][c] = { + "stats": stats.model_dump(), + "verdict": {"passed": False, "dsr": 0.0, "is_sharpe": 0.0, + "oos_sharpe": 0.0, "n_trials": n_trials, + "reasons": ["insufficient data"], "pvalue": 1.0}, + } + continue split = int((1.0 - oos_fraction) * len(r)) is_r, oos_r = r[:split], r[split:] - sr_variance = (1.0 / len(r)) if len(r) > 1 else 0.0 verdict = evaluate( is_r, oos_r, n_trials=n_trials, sr_variance=sr_variance, dsr_min=dsr_min, oos_min_sharpe=oos_min_sharpe, diff --git a/src/fxhnt/domain/equity_backtest.py b/src/fxhnt/domain/equity_backtest.py index f47e1dc..966dd43 100644 --- a/src/fxhnt/domain/equity_backtest.py +++ b/src/fxhnt/domain/equity_backtest.py @@ -9,7 +9,7 @@ from fxhnt.domain.strategies.equity_factor import book_return _SECONDS_PER_DAY = 86_400 -def _epoch_day(iso_date: str) -> int: +def epoch_day(iso_date: str) -> int: """ISO 'YYYY-MM-DD' -> epoch-day (matches DuckDbFeatureStore.read_panel keys).""" d = dt.date.fromisoformat(iso_date) return (d - dt.date(1970, 1, 1)).days diff --git a/tests/integration/test_equity_backtest_runner.py b/tests/integration/test_equity_backtest_runner.py index 43a0c7b..79989fe 100644 --- a/tests/integration/test_equity_backtest_runner.py +++ b/tests/integration/test_equity_backtest_runner.py @@ -1,9 +1,11 @@ import datetime as dt import numpy as np +import pytest from fxhnt.adapters.warehouse.duckdb_feature_store import DuckDbFeatureStore from fxhnt.application.equity_backtest_runner import ( + BacktestRunResult, EquityBacktestRunner, evaluate_constructions, ) @@ -112,3 +114,63 @@ def test_evaluate_constructions_emits_stats_and_verdict_per_construction(tmp_pat assert isinstance(report["rebalance_dates"], list) import json json.dumps(report) # must not raise + + +def test_evaluate_constructions_short_series_emits_clean_insufficient_data_verdict(): + # FIX #2: a construction with len < 3 must NOT call evaluate() (which would + # return dsr=NaN and break json.dumps). Instead emit a clean fallback verdict. + result = BacktestRunResult( + returns_by_construction={c: np.array([0.01, 0.02]) for c in ("long", "ls", "tilt")}, + rebalance_dates=[], + n_names_avg=0.0, + ) + report = evaluate_constructions(result) + assert set(report["constructions"]) == {"long", "ls", "tilt"} + for c, block in report["constructions"].items(): + v = block["verdict"] + assert v["passed"] is False + assert v["reasons"] == ["insufficient data"] + assert v["dsr"] == 0.0 + assert v["pvalue"] == 1.0 + assert v["n_trials"] == 3 + # fallback verdict must have the SAME keys as Verdict.model_dump() + assert set(v) == {"passed", "dsr", "is_sharpe", "oos_sharpe", + "n_trials", "reasons", "pvalue"} + import json + json.dumps(report) # must not raise (no NaN) + + +def test_evaluate_constructions_sr_variance_is_cross_construction_computed_once(monkeypatch): + # FIX #1: sr_variance must be the variance of the per-construction IS Sharpes + # ACROSS the constructions, computed ONCE and shared by every evaluate() call — + # NOT the per-construction (1/len(r)) value computed inside the loop. + import fxhnt.application.equity_backtest_runner as runner_mod + + rng = np.random.default_rng(0) + series = { + "long": rng.normal(0.001, 0.01, 300), + "ls": rng.normal(0.0, 0.02, 300), + "tilt": rng.normal(0.0005, 0.015, 300), + } + result = BacktestRunResult( + returns_by_construction=series, rebalance_dates=[], n_names_avg=0.0, + ) + + seen_sr_variance: list[float] = [] + real_evaluate = runner_mod.evaluate + + def spy_evaluate(*args, **kwargs): + seen_sr_variance.append(kwargs["sr_variance"]) + return real_evaluate(*args, **kwargs) + + monkeypatch.setattr(runner_mod, "evaluate", spy_evaluate) + evaluate_constructions(result, oos_fraction=0.40) + + # one evaluate() call per construction, all sharing the SAME sr_variance value + assert len(seen_sr_variance) == 3 + assert len(set(seen_sr_variance)) == 1 + # and that single value equals the cross-construction variance of IS Sharpes + is_sharpes = [ + runner_mod.per_period_sharpe(r[: int(0.60 * len(r))]) for r in series.values() + ] + assert seen_sr_variance[0] == pytest.approx(float(np.var(is_sharpes))) diff --git a/tests/unit/test_equity_backtest_domain.py b/tests/unit/test_equity_backtest_domain.py index e16d6c4..bc2cb11 100644 --- a/tests/unit/test_equity_backtest_domain.py +++ b/tests/unit/test_equity_backtest_domain.py @@ -1,7 +1,7 @@ from fxhnt.domain.equity_backtest import ( universe_asof, month_end_rebalance_dates, - _epoch_day, + epoch_day, ) MEMBERS = [ @@ -38,8 +38,8 @@ def test_month_end_rebalance_dates_one_per_month(): def test_epoch_day_roundtrip(): - assert _epoch_day("1970-01-01") == 0 - assert _epoch_day("1970-01-02") == 1 + assert epoch_day("1970-01-01") == 0 + assert epoch_day("1970-01-02") == 1 import pytest