fix(b3b): cross-construction sr_variance for DSR; guard short series (no NaN JSON); epoch_day public

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-06-17 12:53:38 +02:00
parent 33ade4356c
commit c7602dbf66
4 changed files with 89 additions and 9 deletions

View File

@@ -13,9 +13,9 @@ import numpy as np
from fxhnt.adapters.warehouse.duckdb_feature_store import DuckDbFeatureStore
from fxhnt.domain.backtest import compute_stats
from fxhnt.domain.gauntlet.core import evaluate
from fxhnt.domain.gauntlet.core import evaluate, per_period_sharpe
from fxhnt.domain.equity_backtest import (
_epoch_day,
epoch_day,
daily_returns_for_weights,
month_end_rebalance_dates,
universe_asof,
@@ -79,7 +79,7 @@ class EquityBacktestRunner:
traded_dates: list[str] = []
for k, rebal in enumerate(rebals):
rebal_day = _epoch_day(rebal)
rebal_day = epoch_day(rebal)
lo = rebal_day - _DOLLAR_VOL_WINDOW_DAYS
dollar_vol: dict[str, float] = {}
for s in syms:
@@ -111,7 +111,7 @@ class EquityBacktestRunner:
scores = composite_price(momentum_score(mom), lowvol_score(vols))
end_day = _epoch_day(rebals[k + 1]) if k + 1 < len(rebals) else all_days[-1]
end_day = epoch_day(rebals[k + 1]) if k + 1 < len(rebals) else all_days[-1]
hold_days = [d for d in all_days if rebal_day <= d <= end_day]
for c in _CONSTRUCTIONS:
@@ -152,11 +152,29 @@ def evaluate_constructions(
"n_trials": n_trials,
},
}
# DSR's sr_variance is the variance of the per-construction IS Sharpe estimates ACROSS the
# trials (the constructions) — the cross-trial Sharpe dispersion, computed ONCE and shared by
# every evaluate() call. Mirrors fxhnt.application.discovery.search (per_period_sharpe over the
# IS slice; np.var when ≥2 trials else 0.0). per_period_sharpe is NaN-safe (0.0 for len<3).
is_sharpes = [
per_period_sharpe(r[: int((1.0 - oos_fraction) * len(r))])
for r in result.returns_by_construction.values()
if len(r[: int((1.0 - oos_fraction) * len(r))]) >= 3
]
sr_variance = float(np.var(is_sharpes)) if len(is_sharpes) > 1 else 0.0
for c, r in result.returns_by_construction.items():
stats = compute_stats(r)
if len(r) < 3:
out["constructions"][c] = {
"stats": stats.model_dump(),
"verdict": {"passed": False, "dsr": 0.0, "is_sharpe": 0.0,
"oos_sharpe": 0.0, "n_trials": n_trials,
"reasons": ["insufficient data"], "pvalue": 1.0},
}
continue
split = int((1.0 - oos_fraction) * len(r))
is_r, oos_r = r[:split], r[split:]
sr_variance = (1.0 / len(r)) if len(r) > 1 else 0.0
verdict = evaluate(
is_r, oos_r, n_trials=n_trials, sr_variance=sr_variance,
dsr_min=dsr_min, oos_min_sharpe=oos_min_sharpe,

View File

@@ -9,7 +9,7 @@ from fxhnt.domain.strategies.equity_factor import book_return
_SECONDS_PER_DAY = 86_400
def _epoch_day(iso_date: str) -> int:
def epoch_day(iso_date: str) -> int:
"""ISO 'YYYY-MM-DD' -> epoch-day (matches DuckDbFeatureStore.read_panel keys)."""
d = dt.date.fromisoformat(iso_date)
return (d - dt.date(1970, 1, 1)).days

View File

@@ -1,9 +1,11 @@
import datetime as dt
import numpy as np
import pytest
from fxhnt.adapters.warehouse.duckdb_feature_store import DuckDbFeatureStore
from fxhnt.application.equity_backtest_runner import (
BacktestRunResult,
EquityBacktestRunner,
evaluate_constructions,
)
@@ -112,3 +114,63 @@ def test_evaluate_constructions_emits_stats_and_verdict_per_construction(tmp_pat
assert isinstance(report["rebalance_dates"], list)
import json
json.dumps(report) # must not raise
def test_evaluate_constructions_short_series_emits_clean_insufficient_data_verdict():
# FIX #2: a construction with len < 3 must NOT call evaluate() (which would
# return dsr=NaN and break json.dumps). Instead emit a clean fallback verdict.
result = BacktestRunResult(
returns_by_construction={c: np.array([0.01, 0.02]) for c in ("long", "ls", "tilt")},
rebalance_dates=[],
n_names_avg=0.0,
)
report = evaluate_constructions(result)
assert set(report["constructions"]) == {"long", "ls", "tilt"}
for c, block in report["constructions"].items():
v = block["verdict"]
assert v["passed"] is False
assert v["reasons"] == ["insufficient data"]
assert v["dsr"] == 0.0
assert v["pvalue"] == 1.0
assert v["n_trials"] == 3
# fallback verdict must have the SAME keys as Verdict.model_dump()
assert set(v) == {"passed", "dsr", "is_sharpe", "oos_sharpe",
"n_trials", "reasons", "pvalue"}
import json
json.dumps(report) # must not raise (no NaN)
def test_evaluate_constructions_sr_variance_is_cross_construction_computed_once(monkeypatch):
# FIX #1: sr_variance must be the variance of the per-construction IS Sharpes
# ACROSS the constructions, computed ONCE and shared by every evaluate() call —
# NOT the per-construction (1/len(r)) value computed inside the loop.
import fxhnt.application.equity_backtest_runner as runner_mod
rng = np.random.default_rng(0)
series = {
"long": rng.normal(0.001, 0.01, 300),
"ls": rng.normal(0.0, 0.02, 300),
"tilt": rng.normal(0.0005, 0.015, 300),
}
result = BacktestRunResult(
returns_by_construction=series, rebalance_dates=[], n_names_avg=0.0,
)
seen_sr_variance: list[float] = []
real_evaluate = runner_mod.evaluate
def spy_evaluate(*args, **kwargs):
seen_sr_variance.append(kwargs["sr_variance"])
return real_evaluate(*args, **kwargs)
monkeypatch.setattr(runner_mod, "evaluate", spy_evaluate)
evaluate_constructions(result, oos_fraction=0.40)
# one evaluate() call per construction, all sharing the SAME sr_variance value
assert len(seen_sr_variance) == 3
assert len(set(seen_sr_variance)) == 1
# and that single value equals the cross-construction variance of IS Sharpes
is_sharpes = [
runner_mod.per_period_sharpe(r[: int(0.60 * len(r))]) for r in series.values()
]
assert seen_sr_variance[0] == pytest.approx(float(np.var(is_sharpes)))

View File

@@ -1,7 +1,7 @@
from fxhnt.domain.equity_backtest import (
universe_asof,
month_end_rebalance_dates,
_epoch_day,
epoch_day,
)
MEMBERS = [
@@ -38,8 +38,8 @@ def test_month_end_rebalance_dates_one_per_month():
def test_epoch_day_roundtrip():
assert _epoch_day("1970-01-01") == 0
assert _epoch_day("1970-01-02") == 1
assert epoch_day("1970-01-01") == 0
assert epoch_day("1970-01-02") == 1
import pytest