diff --git a/src/fxhnt/application/discovery.py b/src/fxhnt/application/discovery.py index bfa12d8..7fa344c 100644 --- a/src/fxhnt/application/discovery.py +++ b/src/fxhnt/application/discovery.py @@ -23,6 +23,7 @@ from fxhnt.domain.backtest import run_backtest from fxhnt.domain.gauntlet import effective_trials, evaluate, per_period_sharpe from fxhnt.domain.models import Market, ResearchRun, StrategySpec, run_id from fxhnt.domain.strategies import get_strategy +from fxhnt.domain.validation import fdr_correct from fxhnt.ports.data import DataProvider from fxhnt.ports.repository import AnalyticalStore, OperationalRepository @@ -55,7 +56,8 @@ class DiscoveryReport(BaseModel): n_candidates: int n_effective_trials: float # correlation-corrected independent trial count (the deflation uses this) sr_variance: float - n_survivors: int + n_passed_gauntlet: int = 0 # passed the per-candidate gauntlet (before across-search FDR) + n_survivors: int = 0 # ALSO survived FDR over the whole search — the final set survivors: list[ResearchRun] = Field(default_factory=list) @@ -100,7 +102,7 @@ class DiscoveryService: # phase 2 — judge each candidate against the effective-search bar g = self._s.gauntlet now = dt.datetime.now(dt.timezone.utc) - survivors: list[ResearchRun] = [] + judged: list[tuple[ResearchRun, object, np.ndarray]] = [] for market, spec, prices, bt, r, split in rows: strategy = get_strategy(spec.kind) verdict = evaluate( @@ -110,12 +112,24 @@ class DiscoveryService: ) run = ResearchRun(run_id=run_id(market, spec), market=market, spec=spec, stats=bt.stats, verdict=verdict, n_trials=n_trials_eff, created_at=now) + judged.append((run, prices, r)) + + # phase 3 — FDR over the WHOLE search: Benjamini-Hochberg on every candidate's p-value at + # alpha = 1 − dsr_min. A candidate survives only if it passes its own gauntlet AND survives FDR — + # the multiple-testing complement to the per-candidate deflated Sharpe. + pvalues = [run.verdict.pvalue for run, _, _ in judged] + fdr = fdr_correct(pvalues, method="bh", alpha=1.0 - g.dsr_min) if len(pvalues) > 1 else None + n_passed = sum(run.verdict.passed for run, _, _ in judged) + survivors: list[ResearchRun] = [] + for i, (run, prices, r) in enumerate(judged): + fdr_ok = fdr.rejected[i] if fdr is not None else True + is_survivor = run.verdict.passed and fdr_ok if persist: self._op.save_run(run) # the verdict (cheap, SQLite) is kept for every candidate — audit - if verdict.passed: + if is_survivor: self._an.save_returns(run.run_id, prices.dates, r) # only survivors keep their timeseries - if verdict.passed: + if is_survivor: survivors.append(run) - return DiscoveryReport(n_candidates=len(candidates), n_effective_trials=n_eff, - sr_variance=sr_variance, n_survivors=len(survivors), survivors=survivors) + return DiscoveryReport(n_candidates=len(candidates), n_effective_trials=n_eff, sr_variance=sr_variance, + n_passed_gauntlet=n_passed, n_survivors=len(survivors), survivors=survivors) diff --git a/src/fxhnt/cli.py b/src/fxhnt/cli.py index e2d3192..a68edb6 100644 --- a/src/fxhnt/cli.py +++ b/src/fxhnt/cli.py @@ -234,7 +234,7 @@ def discover( report = svc.search(GridGenerator(markets, grids)) typer.echo(f"searched {report.n_candidates} candidates (effective independent trials " f"{report.n_effective_trials:.1f}, sr_var={report.sr_variance:.5f})") - typer.echo(f"survivors (deflated over the effective search): {report.n_survivors}") + typer.echo(f"passed the gauntlet: {report.n_passed_gauntlet} → survivors after FDR over the search: {report.n_survivors}") for run in sorted(report.survivors, key=lambda r: -r.verdict.dsr): typer.echo(f" [PASS] {run.market} {run.spec.key():28} | Sharpe {run.stats.sharpe:+.2f} DSR {run.verdict.dsr:.3f} OOS {run.verdict.oos_sharpe:+.2f}") diff --git a/src/fxhnt/domain/gauntlet/core.py b/src/fxhnt/domain/gauntlet/core.py index 6d86407..d2f9111 100644 --- a/src/fxhnt/domain/gauntlet/core.py +++ b/src/fxhnt/domain/gauntlet/core.py @@ -128,4 +128,6 @@ def evaluate( reasons.append("no structural rationale — refusing a purely statistical fit") elif has_economic_rationale is None: reasons.append("⚠ economic rationale UNVERIFIED — justify before deploying") - return Verdict(passed=ok, dsr=dsr, is_sharpe=is_sr, oos_sharpe=oos_sr, n_trials=n_trials, reasons=reasons) + pvalue = max(0.0, min(1.0, 1.0 - dsr)) # for across-search FDR correction + return Verdict(passed=ok, dsr=dsr, is_sharpe=is_sr, oos_sharpe=oos_sr, n_trials=n_trials, + reasons=reasons, pvalue=pvalue) diff --git a/src/fxhnt/domain/models.py b/src/fxhnt/domain/models.py index de4da2b..187f62a 100644 --- a/src/fxhnt/domain/models.py +++ b/src/fxhnt/domain/models.py @@ -95,6 +95,7 @@ class Verdict(BaseModel): oos_sharpe: float n_trials: int reasons: list[str] = Field(default_factory=list) + pvalue: float = 1.0 # one-sided p-value (1 − DSR) — lets a search FDR-correct across candidates def summary(self) -> str: flag = "PASS" if self.passed else "REJECT" diff --git a/tests/integration/test_discovery.py b/tests/integration/test_discovery.py index 3f6ddd3..059ae0a 100644 --- a/tests/integration/test_discovery.py +++ b/tests/integration/test_discovery.py @@ -40,7 +40,9 @@ def test_search_counts_full_matrix_as_trials(tmp_path) -> None: # every run was deflated by the same effective trial count assert len({r.n_trials for r in runs}) == 1 assert 0 <= report.n_survivors <= 6 + assert report.n_survivors <= report.n_passed_gauntlet # FDR over the search can only tighten assert all(r.verdict.passed for r in report.survivors) + assert all(0.0 <= r.verdict.pvalue <= 1.0 for r in svc._op.list_runs()) # noqa: SLF001 def test_effective_trials_below_raw_for_correlated() -> None: