b7ed0d0b1f
FINDING (third this session): fxhnt's existing gauntlet DSR already uses the CANONICAL Bailey-LdP SE ((kurt-1)/4), while the Rust ml-validation SE uses (excess_kurt/4) — slightly non-canonical. So we do NOT swap the DSR (the gauntlet's is already correct). The genuine add from the ported suite is PBO + FDR. Wired FDR in: Verdict gains a pvalue (1−DSR, non-breaking default); DiscoveryService now FDR-corrects (Benjamini-Hochberg at alpha=1−dsr_min) over ALL candidate p-values, and a candidate survives only if it passes its own gauntlet AND survives FDR across the search. DiscoveryReport reports n_passed_gauntlet vs n_survivors (post-FDR). PBO kept available for fold-based CV (future wire). 48/48 tests. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
62 lines
2.9 KiB
Python
62 lines
2.9 KiB
Python
"""Discovery search: the matrix runs through the gauntlet with deflation over the FULL search.
|
||
Fake data + temp stores — proves trial-counting + persistence without network."""
|
||
from __future__ import annotations
|
||
|
||
import numpy as np
|
||
|
||
from fxhnt.adapters.persistence import DuckDbAnalyticalStore, SqlOperationalRepository
|
||
from fxhnt.application import DiscoveryService, GridGenerator
|
||
from fxhnt.config import Settings
|
||
from fxhnt.domain.models import AssetClass, Market, PriceSeries
|
||
|
||
|
||
class FakeData:
|
||
name = "fake"
|
||
|
||
def fetch(self, market: Market, start=None, end=None) -> PriceSeries:
|
||
rng = np.random.default_rng(sum(ord(c) for c in market.symbol)) # deterministic (hash() is randomized)
|
||
close = 100.0 * np.cumprod(1.0 + rng.normal(0.0003, 0.01, 1500))
|
||
return PriceSeries(market=market, dates=tuple(str(i) for i in range(1500)), close=close)
|
||
|
||
|
||
def _svc(tmp_path) -> DiscoveryService:
|
||
settings = Settings(operational_dsn=f"sqlite:///{tmp_path / 'op.db'}", analytical_path=str(tmp_path / "an.duckdb"))
|
||
return DiscoveryService(FakeData(), SqlOperationalRepository(settings.operational_dsn),
|
||
DuckDbAnalyticalStore(settings.analytical_path), settings)
|
||
|
||
|
||
def test_search_counts_full_matrix_as_trials(tmp_path) -> None:
|
||
svc = _svc(tmp_path)
|
||
markets = [Market(symbol=s, asset_class=AssetClass.ETF) for s in ("AAA", "BBB")]
|
||
grids = {"trend": [{"window": 50.0}, {"window": 100.0}, {"window": 150.0}]} # 2 × 3 = 6 candidates
|
||
report = svc.search(GridGenerator(markets, grids))
|
||
|
||
assert report.n_candidates == 6
|
||
# correlated grid params are NOT 6 independent trials -> effective N is below the raw count
|
||
assert 1.0 <= report.n_effective_trials <= 6.0
|
||
assert report.sr_variance >= 0.0
|
||
runs = svc._op.list_runs() # noqa: SLF001
|
||
assert len(runs) == 6
|
||
# every run was deflated by the same effective trial count
|
||
assert len({r.n_trials for r in runs}) == 1
|
||
assert 0 <= report.n_survivors <= 6
|
||
assert report.n_survivors <= report.n_passed_gauntlet # FDR over the search can only tighten
|
||
assert all(r.verdict.passed for r in report.survivors)
|
||
assert all(0.0 <= r.verdict.pvalue <= 1.0 for r in svc._op.list_runs()) # noqa: SLF001
|
||
|
||
|
||
def test_effective_trials_below_raw_for_correlated() -> None:
|
||
"""Effective N < raw N when trials are correlated; ≈ N when independent."""
|
||
from fxhnt.domain.gauntlet import effective_trials
|
||
|
||
rng = np.random.default_rng(3)
|
||
base = rng.normal(0, 0.01, 1000)
|
||
correlated = [base + rng.normal(0, 0.001, 1000) for _ in range(10)] # ~identical
|
||
independent = [rng.normal(0, 0.01, 1000) for _ in range(10)]
|
||
assert effective_trials(correlated) < 3.0
|
||
assert effective_trials(independent) > 7.0
|
||
|
||
|
||
# (the "larger search => stricter deflated bar" property is proven directly in
|
||
# tests/unit/test_gauntlet.py::test_dsr_falls_as_trials_rise — no need to re-prove it through I/O here.)
|