b4275fba30
domain/validation: faithful Python port of Rust ml-validation/statistical.rs + ml-data-validation/fdr.rs. normal_cdf (A&S) + normal_ppf (Acklam) bit-exact; deflated_sharpe_ratio with the skew/kurtosis-ADJUSTED standard error (richer than fxhnt's prior basic DSR); probability_of_backtest_overfitting (CSCV); permutation_test; fdr_correct (Benjamini-Hochberg + Yekutieli). 13 TDD tests (PPF/CDF vs known quantiles, DSR multiple-testing + skew-widens-SE, PBO degenerate/ties, FDR BH-collapse + BY-conservative). 48/48. FINDING (flagged, like the Stoikov one): the Rust permutation_test is DEGENERATE — it shuffles the returns and recomputes Sharpe=mean/std, which is order-invariant, so every permutation gives the same Sharpe (null_std~1e-16, pvalue~1). Faithful-ported for SSOT, but a meaningful version must permute the POSITION SIGNAL against fixed returns. Next: wire the richer DSR+PBO+FDR into the gauntlet's evaluate(). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
80 lines
3.7 KiB
Python
80 lines
3.7 KiB
Python
"""Validation suite — TDD of the López de Prado discipline machinery ported from Rust ml-validation."""
|
|
from __future__ import annotations
|
|
|
|
import numpy as np
|
|
|
|
from fxhnt.domain.validation import (
|
|
deflated_sharpe_ratio,
|
|
fdr_correct,
|
|
normal_cdf,
|
|
normal_ppf,
|
|
permutation_test,
|
|
probability_of_backtest_overfitting,
|
|
sharpe_ratio,
|
|
)
|
|
|
|
|
|
def test_normal_ppf_known_quantiles() -> None:
|
|
assert abs(normal_ppf(0.975) - 1.959963985) < 1e-6
|
|
assert abs(normal_ppf(0.025) + 1.959963985) < 1e-6
|
|
assert normal_ppf(0.5) == 0.0
|
|
|
|
|
|
def test_normal_cdf_known_and_roundtrip() -> None:
|
|
assert abs(normal_cdf(0.0) - 0.5) < 1e-9
|
|
assert abs(normal_cdf(1.959963985) - 0.975) < 1e-4
|
|
for p in (0.1, 0.3, 0.8, 0.95):
|
|
assert abs(normal_cdf(normal_ppf(p)) - p) < 1e-4 # round-trip
|
|
|
|
|
|
def test_sharpe_ratio() -> None:
|
|
assert sharpe_ratio(np.array([1.0, 1.0, 1.0])) == 0.0 # zero variance
|
|
r = np.array([0.01, -0.005, 0.02, 0.0, 0.015])
|
|
assert abs(sharpe_ratio(r) - r.mean() / r.std(ddof=1)) < 1e-12
|
|
|
|
|
|
def test_dsr_more_trials_raises_the_bar() -> None:
|
|
one = deflated_sharpe_ratio(0.5, num_trials=1, sharpe_variance=0.04, skew=0.0, kurt=0.0, num_observations=100)
|
|
fifty = deflated_sharpe_ratio(0.5, num_trials=50, sharpe_variance=0.04, skew=0.0, kurt=0.0, num_observations=100)
|
|
assert fifty.expected_max_sharpe > one.expected_max_sharpe # more trials -> higher null bar
|
|
assert fifty.deflated_sharpe < one.deflated_sharpe # -> lower deflated statistic
|
|
assert fifty.pvalue > one.pvalue # -> less significant
|
|
|
|
|
|
def test_dsr_negative_skew_widens_se() -> None:
|
|
base = deflated_sharpe_ratio(1.5, 10, 0.02, 0.0, 0.0, 800)
|
|
skewed = deflated_sharpe_ratio(1.5, 10, 0.02, -1.0, 0.0, 800) # negative skew penalises a positive SR
|
|
assert skewed.sharpe_std_error > base.sharpe_std_error
|
|
|
|
|
|
def test_pbo_degenerate_and_ties() -> None:
|
|
assert probability_of_backtest_overfitting([1.0, 2.0]).pbo == 0.5 # n < 4
|
|
assert probability_of_backtest_overfitting([1.0, 2.0, 3.0]).pbo == 0.5 # odd n
|
|
assert probability_of_backtest_overfitting([3.0, 3.0, 3.0, 3.0]).pbo == 0.0 # all-equal: IS never beats OOS
|
|
res = probability_of_backtest_overfitting([5.0, 4.0, 3.0, 2.0, 1.0, 0.0])
|
|
assert 0.0 <= res.pbo <= 1.0 and res.num_combinations == 20 # C(6,3)
|
|
|
|
|
|
def test_permutation_test_is_order_invariant_for_sharpe() -> None:
|
|
# FINDING (faithful to Rust, but a latent bug): shuffling returns leaves mean & std — hence Sharpe —
|
|
# unchanged, so this permutation test is degenerate (null_std ~0, pvalue ~1). A meaningful version
|
|
# must permute the POSITION SIGNAL against fixed returns, not the returns themselves.
|
|
r = np.full(300, 0.001) + np.random.default_rng(0).normal(0, 0.0005, 300)
|
|
res = permutation_test(r, num_permutations=500, seed=7)
|
|
assert abs(res.observed_sharpe - sharpe_ratio(r)) < 1e-12
|
|
assert res.null_std < 1e-9 and res.pvalue > 0.5 # degenerate: all perms ~equal Sharpe
|
|
|
|
|
|
def test_fdr_benjamini_hochberg() -> None:
|
|
res = fdr_correct([0.01, 0.02, 0.03, 0.04, 0.05], method="bh")
|
|
assert all(abs(a - 0.05) < 1e-12 for a in res.adjusted_pvalues) # all collapse to 0.05
|
|
res2 = fdr_correct([0.001, 0.5, 0.5, 0.5, 0.5], method="bh", alpha=0.05)
|
|
assert abs(res2.adjusted_pvalues[0] - 0.005) < 1e-12 and res2.rejected[0]
|
|
assert res2.num_significant == 1 # only the strong one
|
|
|
|
|
|
def test_fdr_yekutieli_is_more_conservative() -> None:
|
|
bh = fdr_correct([0.001, 0.5, 0.5, 0.5, 0.5], method="bh")
|
|
by = fdr_correct([0.001, 0.5, 0.5, 0.5, 0.5], method="by")
|
|
assert by.adjusted_pvalues[0] > bh.adjusted_pvalues[0] # BY penalises dependence
|