gitadmin/foxhunt
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(validation): correct DSR formula, PBO methodology, and DQN adapter bugs

Browse Source 
Critical fixes found during pre-GPU-test code audit:
- DSR SE formula: (kurt-1)/4 → kurt/4 for excess kurtosis input
- PBO CSCV: replace circular fold ranking with IS/OOS mean comparison
- DqnStrategy evaluate: fix off-by-one (loop over returns, not bars)
- DqnStrategy action mapping: handle small num_actions (3→5) directly
  instead of FactoredAction decomposition (which maps all to Short100)
- Walk-forward: document rolling window as deliberate design choice

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
jgrusewski
2026-02-20 17:20:41 +01:00
parent 74c605683d
commit 49defb0745
3 changed files with 153 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
ml/src/validation/adapters.rs
View File

@@ -100,26 +100,44 @@ impl ValidatableStrategy for DqnStrategy {
/// Evaluate the DQN on out-of-sample data.
///
/// For each bar, selects an action and maps it to a PnL value:
/// - Exposure index 0-1 (short): PnL = -bar_return
/// - Exposure index 3-4 (long): PnL = +bar_return
/// - Exposure index 2 (flat): PnL = 0
/// For each return period, selects an action and maps it to a PnL value.
/// Returns one PnL value per return in `data.returns` (length = data.len() - 1).
///
/// Returns one PnL value per bar in `data`.
/// Action-to-exposure mapping depends on `num_actions`:
/// - **num_actions <= 5** (simple): direct mapping (0=short, mid=flat, last=long)
/// - **num_actions = 45** (factored): uses FactoredAction exposure decomposition
fn evaluate(&self, data: &TimeSeriesData) -> Result<Vec<f64>, MLError> {
let mut pnl = Vec::with_capacity(data.len());
let num_returns = data.returns.len();
let mut pnl = Vec::with_capacity(num_returns);
for i in 0..data.len() {
let num_actions = self.config.num_actions;
for i in 0..num_returns {
let state = data.features.get(i).cloned().unwrap_or_default();
let action = self.dqn.borrow_mut().select_action(&state)?;
let bar_return = data.returns.get(i).copied().unwrap_or(0.0);
// Map action to directional PnL based on exposure level
let exposure_idx = action.exposure as usize;
let bar_pnl = match exposure_idx {
0 | 1 => -bar_return, // Short exposure
3 | 4 => bar_return, // Long exposure
_ => 0.0, // Flat
let bar_pnl = if num_actions <= 5 {
// Simple action space: map action index directly to direction.
// For num_actions=3: 0=short, 1=flat, 2=long
// For num_actions=5: 0=short100, 1=short50, 2=flat, 3=long50, 4=long100
let action_idx = action.to_index() % num_actions;
let mid = num_actions / 2;
if action_idx < mid {
-bar_return // Short
} else if action_idx > mid {
bar_return // Long
} else {
0.0 // Flat
}
} else {
// Factored action space (45 actions): use exposure level
let exposure_idx = action.exposure as usize;
match exposure_idx {
0 | 1 => -bar_return, // Short exposure
3 | 4 => bar_return, // Long exposure
_ => 0.0, // Flat
}
};
pnl.push(bar_pnl);
@@ -214,12 +232,12 @@ mod tests {
let train_result = strategy.train(&data);
assert!(train_result.is_ok(), "train failed: {:?}", train_result.err());
// Evaluate should return one PnL per bar
// Evaluate should return one PnL per return period (data.len() - 1)
let eval_result = strategy.evaluate(&data);
assert!(eval_result.is_ok(), "evaluate failed: {:?}", eval_result.err());
let pnl = eval_result.unwrap_or_else(|e| panic!("evaluate error: {}", e));
assert_eq!(pnl.len(), 30, "Expected 30 PnL values, got {}", pnl.len());
assert_eq!(pnl.len(), 29, "Expected 29 PnL values (one per return), got {}", pnl.len());
}
#[test]

195
ml/src/validation/statistical.rs
View File

@@ -259,10 +259,11 @@ pub fn deflated_sharpe_ratio(
let expected_max_sharpe = sr_std * (term1 + term2);
// Sharpe ratio standard error:
// SE = sqrt((1 - skew*SR + (kurt-1)/4 * SR^2) / (n-1))
// Sharpe ratio standard error (Bailey & Lopez de Prado, 2014):
// SE = sqrt((1 - γ₃·SR + (γ₄/4)·SR²) / (T-1))
// where γ₃ = skewness, γ₄ = excess kurtosis
let sr_sq = observed_sharpe * observed_sharpe;
let se_inner = 1.0 - skew * observed_sharpe + (kurt - 1.0) / 4.0 * sr_sq;
let se_inner = 1.0 - skew * observed_sharpe + kurt / 4.0 * sr_sq;
let se_inner_clamped = se_inner.max(1e-10); // prevent negative sqrt
let sharpe_std_error = (se_inner_clamped / (obs - 1.0)).sqrt();
@@ -470,13 +471,23 @@ fn generate_combos_recursive(
}
}
/// Probability of Backtest Overfitting (Bailey et al., 2017) via CSCV.
/// Probability of Backtest Overfitting (adapted from Bailey et al., 2017) via CSCV.
///
/// Combinatorially Symmetric Cross-Validation (CSCV):
/// Single-strategy adaptation of Combinatorially Symmetric Cross-Validation:
/// 1. Takes N fold Sharpe ratios and generates C(N, N/2) combinations
/// 2. For each combination, one half is treated as in-sample (IS), the other as out-of-sample (OOS)
/// 3. Finds the IS-best strategy and checks its OOS rank
/// 4. PBO = fraction of combinations where IS-best ranks below the OOS median
/// 2. For each combination, one half is treated as in-sample (IS), the other as OOS
/// 3. Computes mean Sharpe for IS and OOS halves
/// 4. PBO = fraction of combinations where IS mean > OOS mean
///
/// **Interpretation for single strategies:**
/// - PBO ≈ 0.5: performance is consistent across time periods (no overfitting signal)
/// - PBO >> 0.5: performance degrades from IS to OOS (temporal overfitting)
/// - PBO << 0.5: performance improves from IS to OOS (unusual, possible regime shift)
///
/// **Note:** The original CSCV paper tests multiple strategy configurations against
/// each other. This adaptation tests a single strategy's consistency across time.
/// For single strategies, DSR and permutation tests are the primary significance measures;
/// PBO adds value when `num_trials > 1` in a hyperopt setting.
///
/// Requires even N >= 4. Returns `pbo = 0.5` for degenerate inputs.
pub fn probability_of_backtest_overfitting(per_fold_sharpes: &[f64]) -> PboResult {
@@ -507,67 +518,52 @@ pub fn probability_of_backtest_overfitting(per_fold_sharpes: &[f64]) -> PboResul
let mut overfit_count = 0usize;
for combo in &combinations {
// IS indices are in `combo`, OOS indices are the complement
let is_set: std::collections::HashSet<usize> = combo.iter().copied().collect();
let is_sharpes: Vec<f64> = combo
// Compute mean Sharpe for IS and OOS halves
let is_sum: f64 = combo
.iter()
.filter_map(|&i| per_fold_sharpes.get(i).copied())
.collect();
let oos_sharpes: Vec<f64> = (0..n)
.sum();
let oos_sum: f64 = (0..n)
.filter(|i| !is_set.contains(i))
.filter_map(|i| per_fold_sharpes.get(i).copied())
.collect();
.sum();
if is_sharpes.is_empty() || oos_sharpes.is_empty() {
let is_count = combo.len() as f64;
let oos_count = (n - combo.len()) as f64;
if is_count < 1.0 || oos_count < 1.0 {
continue;
}
// Find IS-best: index within combo that has the highest IS Sharpe
let is_best_combo_idx = is_sharpes
.iter()
.enumerate()
.max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
.map(|(i, _)| i)
.unwrap_or(0); // safe: is_sharpes is non-empty
let is_mean = is_sum / is_count;
let oos_mean = oos_sum / oos_count;
// The IS-best strategy's fold index in the original array
let is_best_fold_idx = combo.get(is_best_combo_idx).copied().unwrap_or(0);
// Performance degradation: IS outperforming OOS signals overfitting
let degradation = is_mean - oos_mean;
// Get the IS-best strategy's OOS Sharpe (if it appears in OOS, use the fold's sharpe)
// Actually in CSCV, we check ALL strategies' performance.
// The IS-best strategy has an IS sharpe; we look at its OOS sharpe.
// But the IS-best is in the IS set, not OOS. So we need a different interpretation:
//
// CSCV correct interpretation: we have N strategies (folds serve as performance metrics).
// For each combination split, we rank strategies by IS performance and check if the
// IS-best strategy's OOS performance is below median.
//
// Simplified approach: use the IS-best fold's Sharpe as its "quality",
// then check its rank relative to OOS set.
let is_best_sharpe = per_fold_sharpes.get(is_best_fold_idx).copied().unwrap_or(0.0);
// Rank the IS-best strategy's sharpe within the OOS sharpes
let oos_rank = oos_sharpes
.iter()
.filter(|&&s| s >= is_best_sharpe)
.count();
let oos_len = oos_sharpes.len();
let relative_rank = if oos_len > 0 {
oos_rank as f64 / oos_len as f64
// Logit of relative OOS performance.
// ratio = OOS / (|IS| + |OOS|): 0.5 when equal, <0.5 when IS > OOS
let denom = is_mean.abs() + oos_mean.abs();
let ratio = if denom > 1e-15 {
(oos_mean.abs() / denom).clamp(0.01, 0.99)
} else {
0.5
0.5 // both means ≈ 0
};
// Logit of relative rank (clamped to avoid log(0))
let clamped_rank = relative_rank.clamp(0.01, 0.99);
let logit = (clamped_rank / (1.0 - clamped_rank)).ln();
// Adjust sign: if OOS is negative but IS positive, ratio should reflect underperformance
let signed_ratio = if degradation > 0.0 {
// IS > OOS: overfitting signal, push ratio below 0.5
ratio.min(0.49)
} else {
// OOS >= IS: genuine signal, push ratio above 0.5
ratio.max(0.51)
};
let logit = (signed_ratio / (1.0 - signed_ratio)).ln();
logit_distribution.push(logit);
// Overfit if IS-best ranks below OOS median (relative rank > 0.5)
if relative_rank > 0.5 {
// Count as overfitting if IS mean > OOS mean
if is_mean > oos_mean {
overfit_count = overfit_count.saturating_add(1);
}
}
@@ -697,8 +693,9 @@ mod tests {
#[test]
fn test_dsr_single_trial_not_penalized() {
// SR=2.0, only 1 trial -> no multiple testing penalty -> should be significant
let result = deflated_sharpe_ratio(2.0, 1, 1.0, 0.0, 3.0, 252);
// SR=2.0, only 1 trial, normal returns (excess kurtosis=0)
// -> no multiple testing penalty -> should be significant
let result = deflated_sharpe_ratio(2.0, 1, 1.0, 0.0, 0.0, 252);
assert!(
result.pvalue < 0.05,
"Single trial SR=2.0 should have p<0.05, got {}",
@@ -708,8 +705,9 @@ mod tests {
#[test]
fn test_dsr_many_trials_penalized() {
// SR=1.0, 1000 trials -> heavy penalty -> should NOT be significant
let result = deflated_sharpe_ratio(1.0, 1000, 1.0, 0.0, 3.0, 252);
// SR=1.0, 1000 trials, normal returns (excess kurtosis=0)
// -> heavy penalty -> should NOT be significant
let result = deflated_sharpe_ratio(1.0, 1000, 1.0, 0.0, 0.0, 252);
assert!(
result.pvalue > 0.05,
"1000 trials SR=1.0 should have p>0.05, got {}",
@@ -720,8 +718,8 @@ mod tests {
#[test]
fn test_dsr_higher_sharpe_more_significant() {
// SR=3 should be more significant than SR=1 with same parameters
let r1 = deflated_sharpe_ratio(1.0, 100, 1.0, 0.0, 3.0, 252);
let r3 = deflated_sharpe_ratio(3.0, 100, 1.0, 0.0, 3.0, 252);
let r1 = deflated_sharpe_ratio(1.0, 100, 1.0, 0.0, 0.0, 252);
let r3 = deflated_sharpe_ratio(3.0, 100, 1.0, 0.0, 0.0, 252);
assert!(
r3.pvalue < r1.pvalue,
"SR=3 p-value ({}) should be < SR=1 p-value ({})",
@@ -730,6 +728,20 @@ mod tests {
);
}
#[test]
fn test_dsr_se_formula_normal_returns() {
// For normal returns (skew=0, excess_kurt=0), SE should simplify to:
// SE = sqrt(1 / (T-1)) = 1/sqrt(T-1)
let result = deflated_sharpe_ratio(1.0, 1, 1.0, 0.0, 0.0, 252);
let expected_se = 1.0 / (251.0_f64).sqrt();
assert!(
(result.sharpe_std_error - expected_se).abs() < 1e-6,
"SE for normal returns should be 1/sqrt(T-1)={}, got {}",
expected_se,
result.sharpe_std_error
);
}
// ── Permutation test ────────────────────────────────────────────────
#[test]
@@ -768,34 +780,27 @@ mod tests {
// ── PBO tests ───────────────────────────────────────────────────────
#[test]
fn test_pbo_random_sharpes_high() {
// Wildly inconsistent Sharpe ratios across folds suggest overfitting:
// some folds show strong positive, others strong negative -- classic overfit signature
let sharpes = vec![
3.0, -2.5, 2.8, -2.0, 2.5, -1.8, -2.2, 2.9, -1.5, 2.7, -2.3, 1.9,
];
let result = probability_of_backtest_overfitting(&sharpes);
assert!(
result.pbo > 0.0,
"Varied Sharpes should show some overfitting signal, got PBO={}",
result.pbo
);
assert!(
result.num_combinations > 0,
"Should have evaluated combinations"
);
}
#[test]
fn test_pbo_consistent_sharpes_low() {
// All consistently positive Sharpes -> lower PBO
fn test_pbo_produces_valid_output() {
// 8 folds with varied Sharpes — verify structural correctness
let sharpes = vec![1.0, 1.1, 1.05, 0.95, 1.02, 0.98, 1.08, 1.03];
let result = probability_of_backtest_overfitting(&sharpes);
assert!(
result.pbo < 0.5,
"Consistent positive Sharpes should have PBO < 0.5, got {}",
(0.0..=1.0).contains(&result.pbo),
"PBO must be in [0,1], got {}",
result.pbo
);
// C(8,4) = 70 combinations
assert_eq!(result.num_combinations, 70);
assert!(
!result.logit_distribution.is_empty(),
"Should produce logit distribution"
);
// All logits should be finite
assert!(
result.logit_distribution.iter().all(|l| l.is_finite()),
"All logits should be finite"
);
}
#[test]
@@ -811,6 +816,32 @@ mod tests {
assert_eq!(result.num_combinations, 0);
}
#[test]
fn test_pbo_odd_folds_degenerate() {
// N=5 (odd) -> degenerate result
let sharpes = vec![1.0, 2.0, 3.0, 4.0, 5.0];
let result = probability_of_backtest_overfitting(&sharpes);
assert!(
(result.pbo - 0.5).abs() < 1e-10,
"Odd N should return pbo=0.5, got {}",
result.pbo
);
assert_eq!(result.num_combinations, 0);
}
#[test]
fn test_pbo_identical_sharpes_symmetric() {
// All identical Sharpes → IS_mean = OOS_mean for every combination
// Neither IS > OOS nor OOS > IS → PBO = 0.0 (no overfit count)
let sharpes = vec![1.0, 1.0, 1.0, 1.0];
let result = probability_of_backtest_overfitting(&sharpes);
assert!(
result.pbo < 0.01,
"Identical Sharpes should have PBO ≈ 0.0 (no IS > OOS), got {}",
result.pbo
);
}
#[test]
fn test_binomial_coefficient() {
assert_eq!(binomial_coefficient(8, 4), 70);

7
ml/src/validation/walk_forward.rs
View File

@@ -72,6 +72,13 @@ pub struct Fold {
/// Generate walk-forward cross-validation folds with embargo.
///
/// Uses a **rolling window** design: each fold has a fixed-size training window
/// that slides forward. This is the standard approach for evaluating strategies
/// on recent data (avoids stale early data diluting training signal).
///
/// An expanding window variant (where training grows from an anchor point) may
/// be added later if needed for strategies that benefit from larger datasets.
///
/// Slides a `[train | embargo | test]` window across `num_bars` bar indices,
/// advancing by `config.step_bars` each iteration. A fold is only emitted when:
/// - The training window has at least `config.min_train_samples` bars