diff --git a/ml/src/validation/adapters.rs b/ml/src/validation/adapters.rs index 3aa8b0a8c..357119956 100644 --- a/ml/src/validation/adapters.rs +++ b/ml/src/validation/adapters.rs @@ -100,26 +100,44 @@ impl ValidatableStrategy for DqnStrategy { /// Evaluate the DQN on out-of-sample data. /// - /// For each bar, selects an action and maps it to a PnL value: - /// - Exposure index 0-1 (short): PnL = -bar_return - /// - Exposure index 3-4 (long): PnL = +bar_return - /// - Exposure index 2 (flat): PnL = 0 + /// For each return period, selects an action and maps it to a PnL value. + /// Returns one PnL value per return in `data.returns` (length = data.len() - 1). /// - /// Returns one PnL value per bar in `data`. + /// Action-to-exposure mapping depends on `num_actions`: + /// - **num_actions <= 5** (simple): direct mapping (0=short, mid=flat, last=long) + /// - **num_actions = 45** (factored): uses FactoredAction exposure decomposition fn evaluate(&self, data: &TimeSeriesData) -> Result, MLError> { - let mut pnl = Vec::with_capacity(data.len()); + let num_returns = data.returns.len(); + let mut pnl = Vec::with_capacity(num_returns); - for i in 0..data.len() { + let num_actions = self.config.num_actions; + + for i in 0..num_returns { let state = data.features.get(i).cloned().unwrap_or_default(); let action = self.dqn.borrow_mut().select_action(&state)?; let bar_return = data.returns.get(i).copied().unwrap_or(0.0); - // Map action to directional PnL based on exposure level - let exposure_idx = action.exposure as usize; - let bar_pnl = match exposure_idx { - 0 | 1 => -bar_return, // Short exposure - 3 | 4 => bar_return, // Long exposure - _ => 0.0, // Flat + let bar_pnl = if num_actions <= 5 { + // Simple action space: map action index directly to direction. + // For num_actions=3: 0=short, 1=flat, 2=long + // For num_actions=5: 0=short100, 1=short50, 2=flat, 3=long50, 4=long100 + let action_idx = action.to_index() % num_actions; + let mid = num_actions / 2; + if action_idx < mid { + -bar_return // Short + } else if action_idx > mid { + bar_return // Long + } else { + 0.0 // Flat + } + } else { + // Factored action space (45 actions): use exposure level + let exposure_idx = action.exposure as usize; + match exposure_idx { + 0 | 1 => -bar_return, // Short exposure + 3 | 4 => bar_return, // Long exposure + _ => 0.0, // Flat + } }; pnl.push(bar_pnl); @@ -214,12 +232,12 @@ mod tests { let train_result = strategy.train(&data); assert!(train_result.is_ok(), "train failed: {:?}", train_result.err()); - // Evaluate should return one PnL per bar + // Evaluate should return one PnL per return period (data.len() - 1) let eval_result = strategy.evaluate(&data); assert!(eval_result.is_ok(), "evaluate failed: {:?}", eval_result.err()); let pnl = eval_result.unwrap_or_else(|e| panic!("evaluate error: {}", e)); - assert_eq!(pnl.len(), 30, "Expected 30 PnL values, got {}", pnl.len()); + assert_eq!(pnl.len(), 29, "Expected 29 PnL values (one per return), got {}", pnl.len()); } #[test] diff --git a/ml/src/validation/statistical.rs b/ml/src/validation/statistical.rs index 1fd7f6412..70176f738 100644 --- a/ml/src/validation/statistical.rs +++ b/ml/src/validation/statistical.rs @@ -259,10 +259,11 @@ pub fn deflated_sharpe_ratio( let expected_max_sharpe = sr_std * (term1 + term2); - // Sharpe ratio standard error: - // SE = sqrt((1 - skew*SR + (kurt-1)/4 * SR^2) / (n-1)) + // Sharpe ratio standard error (Bailey & Lopez de Prado, 2014): + // SE = sqrt((1 - γ₃·SR + (γ₄/4)·SR²) / (T-1)) + // where γ₃ = skewness, γ₄ = excess kurtosis let sr_sq = observed_sharpe * observed_sharpe; - let se_inner = 1.0 - skew * observed_sharpe + (kurt - 1.0) / 4.0 * sr_sq; + let se_inner = 1.0 - skew * observed_sharpe + kurt / 4.0 * sr_sq; let se_inner_clamped = se_inner.max(1e-10); // prevent negative sqrt let sharpe_std_error = (se_inner_clamped / (obs - 1.0)).sqrt(); @@ -470,13 +471,23 @@ fn generate_combos_recursive( } } -/// Probability of Backtest Overfitting (Bailey et al., 2017) via CSCV. +/// Probability of Backtest Overfitting (adapted from Bailey et al., 2017) via CSCV. /// -/// Combinatorially Symmetric Cross-Validation (CSCV): +/// Single-strategy adaptation of Combinatorially Symmetric Cross-Validation: /// 1. Takes N fold Sharpe ratios and generates C(N, N/2) combinations -/// 2. For each combination, one half is treated as in-sample (IS), the other as out-of-sample (OOS) -/// 3. Finds the IS-best strategy and checks its OOS rank -/// 4. PBO = fraction of combinations where IS-best ranks below the OOS median +/// 2. For each combination, one half is treated as in-sample (IS), the other as OOS +/// 3. Computes mean Sharpe for IS and OOS halves +/// 4. PBO = fraction of combinations where IS mean > OOS mean +/// +/// **Interpretation for single strategies:** +/// - PBO ≈ 0.5: performance is consistent across time periods (no overfitting signal) +/// - PBO >> 0.5: performance degrades from IS to OOS (temporal overfitting) +/// - PBO << 0.5: performance improves from IS to OOS (unusual, possible regime shift) +/// +/// **Note:** The original CSCV paper tests multiple strategy configurations against +/// each other. This adaptation tests a single strategy's consistency across time. +/// For single strategies, DSR and permutation tests are the primary significance measures; +/// PBO adds value when `num_trials > 1` in a hyperopt setting. /// /// Requires even N >= 4. Returns `pbo = 0.5` for degenerate inputs. pub fn probability_of_backtest_overfitting(per_fold_sharpes: &[f64]) -> PboResult { @@ -507,67 +518,52 @@ pub fn probability_of_backtest_overfitting(per_fold_sharpes: &[f64]) -> PboResul let mut overfit_count = 0usize; for combo in &combinations { - // IS indices are in `combo`, OOS indices are the complement let is_set: std::collections::HashSet = combo.iter().copied().collect(); - let is_sharpes: Vec = combo + // Compute mean Sharpe for IS and OOS halves + let is_sum: f64 = combo .iter() .filter_map(|&i| per_fold_sharpes.get(i).copied()) - .collect(); - - let oos_sharpes: Vec = (0..n) + .sum(); + let oos_sum: f64 = (0..n) .filter(|i| !is_set.contains(i)) .filter_map(|i| per_fold_sharpes.get(i).copied()) - .collect(); + .sum(); - if is_sharpes.is_empty() || oos_sharpes.is_empty() { + let is_count = combo.len() as f64; + let oos_count = (n - combo.len()) as f64; + + if is_count < 1.0 || oos_count < 1.0 { continue; } - // Find IS-best: index within combo that has the highest IS Sharpe - let is_best_combo_idx = is_sharpes - .iter() - .enumerate() - .max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)) - .map(|(i, _)| i) - .unwrap_or(0); // safe: is_sharpes is non-empty + let is_mean = is_sum / is_count; + let oos_mean = oos_sum / oos_count; - // The IS-best strategy's fold index in the original array - let is_best_fold_idx = combo.get(is_best_combo_idx).copied().unwrap_or(0); + // Performance degradation: IS outperforming OOS signals overfitting + let degradation = is_mean - oos_mean; - // Get the IS-best strategy's OOS Sharpe (if it appears in OOS, use the fold's sharpe) - // Actually in CSCV, we check ALL strategies' performance. - // The IS-best strategy has an IS sharpe; we look at its OOS sharpe. - // But the IS-best is in the IS set, not OOS. So we need a different interpretation: - // - // CSCV correct interpretation: we have N strategies (folds serve as performance metrics). - // For each combination split, we rank strategies by IS performance and check if the - // IS-best strategy's OOS performance is below median. - // - // Simplified approach: use the IS-best fold's Sharpe as its "quality", - // then check its rank relative to OOS set. - let is_best_sharpe = per_fold_sharpes.get(is_best_fold_idx).copied().unwrap_or(0.0); - - // Rank the IS-best strategy's sharpe within the OOS sharpes - let oos_rank = oos_sharpes - .iter() - .filter(|&&s| s >= is_best_sharpe) - .count(); - - let oos_len = oos_sharpes.len(); - let relative_rank = if oos_len > 0 { - oos_rank as f64 / oos_len as f64 + // Logit of relative OOS performance. + // ratio = OOS / (|IS| + |OOS|): 0.5 when equal, <0.5 when IS > OOS + let denom = is_mean.abs() + oos_mean.abs(); + let ratio = if denom > 1e-15 { + (oos_mean.abs() / denom).clamp(0.01, 0.99) } else { - 0.5 + 0.5 // both means ≈ 0 }; - - // Logit of relative rank (clamped to avoid log(0)) - let clamped_rank = relative_rank.clamp(0.01, 0.99); - let logit = (clamped_rank / (1.0 - clamped_rank)).ln(); + // Adjust sign: if OOS is negative but IS positive, ratio should reflect underperformance + let signed_ratio = if degradation > 0.0 { + // IS > OOS: overfitting signal, push ratio below 0.5 + ratio.min(0.49) + } else { + // OOS >= IS: genuine signal, push ratio above 0.5 + ratio.max(0.51) + }; + let logit = (signed_ratio / (1.0 - signed_ratio)).ln(); logit_distribution.push(logit); - // Overfit if IS-best ranks below OOS median (relative rank > 0.5) - if relative_rank > 0.5 { + // Count as overfitting if IS mean > OOS mean + if is_mean > oos_mean { overfit_count = overfit_count.saturating_add(1); } } @@ -697,8 +693,9 @@ mod tests { #[test] fn test_dsr_single_trial_not_penalized() { - // SR=2.0, only 1 trial -> no multiple testing penalty -> should be significant - let result = deflated_sharpe_ratio(2.0, 1, 1.0, 0.0, 3.0, 252); + // SR=2.0, only 1 trial, normal returns (excess kurtosis=0) + // -> no multiple testing penalty -> should be significant + let result = deflated_sharpe_ratio(2.0, 1, 1.0, 0.0, 0.0, 252); assert!( result.pvalue < 0.05, "Single trial SR=2.0 should have p<0.05, got {}", @@ -708,8 +705,9 @@ mod tests { #[test] fn test_dsr_many_trials_penalized() { - // SR=1.0, 1000 trials -> heavy penalty -> should NOT be significant - let result = deflated_sharpe_ratio(1.0, 1000, 1.0, 0.0, 3.0, 252); + // SR=1.0, 1000 trials, normal returns (excess kurtosis=0) + // -> heavy penalty -> should NOT be significant + let result = deflated_sharpe_ratio(1.0, 1000, 1.0, 0.0, 0.0, 252); assert!( result.pvalue > 0.05, "1000 trials SR=1.0 should have p>0.05, got {}", @@ -720,8 +718,8 @@ mod tests { #[test] fn test_dsr_higher_sharpe_more_significant() { // SR=3 should be more significant than SR=1 with same parameters - let r1 = deflated_sharpe_ratio(1.0, 100, 1.0, 0.0, 3.0, 252); - let r3 = deflated_sharpe_ratio(3.0, 100, 1.0, 0.0, 3.0, 252); + let r1 = deflated_sharpe_ratio(1.0, 100, 1.0, 0.0, 0.0, 252); + let r3 = deflated_sharpe_ratio(3.0, 100, 1.0, 0.0, 0.0, 252); assert!( r3.pvalue < r1.pvalue, "SR=3 p-value ({}) should be < SR=1 p-value ({})", @@ -730,6 +728,20 @@ mod tests { ); } + #[test] + fn test_dsr_se_formula_normal_returns() { + // For normal returns (skew=0, excess_kurt=0), SE should simplify to: + // SE = sqrt(1 / (T-1)) = 1/sqrt(T-1) + let result = deflated_sharpe_ratio(1.0, 1, 1.0, 0.0, 0.0, 252); + let expected_se = 1.0 / (251.0_f64).sqrt(); + assert!( + (result.sharpe_std_error - expected_se).abs() < 1e-6, + "SE for normal returns should be 1/sqrt(T-1)={}, got {}", + expected_se, + result.sharpe_std_error + ); + } + // ── Permutation test ──────────────────────────────────────────────── #[test] @@ -768,34 +780,27 @@ mod tests { // ── PBO tests ─────────────────────────────────────────────────────── #[test] - fn test_pbo_random_sharpes_high() { - // Wildly inconsistent Sharpe ratios across folds suggest overfitting: - // some folds show strong positive, others strong negative -- classic overfit signature - let sharpes = vec![ - 3.0, -2.5, 2.8, -2.0, 2.5, -1.8, -2.2, 2.9, -1.5, 2.7, -2.3, 1.9, - ]; - let result = probability_of_backtest_overfitting(&sharpes); - assert!( - result.pbo > 0.0, - "Varied Sharpes should show some overfitting signal, got PBO={}", - result.pbo - ); - assert!( - result.num_combinations > 0, - "Should have evaluated combinations" - ); - } - - #[test] - fn test_pbo_consistent_sharpes_low() { - // All consistently positive Sharpes -> lower PBO + fn test_pbo_produces_valid_output() { + // 8 folds with varied Sharpes — verify structural correctness let sharpes = vec![1.0, 1.1, 1.05, 0.95, 1.02, 0.98, 1.08, 1.03]; let result = probability_of_backtest_overfitting(&sharpes); + assert!( - result.pbo < 0.5, - "Consistent positive Sharpes should have PBO < 0.5, got {}", + (0.0..=1.0).contains(&result.pbo), + "PBO must be in [0,1], got {}", result.pbo ); + // C(8,4) = 70 combinations + assert_eq!(result.num_combinations, 70); + assert!( + !result.logit_distribution.is_empty(), + "Should produce logit distribution" + ); + // All logits should be finite + assert!( + result.logit_distribution.iter().all(|l| l.is_finite()), + "All logits should be finite" + ); } #[test] @@ -811,6 +816,32 @@ mod tests { assert_eq!(result.num_combinations, 0); } + #[test] + fn test_pbo_odd_folds_degenerate() { + // N=5 (odd) -> degenerate result + let sharpes = vec![1.0, 2.0, 3.0, 4.0, 5.0]; + let result = probability_of_backtest_overfitting(&sharpes); + assert!( + (result.pbo - 0.5).abs() < 1e-10, + "Odd N should return pbo=0.5, got {}", + result.pbo + ); + assert_eq!(result.num_combinations, 0); + } + + #[test] + fn test_pbo_identical_sharpes_symmetric() { + // All identical Sharpes → IS_mean = OOS_mean for every combination + // Neither IS > OOS nor OOS > IS → PBO = 0.0 (no overfit count) + let sharpes = vec![1.0, 1.0, 1.0, 1.0]; + let result = probability_of_backtest_overfitting(&sharpes); + assert!( + result.pbo < 0.01, + "Identical Sharpes should have PBO ≈ 0.0 (no IS > OOS), got {}", + result.pbo + ); + } + #[test] fn test_binomial_coefficient() { assert_eq!(binomial_coefficient(8, 4), 70); diff --git a/ml/src/validation/walk_forward.rs b/ml/src/validation/walk_forward.rs index 733b263d6..60ef78944 100644 --- a/ml/src/validation/walk_forward.rs +++ b/ml/src/validation/walk_forward.rs @@ -72,6 +72,13 @@ pub struct Fold { /// Generate walk-forward cross-validation folds with embargo. /// +/// Uses a **rolling window** design: each fold has a fixed-size training window +/// that slides forward. This is the standard approach for evaluating strategies +/// on recent data (avoids stale early data diluting training signal). +/// +/// An expanding window variant (where training grows from an anchor point) may +/// be added later if needed for strategies that benefit from larger datasets. +/// /// Slides a `[train | embargo | test]` window across `num_bars` bar indices, /// advancing by `config.step_bars` each iteration. A fold is only emitted when: /// - The training window has at least `config.min_train_samples` bars