diff --git a/backtesting/src/lib.rs b/backtesting/src/lib.rs index aaec4919b..3bcae309f 100644 --- a/backtesting/src/lib.rs +++ b/backtesting/src/lib.rs @@ -68,6 +68,7 @@ use rust_decimal::Decimal; pub mod metrics; pub mod replay_engine; +pub mod slippage; pub mod strategies; pub mod strategy_runner; pub mod strategy_tester; @@ -79,6 +80,10 @@ pub use strategies::{DQNAction, DQNReplayStrategy, PositionState, TradingActionT pub use strategy_runner::{ create_adaptive_strategy_with_config, AdaptiveStrategyConfig, FeatureSettings, RiskSettings, }; +pub use slippage::{ + FixedSlippage, RegimeAwareSlippage, SlippageContext, SlippageModel, SpreadProfile, + VolatilityRegime, VolumeImpactSlippage, +}; pub use strategy_tester::{ PerformanceSnapshot, SignalType, Strategy, StrategyConfig, StrategyContext, StrategyResult, StrategyTester, TradeRecord, TradingSignal, diff --git a/backtesting/src/slippage.rs b/backtesting/src/slippage.rs new file mode 100644 index 000000000..00b9dcfe7 --- /dev/null +++ b/backtesting/src/slippage.rs @@ -0,0 +1,287 @@ +//! Dynamic slippage models for realistic trade execution simulation. +//! +//! Provides pluggable slippage estimation that accounts for order size, +//! market liquidity, bid-ask spreads, and volatility regimes. + +use serde::{Deserialize, Serialize}; + +// --------------------------------------------------------------------------- +// Core types +// --------------------------------------------------------------------------- + +/// Volatility regime classification used to scale slippage estimates. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum VolatilityRegime { + /// Calm markets — tighter spreads, lower impact. + Low, + /// Typical trading conditions. + Normal, + /// Stressed / crisis markets — wide spreads, high impact. + Crisis, +} + +/// Contextual information passed to a [`SlippageModel`] for each fill. +#[derive(Debug, Clone)] +pub struct SlippageContext { + /// Size of the order being filled (contracts / shares). + pub order_size: f64, + /// Average daily volume for the instrument. + pub average_daily_volume: f64, + /// Current bid-ask spread (price units). + pub current_spread: f64, + /// Current volatility regime. + pub volatility_regime: VolatilityRegime, +} + +/// Spread characteristics for a specific instrument. +#[derive(Debug, Clone)] +pub struct SpreadProfile { + /// Human-readable instrument name. + pub name: String, + /// Minimum price increment. + pub tick_size: f64, + /// Typical spread expressed in ticks. + pub typical_spread_ticks: f64, +} + +impl SpreadProfile { + /// E-mini S&P 500 futures (ES) — tick = 0.25, typical spread = 1 tick. + #[must_use] + pub fn es_futures() -> Self { + Self { + name: "ES".to_string(), + tick_size: 0.25, + typical_spread_ticks: 1.0, + } + } + + /// 10-Year Treasury Note futures (ZN) — tick = 1/64, typical spread = 1 tick. + #[must_use] + pub fn zn_futures() -> Self { + Self { + name: "ZN".to_string(), + tick_size: 1.0 / 64.0, + typical_spread_ticks: 1.0, + } + } + + /// USD/CHF (6S / SIXE) futures — tick = 0.0005 (half-pip), typical spread = 2 ticks. + #[must_use] + pub fn sixe_futures() -> Self { + Self { + name: "6S".to_string(), + tick_size: 0.0005, + typical_spread_ticks: 2.0, + } + } + + /// Typical spread in price units (`tick_size * typical_spread_ticks`). + #[must_use] + pub fn typical_spread(&self) -> f64 { + self.tick_size * self.typical_spread_ticks + } +} + +// --------------------------------------------------------------------------- +// Trait +// --------------------------------------------------------------------------- + +/// A model that estimates execution slippage (in price units) for a given +/// order context. Implementations must be `Send + Sync` so they can be +/// shared across async tasks. +pub trait SlippageModel: Send + Sync { + /// Compute the estimated slippage for the given context. + /// + /// Returns a non-negative value in price units that should be *added* to + /// the execution cost (subtracted from fills on buys, added on sells). + fn compute(&self, ctx: &SlippageContext) -> f64; +} + +// --------------------------------------------------------------------------- +// Implementations +// --------------------------------------------------------------------------- + +/// Returns a constant slippage regardless of context. +/// +/// This is the simplest model and serves as a backward-compatible default. +#[derive(Debug, Clone)] +pub struct FixedSlippage { + /// Constant slippage value (price units). + pub slippage: f64, +} + +impl FixedSlippage { + /// Create a new fixed slippage model. + #[must_use] + pub fn new(slippage: f64) -> Self { + Self { slippage } + } +} + +impl SlippageModel for FixedSlippage { + fn compute(&self, _ctx: &SlippageContext) -> f64 { + self.slippage + } +} + +/// Square-root market-impact model: `slippage = k * sqrt(order_size / ADV)`. +/// +/// This is the standard Almgren-Chriss style temporary impact model used +/// widely in execution analytics. +#[derive(Debug, Clone)] +pub struct VolumeImpactSlippage { + /// Impact coefficient (scales the sqrt term). + pub k: f64, +} + +impl VolumeImpactSlippage { + /// Create a new volume-impact model with the given coefficient. + #[must_use] + pub fn new(k: f64) -> Self { + Self { k } + } +} + +impl SlippageModel for VolumeImpactSlippage { + fn compute(&self, ctx: &SlippageContext) -> f64 { + if ctx.average_daily_volume <= 0.0 { + return 0.0; + } + let participation = ctx.order_size / ctx.average_daily_volume; + self.k * participation.sqrt() + } +} + +/// Wraps an inner [`SlippageModel`] and multiplies its output by a factor +/// determined by the current [`VolatilityRegime`]. +/// +/// Regime multipliers: +/// - `Low` — 0.5 +/// - `Normal` — 1.0 +/// - `Crisis` — 3.0 +pub struct RegimeAwareSlippage { + inner: Box, + low_factor: f64, + normal_factor: f64, + crisis_factor: f64, +} + +impl std::fmt::Debug for RegimeAwareSlippage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RegimeAwareSlippage") + .field("low_factor", &self.low_factor) + .field("normal_factor", &self.normal_factor) + .field("crisis_factor", &self.crisis_factor) + .finish_non_exhaustive() + } +} + +impl RegimeAwareSlippage { + /// Wrap `inner` with default regime multipliers (0.5 / 1.0 / 3.0). + pub fn new(inner: Box) -> Self { + Self { + inner, + low_factor: 0.5, + normal_factor: 1.0, + crisis_factor: 3.0, + } + } + + /// Return the multiplier for the given regime. + fn regime_factor(&self, regime: VolatilityRegime) -> f64 { + match regime { + VolatilityRegime::Low => self.low_factor, + VolatilityRegime::Normal => self.normal_factor, + VolatilityRegime::Crisis => self.crisis_factor, + } + } +} + +impl SlippageModel for RegimeAwareSlippage { + fn compute(&self, ctx: &SlippageContext) -> f64 { + let base = self.inner.compute(ctx); + base * self.regime_factor(ctx.volatility_regime) + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fixed_slippage_returns_constant() { + let model = FixedSlippage::new(0.5); + let ctx = SlippageContext { + order_size: 100.0, + average_daily_volume: 1_000_000.0, + current_spread: 0.25, + volatility_regime: VolatilityRegime::Normal, + }; + let result = model.compute(&ctx); + assert!( + (result - 0.5).abs() < f64::EPSILON, + "Fixed slippage should return the constant value, got {result}" + ); + } + + #[test] + fn volume_impact_scales_with_order_size() { + let model = VolumeImpactSlippage::new(1.0); + let small_ctx = SlippageContext { + order_size: 100.0, + average_daily_volume: 1_000_000.0, + current_spread: 0.25, + volatility_regime: VolatilityRegime::Normal, + }; + let large_ctx = SlippageContext { + order_size: 10_000.0, + average_daily_volume: 1_000_000.0, + current_spread: 0.25, + volatility_regime: VolatilityRegime::Normal, + }; + let small_slip = model.compute(&small_ctx); + let large_slip = model.compute(&large_ctx); + assert!( + large_slip > small_slip, + "Larger orders should produce more slippage: large={large_slip}, small={small_slip}" + ); + } + + #[test] + fn regime_aware_crisis_exceeds_normal() { + let inner = Box::new(FixedSlippage::new(1.0)); + let model = RegimeAwareSlippage::new(inner); + + let normal_ctx = SlippageContext { + order_size: 100.0, + average_daily_volume: 1_000_000.0, + current_spread: 0.25, + volatility_regime: VolatilityRegime::Normal, + }; + let crisis_ctx = SlippageContext { + volatility_regime: VolatilityRegime::Crisis, + ..normal_ctx.clone() + }; + + let normal_slip = model.compute(&normal_ctx); + let crisis_slip = model.compute(&crisis_ctx); + assert!( + crisis_slip > normal_slip, + "Crisis slippage ({crisis_slip}) should exceed normal ({normal_slip})" + ); + } + + #[test] + fn es_spread_profile_tick_size() { + let es = SpreadProfile::es_futures(); + assert!( + (es.tick_size - 0.25).abs() < f64::EPSILON, + "ES tick_size should be 0.25, got {}", + es.tick_size + ); + } +} diff --git a/ml/src/dqn/performance_tests.rs b/ml/src/dqn/performance_tests.rs index 49f9b7ade..a0e2aee93 100644 --- a/ml/src/dqn/performance_tests.rs +++ b/ml/src/dqn/performance_tests.rs @@ -232,18 +232,25 @@ async fn test_rainbow_network_performance() -> Result<(), MLError> { let _ = network.forward(&input)?; } - // Measure single inference - let start = Instant::now(); - let _output = network.forward(&input)?; - let latency = start.elapsed(); + // Measure inference (median of 21 runs to avoid flaky single-sample outliers) + let mut latencies: Vec = (0..21) + .map(|_| { + let start = Instant::now(); + let _output = network.forward(&input).unwrap(); + start.elapsed().as_micros() + }) + .collect(); + latencies.sort(); + let median_latency = latencies[latencies.len() / 2]; - println!("Single inference latency: {}μs", latency.as_micros()); + println!("Median inference latency: {}μs (min={}μs, max={}μs)", + median_latency, latencies[0], latencies[latencies.len() - 1]); - // Should be well under 100μs for small networks + // Median should be well under 1ms for small networks assert!( - latency.as_micros() < 1000, - "Inference took too long: {}μs", - latency.as_micros() + median_latency < 1000, + "Median inference too slow: {}μs", + median_latency ); Ok(()) diff --git a/ml/src/validation/degradation.rs b/ml/src/validation/degradation.rs new file mode 100644 index 000000000..804586866 --- /dev/null +++ b/ml/src/validation/degradation.rs @@ -0,0 +1,394 @@ +//! Fold-by-fold performance degradation tracking. +//! +//! [`DegradationTracker`] collects per-fold performance metrics and computes a +//! [`DegradationReport`] with linear regression slope of Sharpe ratios across +//! folds, p-value, coefficient of variation, recency-weighted Sharpe, and an +//! estimated half-life. + +use serde::{Deserialize, Serialize}; + +// --------------------------------------------------------------------------- +// FoldMetrics +// --------------------------------------------------------------------------- + +/// Performance metrics for a single walk-forward fold. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FoldMetrics { + /// Zero-based fold index. + pub fold_index: usize, + /// Annualised Sharpe ratio for this fold. + pub sharpe: f64, + /// Calmar ratio for this fold. + pub calmar: f64, + /// Win rate (fraction of profitable trades). + pub win_rate: f64, + /// Maximum drawdown (positive value, e.g. 0.15 = 15%). + pub max_drawdown: f64, + /// Number of trades executed in this fold. + pub trade_count: usize, +} + +// --------------------------------------------------------------------------- +// DegradationReport +// --------------------------------------------------------------------------- + +/// Summary of performance decay across walk-forward folds. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DegradationReport { + /// OLS slope of Sharpe vs fold index. Negative ⇒ decaying performance. + pub sharpe_slope: f64, + /// Two-tailed p-value for the slope (H0: slope = 0). + pub slope_pvalue: f64, + /// Coefficient of variation of per-fold Sharpe ratios. + pub coefficient_of_variation: f64, + /// Exponentially recency-weighted average Sharpe (lambda = 0.3). + pub recency_weighted_sharpe: f64, + /// Estimated number of folds until Sharpe reaches zero at the current + /// decay rate. `None` if slope >= 0 (no decay) or the last Sharpe <= 0. + pub half_life_folds: Option, + /// The raw fold metrics used to compute this report. + pub fold_metrics: Vec, +} + +// --------------------------------------------------------------------------- +// DegradationTracker +// --------------------------------------------------------------------------- + +/// Collects per-fold metrics and computes a degradation report. +#[derive(Debug, Default)] +pub struct DegradationTracker { + folds: Vec, +} + +impl DegradationTracker { + /// Create a new, empty tracker. + pub fn new() -> Self { + Self { folds: Vec::new() } + } + + /// Record metrics for the next fold. + pub fn add_fold(&mut self, metrics: FoldMetrics) { + self.folds.push(metrics); + } + + /// Number of folds recorded so far. + pub fn num_folds(&self) -> usize { + self.folds.len() + } + + /// Compute the degradation report from all recorded folds. + /// + /// Requires at least 2 folds. With fewer, slopes and p-values are set + /// to 0.0 / 1.0 respectively. + pub fn compute(&self) -> DegradationReport { + let n = self.folds.len(); + let sharpes: Vec = self.folds.iter().map(|f| f.sharpe).collect(); + + if n < 2 { + return DegradationReport { + sharpe_slope: 0.0, + slope_pvalue: 1.0, + coefficient_of_variation: 0.0, + recency_weighted_sharpe: sharpes.first().copied().unwrap_or(0.0), + half_life_folds: None, + fold_metrics: self.folds.clone(), + }; + } + + // --- Linear regression: y = sharpe, x = fold_index --- + let (slope, intercept) = linear_regression(&sharpes); + + // --- P-value via t-test --- + let slope_pvalue = slope_p_value(&sharpes, slope, intercept); + + // --- Coefficient of variation --- + let coefficient_of_variation = coeff_of_variation(&sharpes); + + // --- Recency-weighted Sharpe (exponential decay, lambda = 0.3) --- + let recency_weighted_sharpe = recency_weighted_mean(&sharpes, 0.3); + + // --- Half-life estimate --- + let half_life_folds = if slope < 0.0 { + // Extrapolate from last fold: how many more folds until Sharpe = 0? + let last_sharpe = sharpes.last().copied().unwrap_or(0.0); + if last_sharpe > 0.0 { + Some(last_sharpe / (-slope)) + } else { + None + } + } else { + None + }; + + DegradationReport { + sharpe_slope: slope, + slope_pvalue, + coefficient_of_variation, + recency_weighted_sharpe, + half_life_folds, + fold_metrics: self.folds.clone(), + } + } +} + +// --------------------------------------------------------------------------- +// Statistical helpers +// --------------------------------------------------------------------------- + +/// Ordinary least-squares linear regression of `ys` against `x = 0, 1, 2, ...`. +/// +/// Returns `(slope, intercept)`. +fn linear_regression(ys: &[f64]) -> (f64, f64) { + let n = ys.len() as f64; + if n < 2.0 { + return (0.0, ys.first().copied().unwrap_or(0.0)); + } + + let x_mean = (n - 1.0) / 2.0; + let y_mean = ys.iter().sum::() / n; + + let mut numerator = 0.0_f64; + let mut denominator = 0.0_f64; + + for (i, y) in ys.iter().enumerate() { + let x = i as f64; + numerator += (x - x_mean) * (y - y_mean); + denominator += (x - x_mean) * (x - x_mean); + } + + if denominator.abs() < 1e-30 { + return (0.0, y_mean); + } + + let slope = numerator / denominator; + let intercept = y_mean - slope * x_mean; + (slope, intercept) +} + +/// Two-tailed p-value for the regression slope via a t-test. +/// +/// Uses the Abramowitz & Stegun rational approximation for the normal CDF +/// when degrees of freedom >= 30, and a simple t-to-normal approximation +/// for smaller samples. +fn slope_p_value(ys: &[f64], slope: f64, intercept: f64) -> f64 { + let n = ys.len(); + if n < 3 { + return 1.0; + } + + let n_f = n as f64; + let df = n_f - 2.0; + + // Residual sum of squares + let rss: f64 = ys + .iter() + .enumerate() + .map(|(i, &y)| { + let predicted = intercept + slope * (i as f64); + (y - predicted).powi(2) + }) + .sum(); + + let mse = rss / df; + if mse < 1e-30 { + // Perfect fit → slope is exactly determined + if slope.abs() < 1e-30 { + return 1.0; + } + return 0.0; + } + + // Sum of squared deviations of x from its mean + let x_mean = (n_f - 1.0) / 2.0; + let ss_x: f64 = (0..n).map(|i| (i as f64 - x_mean).powi(2)).sum(); + + if ss_x < 1e-30 { + return 1.0; + } + + let se_slope = (mse / ss_x).sqrt(); + let t_stat = slope / se_slope; + + // Convert t-statistic to p-value using normal approximation + // (adequate for our use case; exact t-distribution needs more machinery) + let p = 2.0 * (1.0 - normal_cdf_approx(t_stat.abs())); + p.clamp(0.0, 1.0) +} + +/// Abramowitz & Stegun approximation to the standard normal CDF. +/// +/// Maximum error < 7.5e-8. Formula 26.2.17 from Handbook of Mathematical +/// Functions. +fn normal_cdf_approx(x: f64) -> f64 { + if x < -8.0 { + return 0.0; + } + if x > 8.0 { + return 1.0; + } + + let a1 = 0.254829592_f64; + let a2 = -0.284496736_f64; + let a3 = 1.421413741_f64; + let a4 = -1.453152027_f64; + let a5 = 1.061405429_f64; + let p = 0.3275911_f64; + + let sign = if x >= 0.0 { 1.0 } else { -1.0 }; + let abs_x = x.abs(); + let t = 1.0 / (1.0 + p * abs_x); + let y = 1.0 - (((((a5 * t + a4) * t) + a3) * t + a2) * t + a1) * t * (-abs_x * abs_x / 2.0).exp(); + + 0.5 * (1.0 + sign * y) +} + +/// Coefficient of variation (std / |mean|). +fn coeff_of_variation(values: &[f64]) -> f64 { + let n = values.len() as f64; + if n < 1.0 { + return 0.0; + } + + let mean = values.iter().sum::() / n; + if mean.abs() < 1e-30 { + return 0.0; + } + + let variance = values.iter().map(|v| (v - mean).powi(2)).sum::() / n; + variance.sqrt() / mean.abs() +} + +/// Exponentially recency-weighted mean. +/// +/// Weight for fold `i` (0-indexed) is `exp(lambda * i)`, so later folds +/// receive higher weight. +fn recency_weighted_mean(values: &[f64], lambda: f64) -> f64 { + if values.is_empty() { + return 0.0; + } + + let mut weight_sum = 0.0_f64; + let mut weighted_sum = 0.0_f64; + + for (i, &v) in values.iter().enumerate() { + let w = (lambda * i as f64).exp(); + weighted_sum += w * v; + weight_sum += w; + } + + if weight_sum.abs() < 1e-30 { + return 0.0; + } + + weighted_sum / weight_sum +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + /// Helper: create a tracker with linearly declining Sharpe ratios. + fn make_declining_tracker(n: usize, start: f64, slope: f64) -> DegradationTracker { + let mut tracker = DegradationTracker::new(); + for i in 0..n { + tracker.add_fold(FoldMetrics { + fold_index: i, + sharpe: start + slope * i as f64, + calmar: 1.0, + win_rate: 0.55, + max_drawdown: 0.10, + trade_count: 100, + }); + } + tracker + } + + #[test] + fn test_negative_slope_for_declining_sharpe() { + let tracker = make_declining_tracker(10, 2.0, -0.15); + let report = tracker.compute(); + + assert!( + report.sharpe_slope < 0.0, + "Expected negative slope, got {}", + report.sharpe_slope, + ); + // The slope should be approximately -0.15 + assert!( + (report.sharpe_slope - (-0.15)).abs() < 1e-10, + "Expected slope ~-0.15, got {}", + report.sharpe_slope, + ); + } + + #[test] + fn test_pvalue_significant_for_strong_trend() { + let tracker = make_declining_tracker(20, 2.0, -0.1); + let report = tracker.compute(); + + // With a perfectly linear decline across 20 folds, p-value should be + // essentially 0 (the fit is exact). + assert!( + report.slope_pvalue < 0.05, + "Expected p-value < 0.05, got {}", + report.slope_pvalue, + ); + } + + #[test] + fn test_half_life_estimated_correctly() { + // Start=2.0, slope=-0.2/fold → after 10 folds, Sharpe=0.0 + // Last Sharpe at fold 9: 2.0 + (-0.2)*9 = 0.2 + // Half-life = 0.2 / 0.2 = 1.0 folds remaining + let tracker = make_declining_tracker(10, 2.0, -0.2); + let report = tracker.compute(); + + assert!(report.half_life_folds.is_some()); + let hl = report.half_life_folds.unwrap_or(f64::NAN); + assert!( + (hl - 1.0).abs() < 1e-10, + "Expected half_life ~1.0, got {}", + hl, + ); + } + + #[test] + fn test_recency_weighting_favours_later_folds() { + // Two folds: fold 0 has Sharpe=0.0, fold 1 has Sharpe=2.0 + // Recency weighting (lambda=0.3) should produce a weighted mean > 1.0 + let mut tracker = DegradationTracker::new(); + tracker.add_fold(FoldMetrics { + fold_index: 0, + sharpe: 0.0, + calmar: 0.5, + win_rate: 0.45, + max_drawdown: 0.20, + trade_count: 50, + }); + tracker.add_fold(FoldMetrics { + fold_index: 1, + sharpe: 2.0, + calmar: 1.5, + win_rate: 0.60, + max_drawdown: 0.08, + trade_count: 80, + }); + + let report = tracker.compute(); + assert!( + report.recency_weighted_sharpe > 1.0, + "Expected recency-weighted Sharpe > 1.0, got {}", + report.recency_weighted_sharpe, + ); + // Simple mean would be 1.0; recency weighting should push above 1.0 + assert!( + report.recency_weighted_sharpe < 2.0, + "Expected recency-weighted Sharpe < 2.0, got {}", + report.recency_weighted_sharpe, + ); + } +} diff --git a/ml/src/validation/harness.rs b/ml/src/validation/harness.rs index 883772b1f..67ffbbfed 100644 --- a/ml/src/validation/harness.rs +++ b/ml/src/validation/harness.rs @@ -15,7 +15,7 @@ use crate::MLError; use super::{ deflated_sharpe_ratio, excess_kurtosis, per_regime_breakdown, permutation_test, probability_of_backtest_overfitting, sharpe_ratio, skewness, walk_forward_split, - DsrResult, PboResult, PermutationResult, RegimeMetrics, TimeSeriesData, + DegradationReport, DsrResult, PboResult, PermutationResult, RegimeMetrics, TimeSeriesData, ValidatableStrategy, WalkForwardConfig, }; @@ -74,6 +74,9 @@ pub struct ValidationReport { pub per_regime_metrics: HashMap, /// Overall verdict. pub verdict: ValidationVerdict, + /// Optional degradation report (fold-by-fold performance decay analysis). + #[serde(skip_serializing_if = "Option::is_none")] + pub degradation: Option, } // --------------------------------------------------------------------------- @@ -245,6 +248,7 @@ impl ValidationHarness { permutation, per_regime_metrics, verdict, + degradation: None, }) } } diff --git a/ml/src/validation/mod.rs b/ml/src/validation/mod.rs index 8344ade45..0f0e2dc5a 100644 --- a/ml/src/validation/mod.rs +++ b/ml/src/validation/mod.rs @@ -5,11 +5,13 @@ //! - Statistical validation (walk-forward, DSR, PBO, permutation tests) pub mod adapters; +pub mod degradation; pub mod financial; pub mod harness; pub mod ppo_adapter; pub mod regime_analysis; pub mod statistical; +pub mod temporal_guard; pub mod types; pub mod walk_forward; @@ -43,3 +45,9 @@ pub use statistical::{ probability_of_backtest_overfitting, sharpe_ratio, skewness, DsrResult, PboResult, PermutationResult, }; + +// Re-export temporal guard types +pub use temporal_guard::{LeakageAuditReport, NormalizationStats, TemporalGuard}; + +// Re-export degradation tracking types +pub use degradation::{DegradationReport, DegradationTracker, FoldMetrics}; diff --git a/ml/src/validation/temporal_guard.rs b/ml/src/validation/temporal_guard.rs new file mode 100644 index 000000000..c8c40983d --- /dev/null +++ b/ml/src/validation/temporal_guard.rs @@ -0,0 +1,403 @@ +//! Temporal guard for preventing forward-looking data leakage. +//! +//! Wraps a [`TimeSeriesData`] reference with a cutoff index that partitions +//! the series into a training region `[0, cutoff)` and a test region +//! `[cutoff, len)`. Any attempt to slice across the boundary is rejected, +//! ensuring that training code never accidentally sees future data. + +use chrono::{DateTime, Utc}; + +use super::TimeSeriesData; +use crate::MLError; + +// --------------------------------------------------------------------------- +// LeakageAuditReport +// --------------------------------------------------------------------------- + +/// Result of a forward-leakage audit on a [`TemporalGuard`]. +#[derive(Debug, Clone)] +pub struct LeakageAuditReport { + /// `true` if any training timestamp is >= any test timestamp. + pub has_future_timestamps: bool, + /// Number of bars in the training partition. + pub training_bars: usize, + /// The timestamp at the cutoff index, if available. + pub cutoff_timestamp: Option>, +} + +// --------------------------------------------------------------------------- +// NormalizationStats +// --------------------------------------------------------------------------- + +/// Feature-wise normalization statistics computed from training data only. +#[derive(Debug, Clone)] +pub struct NormalizationStats { + /// Per-feature means. + pub means: Vec, + /// Per-feature standard deviations. + pub stds: Vec, + /// Number of samples (bars) used. + pub sample_count: usize, +} + +// --------------------------------------------------------------------------- +// TemporalGuard +// --------------------------------------------------------------------------- + +/// Prevents forward-looking data leakage by enforcing a temporal cutoff. +/// +/// The guard partitions the underlying [`TimeSeriesData`] into: +/// - **Training region**: `[0, cutoff_idx)` +/// - **Test region**: `[cutoff_idx, data.len())` +/// +/// Slicing across the boundary is an error. +#[derive(Debug)] +pub struct TemporalGuard<'a> { + data: &'a TimeSeriesData, + cutoff_idx: usize, +} + +impl<'a> TemporalGuard<'a> { + /// Create a new `TemporalGuard`. + /// + /// # Errors + /// + /// Returns [`MLError::InvalidInput`] if `cutoff_idx` is 0, less than 2 + /// (training needs at least 2 bars), or exceeds `data.len()`. + pub fn new(data: &'a TimeSeriesData, cutoff_idx: usize) -> Result { + if cutoff_idx < 2 { + return Err(MLError::InvalidInput(format!( + "TemporalGuard cutoff must be >= 2 for valid training slice, got {}", + cutoff_idx, + ))); + } + if cutoff_idx > data.len() { + return Err(MLError::InvalidInput(format!( + "TemporalGuard cutoff ({}) exceeds data length ({})", + cutoff_idx, + data.len(), + ))); + } + Ok(Self { data, cutoff_idx }) + } + + /// Return the training partition `[0, cutoff)` as a new `TimeSeriesData`. + /// + /// # Errors + /// + /// Propagates errors from [`TimeSeriesData::slice`]. + pub fn training_slice(&self) -> Result { + self.data.slice(0, self.cutoff_idx) + } + + /// Return a test sub-range `[start, end)`. + /// + /// Both `start` and `end` must be at or after the cutoff index. + /// + /// # Errors + /// + /// Returns [`MLError::InvalidInput`] if `start < cutoff_idx`, or + /// propagates errors from [`TimeSeriesData::slice`]. + pub fn test_slice(&self, start: usize, end: usize) -> Result { + if start < self.cutoff_idx { + return Err(MLError::InvalidInput(format!( + "test_slice start ({}) is before cutoff ({}); this would leak training data", + start, self.cutoff_idx, + ))); + } + self.data.slice(start, end) + } + + /// Slice the data within a single partition. + /// + /// The range `[start, end)` must lie entirely within `[0, cutoff)` or + /// entirely within `[cutoff, len)`. Crossing the boundary is an error. + /// + /// # Errors + /// + /// Returns [`MLError::InvalidInput`] if the range crosses the cutoff + /// boundary, or propagates errors from [`TimeSeriesData::slice`]. + pub fn slice(&self, start: usize, end: usize) -> Result { + if start >= end { + return Err(MLError::InvalidInput(format!( + "Invalid slice range: start ({}) must be less than end ({})", + start, end, + ))); + } + // Entirely in training region + let in_train = end <= self.cutoff_idx; + // Entirely in test region + let in_test = start >= self.cutoff_idx; + + if !in_train && !in_test { + return Err(MLError::InvalidInput(format!( + "Slice [{}, {}) crosses the temporal cutoff at index {}; \ + this would leak future data into the training partition", + start, end, self.cutoff_idx, + ))); + } + self.data.slice(start, end) + } + + /// Audit for timestamp-based forward leakage. + /// + /// Checks whether any training timestamp is >= the earliest test + /// timestamp, which would indicate temporal ordering issues. + pub fn audit_leakage(&self) -> LeakageAuditReport { + let cutoff_timestamp = self.data.timestamps.get(self.cutoff_idx).copied(); + + let has_future_timestamps = if self.cutoff_idx < self.data.len() { + // Find the minimum test timestamp + let min_test_ts = self + .data + .timestamps + .get(self.cutoff_idx..) + .unwrap_or_default() + .iter() + .min() + .copied(); + + // Find the maximum training timestamp + let max_train_ts = self + .data + .timestamps + .get(..self.cutoff_idx) + .unwrap_or_default() + .iter() + .max() + .copied(); + + match (max_train_ts, min_test_ts) { + (Some(train_max), Some(test_min)) => train_max >= test_min, + _ => false, + } + } else { + false + }; + + LeakageAuditReport { + has_future_timestamps, + training_bars: self.cutoff_idx, + cutoff_timestamp, + } + } + + /// Compute per-feature normalization statistics from training data only. + /// + /// Returns means and standard deviations for each feature dimension, + /// computed exclusively from `[0, cutoff)`. These can then be applied + /// to the test partition without leaking future information. + pub fn compute_normalization_stats(&self) -> NormalizationStats { + let train_features = self + .data + .features + .get(..self.cutoff_idx) + .unwrap_or_default(); + + let sample_count = train_features.len(); + if sample_count == 0 { + return NormalizationStats { + means: Vec::new(), + stds: Vec::new(), + sample_count: 0, + }; + } + + // Determine feature dimension from first row + let dim = train_features + .first() + .map(|r| r.len()) + .unwrap_or_default(); + + let mut means = vec![0.0_f64; dim]; + let mut sq_sums = vec![0.0_f64; dim]; + + for row in train_features { + for (j, val) in row.iter().enumerate() { + if let Some(m) = means.get_mut(j) { + *m += *val as f64; + } + if let Some(s) = sq_sums.get_mut(j) { + *s += (*val as f64) * (*val as f64); + } + } + } + + let n = sample_count as f64; + for m in &mut means { + *m /= n; + } + + let stds: Vec = means + .iter() + .zip(sq_sums.iter()) + .map(|(&mean, &sq_sum)| { + let variance = (sq_sum / n) - mean * mean; + if variance > 0.0 { + variance.sqrt() + } else { + 0.0 + } + }) + .collect(); + + NormalizationStats { + means, + stds, + sample_count, + } + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeZone; + + /// Helper: create N UTC timestamps starting from 2024-01-01. + fn make_timestamps(n: usize) -> Vec> { + (0..n) + .map(|i| { + Utc.with_ymd_and_hms(2024, 1, 1, 0, 0, 0) + .single() + .unwrap_or_else(Utc::now) + + chrono::Duration::days(i as i64) + }) + .collect() + } + + /// Helper: create N feature rows with a deterministic pattern. + fn make_features(n: usize, dim: usize) -> Vec> { + (0..n) + .map(|i| (0..dim).map(|j| (i * dim + j) as f32).collect()) + .collect() + } + + /// Helper: create a simple ascending-price time series. + fn make_test_data(n: usize) -> TimeSeriesData { + let prices: Vec = (0..n).map(|i| 100.0 + i as f64).collect(); + TimeSeriesData::new(make_timestamps(n), make_features(n, 3), prices) + .unwrap_or_else(|e| { + // We cannot panic due to clippy deny, but this is test code. + // Use a fallback that will never actually be reached. + eprintln!("Test data creation failed: {e}"); + // Return minimal valid data + TimeSeriesData::new( + make_timestamps(2), + make_features(2, 3), + vec![100.0, 101.0], + ) + .unwrap_or_else(|_| std::process::exit(1)) + }) + } + + #[test] + fn test_training_slice_returns_correct_range() { + let data = make_test_data(10); + let guard = + TemporalGuard::new(&data, 5).unwrap_or_else(|e| { + eprintln!("Guard creation failed: {e}"); + std::process::exit(1) + }); + let train = guard.training_slice().unwrap_or_else(|e| { + eprintln!("training_slice failed: {e}"); + std::process::exit(1) + }); + + assert_eq!(train.len(), 5); + // First price should be 100.0, last should be 104.0 + let first = train.prices.first().copied().unwrap_or(0.0); + let last = train.prices.last().copied().unwrap_or(0.0); + assert!((first - 100.0).abs() < 1e-12); + assert!((last - 104.0).abs() < 1e-12); + } + + #[test] + fn test_test_slice_rejects_before_cutoff() { + let data = make_test_data(10); + let guard = + TemporalGuard::new(&data, 5).unwrap_or_else(|e| { + eprintln!("Guard creation failed: {e}"); + std::process::exit(1) + }); + + // start=3 is before cutoff=5 → should fail + let result = guard.test_slice(3, 8); + assert!(result.is_err()); + let err_msg = format!("{}", result.unwrap_err()); + assert!( + err_msg.contains("leak"), + "Expected 'leak' in error, got: {err_msg}", + ); + } + + #[test] + fn test_slice_rejects_cross_boundary() { + let data = make_test_data(10); + let guard = + TemporalGuard::new(&data, 5).unwrap_or_else(|e| { + eprintln!("Guard creation failed: {e}"); + std::process::exit(1) + }); + + // [3, 7) crosses cutoff=5 + let result = guard.slice(3, 7); + assert!(result.is_err()); + let err_msg = format!("{}", result.unwrap_err()); + assert!( + err_msg.contains("crosses"), + "Expected 'crosses' in error, got: {err_msg}", + ); + + // [0, 4) is entirely in training → should succeed + let train_ok = guard.slice(0, 4); + assert!(train_ok.is_ok()); + + // [5, 8) is entirely in test → should succeed + let test_ok = guard.slice(5, 8); + assert!(test_ok.is_ok()); + } + + #[test] + fn test_audit_detects_no_leakage_in_sorted_data() { + let data = make_test_data(10); + let guard = + TemporalGuard::new(&data, 5).unwrap_or_else(|e| { + eprintln!("Guard creation failed: {e}"); + std::process::exit(1) + }); + + let report = guard.audit_leakage(); + assert!(!report.has_future_timestamps); + assert_eq!(report.training_bars, 5); + assert!(report.cutoff_timestamp.is_some()); + } + + #[test] + fn test_normalization_stats_from_training_only() { + let data = make_test_data(10); + let guard = + TemporalGuard::new(&data, 5).unwrap_or_else(|e| { + eprintln!("Guard creation failed: {e}"); + std::process::exit(1) + }); + + let stats = guard.compute_normalization_stats(); + assert_eq!(stats.sample_count, 5); + assert_eq!(stats.means.len(), 3); + assert_eq!(stats.stds.len(), 3); + + // Verify means are computed from training features only (rows 0..5) + // Feature column 0: values 0, 3, 6, 9, 12 → mean = 6.0 + let expected_mean_0 = (0.0 + 3.0 + 6.0 + 9.0 + 12.0) / 5.0; + let got_mean_0 = stats.means.first().copied().unwrap_or(f64::NAN); + assert!( + (got_mean_0 - expected_mean_0).abs() < 1e-10, + "Expected mean_0={expected_mean_0}, got {got_mean_0}", + ); + } +}